From c50d248e97d9516ed9c14885dc60d61f40b1d857 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Fri, 27 Mar 2026 17:23:01 -0700 Subject: [PATCH 01/92] feat: supabase auth users sync background runner in dashboard api --- packages/dashboard-api/internal/cfg/model.go | 8 + .../internal/supabaseauthusersync/config.go | 21 ++ .../supabaseauthusersync/processor.go | 107 +++++++ .../internal/supabaseauthusersync/runner.go | 69 ++++ .../supabaseauthusersync/runner_test.go | 300 ++++++++++++++++++ .../internal/supabaseauthusersync/store.go | 102 ++++++ packages/dashboard-api/main.go | 25 ++ ...ashboard_supabase_auth_user_sync_queue.sql | 111 +++++++ .../supabase_auth_user_sync/ack.sql | 3 + .../supabase_auth_user_sync/claim_batch.sql | 17 + .../supabase_auth_user_sync/dead_letter.sql | 8 + .../delete_public_user.sql | 3 + .../supabase_auth_user_sync/get_auth_user.sql | 4 + .../supabase_auth_user_sync/retry.sql | 8 + .../upsert_public_user.sql | 7 + packages/db/queries/ack.sql.go | 20 ++ packages/db/queries/claim_batch.sql.go | 73 +++++ packages/db/queries/dead_letter.sql.go | 30 ++ packages/db/queries/delete_public_user.sql.go | 22 ++ packages/db/queries/get_auth_user.sql.go | 25 ++ packages/db/queries/models.go | 13 + packages/db/queries/retry.sql.go | 33 ++ packages/db/queries/upsert_public_user.sql.go | 31 ++ 23 files changed, 1040 insertions(+) create mode 100644 packages/dashboard-api/internal/supabaseauthusersync/config.go create mode 100644 packages/dashboard-api/internal/supabaseauthusersync/processor.go create mode 100644 packages/dashboard-api/internal/supabaseauthusersync/runner.go create mode 100644 packages/dashboard-api/internal/supabaseauthusersync/runner_test.go create mode 100644 packages/dashboard-api/internal/supabaseauthusersync/store.go create mode 100644 packages/db/pkg/dashboard/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql create mode 100644 packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/ack.sql create mode 100644 packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/claim_batch.sql create mode 100644 packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/dead_letter.sql create mode 100644 packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/delete_public_user.sql create mode 100644 packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/get_auth_user.sql create mode 100644 packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/retry.sql create mode 100644 packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/upsert_public_user.sql create mode 100644 packages/db/queries/ack.sql.go create mode 100644 packages/db/queries/claim_batch.sql.go create mode 100644 packages/db/queries/dead_letter.sql.go create mode 100644 packages/db/queries/delete_public_user.sql.go create mode 100644 packages/db/queries/get_auth_user.sql.go create mode 100644 packages/db/queries/retry.sql.go create mode 100644 packages/db/queries/upsert_public_user.sql.go diff --git a/packages/dashboard-api/internal/cfg/model.go b/packages/dashboard-api/internal/cfg/model.go index f0d9ad10a1..aafed3776c 100644 --- a/packages/dashboard-api/internal/cfg/model.go +++ b/packages/dashboard-api/internal/cfg/model.go @@ -1,6 +1,8 @@ package cfg import ( + "time" + "github.com/caarlos0/env/v11" ) @@ -12,6 +14,12 @@ type Config struct { AuthDBConnectionString string `env:"AUTH_DB_CONNECTION_STRING"` AuthDBReadReplicaConnectionString string `env:"AUTH_DB_READ_REPLICA_CONNECTION_STRING"` + + SupabaseAuthUserSyncEnabled bool `env:"SUPABASE_AUTH_USER_SYNC_ENABLED" envDefault:"false"` + SupabaseAuthUserSyncBatchSize int32 `env:"SUPABASE_AUTH_USER_SYNC_BATCH_SIZE" envDefault:"50"` + SupabaseAuthUserSyncPollInterval time.Duration `env:"SUPABASE_AUTH_USER_SYNC_POLL_INTERVAL" envDefault:"2s"` + SupabaseAuthUserSyncLockTimeout time.Duration `env:"SUPABASE_AUTH_USER_SYNC_LOCK_TIMEOUT" envDefault:"2m"` + SupabaseAuthUserSyncMaxAttempts int32 `env:"SUPABASE_AUTH_USER_SYNC_MAX_ATTEMPTS" envDefault:"20"` } func Parse() (Config, error) { diff --git a/packages/dashboard-api/internal/supabaseauthusersync/config.go b/packages/dashboard-api/internal/supabaseauthusersync/config.go new file mode 100644 index 0000000000..6883064a9e --- /dev/null +++ b/packages/dashboard-api/internal/supabaseauthusersync/config.go @@ -0,0 +1,21 @@ +package supabaseauthusersync + +import "time" + +type Config struct { + Enabled bool + BatchSize int32 + PollInterval time.Duration + LockTimeout time.Duration + MaxAttempts int32 +} + +func DefaultConfig() Config { + return Config{ + Enabled: false, + BatchSize: 50, + PollInterval: 2 * time.Second, + LockTimeout: 2 * time.Minute, + MaxAttempts: 20, + } +} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/processor.go b/packages/dashboard-api/internal/supabaseauthusersync/processor.go new file mode 100644 index 0000000000..dfd8ae79a8 --- /dev/null +++ b/packages/dashboard-api/internal/supabaseauthusersync/processor.go @@ -0,0 +1,107 @@ +package supabaseauthusersync + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + "go.uber.org/zap" + + "github.com/e2b-dev/infra/packages/shared/pkg/logger" +) + +type Processor struct { + store *Store + maxAttempts int32 + l logger.Logger +} + +func NewProcessor(store *Store, maxAttempts int32, l logger.Logger) *Processor { + return &Processor{ + store: store, + maxAttempts: maxAttempts, + l: l, + } +} + +func (p *Processor) Process(ctx context.Context, item QueueItem) { + err := p.reconcile(ctx, item) + + if err == nil { + if ackErr := p.store.Ack(ctx, item.ID); ackErr != nil { + p.l.Error(ctx, "failed to ack queue item", + zap.Int64("queue_item_id", item.ID), + zap.String("user_id", item.UserID.String()), + zap.Error(ackErr), + ) + } + + return + } + + p.l.Warn(ctx, "failed to process queue item", + zap.Int64("queue_item_id", item.ID), + zap.String("user_id", item.UserID.String()), + zap.Int32("attempt", item.AttemptCount), + zap.Error(err), + ) + + if item.AttemptCount >= p.maxAttempts { + if dlErr := p.store.DeadLetter(ctx, item.ID, err.Error()); dlErr != nil { + p.l.Error(ctx, "failed to dead-letter queue item", + zap.Int64("queue_item_id", item.ID), + zap.Error(dlErr), + ) + } + + return + } + + backoff := retryBackoff(item.AttemptCount) + + if retryErr := p.store.Retry(ctx, item.ID, backoff, err.Error()); retryErr != nil { + p.l.Error(ctx, "failed to retry queue item", + zap.Int64("queue_item_id", item.ID), + zap.Error(retryErr), + ) + } +} + +func (p *Processor) reconcile(ctx context.Context, item QueueItem) error { + authUser, err := p.store.GetAuthUser(ctx, item.UserID) + + if errors.Is(err, pgx.ErrNoRows) { + if delErr := p.store.DeletePublicUser(ctx, item.UserID); delErr != nil { + return fmt.Errorf("delete public.users %s: %w", item.UserID, delErr) + } + + return nil + } + + if err != nil { + return fmt.Errorf("get auth.users %s: %w", item.UserID, err) + } + + if err = p.store.UpsertPublicUser(ctx, authUser.ID, authUser.Email); err != nil { + return fmt.Errorf("upsert public.users %s: %w", authUser.ID, err) + } + + return nil +} + +func retryBackoff(attempt int32) time.Duration { + switch { + case attempt <= 1: + return 5 * time.Second + case attempt <= 3: + return 30 * time.Second + case attempt <= 6: + return 2 * time.Minute + case attempt <= 10: + return 5 * time.Minute + default: + return 15 * time.Minute + } +} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/runner.go b/packages/dashboard-api/internal/supabaseauthusersync/runner.go new file mode 100644 index 0000000000..39de50196c --- /dev/null +++ b/packages/dashboard-api/internal/supabaseauthusersync/runner.go @@ -0,0 +1,69 @@ +package supabaseauthusersync + +import ( + "context" + "time" + + "go.uber.org/zap" + + "github.com/e2b-dev/infra/packages/shared/pkg/logger" +) + +type Runner struct { + cfg Config + store *Store + processor *Processor + lockOwner string + l logger.Logger +} + +func NewRunner(cfg Config, store *Store, lockOwner string, l logger.Logger) *Runner { + return &Runner{ + cfg: cfg, + store: store, + processor: NewProcessor(store, cfg.MaxAttempts, l), + lockOwner: lockOwner, + l: l, + } +} + +func (r *Runner) Run(ctx context.Context) error { + r.l.Info(ctx, "starting supabase auth user sync worker", + zap.String("lock_owner", r.lockOwner), + zap.Duration("poll_interval", r.cfg.PollInterval), + zap.Int32("batch_size", r.cfg.BatchSize), + ) + + ticker := time.NewTicker(r.cfg.PollInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + r.l.Info(ctx, "stopping supabase auth user sync worker") + + return ctx.Err() + case <-ticker.C: + r.poll(ctx) + } + } +} + +func (r *Runner) poll(ctx context.Context) { + items, err := r.store.ClaimBatch(ctx, r.lockOwner, r.cfg.LockTimeout, r.cfg.BatchSize) + if err != nil { + r.l.Error(ctx, "failed to claim queue batch", zap.Error(err)) + + return + } + + if len(items) == 0 { + return + } + + r.l.Debug(ctx, "claimed queue batch", zap.Int("count", len(items))) + + for _, item := range items { + r.processor.Process(ctx, item) + } +} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go b/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go new file mode 100644 index 0000000000..79fe007842 --- /dev/null +++ b/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go @@ -0,0 +1,300 @@ +package supabaseauthusersync + +import ( + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/e2b-dev/infra/packages/db/pkg/testutils" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" +) + +func setupTestDB(t *testing.T) *testutils.Database { + t.Helper() + + db := testutils.SetupDatabase(t) + + repoRoot := gitRoot(t) + migrationSQL := readFile(t, filepath.Join( + repoRoot, + "packages", "db", "pkg", "dashboard", "migrations", + "20260328000000_dashboard_supabase_auth_user_sync_queue.sql", + )) + + upSQL := extractGooseUp(migrationSQL) + err := db.AuthDb.TestsRawSQL(t.Context(), upSQL) + require.NoError(t, err, "failed to apply dashboard auth sync migration") + + return db +} + +func gitRoot(t *testing.T) string { + t.Helper() + + cmd := exec.CommandContext(t.Context(), "git", "rev-parse", "--show-toplevel") + output, err := cmd.Output() + require.NoError(t, err) + + return strings.TrimSpace(string(output)) +} + +func readFile(t *testing.T, path string) string { + t.Helper() + + cmd := exec.CommandContext(t.Context(), "cat", path) + output, err := cmd.Output() + require.NoError(t, err) + + return string(output) +} + +func extractGooseUp(sql string) string { + parts := strings.SplitN(sql, "-- +goose Down", 2) + up := parts[0] + up = strings.ReplaceAll(up, "-- +goose Up", "") + up = strings.ReplaceAll(up, "-- +goose StatementBegin", "") + up = strings.ReplaceAll(up, "-- +goose StatementEnd", "") + + return up +} + +func insertAuthUser(t *testing.T, db *testutils.Database, userID uuid.UUID, email string) { + t.Helper() + err := db.AuthDb.TestsRawSQL(t.Context(), + "INSERT INTO auth.users (id, email) VALUES ($1, $2)", userID, email) + require.NoError(t, err) +} + +func updateAuthUserEmail(t *testing.T, db *testutils.Database, userID uuid.UUID, email string) { + t.Helper() + err := db.AuthDb.TestsRawSQL(t.Context(), + "UPDATE auth.users SET email = $1 WHERE id = $2", email, userID) + require.NoError(t, err) +} + +func deleteAuthUser(t *testing.T, db *testutils.Database, userID uuid.UUID) { + t.Helper() + err := db.AuthDb.TestsRawSQL(t.Context(), + "DELETE FROM auth.users WHERE id = $1", userID) + require.NoError(t, err) +} + +func getPublicUserEmail(t *testing.T, db *testutils.Database, userID uuid.UUID) (string, bool) { + t.Helper() + + var email string + var found bool + + err := db.AuthDb.TestsRawSQLQuery(t.Context(), + "SELECT email FROM public.users WHERE id = $1", + func(rows pgx.Rows) error { + if rows.Next() { + found = true + return rows.Scan(&email) + } + return nil + }, + userID, + ) + require.NoError(t, err) + + return email, found +} + +func queueDepth(t *testing.T, db *testutils.Database) int { + t.Helper() + + var count int + + err := db.AuthDb.TestsRawSQLQuery(t.Context(), + "SELECT count(*) FROM auth.user_sync_queue WHERE dead_lettered_at IS NULL", + func(rows pgx.Rows) error { + if rows.Next() { + return rows.Scan(&count) + } + return nil + }, + ) + require.NoError(t, err) + + return count +} + +func TestInsertAuthUserCreatesQueueRow(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + db := setupTestDB(t) + + userID := uuid.New() + insertAuthUser(t, db, userID, "test@example.com") + + depth := queueDepth(t, db) + assert.Equal(t, 1, depth) +} + +func TestProcessorReconciles_Insert(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + db := setupTestDB(t) + store := NewStore(db.SqlcClient.Queries) + l := logger.NewNopLogger() + proc := NewProcessor(store, 5, l) + + userID := uuid.New() + insertAuthUser(t, db, userID, "alice@example.com") + + items, err := store.ClaimBatch(t.Context(), "test-worker", 2*time.Minute, 10) + require.NoError(t, err) + require.Len(t, items, 1) + + proc.Process(t.Context(), items[0]) + + email, found := getPublicUserEmail(t, db, userID) + assert.True(t, found) + assert.Equal(t, "alice@example.com", email) + + assert.Equal(t, 0, queueDepth(t, db)) +} + +func TestProcessorReconciles_UpdateEmail(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + db := setupTestDB(t) + store := NewStore(db.SqlcClient.Queries) + l := logger.NewNopLogger() + proc := NewProcessor(store, 5, l) + + userID := uuid.New() + insertAuthUser(t, db, userID, "old@example.com") + + items, err := store.ClaimBatch(t.Context(), "test-worker", 2*time.Minute, 10) + require.NoError(t, err) + proc.Process(t.Context(), items[0]) + + updateAuthUserEmail(t, db, userID, "new@example.com") + + items, err = store.ClaimBatch(t.Context(), "test-worker", 2*time.Minute, 10) + require.NoError(t, err) + require.Len(t, items, 1) + proc.Process(t.Context(), items[0]) + + email, found := getPublicUserEmail(t, db, userID) + assert.True(t, found) + assert.Equal(t, "new@example.com", email) +} + +func TestProcessorReconciles_Delete(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + db := setupTestDB(t) + store := NewStore(db.SqlcClient.Queries) + l := logger.NewNopLogger() + proc := NewProcessor(store, 5, l) + + userID := uuid.New() + insertAuthUser(t, db, userID, "doomed@example.com") + + items, err := store.ClaimBatch(t.Context(), "test-worker", 2*time.Minute, 10) + require.NoError(t, err) + proc.Process(t.Context(), items[0]) + + _, found := getPublicUserEmail(t, db, userID) + require.True(t, found) + + deleteAuthUser(t, db, userID) + + items, err = store.ClaimBatch(t.Context(), "test-worker", 2*time.Minute, 10) + require.NoError(t, err) + require.Len(t, items, 1) + proc.Process(t.Context(), items[0]) + + _, found = getPublicUserEmail(t, db, userID) + assert.False(t, found) +} + +func TestDuplicateQueueRowsConverge(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + db := setupTestDB(t) + store := NewStore(db.SqlcClient.Queries) + l := logger.NewNopLogger() + proc := NewProcessor(store, 5, l) + + userID := uuid.New() + insertAuthUser(t, db, userID, "dup@example.com") + + err := db.AuthDb.TestsRawSQL(t.Context(), + "INSERT INTO auth.user_sync_queue (user_id, operation) VALUES ($1, 'upsert')", + userID) + require.NoError(t, err) + + items, err := store.ClaimBatch(t.Context(), "test-worker", 2*time.Minute, 10) + require.NoError(t, err) + assert.GreaterOrEqual(t, len(items), 2) + + for _, item := range items { + proc.Process(t.Context(), item) + } + + email, found := getPublicUserEmail(t, db, userID) + assert.True(t, found) + assert.Equal(t, "dup@example.com", email) + assert.Equal(t, 0, queueDepth(t, db)) +} + +func TestMultiInstanceClaimNoDoubleProcessing(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + db := setupTestDB(t) + + for i := range 10 { + userID := uuid.New() + insertAuthUser(t, db, userID, "user"+string(rune('a'+i))+"@example.com") + } + + store1 := NewStore(db.SqlcClient.Queries) + store2 := NewStore(db.SqlcClient.Queries) + + var claimed1, claimed2 atomic.Int32 + + ctx := t.Context() + + items1, err := store1.ClaimBatch(ctx, "worker-1", 2*time.Minute, 10) + require.NoError(t, err) + claimed1.Store(int32(len(items1))) + + items2, err := store2.ClaimBatch(ctx, "worker-2", 2*time.Minute, 10) + require.NoError(t, err) + claimed2.Store(int32(len(items2))) + + total := claimed1.Load() + claimed2.Load() + assert.Equal(t, int32(10), total, "all items should be claimed exactly once across both workers") + + ids := make(map[int64]bool) + for _, item := range items1 { + ids[item.ID] = true + } + for _, item := range items2 { + assert.False(t, ids[item.ID], "item %d claimed by both workers", item.ID) + } +} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/store.go b/packages/dashboard-api/internal/supabaseauthusersync/store.go new file mode 100644 index 0000000000..019e4bf6c6 --- /dev/null +++ b/packages/dashboard-api/internal/supabaseauthusersync/store.go @@ -0,0 +1,102 @@ +package supabaseauthusersync + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/e2b-dev/infra/packages/db/queries" +) + +type QueueItem struct { + ID int64 + UserID uuid.UUID + Operation string + CreatedAt time.Time + AttemptCount int32 +} + +type AuthUser struct { + ID uuid.UUID + Email string +} + +type Store struct { + q *queries.Queries +} + +func NewStore(q *queries.Queries) *Store { + return &Store{q: q} +} + +func (s *Store) ClaimBatch(ctx context.Context, lockOwner string, lockTimeout time.Duration, batchSize int32) ([]QueueItem, error) { + rows, err := s.q.ClaimUserSyncQueueBatch(ctx, queries.ClaimUserSyncQueueBatchParams{ + LockOwner: lockOwner, + LockTimeout: durationToInterval(lockTimeout), + BatchSize: batchSize, + }) + if err != nil { + return nil, err + } + + items := make([]QueueItem, len(rows)) + for i, r := range rows { + items[i] = QueueItem{ + ID: r.ID, + UserID: r.UserID, + Operation: r.Operation, + CreatedAt: r.CreatedAt, + AttemptCount: r.AttemptCount, + } + } + + return items, nil +} + +func (s *Store) Ack(ctx context.Context, id int64) error { + return s.q.AckUserSyncQueueItem(ctx, id) +} + +func (s *Store) Retry(ctx context.Context, id int64, backoff time.Duration, lastError string) error { + return s.q.RetryUserSyncQueueItem(ctx, queries.RetryUserSyncQueueItemParams{ + ID: id, + Backoff: durationToInterval(backoff), + LastError: lastError, + }) +} + +func (s *Store) DeadLetter(ctx context.Context, id int64, lastError string) error { + return s.q.DeadLetterUserSyncQueueItem(ctx, queries.DeadLetterUserSyncQueueItemParams{ + ID: id, + LastError: lastError, + }) +} + +func (s *Store) GetAuthUser(ctx context.Context, userID uuid.UUID) (*AuthUser, error) { + row, err := s.q.GetAuthUserByID(ctx, userID) + if err != nil { + return nil, err + } + + return &AuthUser{ID: row.ID, Email: row.Email}, nil +} + +func (s *Store) UpsertPublicUser(ctx context.Context, id uuid.UUID, email string) error { + return s.q.UpsertPublicUser(ctx, queries.UpsertPublicUserParams{ + ID: id, + Email: email, + }) +} + +func (s *Store) DeletePublicUser(ctx context.Context, id uuid.UUID) error { + return s.q.DeletePublicUser(ctx, id) +} + +func durationToInterval(d time.Duration) pgtype.Interval { + return pgtype.Interval{ + Microseconds: d.Microseconds(), + Valid: true, + } +} diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 4b40634473..0f0bac4ddb 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -30,6 +30,7 @@ import ( "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" "github.com/e2b-dev/infra/packages/dashboard-api/internal/cfg" "github.com/e2b-dev/infra/packages/dashboard-api/internal/handlers" + "github.com/e2b-dev/infra/packages/dashboard-api/internal/supabaseauthusersync" sqlcdb "github.com/e2b-dev/infra/packages/db/client" authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" "github.com/e2b-dev/infra/packages/db/pkg/pool" @@ -229,6 +230,30 @@ func run() int { wg := sync.WaitGroup{} + if config.SupabaseAuthUserSyncEnabled { + workerLogger := l.With(zap.String("worker", "supabase_auth_user_sync")) + syncStore := supabaseauthusersync.NewStore(db.Queries) + syncRunner := supabaseauthusersync.NewRunner( + supabaseauthusersync.Config{ + Enabled: true, + BatchSize: config.SupabaseAuthUserSyncBatchSize, + PollInterval: config.SupabaseAuthUserSyncPollInterval, + LockTimeout: config.SupabaseAuthUserSyncLockTimeout, + MaxAttempts: config.SupabaseAuthUserSyncMaxAttempts, + }, + syncStore, + serviceInstanceID, + workerLogger, + ) + + wg.Go(func() { + if err := syncRunner.Run(signalCtx); err != nil && !errors.Is(err, context.Canceled) { + l.Error(ctx, "supabase auth user sync worker error", zap.Error(err)) + errorCode.Add(1) + } + }) + } + wg.Go(func() { <-signalCtx.Done() l.Info(ctx, "Shutting down dashboard-api service...") diff --git a/packages/db/pkg/dashboard/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql b/packages/db/pkg/dashboard/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql new file mode 100644 index 0000000000..90ceda9d06 --- /dev/null +++ b/packages/db/pkg/dashboard/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql @@ -0,0 +1,111 @@ +-- +goose Up +-- +goose StatementBegin + +CREATE TABLE auth.user_sync_queue ( + id BIGSERIAL PRIMARY KEY, + user_id UUID NOT NULL, + operation TEXT NOT NULL CHECK (operation IN ('upsert', 'delete')), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), + locked_at TIMESTAMPTZ NULL, + lock_owner TEXT NULL, + attempt_count INT NOT NULL DEFAULT 0, + last_error TEXT NULL, + dead_lettered_at TIMESTAMPTZ NULL +); + +CREATE INDEX auth_user_sync_queue_pending_idx + ON auth.user_sync_queue (id) + WHERE dead_lettered_at IS NULL AND locked_at IS NULL; + +CREATE INDEX auth_user_sync_queue_user_idx + ON auth.user_sync_queue (user_id); + +GRANT INSERT ON auth.user_sync_queue TO trigger_user; +GRANT USAGE, SELECT ON SEQUENCE auth.user_sync_queue_id_seq TO trigger_user; + +-- Replace direct insert-sync with enqueue +CREATE OR REPLACE FUNCTION public.sync_insert_auth_users_to_public_users_trigger() RETURNS TRIGGER +LANGUAGE plpgsql +AS $func$ +BEGIN + INSERT INTO auth.user_sync_queue (user_id, operation) + VALUES (NEW.id, 'upsert'); + RETURN NEW; +END; +$func$ SECURITY DEFINER SET search_path = public; + +-- Replace direct update-sync with enqueue (only when mirrored fields change) +CREATE OR REPLACE FUNCTION public.sync_update_auth_users_to_public_users_trigger() RETURNS TRIGGER +LANGUAGE plpgsql +AS $func$ +BEGIN + IF OLD.email IS DISTINCT FROM NEW.email THEN + INSERT INTO auth.user_sync_queue (user_id, operation) + VALUES (NEW.id, 'upsert'); + END IF; + RETURN NEW; +END; +$func$ SECURITY DEFINER SET search_path = public; + +-- Replace direct delete-sync with enqueue +CREATE OR REPLACE FUNCTION public.sync_delete_auth_users_to_public_users_trigger() RETURNS TRIGGER +LANGUAGE plpgsql +AS $func$ +BEGIN + INSERT INTO auth.user_sync_queue (user_id, operation) + VALUES (OLD.id, 'delete'); + RETURN OLD; +END; +$func$ SECURITY DEFINER SET search_path = public; + +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin + +-- Restore direct insert-sync +CREATE OR REPLACE FUNCTION public.sync_insert_auth_users_to_public_users_trigger() RETURNS TRIGGER +LANGUAGE plpgsql +AS $func$ +BEGIN + INSERT INTO public.users (id, email) + VALUES (NEW.id, NEW.email); + RETURN NEW; +END; +$func$ SECURITY DEFINER SET search_path = public; + +-- Restore direct update-sync +CREATE OR REPLACE FUNCTION public.sync_update_auth_users_to_public_users_trigger() RETURNS TRIGGER +LANGUAGE plpgsql +AS $func$ +BEGIN + UPDATE public.users + SET email = NEW.email, + updated_at = now() + WHERE id = NEW.id; + + IF NOT FOUND THEN + RAISE EXCEPTION 'User with id % does not exist in public.users', NEW.id; + END IF; + + RETURN NEW; +END; +$func$ SECURITY DEFINER SET search_path = public; + +-- Restore direct delete-sync +CREATE OR REPLACE FUNCTION public.sync_delete_auth_users_to_public_users_trigger() RETURNS TRIGGER +LANGUAGE plpgsql +AS $func$ +BEGIN + DELETE FROM public.users WHERE id = OLD.id; + RETURN OLD; +END; +$func$ SECURITY DEFINER SET search_path = public; + +REVOKE INSERT ON auth.user_sync_queue FROM trigger_user; +REVOKE USAGE, SELECT ON SEQUENCE auth.user_sync_queue_id_seq FROM trigger_user; + +DROP TABLE auth.user_sync_queue; + +-- +goose StatementEnd diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/ack.sql b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/ack.sql new file mode 100644 index 0000000000..f2fe8ed889 --- /dev/null +++ b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/ack.sql @@ -0,0 +1,3 @@ +-- name: AckUserSyncQueueItem :exec +DELETE FROM auth.user_sync_queue +WHERE id = sqlc.arg(id)::bigint; diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/claim_batch.sql b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/claim_batch.sql new file mode 100644 index 0000000000..af8c29aeaf --- /dev/null +++ b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/claim_batch.sql @@ -0,0 +1,17 @@ +-- name: ClaimUserSyncQueueBatch :many +UPDATE auth.user_sync_queue +SET + locked_at = now(), + lock_owner = sqlc.arg(lock_owner)::text, + attempt_count = attempt_count + 1 +WHERE id IN ( + SELECT id + FROM auth.user_sync_queue + WHERE dead_lettered_at IS NULL + AND next_attempt_at <= now() + AND (locked_at IS NULL OR locked_at < now() - sqlc.arg(lock_timeout)::interval) + ORDER BY id + FOR UPDATE SKIP LOCKED + LIMIT sqlc.arg(batch_size)::int +) +RETURNING id, user_id, operation, created_at, attempt_count; diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/dead_letter.sql b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/dead_letter.sql new file mode 100644 index 0000000000..fd4d6a87e9 --- /dev/null +++ b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/dead_letter.sql @@ -0,0 +1,8 @@ +-- name: DeadLetterUserSyncQueueItem :exec +UPDATE auth.user_sync_queue +SET + locked_at = NULL, + lock_owner = NULL, + dead_lettered_at = now(), + last_error = sqlc.arg(last_error)::text +WHERE id = sqlc.arg(id)::bigint; diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/delete_public_user.sql b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/delete_public_user.sql new file mode 100644 index 0000000000..492f1051cd --- /dev/null +++ b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/delete_public_user.sql @@ -0,0 +1,3 @@ +-- name: DeletePublicUser :exec +DELETE FROM public.users +WHERE id = sqlc.arg(id)::uuid; diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/get_auth_user.sql b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/get_auth_user.sql new file mode 100644 index 0000000000..414b3fe3cf --- /dev/null +++ b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/get_auth_user.sql @@ -0,0 +1,4 @@ +-- name: GetAuthUserByID :one +SELECT id, email +FROM auth.users +WHERE id = sqlc.arg(user_id)::uuid; diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/retry.sql b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/retry.sql new file mode 100644 index 0000000000..cdc21a34d1 --- /dev/null +++ b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/retry.sql @@ -0,0 +1,8 @@ +-- name: RetryUserSyncQueueItem :exec +UPDATE auth.user_sync_queue +SET + locked_at = NULL, + lock_owner = NULL, + next_attempt_at = now() + sqlc.arg(backoff)::interval, + last_error = sqlc.arg(last_error)::text +WHERE id = sqlc.arg(id)::bigint; diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/upsert_public_user.sql b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/upsert_public_user.sql new file mode 100644 index 0000000000..ebabd969e6 --- /dev/null +++ b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/upsert_public_user.sql @@ -0,0 +1,7 @@ +-- name: UpsertPublicUser :exec +INSERT INTO public.users (id, email) +VALUES (sqlc.arg(id)::uuid, sqlc.arg(email)::text) +ON CONFLICT (id) +DO UPDATE SET + email = EXCLUDED.email, + updated_at = now(); diff --git a/packages/db/queries/ack.sql.go b/packages/db/queries/ack.sql.go new file mode 100644 index 0000000000..66a517f0a5 --- /dev/null +++ b/packages/db/queries/ack.sql.go @@ -0,0 +1,20 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: ack.sql + +package queries + +import ( + "context" +) + +const ackUserSyncQueueItem = `-- name: AckUserSyncQueueItem :exec +DELETE FROM auth.user_sync_queue +WHERE id = $1::bigint +` + +func (q *Queries) AckUserSyncQueueItem(ctx context.Context, id int64) error { + _, err := q.db.Exec(ctx, ackUserSyncQueueItem, id) + return err +} diff --git a/packages/db/queries/claim_batch.sql.go b/packages/db/queries/claim_batch.sql.go new file mode 100644 index 0000000000..f6797712b5 --- /dev/null +++ b/packages/db/queries/claim_batch.sql.go @@ -0,0 +1,73 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: claim_batch.sql + +package queries + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +const claimUserSyncQueueBatch = `-- name: ClaimUserSyncQueueBatch :many +UPDATE auth.user_sync_queue +SET + locked_at = now(), + lock_owner = $1::text, + attempt_count = attempt_count + 1 +WHERE id IN ( + SELECT id + FROM auth.user_sync_queue + WHERE dead_lettered_at IS NULL + AND next_attempt_at <= now() + AND (locked_at IS NULL OR locked_at < now() - $2::interval) + ORDER BY id + FOR UPDATE SKIP LOCKED + LIMIT $3::int +) +RETURNING id, user_id, operation, created_at, attempt_count +` + +type ClaimUserSyncQueueBatchParams struct { + LockOwner string + LockTimeout pgtype.Interval + BatchSize int32 +} + +type ClaimUserSyncQueueBatchRow struct { + ID int64 + UserID uuid.UUID + Operation string + CreatedAt time.Time + AttemptCount int32 +} + +func (q *Queries) ClaimUserSyncQueueBatch(ctx context.Context, arg ClaimUserSyncQueueBatchParams) ([]ClaimUserSyncQueueBatchRow, error) { + rows, err := q.db.Query(ctx, claimUserSyncQueueBatch, arg.LockOwner, arg.LockTimeout, arg.BatchSize) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ClaimUserSyncQueueBatchRow + for rows.Next() { + var i ClaimUserSyncQueueBatchRow + if err := rows.Scan( + &i.ID, + &i.UserID, + &i.Operation, + &i.CreatedAt, + &i.AttemptCount, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/packages/db/queries/dead_letter.sql.go b/packages/db/queries/dead_letter.sql.go new file mode 100644 index 0000000000..3df0f8c511 --- /dev/null +++ b/packages/db/queries/dead_letter.sql.go @@ -0,0 +1,30 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: dead_letter.sql + +package queries + +import ( + "context" +) + +const deadLetterUserSyncQueueItem = `-- name: DeadLetterUserSyncQueueItem :exec +UPDATE auth.user_sync_queue +SET + locked_at = NULL, + lock_owner = NULL, + dead_lettered_at = now(), + last_error = $1::text +WHERE id = $2::bigint +` + +type DeadLetterUserSyncQueueItemParams struct { + LastError string + ID int64 +} + +func (q *Queries) DeadLetterUserSyncQueueItem(ctx context.Context, arg DeadLetterUserSyncQueueItemParams) error { + _, err := q.db.Exec(ctx, deadLetterUserSyncQueueItem, arg.LastError, arg.ID) + return err +} diff --git a/packages/db/queries/delete_public_user.sql.go b/packages/db/queries/delete_public_user.sql.go new file mode 100644 index 0000000000..585d8c977c --- /dev/null +++ b/packages/db/queries/delete_public_user.sql.go @@ -0,0 +1,22 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: delete_public_user.sql + +package queries + +import ( + "context" + + "github.com/google/uuid" +) + +const deletePublicUser = `-- name: DeletePublicUser :exec +DELETE FROM public.users +WHERE id = $1::uuid +` + +func (q *Queries) DeletePublicUser(ctx context.Context, id uuid.UUID) error { + _, err := q.db.Exec(ctx, deletePublicUser, id) + return err +} diff --git a/packages/db/queries/get_auth_user.sql.go b/packages/db/queries/get_auth_user.sql.go new file mode 100644 index 0000000000..4b7c341df2 --- /dev/null +++ b/packages/db/queries/get_auth_user.sql.go @@ -0,0 +1,25 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: get_auth_user.sql + +package queries + +import ( + "context" + + "github.com/google/uuid" +) + +const getAuthUserByID = `-- name: GetAuthUserByID :one +SELECT id, email +FROM auth.users +WHERE id = $1::uuid +` + +func (q *Queries) GetAuthUserByID(ctx context.Context, userID uuid.UUID) (AuthUser, error) { + row := q.db.QueryRow(ctx, getAuthUserByID, userID) + var i AuthUser + err := row.Scan(&i.ID, &i.Email) + return i, err +} diff --git a/packages/db/queries/models.go b/packages/db/queries/models.go index 6c960a7a59..9f0b0cf689 100644 --- a/packages/db/queries/models.go +++ b/packages/db/queries/models.go @@ -54,6 +54,19 @@ type AuthUser struct { Email string } +type AuthUserSyncQueue struct { + ID int64 + UserID uuid.UUID + Operation string + CreatedAt time.Time + NextAttemptAt time.Time + LockedAt *time.Time + LockOwner *string + AttemptCount int32 + LastError *string + DeadLetteredAt *time.Time +} + type BillingSandboxLog struct { SandboxID string EnvID string diff --git a/packages/db/queries/retry.sql.go b/packages/db/queries/retry.sql.go new file mode 100644 index 0000000000..ccbb7da45b --- /dev/null +++ b/packages/db/queries/retry.sql.go @@ -0,0 +1,33 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: retry.sql + +package queries + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const retryUserSyncQueueItem = `-- name: RetryUserSyncQueueItem :exec +UPDATE auth.user_sync_queue +SET + locked_at = NULL, + lock_owner = NULL, + next_attempt_at = now() + $1::interval, + last_error = $2::text +WHERE id = $3::bigint +` + +type RetryUserSyncQueueItemParams struct { + Backoff pgtype.Interval + LastError string + ID int64 +} + +func (q *Queries) RetryUserSyncQueueItem(ctx context.Context, arg RetryUserSyncQueueItemParams) error { + _, err := q.db.Exec(ctx, retryUserSyncQueueItem, arg.Backoff, arg.LastError, arg.ID) + return err +} diff --git a/packages/db/queries/upsert_public_user.sql.go b/packages/db/queries/upsert_public_user.sql.go new file mode 100644 index 0000000000..dc0fd6eba3 --- /dev/null +++ b/packages/db/queries/upsert_public_user.sql.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: upsert_public_user.sql + +package queries + +import ( + "context" + + "github.com/google/uuid" +) + +const upsertPublicUser = `-- name: UpsertPublicUser :exec +INSERT INTO public.users (id, email) +VALUES ($1::uuid, $2::text) +ON CONFLICT (id) +DO UPDATE SET + email = EXCLUDED.email, + updated_at = now() +` + +type UpsertPublicUserParams struct { + ID uuid.UUID + Email string +} + +func (q *Queries) UpsertPublicUser(ctx context.Context, arg UpsertPublicUserParams) error { + _, err := q.db.Exec(ctx, upsertPublicUser, arg.ID, arg.Email) + return err +} From 68a626578943ce40a89aba95493c06179b248093 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 28 Mar 2026 00:25:14 +0000 Subject: [PATCH 02/92] chore: auto-commit generated changes --- packages/dashboard-api/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dashboard-api/go.mod b/packages/dashboard-api/go.mod index cd69e6352b..65de5392b9 100644 --- a/packages/dashboard-api/go.mod +++ b/packages/dashboard-api/go.mod @@ -23,6 +23,7 @@ require ( github.com/jackc/pgx/v5 v5.7.5 github.com/oapi-codegen/gin-middleware v1.0.2 github.com/oapi-codegen/runtime v1.1.1 + github.com/stretchr/testify v1.11.1 go.uber.org/zap v1.27.1 ) @@ -117,7 +118,6 @@ require ( github.com/shirou/gopsutil/v4 v4.25.9 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/stretchr/testify v1.11.1 // indirect github.com/testcontainers/testcontainers-go v0.40.0 // indirect github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 // indirect github.com/tklauser/go-sysconf v0.3.15 // indirect From 9d8fe6359e239e3681b52bcfd549fcb0369837b5 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Mon, 30 Mar 2026 11:53:13 -0700 Subject: [PATCH 03/92] feat(db): enhance auth user sync triggers to retain direct operations while enqueuing for processing --- ...ashboard_supabase_auth_user_sync_queue.sql | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/packages/db/pkg/dashboard/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql b/packages/db/pkg/dashboard/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql index 90ceda9d06..63ac79e1de 100644 --- a/packages/db/pkg/dashboard/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql +++ b/packages/db/pkg/dashboard/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql @@ -24,37 +24,54 @@ CREATE INDEX auth_user_sync_queue_user_idx GRANT INSERT ON auth.user_sync_queue TO trigger_user; GRANT USAGE, SELECT ON SEQUENCE auth.user_sync_queue_id_seq TO trigger_user; --- Replace direct insert-sync with enqueue +-- Keep direct insert-sync and also enqueue CREATE OR REPLACE FUNCTION public.sync_insert_auth_users_to_public_users_trigger() RETURNS TRIGGER LANGUAGE plpgsql AS $func$ BEGIN + INSERT INTO public.users (id, email) + VALUES (NEW.id, NEW.email); + INSERT INTO auth.user_sync_queue (user_id, operation) VALUES (NEW.id, 'upsert'); + RETURN NEW; END; $func$ SECURITY DEFINER SET search_path = public; --- Replace direct update-sync with enqueue (only when mirrored fields change) +-- Keep direct update-sync and also enqueue when mirrored fields change CREATE OR REPLACE FUNCTION public.sync_update_auth_users_to_public_users_trigger() RETURNS TRIGGER LANGUAGE plpgsql AS $func$ BEGIN + UPDATE public.users + SET email = NEW.email, + updated_at = now() + WHERE id = NEW.id; + + IF NOT FOUND THEN + RAISE EXCEPTION 'User with id % does not exist in public.users', NEW.id; + END IF; + IF OLD.email IS DISTINCT FROM NEW.email THEN INSERT INTO auth.user_sync_queue (user_id, operation) VALUES (NEW.id, 'upsert'); END IF; + RETURN NEW; END; $func$ SECURITY DEFINER SET search_path = public; --- Replace direct delete-sync with enqueue +-- Keep direct delete-sync and also enqueue CREATE OR REPLACE FUNCTION public.sync_delete_auth_users_to_public_users_trigger() RETURNS TRIGGER LANGUAGE plpgsql AS $func$ BEGIN + DELETE FROM public.users WHERE id = OLD.id; + INSERT INTO auth.user_sync_queue (user_id, operation) VALUES (OLD.id, 'delete'); + RETURN OLD; END; $func$ SECURITY DEFINER SET search_path = public; From 56154df8b9a33e98d39cb134226137825b073532 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Mon, 30 Mar 2026 14:07:19 -0700 Subject: [PATCH 04/92] feat(sync): implement user sync queue with enhanced error handling and recovery mechanisms - Updated the sync runner to use `RunWithRestart` for improved error recovery. - Introduced a new `UserSyncQueue` model to manage user synchronization tasks. - Added SQL migration for creating the `user_sync_queue` table with necessary triggers. - Implemented tests for the processor and supervisor to ensure robust handling of retries and panics. - Refactored existing queries to target the new `public.user_sync_queue` table. --- .../supabaseauthusersync/processor.go | 34 +++++- .../supabaseauthusersync/processor_test.go | 109 +++++++++++++++++ .../supabaseauthusersync/runner_test.go | 4 +- .../supabaseauthusersync/supervisor.go | 115 ++++++++++++++++++ .../supabaseauthusersync/supervisor_test.go | 71 +++++++++++ packages/dashboard-api/main.go | 2 +- ...ashboard_supabase_auth_user_sync_queue.sql | 33 +++-- packages/db/pkg/auth/queries/models.go | 13 ++ .../supabase_auth_user_sync/ack.sql | 2 +- .../supabase_auth_user_sync/claim_batch.sql | 4 +- .../supabase_auth_user_sync/dead_letter.sql | 2 +- .../supabase_auth_user_sync/retry.sql | 2 +- packages/db/queries/ack.sql.go | 2 +- packages/db/queries/claim_batch.sql.go | 4 +- packages/db/queries/dead_letter.sql.go | 2 +- packages/db/queries/models.go | 26 ++-- packages/db/queries/retry.sql.go | 2 +- 17 files changed, 387 insertions(+), 40 deletions(-) create mode 100644 packages/dashboard-api/internal/supabaseauthusersync/processor_test.go create mode 100644 packages/dashboard-api/internal/supabaseauthusersync/supervisor.go create mode 100644 packages/dashboard-api/internal/supabaseauthusersync/supervisor_test.go rename packages/db/{pkg/dashboard => }/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql (76%) diff --git a/packages/dashboard-api/internal/supabaseauthusersync/processor.go b/packages/dashboard-api/internal/supabaseauthusersync/processor.go index dfd8ae79a8..28b0816f64 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/processor.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/processor.go @@ -4,21 +4,32 @@ import ( "context" "errors" "fmt" + "runtime/debug" "time" + "github.com/google/uuid" "github.com/jackc/pgx/v5" "go.uber.org/zap" "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) +type processorStore interface { + Ack(ctx context.Context, id int64) error + Retry(ctx context.Context, id int64, backoff time.Duration, lastError string) error + DeadLetter(ctx context.Context, id int64, lastError string) error + GetAuthUser(ctx context.Context, userID uuid.UUID) (*AuthUser, error) + UpsertPublicUser(ctx context.Context, id uuid.UUID, email string) error + DeletePublicUser(ctx context.Context, id uuid.UUID) error +} + type Processor struct { - store *Store + store processorStore maxAttempts int32 l logger.Logger } -func NewProcessor(store *Store, maxAttempts int32, l logger.Logger) *Processor { +func NewProcessor(store processorStore, maxAttempts int32, l logger.Logger) *Processor { return &Processor{ store: store, maxAttempts: maxAttempts, @@ -27,7 +38,7 @@ func NewProcessor(store *Store, maxAttempts int32, l logger.Logger) *Processor { } func (p *Processor) Process(ctx context.Context, item QueueItem) { - err := p.reconcile(ctx, item) + err := p.processOnce(ctx, item) if err == nil { if ackErr := p.store.Ack(ctx, item.ID); ackErr != nil { @@ -69,6 +80,23 @@ func (p *Processor) Process(ctx context.Context, item QueueItem) { } } +func (p *Processor) processOnce(ctx context.Context, item QueueItem) (err error) { + defer func() { + if recovered := recover(); recovered != nil { + p.l.Error(ctx, "panic while processing queue item", + zap.Int64("queue_item_id", item.ID), + zap.String("user_id", item.UserID.String()), + zap.String("panic", fmt.Sprint(recovered)), + zap.String("stack", string(debug.Stack())), + ) + + err = fmt.Errorf("panic while processing queue item: %v", recovered) + } + }() + + return p.reconcile(ctx, item) +} + func (p *Processor) reconcile(ctx context.Context, item QueueItem) error { authUser, err := p.store.GetAuthUser(ctx, item.UserID) diff --git a/packages/dashboard-api/internal/supabaseauthusersync/processor_test.go b/packages/dashboard-api/internal/supabaseauthusersync/processor_test.go new file mode 100644 index 0000000000..23072901fa --- /dev/null +++ b/packages/dashboard-api/internal/supabaseauthusersync/processor_test.go @@ -0,0 +1,109 @@ +package supabaseauthusersync + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/e2b-dev/infra/packages/shared/pkg/logger" +) + +type retryCall struct { + id int64 + backoff time.Duration + lastError string +} + +type deadLetterCall struct { + id int64 + lastError string +} + +type fakeProcessorStore struct { + getAuthUserFn func(context.Context, uuid.UUID) (*AuthUser, error) + + ackCalls []int64 + retryCalls []retryCall + deadLetterCalls []deadLetterCall +} + +func (s *fakeProcessorStore) Ack(_ context.Context, id int64) error { + s.ackCalls = append(s.ackCalls, id) + return nil +} + +func (s *fakeProcessorStore) Retry(_ context.Context, id int64, backoff time.Duration, lastError string) error { + s.retryCalls = append(s.retryCalls, retryCall{ + id: id, + backoff: backoff, + lastError: lastError, + }) + return nil +} + +func (s *fakeProcessorStore) DeadLetter(_ context.Context, id int64, lastError string) error { + s.deadLetterCalls = append(s.deadLetterCalls, deadLetterCall{ + id: id, + lastError: lastError, + }) + return nil +} + +func (s *fakeProcessorStore) GetAuthUser(ctx context.Context, userID uuid.UUID) (*AuthUser, error) { + return s.getAuthUserFn(ctx, userID) +} + +func (s *fakeProcessorStore) UpsertPublicUser(_ context.Context, _ uuid.UUID, _ string) error { + return nil +} + +func (s *fakeProcessorStore) DeletePublicUser(_ context.Context, _ uuid.UUID) error { + return nil +} + +func TestProcessorProcessRetriesRecoveredPanic(t *testing.T) { + store := &fakeProcessorStore{ + getAuthUserFn: func(context.Context, uuid.UUID) (*AuthUser, error) { + panic("boom") + }, + } + processor := NewProcessor(store, 3, logger.NewNopLogger()) + item := QueueItem{ + ID: 1, + UserID: uuid.New(), + AttemptCount: 1, + } + + require.NotPanics(t, func() { + processor.Process(context.Background(), item) + }) + require.Empty(t, store.ackCalls) + require.Len(t, store.retryCalls, 1) + require.Contains(t, store.retryCalls[0].lastError, "panic while processing queue item") + require.Empty(t, store.deadLetterCalls) +} + +func TestProcessorProcessDeadLettersRecoveredPanicAtMaxAttempts(t *testing.T) { + store := &fakeProcessorStore{ + getAuthUserFn: func(context.Context, uuid.UUID) (*AuthUser, error) { + panic("boom") + }, + } + processor := NewProcessor(store, 3, logger.NewNopLogger()) + item := QueueItem{ + ID: 1, + UserID: uuid.New(), + AttemptCount: 3, + } + + require.NotPanics(t, func() { + processor.Process(context.Background(), item) + }) + require.Empty(t, store.ackCalls) + require.Empty(t, store.retryCalls) + require.Len(t, store.deadLetterCalls, 1) + require.Contains(t, store.deadLetterCalls[0].lastError, "panic while processing queue item") +} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go b/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go index 79fe007842..b578185745 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go @@ -115,7 +115,7 @@ func queueDepth(t *testing.T, db *testutils.Database) int { var count int err := db.AuthDb.TestsRawSQLQuery(t.Context(), - "SELECT count(*) FROM auth.user_sync_queue WHERE dead_lettered_at IS NULL", + "SELECT count(*) FROM public.user_sync_queue WHERE dead_lettered_at IS NULL", func(rows pgx.Rows) error { if rows.Next() { return rows.Scan(&count) @@ -242,7 +242,7 @@ func TestDuplicateQueueRowsConverge(t *testing.T) { insertAuthUser(t, db, userID, "dup@example.com") err := db.AuthDb.TestsRawSQL(t.Context(), - "INSERT INTO auth.user_sync_queue (user_id, operation) VALUES ($1, 'upsert')", + "INSERT INTO public.user_sync_queue (user_id, operation) VALUES ($1, 'upsert')", userID) require.NoError(t, err) diff --git a/packages/dashboard-api/internal/supabaseauthusersync/supervisor.go b/packages/dashboard-api/internal/supabaseauthusersync/supervisor.go new file mode 100644 index 0000000000..ce8dafb463 --- /dev/null +++ b/packages/dashboard-api/internal/supabaseauthusersync/supervisor.go @@ -0,0 +1,115 @@ +package supabaseauthusersync + +import ( + "context" + "errors" + "fmt" + "runtime/debug" + "time" + + "go.uber.org/zap" + + "github.com/e2b-dev/infra/packages/shared/pkg/logger" +) + +const ( + defaultRestartDelay = time.Second + maxRestartDelay = 30 * time.Second + healthyRunResetThreshold = time.Minute +) + +type supervisorConfig struct { + RestartDelay time.Duration + MaxRestartDelay time.Duration + HealthyRunResetAfter time.Duration +} + +func defaultSupervisorConfig() supervisorConfig { + return supervisorConfig{ + RestartDelay: defaultRestartDelay, + MaxRestartDelay: maxRestartDelay, + HealthyRunResetAfter: healthyRunResetThreshold, + } +} + +func (r *Runner) RunWithRestart(ctx context.Context) error { + return supervise(ctx, r.l, defaultSupervisorConfig(), r.Run) +} + +func supervise(ctx context.Context, l logger.Logger, cfg supervisorConfig, run func(context.Context) error) error { + restartAttempt := 0 + + for { + startedAt := time.Now() + err := runRecovering(ctx, l, run) + runtime := time.Since(startedAt) + + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + if ctx.Err() != nil { + return ctx.Err() + } + + if runtime >= cfg.HealthyRunResetAfter { + restartAttempt = 0 + } + restartAttempt++ + + delay := restartBackoff(restartAttempt, cfg.RestartDelay, cfg.MaxRestartDelay) + l.Error(ctx, "supabase auth user sync worker exited unexpectedly; restarting", + zap.Error(err), + zap.Int("restart_attempt", restartAttempt), + zap.Duration("restart_in", delay), + zap.Duration("runtime", runtime), + ) + + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } +} + +func runRecovering(ctx context.Context, l logger.Logger, run func(context.Context) error) (err error) { + defer func() { + if recovered := recover(); recovered != nil { + l.Error(ctx, "supabase auth user sync worker panicked", + zap.String("panic", fmt.Sprint(recovered)), + zap.String("stack", string(debug.Stack())), + ) + + err = fmt.Errorf("worker panic: %v", recovered) + } + }() + + err = run(ctx) + if err == nil && ctx.Err() == nil { + return errors.New("worker exited without error") + } + + return err +} + +func restartBackoff(attempt int, base time.Duration, max time.Duration) time.Duration { + if base <= 0 { + base = defaultRestartDelay + } + if max < base { + max = base + } + + delay := base + for i := 1; i < attempt; i++ { + if delay >= max/2 { + return max + } + + delay *= 2 + } + + return delay +} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/supervisor_test.go b/packages/dashboard-api/internal/supabaseauthusersync/supervisor_test.go new file mode 100644 index 0000000000..3e26d71a71 --- /dev/null +++ b/packages/dashboard-api/internal/supabaseauthusersync/supervisor_test.go @@ -0,0 +1,71 @@ +package supabaseauthusersync + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/e2b-dev/infra/packages/shared/pkg/logger" +) + +func TestSuperviseRestartsAfterUnexpectedError(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var runs atomic.Int32 + errCh := make(chan error, 1) + + go func() { + errCh <- supervise(ctx, logger.NewNopLogger(), supervisorConfig{ + RestartDelay: time.Millisecond, + MaxRestartDelay: time.Millisecond, + HealthyRunResetAfter: time.Hour, + }, func(ctx context.Context) error { + attempt := runs.Add(1) + if attempt < 3 { + return errors.New("boom") + } + + cancel() + <-ctx.Done() + return ctx.Err() + }) + }() + + err := <-errCh + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, int32(3), runs.Load()) +} + +func TestSuperviseRestartsAfterPanic(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var runs atomic.Int32 + errCh := make(chan error, 1) + + go func() { + errCh <- supervise(ctx, logger.NewNopLogger(), supervisorConfig{ + RestartDelay: time.Millisecond, + MaxRestartDelay: time.Millisecond, + HealthyRunResetAfter: time.Hour, + }, func(ctx context.Context) error { + attempt := runs.Add(1) + if attempt == 1 { + panic("boom") + } + + cancel() + <-ctx.Done() + return ctx.Err() + }) + }() + + err := <-errCh + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, int32(2), runs.Load()) +} diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 0f0bac4ddb..106217091c 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -247,7 +247,7 @@ func run() int { ) wg.Go(func() { - if err := syncRunner.Run(signalCtx); err != nil && !errors.Is(err, context.Canceled) { + if err := syncRunner.RunWithRestart(signalCtx); err != nil && !errors.Is(err, context.Canceled) { l.Error(ctx, "supabase auth user sync worker error", zap.Error(err)) errorCode.Add(1) } diff --git a/packages/db/pkg/dashboard/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql b/packages/db/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql similarity index 76% rename from packages/db/pkg/dashboard/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql rename to packages/db/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql index 63ac79e1de..a250681905 100644 --- a/packages/db/pkg/dashboard/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql +++ b/packages/db/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql @@ -1,7 +1,7 @@ -- +goose Up -- +goose StatementBegin -CREATE TABLE auth.user_sync_queue ( +CREATE TABLE public.user_sync_queue ( id BIGSERIAL PRIMARY KEY, user_id UUID NOT NULL, operation TEXT NOT NULL CHECK (operation IN ('upsert', 'delete')), @@ -14,15 +14,24 @@ CREATE TABLE auth.user_sync_queue ( dead_lettered_at TIMESTAMPTZ NULL ); +ALTER TABLE public.user_sync_queue ENABLE ROW LEVEL SECURITY; + CREATE INDEX auth_user_sync_queue_pending_idx - ON auth.user_sync_queue (id) + ON public.user_sync_queue (id) WHERE dead_lettered_at IS NULL AND locked_at IS NULL; CREATE INDEX auth_user_sync_queue_user_idx - ON auth.user_sync_queue (user_id); + ON public.user_sync_queue (user_id); + +GRANT INSERT ON public.user_sync_queue TO trigger_user; +GRANT USAGE, SELECT ON SEQUENCE public.user_sync_queue_id_seq TO trigger_user; -GRANT INSERT ON auth.user_sync_queue TO trigger_user; -GRANT USAGE, SELECT ON SEQUENCE auth.user_sync_queue_id_seq TO trigger_user; +CREATE POLICY "Allow to create a user sync queue item" + ON public.user_sync_queue + AS PERMISSIVE + FOR INSERT + TO trigger_user + WITH CHECK (TRUE); -- Keep direct insert-sync and also enqueue CREATE OR REPLACE FUNCTION public.sync_insert_auth_users_to_public_users_trigger() RETURNS TRIGGER @@ -32,7 +41,7 @@ BEGIN INSERT INTO public.users (id, email) VALUES (NEW.id, NEW.email); - INSERT INTO auth.user_sync_queue (user_id, operation) + INSERT INTO public.user_sync_queue (user_id, operation) VALUES (NEW.id, 'upsert'); RETURN NEW; @@ -54,7 +63,7 @@ BEGIN END IF; IF OLD.email IS DISTINCT FROM NEW.email THEN - INSERT INTO auth.user_sync_queue (user_id, operation) + INSERT INTO public.user_sync_queue (user_id, operation) VALUES (NEW.id, 'upsert'); END IF; @@ -69,7 +78,7 @@ AS $func$ BEGIN DELETE FROM public.users WHERE id = OLD.id; - INSERT INTO auth.user_sync_queue (user_id, operation) + INSERT INTO public.user_sync_queue (user_id, operation) VALUES (OLD.id, 'delete'); RETURN OLD; @@ -120,9 +129,11 @@ BEGIN END; $func$ SECURITY DEFINER SET search_path = public; -REVOKE INSERT ON auth.user_sync_queue FROM trigger_user; -REVOKE USAGE, SELECT ON SEQUENCE auth.user_sync_queue_id_seq FROM trigger_user; +REVOKE INSERT ON public.user_sync_queue FROM trigger_user; +REVOKE USAGE, SELECT ON SEQUENCE public.user_sync_queue_id_seq FROM trigger_user; + +DROP POLICY IF EXISTS "Allow to create a user sync queue item" ON public.user_sync_queue; -DROP TABLE auth.user_sync_queue; +DROP TABLE public.user_sync_queue; -- +goose StatementEnd diff --git a/packages/db/pkg/auth/queries/models.go b/packages/db/pkg/auth/queries/models.go index de91154ef2..e32050f8ad 100644 --- a/packages/db/pkg/auth/queries/models.go +++ b/packages/db/pkg/auth/queries/models.go @@ -220,6 +220,19 @@ type User struct { Email string } +type UserSyncQueue struct { + ID int64 + UserID uuid.UUID + Operation string + CreatedAt time.Time + NextAttemptAt time.Time + LockedAt *time.Time + LockOwner *string + AttemptCount int32 + LastError *string + DeadLetteredAt *time.Time +} + type UsersTeam struct { ID int64 UserID uuid.UUID diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/ack.sql b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/ack.sql index f2fe8ed889..e0d7354dc9 100644 --- a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/ack.sql +++ b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/ack.sql @@ -1,3 +1,3 @@ -- name: AckUserSyncQueueItem :exec -DELETE FROM auth.user_sync_queue +DELETE FROM public.user_sync_queue WHERE id = sqlc.arg(id)::bigint; diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/claim_batch.sql b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/claim_batch.sql index af8c29aeaf..08f35e662d 100644 --- a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/claim_batch.sql +++ b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/claim_batch.sql @@ -1,12 +1,12 @@ -- name: ClaimUserSyncQueueBatch :many -UPDATE auth.user_sync_queue +UPDATE public.user_sync_queue SET locked_at = now(), lock_owner = sqlc.arg(lock_owner)::text, attempt_count = attempt_count + 1 WHERE id IN ( SELECT id - FROM auth.user_sync_queue + FROM public.user_sync_queue WHERE dead_lettered_at IS NULL AND next_attempt_at <= now() AND (locked_at IS NULL OR locked_at < now() - sqlc.arg(lock_timeout)::interval) diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/dead_letter.sql b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/dead_letter.sql index fd4d6a87e9..a68d8cd363 100644 --- a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/dead_letter.sql +++ b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/dead_letter.sql @@ -1,5 +1,5 @@ -- name: DeadLetterUserSyncQueueItem :exec -UPDATE auth.user_sync_queue +UPDATE public.user_sync_queue SET locked_at = NULL, lock_owner = NULL, diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/retry.sql b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/retry.sql index cdc21a34d1..5386fdce5e 100644 --- a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/retry.sql +++ b/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/retry.sql @@ -1,5 +1,5 @@ -- name: RetryUserSyncQueueItem :exec -UPDATE auth.user_sync_queue +UPDATE public.user_sync_queue SET locked_at = NULL, lock_owner = NULL, diff --git a/packages/db/queries/ack.sql.go b/packages/db/queries/ack.sql.go index 66a517f0a5..b55274a6c9 100644 --- a/packages/db/queries/ack.sql.go +++ b/packages/db/queries/ack.sql.go @@ -10,7 +10,7 @@ import ( ) const ackUserSyncQueueItem = `-- name: AckUserSyncQueueItem :exec -DELETE FROM auth.user_sync_queue +DELETE FROM public.user_sync_queue WHERE id = $1::bigint ` diff --git a/packages/db/queries/claim_batch.sql.go b/packages/db/queries/claim_batch.sql.go index f6797712b5..b36cfb0a39 100644 --- a/packages/db/queries/claim_batch.sql.go +++ b/packages/db/queries/claim_batch.sql.go @@ -14,14 +14,14 @@ import ( ) const claimUserSyncQueueBatch = `-- name: ClaimUserSyncQueueBatch :many -UPDATE auth.user_sync_queue +UPDATE public.user_sync_queue SET locked_at = now(), lock_owner = $1::text, attempt_count = attempt_count + 1 WHERE id IN ( SELECT id - FROM auth.user_sync_queue + FROM public.user_sync_queue WHERE dead_lettered_at IS NULL AND next_attempt_at <= now() AND (locked_at IS NULL OR locked_at < now() - $2::interval) diff --git a/packages/db/queries/dead_letter.sql.go b/packages/db/queries/dead_letter.sql.go index 3df0f8c511..b9b7941d8a 100644 --- a/packages/db/queries/dead_letter.sql.go +++ b/packages/db/queries/dead_letter.sql.go @@ -10,7 +10,7 @@ import ( ) const deadLetterUserSyncQueueItem = `-- name: DeadLetterUserSyncQueueItem :exec -UPDATE auth.user_sync_queue +UPDATE public.user_sync_queue SET locked_at = NULL, lock_owner = NULL, diff --git a/packages/db/queries/models.go b/packages/db/queries/models.go index 9f0b0cf689..f3ec4451c9 100644 --- a/packages/db/queries/models.go +++ b/packages/db/queries/models.go @@ -54,19 +54,6 @@ type AuthUser struct { Email string } -type AuthUserSyncQueue struct { - ID int64 - UserID uuid.UUID - Operation string - CreatedAt time.Time - NextAttemptAt time.Time - LockedAt *time.Time - LockOwner *string - AttemptCount int32 - LastError *string - DeadLetteredAt *time.Time -} - type BillingSandboxLog struct { SandboxID string EnvID string @@ -239,6 +226,19 @@ type User struct { Email string } +type UserSyncQueue struct { + ID int64 + UserID uuid.UUID + Operation string + CreatedAt time.Time + NextAttemptAt time.Time + LockedAt *time.Time + LockOwner *string + AttemptCount int32 + LastError *string + DeadLetteredAt *time.Time +} + type UsersTeam struct { ID int64 UserID uuid.UUID diff --git a/packages/db/queries/retry.sql.go b/packages/db/queries/retry.sql.go index ccbb7da45b..941c96ae18 100644 --- a/packages/db/queries/retry.sql.go +++ b/packages/db/queries/retry.sql.go @@ -12,7 +12,7 @@ import ( ) const retryUserSyncQueueItem = `-- name: RetryUserSyncQueueItem :exec -UPDATE auth.user_sync_queue +UPDATE public.user_sync_queue SET locked_at = NULL, lock_owner = NULL, From 3a431aa1199f1cea4fe48a87e22b29cc60b14e65 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Mon, 30 Mar 2026 21:08:20 -0700 Subject: [PATCH 05/92] feat(sync): add supabase auth user sync configuration and secrets management - Introduced `supabase_auth_user_sync_enabled` variable to control user synchronization. - Updated Nomad job configuration to include the new sync setting. - Added Google Secret Manager resources for managing the sync configuration securely. - Enhanced the dashboard API to utilize the new sync configuration in processing logic. - Refactored related components to improve error handling and logging for the sync process. --- .../job-dashboard-api/jobs/dashboard-api.hcl | 1 + iac/modules/job-dashboard-api/main.tf | 1 + iac/modules/job-dashboard-api/variables.tf | 4 + iac/provider-gcp/api.tf | 34 +++ iac/provider-gcp/main.tf | 52 +++-- iac/provider-gcp/nomad/main.tf | 11 +- iac/provider-gcp/nomad/variables.tf | 8 + packages/dashboard-api/internal/cfg/model.go | 12 +- .../internal/supabaseauthusersync/config.go | 15 +- .../internal/supabaseauthusersync/logging.go | 213 +++++++++++++++++ .../supabaseauthusersync/processor.go | 108 ++++++--- .../internal/supabaseauthusersync/runner.go | 30 ++- .../supabaseauthusersync/supervisor.go | 11 +- packages/dashboard-api/main.go | 10 +- packages/db/Makefile | 7 + .../db/scripts/auth-user-sync-smoke/main.go | 219 ++++++++++++++++++ 16 files changed, 642 insertions(+), 94 deletions(-) create mode 100644 packages/dashboard-api/internal/supabaseauthusersync/logging.go create mode 100644 packages/db/scripts/auth-user-sync-smoke/main.go diff --git a/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl b/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl index f37d681cc9..7bdaca2f24 100644 --- a/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl +++ b/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl @@ -80,6 +80,7 @@ job "dashboard-api" { AUTH_DB_READ_REPLICA_CONNECTION_STRING = "${auth_db_read_replica_connection_string}" CLICKHOUSE_CONNECTION_STRING = "${clickhouse_connection_string}" SUPABASE_JWT_SECRETS = "${supabase_jwt_secrets}" + SUPABASE_AUTH_USER_SYNC_ENABLED = "${supabase_auth_user_sync_enabled}" OTEL_COLLECTOR_GRPC_ENDPOINT = "${otel_collector_grpc_endpoint}" LOGS_COLLECTOR_ADDRESS = "${logs_collector_address}" } diff --git a/iac/modules/job-dashboard-api/main.tf b/iac/modules/job-dashboard-api/main.tf index d1c455e3d1..482ae19d9c 100644 --- a/iac/modules/job-dashboard-api/main.tf +++ b/iac/modules/job-dashboard-api/main.tf @@ -15,6 +15,7 @@ resource "nomad_job" "dashboard_api" { auth_db_read_replica_connection_string = var.auth_db_read_replica_connection_string clickhouse_connection_string = var.clickhouse_connection_string supabase_jwt_secrets = var.supabase_jwt_secrets + supabase_auth_user_sync_enabled = var.supabase_auth_user_sync_enabled subdomain = "dashboard-api" diff --git a/iac/modules/job-dashboard-api/variables.tf b/iac/modules/job-dashboard-api/variables.tf index 8b77e1c9a6..69d16a0419 100644 --- a/iac/modules/job-dashboard-api/variables.tf +++ b/iac/modules/job-dashboard-api/variables.tf @@ -44,6 +44,10 @@ variable "supabase_jwt_secrets" { sensitive = true } +variable "supabase_auth_user_sync_enabled" { + type = string +} + variable "otel_collector_grpc_port" { type = number default = 4317 diff --git a/iac/provider-gcp/api.tf b/iac/provider-gcp/api.tf index 26f1a1b408..736b322090 100644 --- a/iac/provider-gcp/api.tf +++ b/iac/provider-gcp/api.tf @@ -27,6 +27,40 @@ resource "google_secret_manager_secret_version" "postgres_read_replica_connectio } } +resource "google_secret_manager_secret" "auth_db_connection_string" { + secret_id = "${var.prefix}auth-db-connection-string" + + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "auth_db_connection_string" { + secret = google_secret_manager_secret.auth_db_connection_string.name + secret_data = " " + + lifecycle { + ignore_changes = [secret_data] + } +} + +resource "google_secret_manager_secret" "dashboard_api_supabase_auth_user_sync_enabled" { + secret_id = "${var.prefix}dashboard-api-supabase-auth-user-sync-enabled" + + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "dashboard_api_supabase_auth_user_sync_enabled" { + secret = google_secret_manager_secret.dashboard_api_supabase_auth_user_sync_enabled.name + secret_data = "false" + + lifecycle { + ignore_changes = [secret_data] + } +} + resource "random_password" "api_secret" { length = 32 special = false diff --git a/iac/provider-gcp/main.tf b/iac/provider-gcp/main.tf index 8fe538e1fa..32da66604f 100644 --- a/iac/provider-gcp/main.tf +++ b/iac/provider-gcp/main.tf @@ -213,31 +213,33 @@ module "nomad" { additional_traefik_arguments = var.additional_traefik_arguments # API - api_server_count = var.api_server_count - api_resources_cpu_count = var.api_resources_cpu_count - api_resources_memory_mb = var.api_resources_memory_mb - api_machine_count = var.api_cluster_size - api_node_pool = var.api_node_pool - api_port = var.api_port - environment = var.environment - google_service_account_key = module.init.google_service_account_key - api_secret = random_password.api_secret.result - custom_envs_repository_name = google_artifact_registry_repository.custom_environments_repository.name - postgres_connection_string_secret_name = module.init.postgres_connection_string_secret_name - postgres_read_replica_connection_string_secret_version = google_secret_manager_secret_version.postgres_read_replica_connection_string - supabase_jwt_secrets_secret_name = module.init.supabase_jwt_secret_name - posthog_api_key_secret_name = module.init.posthog_api_key_secret_name - analytics_collector_host_secret_name = module.init.analytics_collector_host_secret_name - analytics_collector_api_token_secret_name = module.init.analytics_collector_api_token_secret_name - api_admin_token = random_password.api_admin_secret.result - redis_cluster_url_secret_version = module.init.redis_cluster_url_secret_version - redis_tls_ca_base64_secret_version = module.init.redis_tls_ca_base64_secret_version - sandbox_access_token_hash_seed = random_password.sandbox_access_token_hash_seed.result - sandbox_storage_backend = var.sandbox_storage_backend - db_max_open_connections = var.db_max_open_connections - db_min_idle_connections = var.db_min_idle_connections - auth_db_max_open_connections = var.auth_db_max_open_connections - auth_db_min_idle_connections = var.auth_db_min_idle_connections + api_server_count = var.api_server_count + api_resources_cpu_count = var.api_resources_cpu_count + api_resources_memory_mb = var.api_resources_memory_mb + api_machine_count = var.api_cluster_size + api_node_pool = var.api_node_pool + api_port = var.api_port + environment = var.environment + google_service_account_key = module.init.google_service_account_key + api_secret = random_password.api_secret.result + custom_envs_repository_name = google_artifact_registry_repository.custom_environments_repository.name + postgres_connection_string_secret_name = module.init.postgres_connection_string_secret_name + auth_db_connection_string_secret_version = google_secret_manager_secret_version.auth_db_connection_string + postgres_read_replica_connection_string_secret_version = google_secret_manager_secret_version.postgres_read_replica_connection_string + supabase_jwt_secrets_secret_name = module.init.supabase_jwt_secret_name + dashboard_api_supabase_auth_user_sync_enabled_secret_version = google_secret_manager_secret_version.dashboard_api_supabase_auth_user_sync_enabled + posthog_api_key_secret_name = module.init.posthog_api_key_secret_name + analytics_collector_host_secret_name = module.init.analytics_collector_host_secret_name + analytics_collector_api_token_secret_name = module.init.analytics_collector_api_token_secret_name + api_admin_token = random_password.api_admin_secret.result + redis_cluster_url_secret_version = module.init.redis_cluster_url_secret_version + redis_tls_ca_base64_secret_version = module.init.redis_tls_ca_base64_secret_version + sandbox_access_token_hash_seed = random_password.sandbox_access_token_hash_seed.result + sandbox_storage_backend = var.sandbox_storage_backend + db_max_open_connections = var.db_max_open_connections + db_min_idle_connections = var.db_min_idle_connections + auth_db_max_open_connections = var.auth_db_max_open_connections + auth_db_min_idle_connections = var.auth_db_min_idle_connections # Click Proxy client_proxy_count = var.client_proxy_count diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index f7889e2e35..4a9aae6220 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -10,6 +10,10 @@ data "google_secret_manager_secret_version" "postgres_connection_string" { secret = var.postgres_connection_string_secret_name } +data "google_secret_manager_secret_version" "auth_db_connection_string" { + secret = var.auth_db_connection_string_secret_version.secret +} + data "google_secret_manager_secret_version" "postgres_read_replica_connection_string" { secret = var.postgres_read_replica_connection_string_secret_version.secret } @@ -18,6 +22,10 @@ data "google_secret_manager_secret_version" "supabase_jwt_secrets" { secret = var.supabase_jwt_secrets_secret_name } +data "google_secret_manager_secret_version" "dashboard_api_supabase_auth_user_sync_enabled" { + secret = var.dashboard_api_supabase_auth_user_sync_enabled_secret_version.secret +} + data "google_secret_manager_secret_version" "posthog_api_key" { secret = var.posthog_api_key_secret_name } @@ -131,10 +139,11 @@ module "dashboard_api" { image = data.google_artifact_registry_docker_image.dashboard_api_image[0].self_link postgres_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data - auth_db_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data + auth_db_connection_string = trimspace(data.google_secret_manager_secret_version.auth_db_connection_string.secret_data) auth_db_read_replica_connection_string = trimspace(data.google_secret_manager_secret_version.postgres_read_replica_connection_string.secret_data) clickhouse_connection_string = local.clickhouse_connection_string supabase_jwt_secrets = trimspace(data.google_secret_manager_secret_version.supabase_jwt_secrets.secret_data) + supabase_auth_user_sync_enabled = trimspace(data.google_secret_manager_secret_version.dashboard_api_supabase_auth_user_sync_enabled.secret_data) otel_collector_grpc_port = var.otel_collector_grpc_port logs_proxy_port = var.logs_proxy_port diff --git a/iac/provider-gcp/nomad/variables.tf b/iac/provider-gcp/nomad/variables.tf index 54ead44500..2bbbb9494f 100644 --- a/iac/provider-gcp/nomad/variables.tf +++ b/iac/provider-gcp/nomad/variables.tf @@ -175,6 +175,10 @@ variable "postgres_connection_string_secret_name" { type = string } +variable "auth_db_connection_string_secret_version" { + type = any +} + variable "postgres_read_replica_connection_string_secret_version" { type = any } @@ -183,6 +187,10 @@ variable "supabase_jwt_secrets_secret_name" { type = string } +variable "dashboard_api_supabase_auth_user_sync_enabled_secret_version" { + type = any +} + variable "client_proxy_count" { type = number } diff --git a/packages/dashboard-api/internal/cfg/model.go b/packages/dashboard-api/internal/cfg/model.go index aafed3776c..bbd83194f6 100644 --- a/packages/dashboard-api/internal/cfg/model.go +++ b/packages/dashboard-api/internal/cfg/model.go @@ -1,10 +1,6 @@ package cfg -import ( - "time" - - "github.com/caarlos0/env/v11" -) +import "github.com/caarlos0/env/v11" type Config struct { Port int `env:"PORT" envDefault:"3010"` @@ -15,11 +11,7 @@ type Config struct { AuthDBConnectionString string `env:"AUTH_DB_CONNECTION_STRING"` AuthDBReadReplicaConnectionString string `env:"AUTH_DB_READ_REPLICA_CONNECTION_STRING"` - SupabaseAuthUserSyncEnabled bool `env:"SUPABASE_AUTH_USER_SYNC_ENABLED" envDefault:"false"` - SupabaseAuthUserSyncBatchSize int32 `env:"SUPABASE_AUTH_USER_SYNC_BATCH_SIZE" envDefault:"50"` - SupabaseAuthUserSyncPollInterval time.Duration `env:"SUPABASE_AUTH_USER_SYNC_POLL_INTERVAL" envDefault:"2s"` - SupabaseAuthUserSyncLockTimeout time.Duration `env:"SUPABASE_AUTH_USER_SYNC_LOCK_TIMEOUT" envDefault:"2m"` - SupabaseAuthUserSyncMaxAttempts int32 `env:"SUPABASE_AUTH_USER_SYNC_MAX_ATTEMPTS" envDefault:"20"` + SupabaseAuthUserSyncEnabled bool `env:"SUPABASE_AUTH_USER_SYNC_ENABLED" envDefault:"false"` } func Parse() (Config, error) { diff --git a/packages/dashboard-api/internal/supabaseauthusersync/config.go b/packages/dashboard-api/internal/supabaseauthusersync/config.go index 6883064a9e..778be6d592 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/config.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/config.go @@ -2,6 +2,13 @@ package supabaseauthusersync import "time" +const ( + defaultBatchSize int32 = 50 + defaultPollInterval time.Duration = 2 * time.Second + defaultLockTimeout time.Duration = 2 * time.Minute + defaultMaxAttempts int32 = 20 +) + type Config struct { Enabled bool BatchSize int32 @@ -13,9 +20,9 @@ type Config struct { func DefaultConfig() Config { return Config{ Enabled: false, - BatchSize: 50, - PollInterval: 2 * time.Second, - LockTimeout: 2 * time.Minute, - MaxAttempts: 20, + BatchSize: defaultBatchSize, + PollInterval: defaultPollInterval, + LockTimeout: defaultLockTimeout, + MaxAttempts: defaultMaxAttempts, } } diff --git a/packages/dashboard-api/internal/supabaseauthusersync/logging.go b/packages/dashboard-api/internal/supabaseauthusersync/logging.go new file mode 100644 index 0000000000..c60d02a0c5 --- /dev/null +++ b/packages/dashboard-api/internal/supabaseauthusersync/logging.go @@ -0,0 +1,213 @@ +package supabaseauthusersync + +import ( + "sort" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/e2b-dev/infra/packages/shared/pkg/logger" +) + +type processOutcome string + +const ( + processOutcomeAcked processOutcome = "acked" + processOutcomeAckFailed processOutcome = "ack_failed" + processOutcomeRetried processOutcome = "retried" + processOutcomeRetryFailed processOutcome = "retry_failed" + processOutcomeDeadLettered processOutcome = "dead_lettered" + processOutcomeDeadLetterFailed processOutcome = "dead_letter_failed" +) + +type reconcileAction string + +const ( + reconcileActionUpsertPublicUser reconcileAction = "upsert_public_user" + reconcileActionDeletePublicUser reconcileAction = "delete_public_user" +) + +type processResult struct { + Outcome processOutcome + Action reconcileAction + Duration time.Duration + Backoff time.Duration +} + +type batchSummary struct { + ClaimedCount int + AckedCount int + AckFailedCount int + RetriedCount int + RetryFailedCount int + DeadLetteredCount int + DeadLetterFailedCount int + MaxAttemptCount int32 + OldestCreatedAt time.Time + NewestCreatedAt time.Time + OldestItemAge time.Duration + NewestItemAge time.Duration + OperationCounts map[string]int + ActionCounts map[string]int +} + +func newBatchSummary(items []QueueItem, now time.Time) batchSummary { + summary := batchSummary{ + ClaimedCount: len(items), + OperationCounts: make(map[string]int), + ActionCounts: make(map[string]int), + } + + for i, item := range items { + if i == 0 || item.AttemptCount > summary.MaxAttemptCount { + summary.MaxAttemptCount = item.AttemptCount + } + + summary.OperationCounts[item.Operation]++ + + if item.CreatedAt.IsZero() { + continue + } + + if summary.OldestCreatedAt.IsZero() || item.CreatedAt.Before(summary.OldestCreatedAt) { + summary.OldestCreatedAt = item.CreatedAt + } + if summary.NewestCreatedAt.IsZero() || item.CreatedAt.After(summary.NewestCreatedAt) { + summary.NewestCreatedAt = item.CreatedAt + } + } + + if !summary.OldestCreatedAt.IsZero() { + summary.OldestItemAge = ageSince(summary.OldestCreatedAt, now) + summary.NewestItemAge = ageSince(summary.NewestCreatedAt, now) + } + + return summary +} + +func (s *batchSummary) Add(result processResult) { + switch result.Outcome { + case processOutcomeAcked: + s.AckedCount++ + case processOutcomeAckFailed: + s.AckFailedCount++ + case processOutcomeRetried: + s.RetriedCount++ + case processOutcomeRetryFailed: + s.RetryFailedCount++ + case processOutcomeDeadLettered: + s.DeadLetteredCount++ + case processOutcomeDeadLetterFailed: + s.DeadLetterFailedCount++ + } + + if result.Action != "" { + s.ActionCounts[string(result.Action)]++ + } +} + +func (s batchSummary) Fields(totalDuration time.Duration) []zap.Field { + fields := []zap.Field{ + zap.Int("queue_batch.claimed_count", s.ClaimedCount), + zap.Int("queue_batch.acked_count", s.AckedCount), + zap.Int("queue_batch.ack_failed_count", s.AckFailedCount), + zap.Int("queue_batch.retried_count", s.RetriedCount), + zap.Int("queue_batch.retry_failed_count", s.RetryFailedCount), + zap.Int("queue_batch.dead_lettered_count", s.DeadLetteredCount), + zap.Int("queue_batch.dead_letter_failed_count", s.DeadLetterFailedCount), + zap.Int32("queue_batch.max_attempt", s.MaxAttemptCount), + zap.Duration("queue_batch.duration", totalDuration), + } + + if !s.OldestCreatedAt.IsZero() { + fields = append(fields, + logger.Time("queue_batch.oldest_item_created_at", s.OldestCreatedAt), + logger.Time("queue_batch.newest_item_created_at", s.NewestCreatedAt), + zap.Duration("queue_batch.oldest_item_age", s.OldestItemAge), + zap.Duration("queue_batch.newest_item_age", s.NewestItemAge), + ) + } + + if len(s.OperationCounts) > 0 { + fields = append(fields, zap.Object("queue_batch.operation_counts", countsField(s.OperationCounts))) + } + if len(s.ActionCounts) > 0 { + fields = append(fields, zap.Object("queue_batch.action_counts", countsField(s.ActionCounts))) + } + + return fields +} + +func (s batchSummary) Level() zapcore.Level { + if s.AckFailedCount > 0 || s.RetryFailedCount > 0 || s.DeadLetteredCount > 0 || s.DeadLetterFailedCount > 0 { + return zap.ErrorLevel + } + if s.RetriedCount > 0 { + return zap.WarnLevel + } + + return zap.InfoLevel +} + +func processResultFields(item QueueItem, result processResult, now time.Time) []zap.Field { + fields := queueItemFields(item, now) + fields = append(fields, + zap.String("queue_item.outcome", string(result.Outcome)), + zap.Duration("queue_item.duration", result.Duration), + ) + + if result.Action != "" { + fields = append(fields, zap.String("queue_item.action", string(result.Action))) + } + if result.Backoff > 0 { + fields = append(fields, + zap.Duration("queue_item.retry_backoff", result.Backoff), + zap.Int32("queue_item.next_attempt", item.AttemptCount+1), + ) + } + + return fields +} + +func queueItemFields(item QueueItem, now time.Time) []zap.Field { + fields := []zap.Field{ + zap.Int64("queue_item.id", item.ID), + logger.WithUserID(item.UserID.String()), + zap.String("queue_item.operation", item.Operation), + zap.Int32("queue_item.attempt", item.AttemptCount), + } + + if !item.CreatedAt.IsZero() { + fields = append(fields, + logger.Time("queue_item.created_at", item.CreatedAt), + zap.Duration("queue_item.age", ageSince(item.CreatedAt, now)), + ) + } + + return fields +} + +func ageSince(createdAt time.Time, now time.Time) time.Duration { + if createdAt.IsZero() || now.Before(createdAt) { + return 0 + } + + return now.Sub(createdAt) +} + +type countsField map[string]int + +func (f countsField) MarshalLogObject(enc zapcore.ObjectEncoder) error { + keys := make([]string, 0, len(f)) + for key := range f { + keys = append(keys, key) + } + sort.Strings(keys) + + for _, key := range keys { + enc.AddInt(key, f[key]) + } + + return nil +} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/processor.go b/packages/dashboard-api/internal/supabaseauthusersync/processor.go index 28b0816f64..1d562c8288 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/processor.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/processor.go @@ -37,57 +37,99 @@ func NewProcessor(store processorStore, maxAttempts int32, l logger.Logger) *Pro } } -func (p *Processor) Process(ctx context.Context, item QueueItem) { - err := p.processOnce(ctx, item) +func (p *Processor) Process(ctx context.Context, item QueueItem) processResult { + startedAt := time.Now() + action, err := p.processOnce(ctx, item) + result := processResult{ + Action: action, + Duration: time.Since(startedAt), + } if err == nil { if ackErr := p.store.Ack(ctx, item.ID); ackErr != nil { - p.l.Error(ctx, "failed to ack queue item", - zap.Int64("queue_item_id", item.ID), - zap.String("user_id", item.UserID.String()), - zap.Error(ackErr), + result.Outcome = processOutcomeAckFailed + + p.l.Error(ctx, "processed supabase auth sync queue item but failed to ack", + append( + processResultFields(item, result, time.Now()), + zap.NamedError("ack_error", ackErr), + )..., ) + + return result } - return - } + result.Outcome = processOutcomeAcked + p.l.Info(ctx, "processed supabase auth sync queue item", processResultFields(item, result, time.Now())...) - p.l.Warn(ctx, "failed to process queue item", - zap.Int64("queue_item_id", item.ID), - zap.String("user_id", item.UserID.String()), - zap.Int32("attempt", item.AttemptCount), - zap.Error(err), - ) + return result + } if item.AttemptCount >= p.maxAttempts { if dlErr := p.store.DeadLetter(ctx, item.ID, err.Error()); dlErr != nil { - p.l.Error(ctx, "failed to dead-letter queue item", - zap.Int64("queue_item_id", item.ID), - zap.Error(dlErr), + result.Outcome = processOutcomeDeadLetterFailed + + p.l.Error(ctx, "failed to dead-letter supabase auth sync queue item", + append( + processResultFields(item, result, time.Now()), + zap.Int32("queue_item.max_attempts", p.maxAttempts), + zap.NamedError("processing_error", err), + zap.NamedError("dead_letter_error", dlErr), + )..., ) + + return result } - return + result.Outcome = processOutcomeDeadLettered + p.l.Error(ctx, "dead-lettered supabase auth sync queue item after max attempts", + append( + processResultFields(item, result, time.Now()), + zap.Int32("queue_item.max_attempts", p.maxAttempts), + zap.NamedError("processing_error", err), + )..., + ) + + return result } backoff := retryBackoff(item.AttemptCount) + result.Outcome = processOutcomeRetried + result.Backoff = backoff if retryErr := p.store.Retry(ctx, item.ID, backoff, err.Error()); retryErr != nil { - p.l.Error(ctx, "failed to retry queue item", - zap.Int64("queue_item_id", item.ID), - zap.Error(retryErr), + result.Outcome = processOutcomeRetryFailed + + p.l.Error(ctx, "failed to schedule supabase auth sync queue item retry", + append( + processResultFields(item, result, time.Now()), + zap.NamedError("processing_error", err), + zap.NamedError("retry_error", retryErr), + )..., ) + + return result } + + p.l.Warn(ctx, "retrying supabase auth sync queue item after processing error", + append( + processResultFields(item, result, time.Now()), + zap.NamedError("processing_error", err), + )..., + ) + + return result } -func (p *Processor) processOnce(ctx context.Context, item QueueItem) (err error) { +func (p *Processor) processOnce(ctx context.Context, item QueueItem) (action reconcileAction, err error) { defer func() { if recovered := recover(); recovered != nil { - p.l.Error(ctx, "panic while processing queue item", - zap.Int64("queue_item_id", item.ID), - zap.String("user_id", item.UserID.String()), - zap.String("panic", fmt.Sprint(recovered)), - zap.String("stack", string(debug.Stack())), + p.l.Error(ctx, "panic while processing supabase auth sync queue item", + append( + queueItemFields(item, time.Now()), + zap.String("worker.panic", fmt.Sprint(recovered)), + zap.String("worker.stack", string(debug.Stack())), + )..., ) err = fmt.Errorf("panic while processing queue item: %v", recovered) @@ -97,26 +139,26 @@ func (p *Processor) processOnce(ctx context.Context, item QueueItem) (err error) return p.reconcile(ctx, item) } -func (p *Processor) reconcile(ctx context.Context, item QueueItem) error { +func (p *Processor) reconcile(ctx context.Context, item QueueItem) (reconcileAction, error) { authUser, err := p.store.GetAuthUser(ctx, item.UserID) if errors.Is(err, pgx.ErrNoRows) { if delErr := p.store.DeletePublicUser(ctx, item.UserID); delErr != nil { - return fmt.Errorf("delete public.users %s: %w", item.UserID, delErr) + return "", fmt.Errorf("delete public.users %s: %w", item.UserID, delErr) } - return nil + return reconcileActionDeletePublicUser, nil } if err != nil { - return fmt.Errorf("get auth.users %s: %w", item.UserID, err) + return "", fmt.Errorf("get auth.users %s: %w", item.UserID, err) } if err = p.store.UpsertPublicUser(ctx, authUser.ID, authUser.Email); err != nil { - return fmt.Errorf("upsert public.users %s: %w", authUser.ID, err) + return "", fmt.Errorf("upsert public.users %s: %w", authUser.ID, err) } - return nil + return reconcileActionUpsertPublicUser, nil } func retryBackoff(attempt int32) time.Duration { diff --git a/packages/dashboard-api/internal/supabaseauthusersync/runner.go b/packages/dashboard-api/internal/supabaseauthusersync/runner.go index 39de50196c..1acd35d480 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/runner.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/runner.go @@ -18,20 +18,24 @@ type Runner struct { } func NewRunner(cfg Config, store *Store, lockOwner string, l logger.Logger) *Runner { + workerLogger := l.With(logger.WithServiceInstanceID(lockOwner)) + return &Runner{ cfg: cfg, store: store, - processor: NewProcessor(store, cfg.MaxAttempts, l), + processor: NewProcessor(store, cfg.MaxAttempts, workerLogger), lockOwner: lockOwner, - l: l, + l: workerLogger, } } func (r *Runner) Run(ctx context.Context) error { r.l.Info(ctx, "starting supabase auth user sync worker", - zap.String("lock_owner", r.lockOwner), - zap.Duration("poll_interval", r.cfg.PollInterval), - zap.Int32("batch_size", r.cfg.BatchSize), + zap.String("worker.lock_owner", r.lockOwner), + zap.Duration("worker.poll_interval", r.cfg.PollInterval), + zap.Int32("worker.batch_size", r.cfg.BatchSize), + zap.Duration("worker.lock_timeout", r.cfg.LockTimeout), + zap.Int32("worker.max_attempts", r.cfg.MaxAttempts), ) ticker := time.NewTicker(r.cfg.PollInterval) @@ -40,7 +44,7 @@ func (r *Runner) Run(ctx context.Context) error { for { select { case <-ctx.Done(): - r.l.Info(ctx, "stopping supabase auth user sync worker") + r.l.Info(ctx, "stopping supabase auth user sync worker", zap.Error(ctx.Err())) return ctx.Err() case <-ticker.C: @@ -50,9 +54,15 @@ func (r *Runner) Run(ctx context.Context) error { } func (r *Runner) poll(ctx context.Context) { + claimedAt := time.Now() items, err := r.store.ClaimBatch(ctx, r.lockOwner, r.cfg.LockTimeout, r.cfg.BatchSize) if err != nil { - r.l.Error(ctx, "failed to claim queue batch", zap.Error(err)) + r.l.Error(ctx, "failed to claim supabase auth sync queue batch", + zap.String("worker.lock_owner", r.lockOwner), + zap.Duration("worker.lock_timeout", r.cfg.LockTimeout), + zap.Int32("worker.batch_size", r.cfg.BatchSize), + zap.Error(err), + ) return } @@ -61,9 +71,11 @@ func (r *Runner) poll(ctx context.Context) { return } - r.l.Debug(ctx, "claimed queue batch", zap.Int("count", len(items))) + summary := newBatchSummary(items, claimedAt) for _, item := range items { - r.processor.Process(ctx, item) + summary.Add(r.processor.Process(ctx, item)) } + + r.l.Log(ctx, summary.Level(), "processed supabase auth sync queue batch", summary.Fields(time.Since(claimedAt))...) } diff --git a/packages/dashboard-api/internal/supabaseauthusersync/supervisor.go b/packages/dashboard-api/internal/supabaseauthusersync/supervisor.go index ce8dafb463..24269f351e 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/supervisor.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/supervisor.go @@ -59,9 +59,10 @@ func supervise(ctx context.Context, l logger.Logger, cfg supervisorConfig, run f delay := restartBackoff(restartAttempt, cfg.RestartDelay, cfg.MaxRestartDelay) l.Error(ctx, "supabase auth user sync worker exited unexpectedly; restarting", zap.Error(err), - zap.Int("restart_attempt", restartAttempt), - zap.Duration("restart_in", delay), - zap.Duration("runtime", runtime), + zap.Int("worker.restart_attempt", restartAttempt), + zap.Duration("worker.restart_in", delay), + zap.Duration("worker.runtime", runtime), + zap.Duration("worker.healthy_run_reset_after", cfg.HealthyRunResetAfter), ) timer := time.NewTimer(delay) @@ -78,8 +79,8 @@ func runRecovering(ctx context.Context, l logger.Logger, run func(context.Contex defer func() { if recovered := recover(); recovered != nil { l.Error(ctx, "supabase auth user sync worker panicked", - zap.String("panic", fmt.Sprint(recovered)), - zap.String("stack", string(debug.Stack())), + zap.String("worker.panic", fmt.Sprint(recovered)), + zap.String("worker.stack", string(debug.Stack())), ) err = fmt.Errorf("worker panic: %v", recovered) diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 106217091c..22bca86bf3 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -233,14 +233,10 @@ func run() int { if config.SupabaseAuthUserSyncEnabled { workerLogger := l.With(zap.String("worker", "supabase_auth_user_sync")) syncStore := supabaseauthusersync.NewStore(db.Queries) + syncConfig := supabaseauthusersync.DefaultConfig() + syncConfig.Enabled = true syncRunner := supabaseauthusersync.NewRunner( - supabaseauthusersync.Config{ - Enabled: true, - BatchSize: config.SupabaseAuthUserSyncBatchSize, - PollInterval: config.SupabaseAuthUserSyncPollInterval, - LockTimeout: config.SupabaseAuthUserSyncLockTimeout, - MaxAttempts: config.SupabaseAuthUserSyncMaxAttempts, - }, + syncConfig, syncStore, serviceInstanceID, workerLogger, diff --git a/packages/db/Makefile b/packages/db/Makefile index e5f1d67439..acd52c6d0e 100644 --- a/packages/db/Makefile +++ b/packages/db/Makefile @@ -5,6 +5,9 @@ PREFIX := $(strip $(subst ",,$(PREFIX))) goose := GOOSE_DRIVER=postgres GOOSE_DBSTRING=$(POSTGRES_CONNECTION_STRING) go tool goose -table "_migrations" -dir "migrations" goose-local := GOOSE_DBSTRING=postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable go tool goose -table "_migrations" -dir "migrations" postgres +AUTH_USER_SYNC_SMOKE_COUNT ?= 15 +AUTH_USER_SYNC_SMOKE_WAIT ?= 10s + .PHONY: migrate migrate: @@ -61,3 +64,7 @@ test: build-tools seed-db: @echo "Seeding database..." @POSTGRES_CONNECTION_STRING=$(POSTGRES_CONNECTION_STRING) go run ./scripts/seed/postgres/seed-db.go + +.PHONY: auth-user-sync-smoke +auth-user-sync-smoke: + @AUTH_DB_CONNECTION_STRING="$(AUTH_DB_CONNECTION_STRING)" POSTGRES_CONNECTION_STRING="$(POSTGRES_CONNECTION_STRING)" AUTH_USER_SYNC_SMOKE_COUNT="$(AUTH_USER_SYNC_SMOKE_COUNT)" AUTH_USER_SYNC_SMOKE_WAIT="$(AUTH_USER_SYNC_SMOKE_WAIT)" go run ./scripts/auth-user-sync-smoke diff --git a/packages/db/scripts/auth-user-sync-smoke/main.go b/packages/db/scripts/auth-user-sync-smoke/main.go new file mode 100644 index 0000000000..6ba5cb807a --- /dev/null +++ b/packages/db/scripts/auth-user-sync-smoke/main.go @@ -0,0 +1,219 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + "github.com/google/uuid" + + authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" +) + +const ( + defaultCount = 1 + defaultWait = 30 * time.Second +) + +type config struct { + ConnectionString string + Count int + Wait time.Duration +} + +type authUser struct { + ID uuid.UUID + Email string +} + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + err := run(ctx) + stop() + + if err != nil { + fmt.Fprintf(os.Stderr, "auth user sync smoke failed: %v\n", err) + os.Exit(1) + } +} + +func run(ctx context.Context) error { + cfg, err := loadConfig() + if err != nil { + return err + } + + db, err := authdb.NewClient(ctx, cfg.ConnectionString, "") + if err != nil { + return fmt.Errorf("create auth db client: %w", err) + } + defer func() { + if closeErr := db.Close(); closeErr != nil { + fmt.Fprintf(os.Stderr, "close auth db client: %v\n", closeErr) + } + }() + + users := newAuthUsers(cfg.Count) + insertedUsers := make([]authUser, 0, len(users)) + + defer func() { + if len(insertedUsers) == 0 { + return + } + + if cleanupErr := cleanupInsertedUsers(ctx, db, insertedUsers); cleanupErr != nil { + fmt.Fprintf(os.Stderr, "cleanup auth users: %v\n", cleanupErr) + + return + } + + fmt.Fprintf(os.Stdout, "cleaned up %d auth.users rows\n", len(insertedUsers)) + }() + + if err := insertUsers(ctx, db, users, &insertedUsers); err != nil { + return err + } + + fmt.Fprintf(os.Stdout, "created %d auth.users rows\n", len(insertedUsers)) + for _, user := range insertedUsers { + fmt.Fprintf(os.Stdout, " %s %s\n", user.ID.String(), user.Email) + } + + fmt.Fprintf(os.Stdout, "waiting %s before delete\n", cfg.Wait) + + timer := time.NewTimer(cfg.Wait) + defer timer.Stop() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } + + if err := cleanupInsertedUsers(ctx, db, insertedUsers); err != nil { + return fmt.Errorf("delete auth users: %w", err) + } + + insertedUsers = nil + fmt.Fprintln(os.Stdout, "deleted auth.users rows") + + return nil +} + +func loadConfig() (config, error) { + connectionString := strings.TrimSpace(os.Getenv("AUTH_DB_CONNECTION_STRING")) + if connectionString == "" { + connectionString = strings.TrimSpace(os.Getenv("POSTGRES_CONNECTION_STRING")) + } + if connectionString == "" { + return config{}, fmt.Errorf("AUTH_DB_CONNECTION_STRING or POSTGRES_CONNECTION_STRING must be set") + } + + count, err := loadCount() + if err != nil { + return config{}, err + } + + wait, err := loadWait() + if err != nil { + return config{}, err + } + + return config{ + ConnectionString: connectionString, + Count: count, + Wait: wait, + }, nil +} + +func loadCount() (int, error) { + rawCount := strings.TrimSpace(os.Getenv("AUTH_USER_SYNC_SMOKE_COUNT")) + if rawCount == "" { + return defaultCount, nil + } + + count, err := strconv.Atoi(rawCount) + if err != nil { + return 0, fmt.Errorf("parse AUTH_USER_SYNC_SMOKE_COUNT: %w", err) + } + if count < 1 { + return 0, fmt.Errorf("AUTH_USER_SYNC_SMOKE_COUNT must be at least 1") + } + + return count, nil +} + +func loadWait() (time.Duration, error) { + rawWait := strings.TrimSpace(os.Getenv("AUTH_USER_SYNC_SMOKE_WAIT")) + if rawWait == "" { + return defaultWait, nil + } + + wait, err := time.ParseDuration(rawWait) + if err != nil { + return 0, fmt.Errorf("parse AUTH_USER_SYNC_SMOKE_WAIT: %w", err) + } + if wait <= 0 { + return 0, fmt.Errorf("AUTH_USER_SYNC_SMOKE_WAIT must be greater than 0") + } + + return wait, nil +} + +func newAuthUsers(count int) []authUser { + runID := strings.ReplaceAll(uuid.NewString(), "-", "") + users := make([]authUser, 0, count) + + for i := range count { + userID := uuid.New() + email := fmt.Sprintf("auth-sync-smoke-%s-%02d@example.com", runID[:12], i+1) + users = append(users, authUser{ + ID: userID, + Email: email, + }) + } + + return users +} + +func cleanupInsertedUsers(ctx context.Context, db *authdb.Client, users []authUser) error { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + + return deleteUsers(cleanupCtx, db, users) +} + +func insertUsers(ctx context.Context, db *authdb.Client, users []authUser, insertedUsers *[]authUser) error { + for _, user := range users { + err := db.TestsRawSQL(ctx, ` +INSERT INTO auth.users (id, email) +VALUES ($1, $2) +`, user.ID, user.Email) + if err != nil { + return fmt.Errorf("insert auth user %s: %w", user.Email, err) + } + + *insertedUsers = append(*insertedUsers, user) + } + + return nil +} + +func deleteUsers(ctx context.Context, db *authdb.Client, users []authUser) error { + for _, user := range users { + err := db.TestsRawSQL(ctx, ` +DELETE FROM auth.users +WHERE id = $1 +`, user.ID) + if err != nil { + return fmt.Errorf("delete auth user %s: %w", user.Email, err) + } + } + + return nil +} From eb69678058f7258ada2feb6fe686fdced8c7f6c7 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Tue, 31 Mar 2026 13:19:05 -0700 Subject: [PATCH 06/92] chore: remove smoke test --- packages/db/Makefile | 7 - .../db/scripts/auth-user-sync-smoke/main.go | 219 ------------------ 2 files changed, 226 deletions(-) delete mode 100644 packages/db/scripts/auth-user-sync-smoke/main.go diff --git a/packages/db/Makefile b/packages/db/Makefile index acd52c6d0e..e5f1d67439 100644 --- a/packages/db/Makefile +++ b/packages/db/Makefile @@ -5,9 +5,6 @@ PREFIX := $(strip $(subst ",,$(PREFIX))) goose := GOOSE_DRIVER=postgres GOOSE_DBSTRING=$(POSTGRES_CONNECTION_STRING) go tool goose -table "_migrations" -dir "migrations" goose-local := GOOSE_DBSTRING=postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable go tool goose -table "_migrations" -dir "migrations" postgres -AUTH_USER_SYNC_SMOKE_COUNT ?= 15 -AUTH_USER_SYNC_SMOKE_WAIT ?= 10s - .PHONY: migrate migrate: @@ -64,7 +61,3 @@ test: build-tools seed-db: @echo "Seeding database..." @POSTGRES_CONNECTION_STRING=$(POSTGRES_CONNECTION_STRING) go run ./scripts/seed/postgres/seed-db.go - -.PHONY: auth-user-sync-smoke -auth-user-sync-smoke: - @AUTH_DB_CONNECTION_STRING="$(AUTH_DB_CONNECTION_STRING)" POSTGRES_CONNECTION_STRING="$(POSTGRES_CONNECTION_STRING)" AUTH_USER_SYNC_SMOKE_COUNT="$(AUTH_USER_SYNC_SMOKE_COUNT)" AUTH_USER_SYNC_SMOKE_WAIT="$(AUTH_USER_SYNC_SMOKE_WAIT)" go run ./scripts/auth-user-sync-smoke diff --git a/packages/db/scripts/auth-user-sync-smoke/main.go b/packages/db/scripts/auth-user-sync-smoke/main.go deleted file mode 100644 index 6ba5cb807a..0000000000 --- a/packages/db/scripts/auth-user-sync-smoke/main.go +++ /dev/null @@ -1,219 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "os/signal" - "strconv" - "strings" - "syscall" - "time" - - "github.com/google/uuid" - - authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" -) - -const ( - defaultCount = 1 - defaultWait = 30 * time.Second -) - -type config struct { - ConnectionString string - Count int - Wait time.Duration -} - -type authUser struct { - ID uuid.UUID - Email string -} - -func main() { - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - err := run(ctx) - stop() - - if err != nil { - fmt.Fprintf(os.Stderr, "auth user sync smoke failed: %v\n", err) - os.Exit(1) - } -} - -func run(ctx context.Context) error { - cfg, err := loadConfig() - if err != nil { - return err - } - - db, err := authdb.NewClient(ctx, cfg.ConnectionString, "") - if err != nil { - return fmt.Errorf("create auth db client: %w", err) - } - defer func() { - if closeErr := db.Close(); closeErr != nil { - fmt.Fprintf(os.Stderr, "close auth db client: %v\n", closeErr) - } - }() - - users := newAuthUsers(cfg.Count) - insertedUsers := make([]authUser, 0, len(users)) - - defer func() { - if len(insertedUsers) == 0 { - return - } - - if cleanupErr := cleanupInsertedUsers(ctx, db, insertedUsers); cleanupErr != nil { - fmt.Fprintf(os.Stderr, "cleanup auth users: %v\n", cleanupErr) - - return - } - - fmt.Fprintf(os.Stdout, "cleaned up %d auth.users rows\n", len(insertedUsers)) - }() - - if err := insertUsers(ctx, db, users, &insertedUsers); err != nil { - return err - } - - fmt.Fprintf(os.Stdout, "created %d auth.users rows\n", len(insertedUsers)) - for _, user := range insertedUsers { - fmt.Fprintf(os.Stdout, " %s %s\n", user.ID.String(), user.Email) - } - - fmt.Fprintf(os.Stdout, "waiting %s before delete\n", cfg.Wait) - - timer := time.NewTimer(cfg.Wait) - defer timer.Stop() - - select { - case <-ctx.Done(): - return ctx.Err() - case <-timer.C: - } - - if err := cleanupInsertedUsers(ctx, db, insertedUsers); err != nil { - return fmt.Errorf("delete auth users: %w", err) - } - - insertedUsers = nil - fmt.Fprintln(os.Stdout, "deleted auth.users rows") - - return nil -} - -func loadConfig() (config, error) { - connectionString := strings.TrimSpace(os.Getenv("AUTH_DB_CONNECTION_STRING")) - if connectionString == "" { - connectionString = strings.TrimSpace(os.Getenv("POSTGRES_CONNECTION_STRING")) - } - if connectionString == "" { - return config{}, fmt.Errorf("AUTH_DB_CONNECTION_STRING or POSTGRES_CONNECTION_STRING must be set") - } - - count, err := loadCount() - if err != nil { - return config{}, err - } - - wait, err := loadWait() - if err != nil { - return config{}, err - } - - return config{ - ConnectionString: connectionString, - Count: count, - Wait: wait, - }, nil -} - -func loadCount() (int, error) { - rawCount := strings.TrimSpace(os.Getenv("AUTH_USER_SYNC_SMOKE_COUNT")) - if rawCount == "" { - return defaultCount, nil - } - - count, err := strconv.Atoi(rawCount) - if err != nil { - return 0, fmt.Errorf("parse AUTH_USER_SYNC_SMOKE_COUNT: %w", err) - } - if count < 1 { - return 0, fmt.Errorf("AUTH_USER_SYNC_SMOKE_COUNT must be at least 1") - } - - return count, nil -} - -func loadWait() (time.Duration, error) { - rawWait := strings.TrimSpace(os.Getenv("AUTH_USER_SYNC_SMOKE_WAIT")) - if rawWait == "" { - return defaultWait, nil - } - - wait, err := time.ParseDuration(rawWait) - if err != nil { - return 0, fmt.Errorf("parse AUTH_USER_SYNC_SMOKE_WAIT: %w", err) - } - if wait <= 0 { - return 0, fmt.Errorf("AUTH_USER_SYNC_SMOKE_WAIT must be greater than 0") - } - - return wait, nil -} - -func newAuthUsers(count int) []authUser { - runID := strings.ReplaceAll(uuid.NewString(), "-", "") - users := make([]authUser, 0, count) - - for i := range count { - userID := uuid.New() - email := fmt.Sprintf("auth-sync-smoke-%s-%02d@example.com", runID[:12], i+1) - users = append(users, authUser{ - ID: userID, - Email: email, - }) - } - - return users -} - -func cleanupInsertedUsers(ctx context.Context, db *authdb.Client, users []authUser) error { - cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - defer cancel() - - return deleteUsers(cleanupCtx, db, users) -} - -func insertUsers(ctx context.Context, db *authdb.Client, users []authUser, insertedUsers *[]authUser) error { - for _, user := range users { - err := db.TestsRawSQL(ctx, ` -INSERT INTO auth.users (id, email) -VALUES ($1, $2) -`, user.ID, user.Email) - if err != nil { - return fmt.Errorf("insert auth user %s: %w", user.Email, err) - } - - *insertedUsers = append(*insertedUsers, user) - } - - return nil -} - -func deleteUsers(ctx context.Context, db *authdb.Client, users []authUser) error { - for _, user := range users { - err := db.TestsRawSQL(ctx, ` -DELETE FROM auth.users -WHERE id = $1 -`, user.ID) - if err != nil { - return fmt.Errorf("delete auth user %s: %w", user.Email, err) - } - } - - return nil -} From bb9fa39df3ab25cc5c97fc1da4771d43ca006296 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Tue, 31 Mar 2026 13:49:02 -0700 Subject: [PATCH 07/92] add: e2e runner test --- .../supabaseauthusersync/runner_test.go | 579 ++++++++++++------ 1 file changed, 377 insertions(+), 202 deletions(-) diff --git a/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go b/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go index b578185745..1429c2a605 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go @@ -1,10 +1,9 @@ package supabaseauthusersync import ( - "os/exec" - "path/filepath" - "strings" - "sync/atomic" + "context" + "fmt" + "sync" "testing" "time" @@ -17,284 +16,460 @@ import ( "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) -func setupTestDB(t *testing.T) *testutils.Database { - t.Helper() +const ( + testRunnerPollInterval = 20 * time.Millisecond + testRunnerLockTimeout = 150 * time.Millisecond + testEventuallyTimeout = 8 * time.Second + testEventuallyTick = 25 * time.Millisecond + testRunnerStopTimeout = 2 * time.Second +) + +type runnerProcess struct { + cancel context.CancelFunc + done chan error + stopOnce sync.Once +} + +type userExpectation struct { + Email string + Exists bool +} + +type queueSnapshot struct { + Total int + DeadLettered int +} +func TestSupabaseAuthUserSyncRunner_EndToEnd(t *testing.T) { db := testutils.SetupDatabase(t) - repoRoot := gitRoot(t) - migrationSQL := readFile(t, filepath.Join( - repoRoot, - "packages", "db", "pkg", "dashboard", "migrations", - "20260328000000_dashboard_supabase_auth_user_sync_queue.sql", - )) + t.Run("repairs_insert_update_delete_drift", func(t *testing.T) { + ctx := t.Context() + userID := uuid.New() + initialEmail := fmt.Sprintf("auth-sync-%s-initial@example.com", userID.String()[:8]) + updatedEmail := fmt.Sprintf("auth-sync-%s-updated@example.com", userID.String()[:8]) + + insertAuthUser(t, ctx, db, userID, initialEmail) + deletePublicUser(t, ctx, db, userID) + assertQueueBacklog(t, ctx, db, 1) + + insertRunner := startRunnerProcess(t, db, newTestRunnerConfig(4), "repair-insert") + t.Cleanup(func() { + insertRunner.Stop(t) + }) + waitForPublicUsers(t, ctx, db, map[uuid.UUID]userExpectation{ + userID: { + Email: initialEmail, + Exists: true, + }, + }) + waitForQueueDrain(t, ctx, db) + insertRunner.Stop(t) + + updateAuthUserEmail(t, ctx, db, userID, updatedEmail) + setPublicUserEmail(t, ctx, db, userID, "stale@example.com") + assertQueueBacklog(t, ctx, db, 1) + + updateRunner := startRunnerProcess(t, db, newTestRunnerConfig(4), "repair-update") + t.Cleanup(func() { + updateRunner.Stop(t) + }) + waitForPublicUsers(t, ctx, db, map[uuid.UUID]userExpectation{ + userID: { + Email: updatedEmail, + Exists: true, + }, + }) + waitForQueueDrain(t, ctx, db) + updateRunner.Stop(t) + + deleteAuthUser(t, ctx, db, userID) + insertPublicUser(t, ctx, db, userID, "ghost@example.com") + assertQueueBacklog(t, ctx, db, 1) + + deleteRunner := startRunnerProcess(t, db, newTestRunnerConfig(4), "repair-delete") + t.Cleanup(func() { + deleteRunner.Stop(t) + }) + waitForPublicUsers(t, ctx, db, map[uuid.UUID]userExpectation{ + userID: { + Exists: false, + }, + }) + waitForQueueDrain(t, ctx, db) + deleteRunner.Stop(t) + }) + + t.Run("reclaims_stale_queue_locks", func(t *testing.T) { + ctx := t.Context() + userID := uuid.New() + email := fmt.Sprintf("auth-sync-%s-locked@example.com", userID.String()[:8]) + + insertAuthUser(t, ctx, db, userID, email) + deletePublicUser(t, ctx, db, userID) + lockQueueItems(t, ctx, db, userID, time.Now().Add(-time.Minute), "stale-worker") + assertQueueBacklog(t, ctx, db, 1) + + runner := startRunnerProcess(t, db, newTestRunnerConfig(2), "lock-reclaimer") + t.Cleanup(func() { + runner.Stop(t) + }) + + waitForPublicUsers(t, ctx, db, map[uuid.UUID]userExpectation{ + userID: { + Email: email, + Exists: true, + }, + }) + waitForQueueDrain(t, ctx, db) + runner.Stop(t) + }) + + t.Run("drains_burst_backlog_with_multiple_runners", func(t *testing.T) { + ctx := t.Context() + const userCount = 60 + + userIDs := make([]uuid.UUID, 0, userCount) + + for i := 0; i < userCount; i++ { + userID := uuid.New() + userIDs = append(userIDs, userID) + + initialEmail := fmt.Sprintf("auth-sync-burst-%02d-initial@example.com", i) + insertAuthUser(t, ctx, db, userID, initialEmail) + + if i%2 == 0 { + updateAuthUserEmail(t, ctx, db, userID, fmt.Sprintf("auth-sync-burst-%02d-v2@example.com", i)) + } + if i%5 == 0 { + updateAuthUserEmail(t, ctx, db, userID, fmt.Sprintf("auth-sync-burst-%02d-v3@example.com", i)) + } - upSQL := extractGooseUp(migrationSQL) - err := db.AuthDb.TestsRawSQL(t.Context(), upSQL) - require.NoError(t, err, "failed to apply dashboard auth sync migration") + if i%3 == 0 { + deleteAuthUser(t, ctx, db, userID) + enqueueUserSyncItem(t, ctx, db, userID, "delete") + if i%6 == 0 { + insertPublicUser(t, ctx, db, userID, fmt.Sprintf("ghost-%02d@example.com", i)) + } - return db -} + continue + } -func gitRoot(t *testing.T) string { - t.Helper() + if i%8 == 0 { + deletePublicUser(t, ctx, db, userID) + } else if i%7 == 0 { + setPublicUserEmail(t, ctx, db, userID, fmt.Sprintf("stale-%02d@example.com", i)) + } - cmd := exec.CommandContext(t.Context(), "git", "rev-parse", "--show-toplevel") - output, err := cmd.Output() - require.NoError(t, err) + if i%4 == 0 { + enqueueUserSyncItem(t, ctx, db, userID, "upsert") + } + if i%9 == 0 { + enqueueUserSyncItem(t, ctx, db, userID, "upsert") + } + } - return strings.TrimSpace(string(output)) -} + authUsers, err := loadAuthUsers(ctx, db) + require.NoError(t, err) -func readFile(t *testing.T, path string) string { - t.Helper() + want := expectedUsersForIDs(userIDs, authUsers) + assertQueueBacklog(t, ctx, db, userCount) - cmd := exec.CommandContext(t.Context(), "cat", path) - output, err := cmd.Output() - require.NoError(t, err) + runnerA := startRunnerProcess(t, db, newTestRunnerConfig(5), "burst-a") + runnerB := startRunnerProcess(t, db, newTestRunnerConfig(5), "burst-b") + t.Cleanup(func() { + runnerA.Stop(t) + runnerB.Stop(t) + }) - return string(output) + waitForPublicUsers(t, ctx, db, want) + waitForQueueDrain(t, ctx, db) + + runnerA.Stop(t) + runnerB.Stop(t) + }) } -func extractGooseUp(sql string) string { - parts := strings.SplitN(sql, "-- +goose Down", 2) - up := parts[0] - up = strings.ReplaceAll(up, "-- +goose Up", "") - up = strings.ReplaceAll(up, "-- +goose StatementBegin", "") - up = strings.ReplaceAll(up, "-- +goose StatementEnd", "") +func newTestRunnerConfig(batchSize int32) Config { + cfg := DefaultConfig() + cfg.Enabled = true + cfg.BatchSize = batchSize + cfg.PollInterval = testRunnerPollInterval + cfg.LockTimeout = testRunnerLockTimeout + cfg.MaxAttempts = 5 - return up + return cfg } -func insertAuthUser(t *testing.T, db *testutils.Database, userID uuid.UUID, email string) { +func startRunnerProcess(t *testing.T, db *testutils.Database, cfg Config, lockOwner string) *runnerProcess { t.Helper() - err := db.AuthDb.TestsRawSQL(t.Context(), - "INSERT INTO auth.users (id, email) VALUES ($1, $2)", userID, email) - require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + runner := NewRunner(cfg, NewStore(db.SqlcClient.Queries), lockOwner, logger.NewNopLogger()) + + go func() { + done <- runner.Run(ctx) + }() + + return &runnerProcess{ + cancel: cancel, + done: done, + } } -func updateAuthUserEmail(t *testing.T, db *testutils.Database, userID uuid.UUID, email string) { +func (p *runnerProcess) Stop(t *testing.T) { t.Helper() - err := db.AuthDb.TestsRawSQL(t.Context(), - "UPDATE auth.users SET email = $1 WHERE id = $2", email, userID) - require.NoError(t, err) + + p.stopOnce.Do(func() { + p.cancel() + + select { + case err := <-p.done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(testRunnerStopTimeout): + t.Fatalf("runner did not stop within %s", testRunnerStopTimeout) + } + }) } -func deleteAuthUser(t *testing.T, db *testutils.Database, userID uuid.UUID) { +func insertAuthUser(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, email string) { t.Helper() - err := db.AuthDb.TestsRawSQL(t.Context(), - "DELETE FROM auth.users WHERE id = $1", userID) + + err := db.AuthDb.TestsRawSQL(ctx, + "INSERT INTO auth.users (id, email) VALUES ($1, $2)", + userID, + email, + ) require.NoError(t, err) } -func getPublicUserEmail(t *testing.T, db *testutils.Database, userID uuid.UUID) (string, bool) { +func updateAuthUserEmail(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, email string) { t.Helper() - var email string - var found bool - - err := db.AuthDb.TestsRawSQLQuery(t.Context(), - "SELECT email FROM public.users WHERE id = $1", - func(rows pgx.Rows) error { - if rows.Next() { - found = true - return rows.Scan(&email) - } - return nil - }, + err := db.AuthDb.TestsRawSQL(ctx, + "UPDATE auth.users SET email = $1 WHERE id = $2", + email, userID, ) require.NoError(t, err) - - return email, found } -func queueDepth(t *testing.T, db *testutils.Database) int { +func deleteAuthUser(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID) { t.Helper() - var count int - - err := db.AuthDb.TestsRawSQLQuery(t.Context(), - "SELECT count(*) FROM public.user_sync_queue WHERE dead_lettered_at IS NULL", - func(rows pgx.Rows) error { - if rows.Next() { - return rows.Scan(&count) - } - return nil - }, + err := db.AuthDb.TestsRawSQL(ctx, + "DELETE FROM auth.users WHERE id = $1", + userID, ) require.NoError(t, err) - - return count } -func TestInsertAuthUserCreatesQueueRow(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } - - db := setupTestDB(t) - - userID := uuid.New() - insertAuthUser(t, db, userID, "test@example.com") +func deletePublicUser(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID) { + t.Helper() - depth := queueDepth(t, db) - assert.Equal(t, 1, depth) + err := db.AuthDb.TestsRawSQL(ctx, + "DELETE FROM public.users WHERE id = $1", + userID, + ) + require.NoError(t, err) } -func TestProcessorReconciles_Insert(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } - - db := setupTestDB(t) - store := NewStore(db.SqlcClient.Queries) - l := logger.NewNopLogger() - proc := NewProcessor(store, 5, l) - - userID := uuid.New() - insertAuthUser(t, db, userID, "alice@example.com") +func insertPublicUser(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, email string) { + t.Helper() - items, err := store.ClaimBatch(t.Context(), "test-worker", 2*time.Minute, 10) + err := db.AuthDb.TestsRawSQL(ctx, ` +INSERT INTO public.users (id, email) +VALUES ($1, $2) +ON CONFLICT (id) DO UPDATE +SET email = EXCLUDED.email, + updated_at = now() +`, + userID, + email, + ) require.NoError(t, err) - require.Len(t, items, 1) - - proc.Process(t.Context(), items[0]) - - email, found := getPublicUserEmail(t, db, userID) - assert.True(t, found) - assert.Equal(t, "alice@example.com", email) - - assert.Equal(t, 0, queueDepth(t, db)) } -func TestProcessorReconciles_UpdateEmail(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } +func setPublicUserEmail(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, email string) { + t.Helper() - db := setupTestDB(t) - store := NewStore(db.SqlcClient.Queries) - l := logger.NewNopLogger() - proc := NewProcessor(store, 5, l) + err := db.AuthDb.TestsRawSQL(ctx, + "UPDATE public.users SET email = $1, updated_at = now() WHERE id = $2", + email, + userID, + ) + require.NoError(t, err) +} - userID := uuid.New() - insertAuthUser(t, db, userID, "old@example.com") +func enqueueUserSyncItem(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, operation string) { + t.Helper() - items, err := store.ClaimBatch(t.Context(), "test-worker", 2*time.Minute, 10) + err := db.AuthDb.TestsRawSQL(ctx, + "INSERT INTO public.user_sync_queue (user_id, operation) VALUES ($1, $2)", + userID, + operation, + ) require.NoError(t, err) - proc.Process(t.Context(), items[0]) +} - updateAuthUserEmail(t, db, userID, "new@example.com") +func lockQueueItems(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, lockedAt time.Time, lockOwner string) { + t.Helper() - items, err = store.ClaimBatch(t.Context(), "test-worker", 2*time.Minute, 10) + err := db.AuthDb.TestsRawSQL(ctx, ` +UPDATE public.user_sync_queue +SET locked_at = $2, + lock_owner = $3 +WHERE user_id = $1 +`, + userID, + lockedAt, + lockOwner, + ) require.NoError(t, err) - require.Len(t, items, 1) - proc.Process(t.Context(), items[0]) - - email, found := getPublicUserEmail(t, db, userID) - assert.True(t, found) - assert.Equal(t, "new@example.com", email) } -func TestProcessorReconciles_Delete(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } +func loadPublicUsers(ctx context.Context, db *testutils.Database) (map[uuid.UUID]string, error) { + users := make(map[uuid.UUID]string) - db := setupTestDB(t) - store := NewStore(db.SqlcClient.Queries) - l := logger.NewNopLogger() - proc := NewProcessor(store, 5, l) + err := db.AuthDb.TestsRawSQLQuery(ctx, + "SELECT id, email FROM public.users", + func(rows pgx.Rows) error { + for rows.Next() { + var userID uuid.UUID + var email string + if err := rows.Scan(&userID, &email); err != nil { + return err + } + + users[userID] = email + } - userID := uuid.New() - insertAuthUser(t, db, userID, "doomed@example.com") + return rows.Err() + }, + ) + if err != nil { + return nil, err + } - items, err := store.ClaimBatch(t.Context(), "test-worker", 2*time.Minute, 10) - require.NoError(t, err) - proc.Process(t.Context(), items[0]) + return users, nil +} - _, found := getPublicUserEmail(t, db, userID) - require.True(t, found) +func loadAuthUsers(ctx context.Context, db *testutils.Database) (map[uuid.UUID]string, error) { + users := make(map[uuid.UUID]string) - deleteAuthUser(t, db, userID) + err := db.AuthDb.TestsRawSQLQuery(ctx, + "SELECT id, email FROM auth.users", + func(rows pgx.Rows) error { + for rows.Next() { + var userID uuid.UUID + var email string + if err := rows.Scan(&userID, &email); err != nil { + return err + } + + users[userID] = email + } - items, err = store.ClaimBatch(t.Context(), "test-worker", 2*time.Minute, 10) - require.NoError(t, err) - require.Len(t, items, 1) - proc.Process(t.Context(), items[0]) + return rows.Err() + }, + ) + if err != nil { + return nil, err + } - _, found = getPublicUserEmail(t, db, userID) - assert.False(t, found) + return users, nil } -func TestDuplicateQueueRowsConverge(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } +func loadQueueSnapshot(ctx context.Context, db *testutils.Database) (queueSnapshot, error) { + var snapshot queueSnapshot - db := setupTestDB(t) - store := NewStore(db.SqlcClient.Queries) - l := logger.NewNopLogger() - proc := NewProcessor(store, 5, l) + err := db.AuthDb.TestsRawSQLQuery(ctx, ` +SELECT + count(*)::int AS total, + count(*) FILTER (WHERE dead_lettered_at IS NOT NULL)::int AS dead_lettered +FROM public.user_sync_queue +`, + func(rows pgx.Rows) error { + if !rows.Next() { + return nil + } - userID := uuid.New() - insertAuthUser(t, db, userID, "dup@example.com") + return rows.Scan(&snapshot.Total, &snapshot.DeadLettered) + }, + ) + if err != nil { + return queueSnapshot{}, err + } - err := db.AuthDb.TestsRawSQL(t.Context(), - "INSERT INTO public.user_sync_queue (user_id, operation) VALUES ($1, 'upsert')", - userID) - require.NoError(t, err) + return snapshot, nil +} - items, err := store.ClaimBatch(t.Context(), "test-worker", 2*time.Minute, 10) - require.NoError(t, err) - assert.GreaterOrEqual(t, len(items), 2) +func expectedUsersForIDs(userIDs []uuid.UUID, authUsers map[uuid.UUID]string) map[uuid.UUID]userExpectation { + want := make(map[uuid.UUID]userExpectation, len(userIDs)) - for _, item := range items { - proc.Process(t.Context(), item) + for _, userID := range userIDs { + email, ok := authUsers[userID] + want[userID] = userExpectation{ + Email: email, + Exists: ok, + } } - email, found := getPublicUserEmail(t, db, userID) - assert.True(t, found) - assert.Equal(t, "dup@example.com", email) - assert.Equal(t, 0, queueDepth(t, db)) + return want } -func TestMultiInstanceClaimNoDoubleProcessing(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } +func assertQueueBacklog(t *testing.T, ctx context.Context, db *testutils.Database, minimum int) { + t.Helper() - db := setupTestDB(t) + snapshot, err := loadQueueSnapshot(ctx, db) + require.NoError(t, err) + require.GreaterOrEqual(t, snapshot.Total, minimum) +} - for i := range 10 { - userID := uuid.New() - insertAuthUser(t, db, userID, "user"+string(rune('a'+i))+"@example.com") - } +func waitForQueueDrain(t *testing.T, ctx context.Context, db *testutils.Database) { + t.Helper() - store1 := NewStore(db.SqlcClient.Queries) - store2 := NewStore(db.SqlcClient.Queries) + require.EventuallyWithT(t, func(c *assert.CollectT) { + snapshot, err := loadQueueSnapshot(ctx, db) + if !assert.NoError(c, err) { + return + } - var claimed1, claimed2 atomic.Int32 + assert.Equal(c, 0, snapshot.Total) + assert.Equal(c, 0, snapshot.DeadLettered) + }, testEventuallyTimeout, testEventuallyTick) +} - ctx := t.Context() +func waitForPublicUsers(t *testing.T, ctx context.Context, db *testutils.Database, want map[uuid.UUID]userExpectation) { + t.Helper() - items1, err := store1.ClaimBatch(ctx, "worker-1", 2*time.Minute, 10) - require.NoError(t, err) - claimed1.Store(int32(len(items1))) + require.EventuallyWithT(t, func(c *assert.CollectT) { + got, err := loadPublicUsers(ctx, db) + if !assert.NoError(c, err) { + return + } - items2, err := store2.ClaimBatch(ctx, "worker-2", 2*time.Minute, 10) - require.NoError(t, err) - claimed2.Store(int32(len(items2))) + var gotExisting int + var wantExisting int - total := claimed1.Load() + claimed2.Load() - assert.Equal(t, int32(10), total, "all items should be claimed exactly once across both workers") + for userID, expectation := range want { + email, ok := got[userID] + if ok { + gotExisting++ + } + if expectation.Exists { + wantExisting++ + } - ids := make(map[int64]bool) - for _, item := range items1 { - ids[item.ID] = true - } - for _, item := range items2 { - assert.False(t, ids[item.ID], "item %d claimed by both workers", item.ID) - } + if !assert.Equalf(c, expectation.Exists, ok, "public.users presence for %s", userID) { + continue + } + if expectation.Exists { + assert.Equalf(c, expectation.Email, email, "public.users email for %s", userID) + } + } + + assert.Equal(c, wantExisting, gotExisting) + }, testEventuallyTimeout, testEventuallyTick) } From 461d2ecaadce1b56a51939c17b405005309f671f Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Tue, 31 Mar 2026 14:20:01 -0700 Subject: [PATCH 08/92] chore: change dashboard-api env variable management --- .env.gcp.template | 3 + .../job-dashboard-api/jobs/dashboard-api.hcl | 5 +- iac/modules/job-dashboard-api/main.tf | 6 +- iac/modules/job-dashboard-api/variables.tf | 5 +- iac/provider-gcp/Makefile | 1 + iac/provider-gcp/api.tf | 17 ------ iac/provider-gcp/main.tf | 56 +++++++++---------- iac/provider-gcp/nomad/main.tf | 6 +- iac/provider-gcp/nomad/variables.tf | 9 +-- iac/provider-gcp/variables.tf | 5 ++ 10 files changed, 55 insertions(+), 58 deletions(-) diff --git a/.env.gcp.template b/.env.gcp.template index 6c5bced521..6f72d2d77c 100644 --- a/.env.gcp.template +++ b/.env.gcp.template @@ -77,6 +77,9 @@ CLICKHOUSE_CLUSTER_SIZE=1 # Dashboard API instance count (default: 0) DASHBOARD_API_COUNT= +# Additional dashboard-api env vars passed directly to the Nomad job (default: {}) +# Example: '{"SUPABASE_AUTH_USER_SYNC_ENABLED":"true"}' +DASHBOARD_API_ENV_VARS= # Filestore cache for builds shared across cluster (default:false) FILESTORE_CACHE_ENABLED= diff --git a/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl b/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl index 7bdaca2f24..86a72cab50 100644 --- a/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl +++ b/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl @@ -80,9 +80,12 @@ job "dashboard-api" { AUTH_DB_READ_REPLICA_CONNECTION_STRING = "${auth_db_read_replica_connection_string}" CLICKHOUSE_CONNECTION_STRING = "${clickhouse_connection_string}" SUPABASE_JWT_SECRETS = "${supabase_jwt_secrets}" - SUPABASE_AUTH_USER_SYNC_ENABLED = "${supabase_auth_user_sync_enabled}" OTEL_COLLECTOR_GRPC_ENDPOINT = "${otel_collector_grpc_endpoint}" LOGS_COLLECTOR_ADDRESS = "${logs_collector_address}" + + %{ for key, val in env } + ${ key } = "${ val }" + %{ endfor } } config { diff --git a/iac/modules/job-dashboard-api/main.tf b/iac/modules/job-dashboard-api/main.tf index 482ae19d9c..aa577bfd2f 100644 --- a/iac/modules/job-dashboard-api/main.tf +++ b/iac/modules/job-dashboard-api/main.tf @@ -1,3 +1,7 @@ +locals { + env = { for key, value in var.env : key => value if value != null && value != "" } +} + resource "nomad_job" "dashboard_api" { jobspec = templatefile("${path.module}/jobs/dashboard-api.hcl", { update_stanza = var.update_stanza @@ -15,7 +19,7 @@ resource "nomad_job" "dashboard_api" { auth_db_read_replica_connection_string = var.auth_db_read_replica_connection_string clickhouse_connection_string = var.clickhouse_connection_string supabase_jwt_secrets = var.supabase_jwt_secrets - supabase_auth_user_sync_enabled = var.supabase_auth_user_sync_enabled + env = local.env subdomain = "dashboard-api" diff --git a/iac/modules/job-dashboard-api/variables.tf b/iac/modules/job-dashboard-api/variables.tf index 69d16a0419..625da7c82b 100644 --- a/iac/modules/job-dashboard-api/variables.tf +++ b/iac/modules/job-dashboard-api/variables.tf @@ -44,8 +44,9 @@ variable "supabase_jwt_secrets" { sensitive = true } -variable "supabase_auth_user_sync_enabled" { - type = string +variable "env" { + type = map(string) + default = {} } variable "otel_collector_grpc_port" { diff --git a/iac/provider-gcp/Makefile b/iac/provider-gcp/Makefile index f1a44e7975..a8f2db309c 100644 --- a/iac/provider-gcp/Makefile +++ b/iac/provider-gcp/Makefile @@ -75,6 +75,7 @@ tf_vars := \ $(call tfvar, LOKI_BOOT_DISK_TYPE) \ $(call tfvar, LOKI_USE_V13_SCHEMA_FROM) \ $(call tfvar, DASHBOARD_API_COUNT) \ + $(call tfvar, DASHBOARD_API_ENV_VARS) \ $(call tfvar, DEFAULT_PERSISTENT_VOLUME_TYPE) \ $(call tfvar, PERSISTENT_VOLUME_TYPES) \ $(call tfvar, DB_MAX_OPEN_CONNECTIONS) \ diff --git a/iac/provider-gcp/api.tf b/iac/provider-gcp/api.tf index 736b322090..c8b5586c96 100644 --- a/iac/provider-gcp/api.tf +++ b/iac/provider-gcp/api.tf @@ -44,23 +44,6 @@ resource "google_secret_manager_secret_version" "auth_db_connection_string" { } } -resource "google_secret_manager_secret" "dashboard_api_supabase_auth_user_sync_enabled" { - secret_id = "${var.prefix}dashboard-api-supabase-auth-user-sync-enabled" - - replication { - auto {} - } -} - -resource "google_secret_manager_secret_version" "dashboard_api_supabase_auth_user_sync_enabled" { - secret = google_secret_manager_secret.dashboard_api_supabase_auth_user_sync_enabled.name - secret_data = "false" - - lifecycle { - ignore_changes = [secret_data] - } -} - resource "random_password" "api_secret" { length = 32 special = false diff --git a/iac/provider-gcp/main.tf b/iac/provider-gcp/main.tf index 32da66604f..90294e554b 100644 --- a/iac/provider-gcp/main.tf +++ b/iac/provider-gcp/main.tf @@ -213,33 +213,32 @@ module "nomad" { additional_traefik_arguments = var.additional_traefik_arguments # API - api_server_count = var.api_server_count - api_resources_cpu_count = var.api_resources_cpu_count - api_resources_memory_mb = var.api_resources_memory_mb - api_machine_count = var.api_cluster_size - api_node_pool = var.api_node_pool - api_port = var.api_port - environment = var.environment - google_service_account_key = module.init.google_service_account_key - api_secret = random_password.api_secret.result - custom_envs_repository_name = google_artifact_registry_repository.custom_environments_repository.name - postgres_connection_string_secret_name = module.init.postgres_connection_string_secret_name - auth_db_connection_string_secret_version = google_secret_manager_secret_version.auth_db_connection_string - postgres_read_replica_connection_string_secret_version = google_secret_manager_secret_version.postgres_read_replica_connection_string - supabase_jwt_secrets_secret_name = module.init.supabase_jwt_secret_name - dashboard_api_supabase_auth_user_sync_enabled_secret_version = google_secret_manager_secret_version.dashboard_api_supabase_auth_user_sync_enabled - posthog_api_key_secret_name = module.init.posthog_api_key_secret_name - analytics_collector_host_secret_name = module.init.analytics_collector_host_secret_name - analytics_collector_api_token_secret_name = module.init.analytics_collector_api_token_secret_name - api_admin_token = random_password.api_admin_secret.result - redis_cluster_url_secret_version = module.init.redis_cluster_url_secret_version - redis_tls_ca_base64_secret_version = module.init.redis_tls_ca_base64_secret_version - sandbox_access_token_hash_seed = random_password.sandbox_access_token_hash_seed.result - sandbox_storage_backend = var.sandbox_storage_backend - db_max_open_connections = var.db_max_open_connections - db_min_idle_connections = var.db_min_idle_connections - auth_db_max_open_connections = var.auth_db_max_open_connections - auth_db_min_idle_connections = var.auth_db_min_idle_connections + api_server_count = var.api_server_count + api_resources_cpu_count = var.api_resources_cpu_count + api_resources_memory_mb = var.api_resources_memory_mb + api_machine_count = var.api_cluster_size + api_node_pool = var.api_node_pool + api_port = var.api_port + environment = var.environment + google_service_account_key = module.init.google_service_account_key + api_secret = random_password.api_secret.result + custom_envs_repository_name = google_artifact_registry_repository.custom_environments_repository.name + postgres_connection_string_secret_name = module.init.postgres_connection_string_secret_name + auth_db_connection_string_secret_version = google_secret_manager_secret_version.auth_db_connection_string + postgres_read_replica_connection_string_secret_version = google_secret_manager_secret_version.postgres_read_replica_connection_string + supabase_jwt_secrets_secret_name = module.init.supabase_jwt_secret_name + posthog_api_key_secret_name = module.init.posthog_api_key_secret_name + analytics_collector_host_secret_name = module.init.analytics_collector_host_secret_name + analytics_collector_api_token_secret_name = module.init.analytics_collector_api_token_secret_name + api_admin_token = random_password.api_admin_secret.result + redis_cluster_url_secret_version = module.init.redis_cluster_url_secret_version + redis_tls_ca_base64_secret_version = module.init.redis_tls_ca_base64_secret_version + sandbox_access_token_hash_seed = random_password.sandbox_access_token_hash_seed.result + sandbox_storage_backend = var.sandbox_storage_backend + db_max_open_connections = var.db_max_open_connections + db_min_idle_connections = var.db_min_idle_connections + auth_db_max_open_connections = var.auth_db_max_open_connections + auth_db_min_idle_connections = var.auth_db_min_idle_connections # Click Proxy client_proxy_count = var.client_proxy_count @@ -266,7 +265,8 @@ module "nomad" { otel_collector_resources_cpu_count = var.otel_collector_resources_cpu_count # Dashboard API - dashboard_api_count = var.dashboard_api_count + dashboard_api_count = var.dashboard_api_count + dashboard_api_env_vars = var.dashboard_api_env_vars # Docker reverse proxy docker_reverse_proxy_port = var.docker_reverse_proxy_port diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index 4a9aae6220..7d05bf3dec 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -22,10 +22,6 @@ data "google_secret_manager_secret_version" "supabase_jwt_secrets" { secret = var.supabase_jwt_secrets_secret_name } -data "google_secret_manager_secret_version" "dashboard_api_supabase_auth_user_sync_enabled" { - secret = var.dashboard_api_supabase_auth_user_sync_enabled_secret_version.secret -} - data "google_secret_manager_secret_version" "posthog_api_key" { secret = var.posthog_api_key_secret_name } @@ -143,7 +139,7 @@ module "dashboard_api" { auth_db_read_replica_connection_string = trimspace(data.google_secret_manager_secret_version.postgres_read_replica_connection_string.secret_data) clickhouse_connection_string = local.clickhouse_connection_string supabase_jwt_secrets = trimspace(data.google_secret_manager_secret_version.supabase_jwt_secrets.secret_data) - supabase_auth_user_sync_enabled = trimspace(data.google_secret_manager_secret_version.dashboard_api_supabase_auth_user_sync_enabled.secret_data) + env = var.dashboard_api_env_vars otel_collector_grpc_port = var.otel_collector_grpc_port logs_proxy_port = var.logs_proxy_port diff --git a/iac/provider-gcp/nomad/variables.tf b/iac/provider-gcp/nomad/variables.tf index 2bbbb9494f..64e9f7dd75 100644 --- a/iac/provider-gcp/nomad/variables.tf +++ b/iac/provider-gcp/nomad/variables.tf @@ -187,10 +187,6 @@ variable "supabase_jwt_secrets_secret_name" { type = string } -variable "dashboard_api_supabase_auth_user_sync_enabled_secret_version" { - type = any -} - variable "client_proxy_count" { type = number } @@ -461,6 +457,11 @@ variable "dashboard_api_count" { default = 0 } +variable "dashboard_api_env_vars" { + type = map(string) + default = {} +} + variable "volume_token_issuer" { type = string } diff --git a/iac/provider-gcp/variables.tf b/iac/provider-gcp/variables.tf index b8aee2ffaf..f4d8cfbf48 100644 --- a/iac/provider-gcp/variables.tf +++ b/iac/provider-gcp/variables.tf @@ -228,6 +228,11 @@ variable "dashboard_api_count" { default = 0 } +variable "dashboard_api_env_vars" { + type = map(string) + default = {} +} + variable "docker_reverse_proxy_port" { type = object({ name = string From ce0cbd1c8e578e2c8e25538e6dfaabd63c057424 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Tue, 31 Mar 2026 16:04:46 -0700 Subject: [PATCH 09/92] refactor(sync): update user sync logic to utilize new database structure - Replaced the previous `Store` implementation with a new structure that integrates both authentication and main database queries. - Updated the `Runner` and `NewRunner` functions to accommodate the new database client structure. - Removed obsolete SQL queries and migration files related to the `user_sync_queue` table. - Enhanced the test suite to reflect changes in the runner's initialization and database interactions. --- .../internal/supabaseauthusersync/runner.go | 16 +++- .../supabaseauthusersync/runner_test.go | 8 +- .../internal/supabaseauthusersync/store.go | 29 ++++--- packages/dashboard-api/main.go | 4 +- ...shboard_supabase_auth_user_sync_queue.sql} | 79 +++++++++++-------- packages/db/{ => pkg/auth}/queries/ack.sql.go | 2 +- .../{ => pkg/auth}/queries/claim_batch.sql.go | 2 +- .../{ => pkg/auth}/queries/dead_letter.sql.go | 2 +- .../auth}/queries/get_auth_user.sql.go | 2 +- .../db/{ => pkg/auth}/queries/retry.sql.go | 2 +- .../supabase_auth_user_sync/ack.sql | 0 .../supabase_auth_user_sync/claim_batch.sql | 0 .../supabase_auth_user_sync/dead_letter.sql | 0 .../supabase_auth_user_sync/get_auth_user.sql | 0 .../supabase_auth_user_sync/retry.sql | 0 packages/db/pkg/testutils/db.go | 19 +++-- packages/db/queries/models.go | 13 --- .../users}/delete_public_user.sql | 0 .../users}/upsert_public_user.sql | 0 packages/db/sqlc.yaml | 1 + 20 files changed, 108 insertions(+), 71 deletions(-) rename packages/db/{migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql => pkg/auth/migrations/20260328000001_dashboard_supabase_auth_user_sync_queue.sql} (55%) rename packages/db/{ => pkg/auth}/queries/ack.sql.go (95%) rename packages/db/{ => pkg/auth}/queries/claim_batch.sql.go (98%) rename packages/db/{ => pkg/auth}/queries/dead_letter.sql.go (97%) rename packages/db/{ => pkg/auth}/queries/get_auth_user.sql.go (95%) rename packages/db/{ => pkg/auth}/queries/retry.sql.go (97%) rename packages/db/pkg/{dashboard => auth}/sql_queries/supabase_auth_user_sync/ack.sql (100%) rename packages/db/pkg/{dashboard => auth}/sql_queries/supabase_auth_user_sync/claim_batch.sql (100%) rename packages/db/pkg/{dashboard => auth}/sql_queries/supabase_auth_user_sync/dead_letter.sql (100%) rename packages/db/pkg/{dashboard => auth}/sql_queries/supabase_auth_user_sync/get_auth_user.sql (100%) rename packages/db/pkg/{dashboard => auth}/sql_queries/supabase_auth_user_sync/retry.sql (100%) rename packages/db/{pkg/dashboard/sql_queries/supabase_auth_user_sync => queries/users}/delete_public_user.sql (100%) rename packages/db/{pkg/dashboard/sql_queries/supabase_auth_user_sync => queries/users}/upsert_public_user.sql (100%) diff --git a/packages/dashboard-api/internal/supabaseauthusersync/runner.go b/packages/dashboard-api/internal/supabaseauthusersync/runner.go index 1acd35d480..fd7cfa3477 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/runner.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/runner.go @@ -6,19 +6,31 @@ import ( "go.uber.org/zap" + sqlcdb "github.com/e2b-dev/infra/packages/db/client" + authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) +type runnerStore interface { + ClaimBatch(ctx context.Context, lockOwner string, lockTimeout time.Duration, batchSize int32) ([]QueueItem, error) +} + +type workerStore interface { + runnerStore + processorStore +} + type Runner struct { cfg Config - store *Store + store runnerStore processor *Processor lockOwner string l logger.Logger } -func NewRunner(cfg Config, store *Store, lockOwner string, l logger.Logger) *Runner { +func NewRunner(cfg Config, authDB *authdb.Client, mainDB *sqlcdb.Client, lockOwner string, l logger.Logger) *Runner { workerLogger := l.With(logger.WithServiceInstanceID(lockOwner)) + store := NewStore(authDB, mainDB) return &Runner{ cfg: cfg, diff --git a/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go b/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go index 1429c2a605..064d35c77e 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go @@ -206,7 +206,13 @@ func startRunnerProcess(t *testing.T, db *testutils.Database, cfg Config, lockOw ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) - runner := NewRunner(cfg, NewStore(db.SqlcClient.Queries), lockOwner, logger.NewNopLogger()) + runner := NewRunner( + cfg, + db.AuthDb, + db.SqlcClient, + lockOwner, + logger.NewNopLogger(), + ) go func() { done <- runner.Run(ctx) diff --git a/packages/dashboard-api/internal/supabaseauthusersync/store.go b/packages/dashboard-api/internal/supabaseauthusersync/store.go index 019e4bf6c6..9eee64ca93 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/store.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/store.go @@ -7,6 +7,9 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" + sqlcdb "github.com/e2b-dev/infra/packages/db/client" + authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" + authqueries "github.com/e2b-dev/infra/packages/db/pkg/auth/queries" "github.com/e2b-dev/infra/packages/db/queries" ) @@ -24,15 +27,21 @@ type AuthUser struct { } type Store struct { - q *queries.Queries + authQueries *authqueries.Queries + mainQueries *queries.Queries } -func NewStore(q *queries.Queries) *Store { - return &Store{q: q} +var _ workerStore = (*Store)(nil) + +func NewStore(authDB *authdb.Client, mainDB *sqlcdb.Client) *Store { + return &Store{ + authQueries: authDB.Write, + mainQueries: mainDB.Queries, + } } func (s *Store) ClaimBatch(ctx context.Context, lockOwner string, lockTimeout time.Duration, batchSize int32) ([]QueueItem, error) { - rows, err := s.q.ClaimUserSyncQueueBatch(ctx, queries.ClaimUserSyncQueueBatchParams{ + rows, err := s.authQueries.ClaimUserSyncQueueBatch(ctx, authqueries.ClaimUserSyncQueueBatchParams{ LockOwner: lockOwner, LockTimeout: durationToInterval(lockTimeout), BatchSize: batchSize, @@ -56,11 +65,11 @@ func (s *Store) ClaimBatch(ctx context.Context, lockOwner string, lockTimeout ti } func (s *Store) Ack(ctx context.Context, id int64) error { - return s.q.AckUserSyncQueueItem(ctx, id) + return s.authQueries.AckUserSyncQueueItem(ctx, id) } func (s *Store) Retry(ctx context.Context, id int64, backoff time.Duration, lastError string) error { - return s.q.RetryUserSyncQueueItem(ctx, queries.RetryUserSyncQueueItemParams{ + return s.authQueries.RetryUserSyncQueueItem(ctx, authqueries.RetryUserSyncQueueItemParams{ ID: id, Backoff: durationToInterval(backoff), LastError: lastError, @@ -68,14 +77,14 @@ func (s *Store) Retry(ctx context.Context, id int64, backoff time.Duration, last } func (s *Store) DeadLetter(ctx context.Context, id int64, lastError string) error { - return s.q.DeadLetterUserSyncQueueItem(ctx, queries.DeadLetterUserSyncQueueItemParams{ + return s.authQueries.DeadLetterUserSyncQueueItem(ctx, authqueries.DeadLetterUserSyncQueueItemParams{ ID: id, LastError: lastError, }) } func (s *Store) GetAuthUser(ctx context.Context, userID uuid.UUID) (*AuthUser, error) { - row, err := s.q.GetAuthUserByID(ctx, userID) + row, err := s.authQueries.GetAuthUserByID(ctx, userID) if err != nil { return nil, err } @@ -84,14 +93,14 @@ func (s *Store) GetAuthUser(ctx context.Context, userID uuid.UUID) (*AuthUser, e } func (s *Store) UpsertPublicUser(ctx context.Context, id uuid.UUID, email string) error { - return s.q.UpsertPublicUser(ctx, queries.UpsertPublicUserParams{ + return s.mainQueries.UpsertPublicUser(ctx, queries.UpsertPublicUserParams{ ID: id, Email: email, }) } func (s *Store) DeletePublicUser(ctx context.Context, id uuid.UUID) error { - return s.q.DeletePublicUser(ctx, id) + return s.mainQueries.DeletePublicUser(ctx, id) } func durationToInterval(d time.Duration) pgtype.Interval { diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 22bca86bf3..f1a75eae72 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -232,12 +232,12 @@ func run() int { if config.SupabaseAuthUserSyncEnabled { workerLogger := l.With(zap.String("worker", "supabase_auth_user_sync")) - syncStore := supabaseauthusersync.NewStore(db.Queries) syncConfig := supabaseauthusersync.DefaultConfig() syncConfig.Enabled = true syncRunner := supabaseauthusersync.NewRunner( syncConfig, - syncStore, + authDB, + db, serviceInstanceID, workerLogger, ) diff --git a/packages/db/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql b/packages/db/pkg/auth/migrations/20260328000001_dashboard_supabase_auth_user_sync_queue.sql similarity index 55% rename from packages/db/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql rename to packages/db/pkg/auth/migrations/20260328000001_dashboard_supabase_auth_user_sync_queue.sql index a250681905..e300948803 100644 --- a/packages/db/migrations/20260328000000_dashboard_supabase_auth_user_sync_queue.sql +++ b/packages/db/pkg/auth/migrations/20260328000001_dashboard_supabase_auth_user_sync_queue.sql @@ -1,16 +1,15 @@ -- +goose Up -- +goose StatementBegin - CREATE TABLE public.user_sync_queue ( - id BIGSERIAL PRIMARY KEY, - user_id UUID NOT NULL, - operation TEXT NOT NULL CHECK (operation IN ('upsert', 'delete')), - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), - locked_at TIMESTAMPTZ NULL, - lock_owner TEXT NULL, - attempt_count INT NOT NULL DEFAULT 0, - last_error TEXT NULL, + id BIGSERIAL PRIMARY KEY, + user_id UUID NOT NULL, + operation TEXT NOT NULL CHECK (operation IN ('upsert', 'delete')), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), + locked_at TIMESTAMPTZ NULL, + lock_owner TEXT NULL, + attempt_count INT NOT NULL DEFAULT 0, + last_error TEXT NULL, dead_lettered_at TIMESTAMPTZ NULL ); @@ -33,14 +32,10 @@ CREATE POLICY "Allow to create a user sync queue item" TO trigger_user WITH CHECK (TRUE); --- Keep direct insert-sync and also enqueue CREATE OR REPLACE FUNCTION public.sync_insert_auth_users_to_public_users_trigger() RETURNS TRIGGER LANGUAGE plpgsql AS $func$ BEGIN - INSERT INTO public.users (id, email) - VALUES (NEW.id, NEW.email); - INSERT INTO public.user_sync_queue (user_id, operation) VALUES (NEW.id, 'upsert'); @@ -48,20 +43,10 @@ BEGIN END; $func$ SECURITY DEFINER SET search_path = public; --- Keep direct update-sync and also enqueue when mirrored fields change CREATE OR REPLACE FUNCTION public.sync_update_auth_users_to_public_users_trigger() RETURNS TRIGGER LANGUAGE plpgsql AS $func$ BEGIN - UPDATE public.users - SET email = NEW.email, - updated_at = now() - WHERE id = NEW.id; - - IF NOT FOUND THEN - RAISE EXCEPTION 'User with id % does not exist in public.users', NEW.id; - END IF; - IF OLD.email IS DISTINCT FROM NEW.email THEN INSERT INTO public.user_sync_queue (user_id, operation) VALUES (NEW.id, 'upsert'); @@ -71,13 +56,10 @@ BEGIN END; $func$ SECURITY DEFINER SET search_path = public; --- Keep direct delete-sync and also enqueue CREATE OR REPLACE FUNCTION public.sync_delete_auth_users_to_public_users_trigger() RETURNS TRIGGER LANGUAGE plpgsql AS $func$ BEGIN - DELETE FROM public.users WHERE id = OLD.id; - INSERT INTO public.user_sync_queue (user_id, operation) VALUES (OLD.id, 'delete'); @@ -85,23 +67,43 @@ BEGIN END; $func$ SECURITY DEFINER SET search_path = public; +ALTER FUNCTION public.sync_insert_auth_users_to_public_users_trigger() OWNER TO trigger_user; +ALTER FUNCTION public.sync_update_auth_users_to_public_users_trigger() OWNER TO trigger_user; +ALTER FUNCTION public.sync_delete_auth_users_to_public_users_trigger() OWNER TO trigger_user; + +DROP TRIGGER IF EXISTS sync_inserts_to_public_users ON auth.users; +CREATE TRIGGER sync_inserts_to_public_users + AFTER INSERT ON auth.users + FOR EACH ROW EXECUTE FUNCTION public.sync_insert_auth_users_to_public_users_trigger(); + +DROP TRIGGER IF EXISTS sync_updates_to_public_users ON auth.users; +CREATE TRIGGER sync_updates_to_public_users + AFTER UPDATE ON auth.users + FOR EACH ROW EXECUTE FUNCTION public.sync_update_auth_users_to_public_users_trigger(); + +DROP TRIGGER IF EXISTS sync_deletes_to_public_users ON auth.users; +CREATE TRIGGER sync_deletes_to_public_users + AFTER DELETE ON auth.users + FOR EACH ROW EXECUTE FUNCTION public.sync_delete_auth_users_to_public_users_trigger(); -- +goose StatementEnd -- +goose Down -- +goose StatementBegin +DROP TRIGGER IF EXISTS sync_inserts_to_public_users ON auth.users; +DROP TRIGGER IF EXISTS sync_updates_to_public_users ON auth.users; +DROP TRIGGER IF EXISTS sync_deletes_to_public_users ON auth.users; --- Restore direct insert-sync CREATE OR REPLACE FUNCTION public.sync_insert_auth_users_to_public_users_trigger() RETURNS TRIGGER LANGUAGE plpgsql AS $func$ BEGIN INSERT INTO public.users (id, email) VALUES (NEW.id, NEW.email); + RETURN NEW; END; $func$ SECURITY DEFINER SET search_path = public; --- Restore direct update-sync CREATE OR REPLACE FUNCTION public.sync_update_auth_users_to_public_users_trigger() RETURNS TRIGGER LANGUAGE plpgsql AS $func$ @@ -119,21 +121,36 @@ BEGIN END; $func$ SECURITY DEFINER SET search_path = public; --- Restore direct delete-sync CREATE OR REPLACE FUNCTION public.sync_delete_auth_users_to_public_users_trigger() RETURNS TRIGGER LANGUAGE plpgsql AS $func$ BEGIN DELETE FROM public.users WHERE id = OLD.id; + RETURN OLD; END; $func$ SECURITY DEFINER SET search_path = public; +ALTER FUNCTION public.sync_insert_auth_users_to_public_users_trigger() OWNER TO trigger_user; +ALTER FUNCTION public.sync_update_auth_users_to_public_users_trigger() OWNER TO trigger_user; +ALTER FUNCTION public.sync_delete_auth_users_to_public_users_trigger() OWNER TO trigger_user; + +CREATE TRIGGER sync_inserts_to_public_users + AFTER INSERT ON auth.users + FOR EACH ROW EXECUTE FUNCTION public.sync_insert_auth_users_to_public_users_trigger(); + +CREATE TRIGGER sync_updates_to_public_users + AFTER UPDATE ON auth.users + FOR EACH ROW EXECUTE FUNCTION public.sync_update_auth_users_to_public_users_trigger(); + +CREATE TRIGGER sync_deletes_to_public_users + AFTER DELETE ON auth.users + FOR EACH ROW EXECUTE FUNCTION public.sync_delete_auth_users_to_public_users_trigger(); + REVOKE INSERT ON public.user_sync_queue FROM trigger_user; REVOKE USAGE, SELECT ON SEQUENCE public.user_sync_queue_id_seq FROM trigger_user; DROP POLICY IF EXISTS "Allow to create a user sync queue item" ON public.user_sync_queue; DROP TABLE public.user_sync_queue; - -- +goose StatementEnd diff --git a/packages/db/queries/ack.sql.go b/packages/db/pkg/auth/queries/ack.sql.go similarity index 95% rename from packages/db/queries/ack.sql.go rename to packages/db/pkg/auth/queries/ack.sql.go index b55274a6c9..2102f263db 100644 --- a/packages/db/queries/ack.sql.go +++ b/packages/db/pkg/auth/queries/ack.sql.go @@ -3,7 +3,7 @@ // sqlc v1.29.0 // source: ack.sql -package queries +package authqueries import ( "context" diff --git a/packages/db/queries/claim_batch.sql.go b/packages/db/pkg/auth/queries/claim_batch.sql.go similarity index 98% rename from packages/db/queries/claim_batch.sql.go rename to packages/db/pkg/auth/queries/claim_batch.sql.go index b36cfb0a39..6c1555549f 100644 --- a/packages/db/queries/claim_batch.sql.go +++ b/packages/db/pkg/auth/queries/claim_batch.sql.go @@ -3,7 +3,7 @@ // sqlc v1.29.0 // source: claim_batch.sql -package queries +package authqueries import ( "context" diff --git a/packages/db/queries/dead_letter.sql.go b/packages/db/pkg/auth/queries/dead_letter.sql.go similarity index 97% rename from packages/db/queries/dead_letter.sql.go rename to packages/db/pkg/auth/queries/dead_letter.sql.go index b9b7941d8a..de2bcb3e80 100644 --- a/packages/db/queries/dead_letter.sql.go +++ b/packages/db/pkg/auth/queries/dead_letter.sql.go @@ -3,7 +3,7 @@ // sqlc v1.29.0 // source: dead_letter.sql -package queries +package authqueries import ( "context" diff --git a/packages/db/queries/get_auth_user.sql.go b/packages/db/pkg/auth/queries/get_auth_user.sql.go similarity index 95% rename from packages/db/queries/get_auth_user.sql.go rename to packages/db/pkg/auth/queries/get_auth_user.sql.go index 4b7c341df2..4c34da8bdf 100644 --- a/packages/db/queries/get_auth_user.sql.go +++ b/packages/db/pkg/auth/queries/get_auth_user.sql.go @@ -3,7 +3,7 @@ // sqlc v1.29.0 // source: get_auth_user.sql -package queries +package authqueries import ( "context" diff --git a/packages/db/queries/retry.sql.go b/packages/db/pkg/auth/queries/retry.sql.go similarity index 97% rename from packages/db/queries/retry.sql.go rename to packages/db/pkg/auth/queries/retry.sql.go index 941c96ae18..297fbd7c76 100644 --- a/packages/db/queries/retry.sql.go +++ b/packages/db/pkg/auth/queries/retry.sql.go @@ -3,7 +3,7 @@ // sqlc v1.29.0 // source: retry.sql -package queries +package authqueries import ( "context" diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/ack.sql b/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/ack.sql similarity index 100% rename from packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/ack.sql rename to packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/ack.sql diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/claim_batch.sql b/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/claim_batch.sql similarity index 100% rename from packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/claim_batch.sql rename to packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/claim_batch.sql diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/dead_letter.sql b/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/dead_letter.sql similarity index 100% rename from packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/dead_letter.sql rename to packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/dead_letter.sql diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/get_auth_user.sql b/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/get_auth_user.sql similarity index 100% rename from packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/get_auth_user.sql rename to packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/get_auth_user.sql diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/retry.sql b/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/retry.sql similarity index 100% rename from packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/retry.sql rename to packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/retry.sql diff --git a/packages/db/pkg/testutils/db.go b/packages/db/pkg/testutils/db.go index ecdf856488..86708c18c4 100644 --- a/packages/db/pkg/testutils/db.go +++ b/packages/db/pkg/testutils/db.go @@ -123,12 +123,17 @@ func runDatabaseMigrations(t *testing.T, connStr string) { }) // run the db migration - err = goose.RunWithOptionsContext( - t.Context(), - "up", - db, + for _, migrationsDir := range []string{ filepath.Join(repoRoot, "packages", "db", "migrations"), - nil, - ) - require.NoError(t, err) + filepath.Join(repoRoot, "packages", "db", "pkg", "auth", "migrations"), + } { + err = goose.RunWithOptionsContext( + t.Context(), + "up", + db, + migrationsDir, + nil, + ) + require.NoError(t, err) + } } diff --git a/packages/db/queries/models.go b/packages/db/queries/models.go index f3ec4451c9..6c960a7a59 100644 --- a/packages/db/queries/models.go +++ b/packages/db/queries/models.go @@ -226,19 +226,6 @@ type User struct { Email string } -type UserSyncQueue struct { - ID int64 - UserID uuid.UUID - Operation string - CreatedAt time.Time - NextAttemptAt time.Time - LockedAt *time.Time - LockOwner *string - AttemptCount int32 - LastError *string - DeadLetteredAt *time.Time -} - type UsersTeam struct { ID int64 UserID uuid.UUID diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/delete_public_user.sql b/packages/db/queries/users/delete_public_user.sql similarity index 100% rename from packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/delete_public_user.sql rename to packages/db/queries/users/delete_public_user.sql diff --git a/packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/upsert_public_user.sql b/packages/db/queries/users/upsert_public_user.sql similarity index 100% rename from packages/db/pkg/dashboard/sql_queries/supabase_auth_user_sync/upsert_public_user.sql rename to packages/db/queries/users/upsert_public_user.sql diff --git a/packages/db/sqlc.yaml b/packages/db/sqlc.yaml index 206c468907..4959602d19 100644 --- a/packages/db/sqlc.yaml +++ b/packages/db/sqlc.yaml @@ -63,6 +63,7 @@ sql: schema: - "migrations" - "schema" + - "pkg/auth/migrations" gen: go: emit_pointers_for_null_types: true From 4d622ceb0e262cb16e0fc394dde7b0f48ccdc41a Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Tue, 31 Mar 2026 16:11:56 -0700 Subject: [PATCH 10/92] fix: lint --- .../internal/supabaseauthusersync/logging.go | 4 +- .../supabaseauthusersync/processor.go | 2 +- .../supabaseauthusersync/processor_test.go | 11 +- .../internal/supabaseauthusersync/runner.go | 2 +- .../supabaseauthusersync/runner_test.go | 244 +++++++++--------- .../supabaseauthusersync/supervisor.go | 11 +- .../supabaseauthusersync/supervisor_test.go | 6 + 7 files changed, 153 insertions(+), 127 deletions(-) diff --git a/packages/dashboard-api/internal/supabaseauthusersync/logging.go b/packages/dashboard-api/internal/supabaseauthusersync/logging.go index c60d02a0c5..b5730330f4 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/logging.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/logging.go @@ -107,7 +107,7 @@ func (s *batchSummary) Add(result processResult) { } } -func (s batchSummary) Fields(totalDuration time.Duration) []zap.Field { +func (s *batchSummary) Fields(totalDuration time.Duration) []zap.Field { fields := []zap.Field{ zap.Int("queue_batch.claimed_count", s.ClaimedCount), zap.Int("queue_batch.acked_count", s.AckedCount), @@ -139,7 +139,7 @@ func (s batchSummary) Fields(totalDuration time.Duration) []zap.Field { return fields } -func (s batchSummary) Level() zapcore.Level { +func (s *batchSummary) Level() zapcore.Level { if s.AckFailedCount > 0 || s.RetryFailedCount > 0 || s.DeadLetteredCount > 0 || s.DeadLetterFailedCount > 0 { return zap.ErrorLevel } diff --git a/packages/dashboard-api/internal/supabaseauthusersync/processor.go b/packages/dashboard-api/internal/supabaseauthusersync/processor.go index 1d562c8288..e25d2a0c6c 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/processor.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/processor.go @@ -37,7 +37,7 @@ func NewProcessor(store processorStore, maxAttempts int32, l logger.Logger) *Pro } } -func (p *Processor) Process(ctx context.Context, item QueueItem) processResult { +func (p *Processor) process(ctx context.Context, item QueueItem) processResult { startedAt := time.Now() action, err := p.processOnce(ctx, item) result := processResult{ diff --git a/packages/dashboard-api/internal/supabaseauthusersync/processor_test.go b/packages/dashboard-api/internal/supabaseauthusersync/processor_test.go index 23072901fa..125222d5ee 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/processor_test.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/processor_test.go @@ -32,6 +32,7 @@ type fakeProcessorStore struct { func (s *fakeProcessorStore) Ack(_ context.Context, id int64) error { s.ackCalls = append(s.ackCalls, id) + return nil } @@ -41,6 +42,7 @@ func (s *fakeProcessorStore) Retry(_ context.Context, id int64, backoff time.Dur backoff: backoff, lastError: lastError, }) + return nil } @@ -49,6 +51,7 @@ func (s *fakeProcessorStore) DeadLetter(_ context.Context, id int64, lastError s id: id, lastError: lastError, }) + return nil } @@ -65,6 +68,8 @@ func (s *fakeProcessorStore) DeletePublicUser(_ context.Context, _ uuid.UUID) er } func TestProcessorProcessRetriesRecoveredPanic(t *testing.T) { + t.Parallel() + store := &fakeProcessorStore{ getAuthUserFn: func(context.Context, uuid.UUID) (*AuthUser, error) { panic("boom") @@ -78,7 +83,7 @@ func TestProcessorProcessRetriesRecoveredPanic(t *testing.T) { } require.NotPanics(t, func() { - processor.Process(context.Background(), item) + processor.process(context.Background(), item) }) require.Empty(t, store.ackCalls) require.Len(t, store.retryCalls, 1) @@ -87,6 +92,8 @@ func TestProcessorProcessRetriesRecoveredPanic(t *testing.T) { } func TestProcessorProcessDeadLettersRecoveredPanicAtMaxAttempts(t *testing.T) { + t.Parallel() + store := &fakeProcessorStore{ getAuthUserFn: func(context.Context, uuid.UUID) (*AuthUser, error) { panic("boom") @@ -100,7 +107,7 @@ func TestProcessorProcessDeadLettersRecoveredPanicAtMaxAttempts(t *testing.T) { } require.NotPanics(t, func() { - processor.Process(context.Background(), item) + processor.process(context.Background(), item) }) require.Empty(t, store.ackCalls) require.Empty(t, store.retryCalls) diff --git a/packages/dashboard-api/internal/supabaseauthusersync/runner.go b/packages/dashboard-api/internal/supabaseauthusersync/runner.go index fd7cfa3477..174c976a84 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/runner.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/runner.go @@ -86,7 +86,7 @@ func (r *Runner) poll(ctx context.Context) { summary := newBatchSummary(items, claimedAt) for _, item := range items { - summary.Add(r.processor.Process(ctx, item)) + summary.Add(r.processor.process(ctx, item)) } r.l.Log(ctx, summary.Level(), "processed supabase auth sync queue batch", summary.Fields(time.Since(claimedAt))...) diff --git a/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go b/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go index 064d35c77e..a99e4ad3b2 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go @@ -41,153 +41,165 @@ type queueSnapshot struct { } func TestSupabaseAuthUserSyncRunner_EndToEnd(t *testing.T) { + t.Parallel() + db := testutils.SetupDatabase(t) - t.Run("repairs_insert_update_delete_drift", func(t *testing.T) { - ctx := t.Context() - userID := uuid.New() - initialEmail := fmt.Sprintf("auth-sync-%s-initial@example.com", userID.String()[:8]) - updatedEmail := fmt.Sprintf("auth-sync-%s-updated@example.com", userID.String()[:8]) + runRepairsInsertUpdateDeleteDrift(t, db) + runReclaimsStaleQueueLocks(t, db) + runDrainsBurstBacklogWithMultipleRunners(t, db) +} - insertAuthUser(t, ctx, db, userID, initialEmail) - deletePublicUser(t, ctx, db, userID) - assertQueueBacklog(t, ctx, db, 1) - - insertRunner := startRunnerProcess(t, db, newTestRunnerConfig(4), "repair-insert") - t.Cleanup(func() { - insertRunner.Stop(t) - }) - waitForPublicUsers(t, ctx, db, map[uuid.UUID]userExpectation{ - userID: { - Email: initialEmail, - Exists: true, - }, - }) - waitForQueueDrain(t, ctx, db) +func runRepairsInsertUpdateDeleteDrift(t *testing.T, db *testutils.Database) { + t.Helper() + + ctx := t.Context() + userID := uuid.New() + initialEmail := fmt.Sprintf("auth-sync-%s-initial@example.com", userID.String()[:8]) + updatedEmail := fmt.Sprintf("auth-sync-%s-updated@example.com", userID.String()[:8]) + + insertAuthUser(t, ctx, db, userID, initialEmail) + deletePublicUser(t, ctx, db, userID) + assertQueueBacklog(t, ctx, db, 1) + + insertRunner := startRunnerProcess(t, db, newTestRunnerConfig(4), "repair-insert") + t.Cleanup(func() { insertRunner.Stop(t) + }) + waitForPublicUsers(t, ctx, db, map[uuid.UUID]userExpectation{ + userID: { + Email: initialEmail, + Exists: true, + }, + }) + waitForQueueDrain(t, ctx, db) + insertRunner.Stop(t) + + updateAuthUserEmail(t, ctx, db, userID, updatedEmail) + setPublicUserEmail(t, ctx, db, userID, "stale@example.com") + assertQueueBacklog(t, ctx, db, 1) - updateAuthUserEmail(t, ctx, db, userID, updatedEmail) - setPublicUserEmail(t, ctx, db, userID, "stale@example.com") - assertQueueBacklog(t, ctx, db, 1) - - updateRunner := startRunnerProcess(t, db, newTestRunnerConfig(4), "repair-update") - t.Cleanup(func() { - updateRunner.Stop(t) - }) - waitForPublicUsers(t, ctx, db, map[uuid.UUID]userExpectation{ - userID: { - Email: updatedEmail, - Exists: true, - }, - }) - waitForQueueDrain(t, ctx, db) + updateRunner := startRunnerProcess(t, db, newTestRunnerConfig(4), "repair-update") + t.Cleanup(func() { updateRunner.Stop(t) + }) + waitForPublicUsers(t, ctx, db, map[uuid.UUID]userExpectation{ + userID: { + Email: updatedEmail, + Exists: true, + }, + }) + waitForQueueDrain(t, ctx, db) + updateRunner.Stop(t) - deleteAuthUser(t, ctx, db, userID) - insertPublicUser(t, ctx, db, userID, "ghost@example.com") - assertQueueBacklog(t, ctx, db, 1) - - deleteRunner := startRunnerProcess(t, db, newTestRunnerConfig(4), "repair-delete") - t.Cleanup(func() { - deleteRunner.Stop(t) - }) - waitForPublicUsers(t, ctx, db, map[uuid.UUID]userExpectation{ - userID: { - Exists: false, - }, - }) - waitForQueueDrain(t, ctx, db) + deleteAuthUser(t, ctx, db, userID) + insertPublicUser(t, ctx, db, userID, "ghost@example.com") + assertQueueBacklog(t, ctx, db, 1) + + deleteRunner := startRunnerProcess(t, db, newTestRunnerConfig(4), "repair-delete") + t.Cleanup(func() { deleteRunner.Stop(t) }) + waitForPublicUsers(t, ctx, db, map[uuid.UUID]userExpectation{ + userID: { + Exists: false, + }, + }) + waitForQueueDrain(t, ctx, db) + deleteRunner.Stop(t) +} - t.Run("reclaims_stale_queue_locks", func(t *testing.T) { - ctx := t.Context() - userID := uuid.New() - email := fmt.Sprintf("auth-sync-%s-locked@example.com", userID.String()[:8]) - - insertAuthUser(t, ctx, db, userID, email) - deletePublicUser(t, ctx, db, userID) - lockQueueItems(t, ctx, db, userID, time.Now().Add(-time.Minute), "stale-worker") - assertQueueBacklog(t, ctx, db, 1) - - runner := startRunnerProcess(t, db, newTestRunnerConfig(2), "lock-reclaimer") - t.Cleanup(func() { - runner.Stop(t) - }) - - waitForPublicUsers(t, ctx, db, map[uuid.UUID]userExpectation{ - userID: { - Email: email, - Exists: true, - }, - }) - waitForQueueDrain(t, ctx, db) +func runReclaimsStaleQueueLocks(t *testing.T, db *testutils.Database) { + t.Helper() + + ctx := t.Context() + userID := uuid.New() + email := fmt.Sprintf("auth-sync-%s-locked@example.com", userID.String()[:8]) + + insertAuthUser(t, ctx, db, userID, email) + deletePublicUser(t, ctx, db, userID) + lockQueueItems(t, ctx, db, userID, time.Now().Add(-time.Minute), "stale-worker") + assertQueueBacklog(t, ctx, db, 1) + + runner := startRunnerProcess(t, db, newTestRunnerConfig(2), "lock-reclaimer") + t.Cleanup(func() { runner.Stop(t) }) - t.Run("drains_burst_backlog_with_multiple_runners", func(t *testing.T) { - ctx := t.Context() - const userCount = 60 + waitForPublicUsers(t, ctx, db, map[uuid.UUID]userExpectation{ + userID: { + Email: email, + Exists: true, + }, + }) + waitForQueueDrain(t, ctx, db) + runner.Stop(t) +} - userIDs := make([]uuid.UUID, 0, userCount) +func runDrainsBurstBacklogWithMultipleRunners(t *testing.T, db *testutils.Database) { + t.Helper() - for i := 0; i < userCount; i++ { - userID := uuid.New() - userIDs = append(userIDs, userID) + ctx := t.Context() + const userCount = 60 - initialEmail := fmt.Sprintf("auth-sync-burst-%02d-initial@example.com", i) - insertAuthUser(t, ctx, db, userID, initialEmail) + userIDs := make([]uuid.UUID, 0, userCount) - if i%2 == 0 { - updateAuthUserEmail(t, ctx, db, userID, fmt.Sprintf("auth-sync-burst-%02d-v2@example.com", i)) - } - if i%5 == 0 { - updateAuthUserEmail(t, ctx, db, userID, fmt.Sprintf("auth-sync-burst-%02d-v3@example.com", i)) - } + for i := range userCount { + userID := uuid.New() + userIDs = append(userIDs, userID) - if i%3 == 0 { - deleteAuthUser(t, ctx, db, userID) - enqueueUserSyncItem(t, ctx, db, userID, "delete") - if i%6 == 0 { - insertPublicUser(t, ctx, db, userID, fmt.Sprintf("ghost-%02d@example.com", i)) - } + initialEmail := fmt.Sprintf("auth-sync-burst-%02d-initial@example.com", i) + insertAuthUser(t, ctx, db, userID, initialEmail) - continue - } + if i%2 == 0 { + updateAuthUserEmail(t, ctx, db, userID, fmt.Sprintf("auth-sync-burst-%02d-v2@example.com", i)) + } + if i%5 == 0 { + updateAuthUserEmail(t, ctx, db, userID, fmt.Sprintf("auth-sync-burst-%02d-v3@example.com", i)) + } - if i%8 == 0 { - deletePublicUser(t, ctx, db, userID) - } else if i%7 == 0 { - setPublicUserEmail(t, ctx, db, userID, fmt.Sprintf("stale-%02d@example.com", i)) + if i%3 == 0 { + deleteAuthUser(t, ctx, db, userID) + enqueueUserSyncItem(t, ctx, db, userID, "delete") + if i%6 == 0 { + insertPublicUser(t, ctx, db, userID, fmt.Sprintf("ghost-%02d@example.com", i)) } - if i%4 == 0 { - enqueueUserSyncItem(t, ctx, db, userID, "upsert") - } - if i%9 == 0 { - enqueueUserSyncItem(t, ctx, db, userID, "upsert") - } + continue } - authUsers, err := loadAuthUsers(ctx, db) - require.NoError(t, err) + if i%8 == 0 { + deletePublicUser(t, ctx, db, userID) + } else if i%7 == 0 { + setPublicUserEmail(t, ctx, db, userID, fmt.Sprintf("stale-%02d@example.com", i)) + } - want := expectedUsersForIDs(userIDs, authUsers) - assertQueueBacklog(t, ctx, db, userCount) + if i%4 == 0 { + enqueueUserSyncItem(t, ctx, db, userID, "upsert") + } + if i%9 == 0 { + enqueueUserSyncItem(t, ctx, db, userID, "upsert") + } + } - runnerA := startRunnerProcess(t, db, newTestRunnerConfig(5), "burst-a") - runnerB := startRunnerProcess(t, db, newTestRunnerConfig(5), "burst-b") - t.Cleanup(func() { - runnerA.Stop(t) - runnerB.Stop(t) - }) + authUsers, err := loadAuthUsers(ctx, db) + require.NoError(t, err) - waitForPublicUsers(t, ctx, db, want) - waitForQueueDrain(t, ctx, db) + want := expectedUsersForIDs(userIDs, authUsers) + assertQueueBacklog(t, ctx, db, userCount) + runnerA := startRunnerProcess(t, db, newTestRunnerConfig(5), "burst-a") + runnerB := startRunnerProcess(t, db, newTestRunnerConfig(5), "burst-b") + t.Cleanup(func() { runnerA.Stop(t) runnerB.Stop(t) }) + + waitForPublicUsers(t, ctx, db, want) + waitForQueueDrain(t, ctx, db) + + runnerA.Stop(t) + runnerB.Stop(t) } func newTestRunnerConfig(batchSize int32) Config { diff --git a/packages/dashboard-api/internal/supabaseauthusersync/supervisor.go b/packages/dashboard-api/internal/supabaseauthusersync/supervisor.go index 24269f351e..de60dda43a 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/supervisor.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/supervisor.go @@ -69,6 +69,7 @@ func supervise(ctx context.Context, l logger.Logger, cfg supervisorConfig, run f select { case <-ctx.Done(): timer.Stop() + return ctx.Err() case <-timer.C: } @@ -95,18 +96,18 @@ func runRecovering(ctx context.Context, l logger.Logger, run func(context.Contex return err } -func restartBackoff(attempt int, base time.Duration, max time.Duration) time.Duration { +func restartBackoff(attempt int, base time.Duration, maxDelay time.Duration) time.Duration { if base <= 0 { base = defaultRestartDelay } - if max < base { - max = base + if maxDelay < base { + maxDelay = base } delay := base for i := 1; i < attempt; i++ { - if delay >= max/2 { - return max + if delay >= maxDelay/2 { + return maxDelay } delay *= 2 diff --git a/packages/dashboard-api/internal/supabaseauthusersync/supervisor_test.go b/packages/dashboard-api/internal/supabaseauthusersync/supervisor_test.go index 3e26d71a71..fc53f77ba4 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/supervisor_test.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/supervisor_test.go @@ -13,6 +13,8 @@ import ( ) func TestSuperviseRestartsAfterUnexpectedError(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -32,6 +34,7 @@ func TestSuperviseRestartsAfterUnexpectedError(t *testing.T) { cancel() <-ctx.Done() + return ctx.Err() }) }() @@ -42,6 +45,8 @@ func TestSuperviseRestartsAfterUnexpectedError(t *testing.T) { } func TestSuperviseRestartsAfterPanic(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -61,6 +66,7 @@ func TestSuperviseRestartsAfterPanic(t *testing.T) { cancel() <-ctx.Done() + return ctx.Err() }) }() From 84ecd065e44439010822b5d49482178e65f54f1c Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Tue, 31 Mar 2026 16:52:27 -0700 Subject: [PATCH 11/92] test: apply database migrations in end-to-end test setup - Updated the `TestSupabaseAuthUserSyncRunner_EndToEnd` to apply necessary database migrations before running tests. - Refactored the `SetupDatabase` function to include a new method `ApplyMigrations` for better migration management. --- .../supabaseauthusersync/runner_test.go | 1 + packages/db/pkg/testutils/db.go | 27 +++++++++++-------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go b/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go index a99e4ad3b2..25d2327af1 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go @@ -44,6 +44,7 @@ func TestSupabaseAuthUserSyncRunner_EndToEnd(t *testing.T) { t.Parallel() db := testutils.SetupDatabase(t) + db.ApplyMigrations(t, "packages/db/pkg/auth/migrations") runRepairsInsertUpdateDeleteDrift(t, db) runReclaimsStaleQueueLocks(t, db) diff --git a/packages/db/pkg/testutils/db.go b/packages/db/pkg/testutils/db.go index 86708c18c4..68fa814977 100644 --- a/packages/db/pkg/testutils/db.go +++ b/packages/db/pkg/testutils/db.go @@ -38,6 +38,7 @@ type Database struct { SqlcClient *db.Client AuthDb *authdb.Client TestQueries *queries.Queries + connStr string } // SetupDatabase creates a fresh PostgreSQL container with migrations applied @@ -103,11 +104,11 @@ func SetupDatabase(t *testing.T) *Database { SqlcClient: sqlcClient, AuthDb: authDb, TestQueries: testQueries, + connStr: connStr, } } -// runDatabaseMigrations executes all required database migrations -func runDatabaseMigrations(t *testing.T, connStr string) { +func (db *Database) ApplyMigrations(t *testing.T, migrationDirs ...string) { t.Helper() cmd := exec.CommandContext(t.Context(), "git", "rev-parse", "--show-toplevel") @@ -115,25 +116,29 @@ func runDatabaseMigrations(t *testing.T, connStr string) { require.NoError(t, err, "Failed to find git root") repoRoot := strings.TrimSpace(string(output)) - db, err := goose.OpenDBWithDriver("pgx", connStr) + sqlDB, err := goose.OpenDBWithDriver("pgx", db.connStr) require.NoError(t, err) t.Cleanup(func() { - err := db.Close() + err := sqlDB.Close() assert.NoError(t, err) }) - // run the db migration - for _, migrationsDir := range []string{ - filepath.Join(repoRoot, "packages", "db", "migrations"), - filepath.Join(repoRoot, "packages", "db", "pkg", "auth", "migrations"), - } { + for _, migrationsDir := range migrationDirs { err = goose.RunWithOptionsContext( t.Context(), "up", - db, - migrationsDir, + sqlDB, + filepath.Join(repoRoot, migrationsDir), nil, ) require.NoError(t, err) } } + +// runDatabaseMigrations executes all required database migrations +func runDatabaseMigrations(t *testing.T, connStr string) { + t.Helper() + + db := &Database{connStr: connStr} + db.ApplyMigrations(t, filepath.Join("packages", "db", "migrations")) +} From 386d0c692dd30df5af8f621e16222e6b7cec196a Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Tue, 31 Mar 2026 18:08:18 -0700 Subject: [PATCH 12/92] feat(sync): enhance user sync processing and acknowledgment - Introduced a new process outcome `ready_to_ack` to streamline acknowledgment handling. - Refactored the `process` method to prepare for batch acknowledgment of processed items. - Added a new `AckBatch` method in the store to handle multiple acknowledgments efficiently. - Updated the `Runner` to process items in batches and finalize acknowledgments accordingly. - Removed obsolete SQL query for single item acknowledgment as part of the refactor. - Enhanced tests to cover new deletion logic and acknowledgment scenarios. --- .../internal/supabaseauthusersync/logging.go | 1 + .../supabaseauthusersync/processor.go | 25 ++---- .../supabaseauthusersync/processor_test.go | 43 ++++++--- .../internal/supabaseauthusersync/runner.go | 87 +++++++++++++++++-- .../internal/supabaseauthusersync/store.go | 8 +- packages/db/pkg/auth/queries/ack_batch.sql.go | 20 +++++ .../supabase_auth_user_sync/ack.sql | 3 - .../supabase_auth_user_sync/ack_batch.sql | 3 + 8 files changed, 149 insertions(+), 41 deletions(-) create mode 100644 packages/db/pkg/auth/queries/ack_batch.sql.go delete mode 100644 packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/ack.sql create mode 100644 packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/ack_batch.sql diff --git a/packages/dashboard-api/internal/supabaseauthusersync/logging.go b/packages/dashboard-api/internal/supabaseauthusersync/logging.go index b5730330f4..06759556e6 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/logging.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/logging.go @@ -13,6 +13,7 @@ import ( type processOutcome string const ( + processOutcomeReadyToAck processOutcome = "ready_to_ack" processOutcomeAcked processOutcome = "acked" processOutcomeAckFailed processOutcome = "ack_failed" processOutcomeRetried processOutcome = "retried" diff --git a/packages/dashboard-api/internal/supabaseauthusersync/processor.go b/packages/dashboard-api/internal/supabaseauthusersync/processor.go index e25d2a0c6c..0a6e9ab369 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/processor.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/processor.go @@ -15,7 +15,6 @@ import ( ) type processorStore interface { - Ack(ctx context.Context, id int64) error Retry(ctx context.Context, id int64, backoff time.Duration, lastError string) error DeadLetter(ctx context.Context, id int64, lastError string) error GetAuthUser(ctx context.Context, userID uuid.UUID) (*AuthUser, error) @@ -46,21 +45,7 @@ func (p *Processor) process(ctx context.Context, item QueueItem) processResult { } if err == nil { - if ackErr := p.store.Ack(ctx, item.ID); ackErr != nil { - result.Outcome = processOutcomeAckFailed - - p.l.Error(ctx, "processed supabase auth sync queue item but failed to ack", - append( - processResultFields(item, result, time.Now()), - zap.NamedError("ack_error", ackErr), - )..., - ) - - return result - } - - result.Outcome = processOutcomeAcked - p.l.Info(ctx, "processed supabase auth sync queue item", processResultFields(item, result, time.Now())...) + result.Outcome = processOutcomeReadyToAck return result } @@ -140,6 +125,14 @@ func (p *Processor) processOnce(ctx context.Context, item QueueItem) (action rec } func (p *Processor) reconcile(ctx context.Context, item QueueItem) (reconcileAction, error) { + if item.Operation == "delete" { + if err := p.store.DeletePublicUser(ctx, item.UserID); err != nil { + return "", fmt.Errorf("delete public.users %s: %w", item.UserID, err) + } + + return reconcileActionDeletePublicUser, nil + } + authUser, err := p.store.GetAuthUser(ctx, item.UserID) if errors.Is(err, pgx.ErrNoRows) { diff --git a/packages/dashboard-api/internal/supabaseauthusersync/processor_test.go b/packages/dashboard-api/internal/supabaseauthusersync/processor_test.go index 125222d5ee..446653a3d0 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/processor_test.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/processor_test.go @@ -25,15 +25,9 @@ type deadLetterCall struct { type fakeProcessorStore struct { getAuthUserFn func(context.Context, uuid.UUID) (*AuthUser, error) - ackCalls []int64 - retryCalls []retryCall - deadLetterCalls []deadLetterCall -} - -func (s *fakeProcessorStore) Ack(_ context.Context, id int64) error { - s.ackCalls = append(s.ackCalls, id) - - return nil + deletePublicUserCalls int + retryCalls []retryCall + deadLetterCalls []deadLetterCall } func (s *fakeProcessorStore) Retry(_ context.Context, id int64, backoff time.Duration, lastError string) error { @@ -64,6 +58,8 @@ func (s *fakeProcessorStore) UpsertPublicUser(_ context.Context, _ uuid.UUID, _ } func (s *fakeProcessorStore) DeletePublicUser(_ context.Context, _ uuid.UUID) error { + s.deletePublicUserCalls++ + return nil } @@ -85,7 +81,6 @@ func TestProcessorProcessRetriesRecoveredPanic(t *testing.T) { require.NotPanics(t, func() { processor.process(context.Background(), item) }) - require.Empty(t, store.ackCalls) require.Len(t, store.retryCalls, 1) require.Contains(t, store.retryCalls[0].lastError, "panic while processing queue item") require.Empty(t, store.deadLetterCalls) @@ -109,8 +104,34 @@ func TestProcessorProcessDeadLettersRecoveredPanicAtMaxAttempts(t *testing.T) { require.NotPanics(t, func() { processor.process(context.Background(), item) }) - require.Empty(t, store.ackCalls) require.Empty(t, store.retryCalls) require.Len(t, store.deadLetterCalls, 1) require.Contains(t, store.deadLetterCalls[0].lastError, "panic while processing queue item") } + +func TestProcessorProcessDeleteSkipsAuthLookup(t *testing.T) { + t.Parallel() + + getAuthUserCalled := false + store := &fakeProcessorStore{ + getAuthUserFn: func(context.Context, uuid.UUID) (*AuthUser, error) { + getAuthUserCalled = true + + return nil, nil + }, + } + processor := NewProcessor(store, 3, logger.NewNopLogger()) + item := QueueItem{ + ID: 1, + UserID: uuid.New(), + Operation: "delete", + AttemptCount: 1, + } + + result := processor.process(context.Background(), item) + + require.False(t, getAuthUserCalled) + require.Equal(t, 1, store.deletePublicUserCalls) + require.Equal(t, processOutcomeReadyToAck, result.Outcome) + require.Equal(t, reconcileActionDeletePublicUser, result.Action) +} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/runner.go b/packages/dashboard-api/internal/supabaseauthusersync/runner.go index 174c976a84..37d014bcc6 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/runner.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/runner.go @@ -13,6 +13,7 @@ import ( type runnerStore interface { ClaimBatch(ctx context.Context, lockOwner string, lockTimeout time.Duration, batchSize int32) ([]QueueItem, error) + AckBatch(ctx context.Context, ids []int64) error } type workerStore interface { @@ -28,6 +29,11 @@ type Runner struct { l logger.Logger } +type ackCandidate struct { + item QueueItem + result processResult +} + func NewRunner(cfg Config, authDB *authdb.Client, mainDB *sqlcdb.Client, lockOwner string, l logger.Logger) *Runner { workerLogger := l.With(logger.WithServiceInstanceID(lockOwner)) store := NewStore(authDB, mainDB) @@ -50,22 +56,39 @@ func (r *Runner) Run(ctx context.Context) error { zap.Int32("worker.max_attempts", r.cfg.MaxAttempts), ) - ticker := time.NewTicker(r.cfg.PollInterval) - defer ticker.Stop() - for { + r.drain(ctx) + if ctx.Err() != nil { + r.l.Info(ctx, "stopping supabase auth user sync worker", zap.Error(ctx.Err())) + + return ctx.Err() + } + + timer := time.NewTimer(r.cfg.PollInterval) select { case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } + r.l.Info(ctx, "stopping supabase auth user sync worker", zap.Error(ctx.Err())) return ctx.Err() - case <-ticker.C: - r.poll(ctx) + case <-timer.C: + } + } +} + +func (r *Runner) drain(ctx context.Context) { + for { + processed := r.pollOnce(ctx) + if processed == 0 { + return } } } -func (r *Runner) poll(ctx context.Context) { +func (r *Runner) pollOnce(ctx context.Context) int { claimedAt := time.Now() items, err := r.store.ClaimBatch(ctx, r.lockOwner, r.cfg.LockTimeout, r.cfg.BatchSize) if err != nil { @@ -76,18 +99,64 @@ func (r *Runner) poll(ctx context.Context) { zap.Error(err), ) - return + return 0 } if len(items) == 0 { - return + return 0 } summary := newBatchSummary(items, claimedAt) + ackCandidates := make([]ackCandidate, 0, len(items)) for _, item := range items { - summary.Add(r.processor.process(ctx, item)) + result := r.processor.process(ctx, item) + if result.Outcome == processOutcomeReadyToAck { + ackCandidates = append(ackCandidates, ackCandidate{ + item: item, + result: result, + }) + + continue + } + + summary.Add(result) + } + + if len(ackCandidates) > 0 { + r.finalizeAcks(ctx, ackCandidates, &summary) } r.l.Log(ctx, summary.Level(), "processed supabase auth sync queue batch", summary.Fields(time.Since(claimedAt))...) + + return len(items) +} + +func (r *Runner) finalizeAcks(ctx context.Context, candidates []ackCandidate, summary *batchSummary) { + ids := make([]int64, 0, len(candidates)) + for _, candidate := range candidates { + ids = append(ids, candidate.item.ID) + } + + if err := r.store.AckBatch(ctx, ids); err != nil { + for _, candidate := range candidates { + candidate.result.Outcome = processOutcomeAckFailed + summary.Add(candidate.result) + + r.l.Error(ctx, "processed supabase auth sync queue item but failed to ack", + append( + processResultFields(candidate.item, candidate.result, time.Now()), + zap.NamedError("ack_error", err), + )..., + ) + } + + return + } + + for _, candidate := range candidates { + candidate.result.Outcome = processOutcomeAcked + summary.Add(candidate.result) + r.l.Info(ctx, "processed supabase auth sync queue item", processResultFields(candidate.item, candidate.result, time.Now())...) + } } diff --git a/packages/dashboard-api/internal/supabaseauthusersync/store.go b/packages/dashboard-api/internal/supabaseauthusersync/store.go index 9eee64ca93..63ebc988d0 100644 --- a/packages/dashboard-api/internal/supabaseauthusersync/store.go +++ b/packages/dashboard-api/internal/supabaseauthusersync/store.go @@ -64,8 +64,12 @@ func (s *Store) ClaimBatch(ctx context.Context, lockOwner string, lockTimeout ti return items, nil } -func (s *Store) Ack(ctx context.Context, id int64) error { - return s.authQueries.AckUserSyncQueueItem(ctx, id) +func (s *Store) AckBatch(ctx context.Context, ids []int64) error { + if len(ids) == 0 { + return nil + } + + return s.authQueries.AckUserSyncQueueItems(ctx, ids) } func (s *Store) Retry(ctx context.Context, id int64, backoff time.Duration, lastError string) error { diff --git a/packages/db/pkg/auth/queries/ack_batch.sql.go b/packages/db/pkg/auth/queries/ack_batch.sql.go new file mode 100644 index 0000000000..1b0d49a0af --- /dev/null +++ b/packages/db/pkg/auth/queries/ack_batch.sql.go @@ -0,0 +1,20 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: ack_batch.sql + +package authqueries + +import ( + "context" +) + +const ackUserSyncQueueItems = `-- name: AckUserSyncQueueItems :exec +DELETE FROM public.user_sync_queue +WHERE id = ANY($1::bigint[]) +` + +func (q *Queries) AckUserSyncQueueItems(ctx context.Context, ids []int64) error { + _, err := q.db.Exec(ctx, ackUserSyncQueueItems, ids) + return err +} diff --git a/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/ack.sql b/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/ack.sql deleted file mode 100644 index e0d7354dc9..0000000000 --- a/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/ack.sql +++ /dev/null @@ -1,3 +0,0 @@ --- name: AckUserSyncQueueItem :exec -DELETE FROM public.user_sync_queue -WHERE id = sqlc.arg(id)::bigint; diff --git a/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/ack_batch.sql b/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/ack_batch.sql new file mode 100644 index 0000000000..45dd7f6e49 --- /dev/null +++ b/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/ack_batch.sql @@ -0,0 +1,3 @@ +-- name: AckUserSyncQueueItems :exec +DELETE FROM public.user_sync_queue +WHERE id = ANY(sqlc.arg(ids)::bigint[]); From 3e3fa3cd70ad19afdc83c30c073574d922691539 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Tue, 31 Mar 2026 19:00:41 -0700 Subject: [PATCH 13/92] refactor(gcp): remove auth_db_connection_string resources and update references - Deleted the `auth_db_connection_string` secret and its version from the GCP configuration. - Updated references in `main.tf` and `nomad/main.tf` to use the `postgres_connection_string` instead. - Removed the corresponding variable declaration from `variables.tf` to clean up unused configurations. --- iac/provider-gcp/api.tf | 17 ----------------- iac/provider-gcp/main.tf | 1 - iac/provider-gcp/nomad/main.tf | 6 +----- iac/provider-gcp/nomad/variables.tf | 4 ---- 4 files changed, 1 insertion(+), 27 deletions(-) diff --git a/iac/provider-gcp/api.tf b/iac/provider-gcp/api.tf index c8b5586c96..26f1a1b408 100644 --- a/iac/provider-gcp/api.tf +++ b/iac/provider-gcp/api.tf @@ -27,23 +27,6 @@ resource "google_secret_manager_secret_version" "postgres_read_replica_connectio } } -resource "google_secret_manager_secret" "auth_db_connection_string" { - secret_id = "${var.prefix}auth-db-connection-string" - - replication { - auto {} - } -} - -resource "google_secret_manager_secret_version" "auth_db_connection_string" { - secret = google_secret_manager_secret.auth_db_connection_string.name - secret_data = " " - - lifecycle { - ignore_changes = [secret_data] - } -} - resource "random_password" "api_secret" { length = 32 special = false diff --git a/iac/provider-gcp/main.tf b/iac/provider-gcp/main.tf index 90294e554b..04f502e8a7 100644 --- a/iac/provider-gcp/main.tf +++ b/iac/provider-gcp/main.tf @@ -224,7 +224,6 @@ module "nomad" { api_secret = random_password.api_secret.result custom_envs_repository_name = google_artifact_registry_repository.custom_environments_repository.name postgres_connection_string_secret_name = module.init.postgres_connection_string_secret_name - auth_db_connection_string_secret_version = google_secret_manager_secret_version.auth_db_connection_string postgres_read_replica_connection_string_secret_version = google_secret_manager_secret_version.postgres_read_replica_connection_string supabase_jwt_secrets_secret_name = module.init.supabase_jwt_secret_name posthog_api_key_secret_name = module.init.posthog_api_key_secret_name diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index 7d05bf3dec..87f3593b7d 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -10,10 +10,6 @@ data "google_secret_manager_secret_version" "postgres_connection_string" { secret = var.postgres_connection_string_secret_name } -data "google_secret_manager_secret_version" "auth_db_connection_string" { - secret = var.auth_db_connection_string_secret_version.secret -} - data "google_secret_manager_secret_version" "postgres_read_replica_connection_string" { secret = var.postgres_read_replica_connection_string_secret_version.secret } @@ -135,7 +131,7 @@ module "dashboard_api" { image = data.google_artifact_registry_docker_image.dashboard_api_image[0].self_link postgres_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data - auth_db_connection_string = trimspace(data.google_secret_manager_secret_version.auth_db_connection_string.secret_data) + auth_db_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data auth_db_read_replica_connection_string = trimspace(data.google_secret_manager_secret_version.postgres_read_replica_connection_string.secret_data) clickhouse_connection_string = local.clickhouse_connection_string supabase_jwt_secrets = trimspace(data.google_secret_manager_secret_version.supabase_jwt_secrets.secret_data) diff --git a/iac/provider-gcp/nomad/variables.tf b/iac/provider-gcp/nomad/variables.tf index 64e9f7dd75..6d179f5c86 100644 --- a/iac/provider-gcp/nomad/variables.tf +++ b/iac/provider-gcp/nomad/variables.tf @@ -175,10 +175,6 @@ variable "postgres_connection_string_secret_name" { type = string } -variable "auth_db_connection_string_secret_version" { - type = any -} - variable "postgres_read_replica_connection_string_secret_version" { type = any } From 5e0972bf35b6f70443c4374ebf1422cdbe864967 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Tue, 31 Mar 2026 19:37:58 -0700 Subject: [PATCH 14/92] chore(dashboard-api): refactor environment variable management - Updated the dashboard API module to separate base and extra environment variables. - Introduced a precondition to prevent conflicts with reserved keys in extra environment variables. - Modified the HCL job configuration to iterate over environment variables dynamically. - Adjusted variable declarations to reflect the new structure for extra environment variables. --- .env.gcp.template | 3 +- .../job-dashboard-api/jobs/dashboard-api.hcl | 18 ++------ iac/modules/job-dashboard-api/main.tf | 43 ++++++++++++++----- iac/modules/job-dashboard-api/variables.tf | 2 +- iac/provider-gcp/nomad/main.tf | 2 +- 5 files changed, 40 insertions(+), 28 deletions(-) diff --git a/.env.gcp.template b/.env.gcp.template index 6f72d2d77c..5269692a3e 100644 --- a/.env.gcp.template +++ b/.env.gcp.template @@ -77,7 +77,8 @@ CLICKHOUSE_CLUSTER_SIZE=1 # Dashboard API instance count (default: 0) DASHBOARD_API_COUNT= -# Additional dashboard-api env vars passed directly to the Nomad job (default: {}) +# Additional non-reserved dashboard-api env vars passed directly to the Nomad job (default: {}) +# Reserved keys managed by the module cannot be overridden here. # Example: '{"SUPABASE_AUTH_USER_SYNC_ENABLED":"true"}' DASHBOARD_API_ENV_VARS= diff --git a/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl b/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl index 86a72cab50..f05aadcbb5 100644 --- a/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl +++ b/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl @@ -71,21 +71,9 @@ job "dashboard-api" { } env { - GIN_MODE = "release" - ENVIRONMENT = "${environment}" - NODE_ID = "$${node.unique.id}" - PORT = "$${NOMAD_PORT_api}" - POSTGRES_CONNECTION_STRING = "${postgres_connection_string}" - AUTH_DB_CONNECTION_STRING = "${auth_db_connection_string}" - AUTH_DB_READ_REPLICA_CONNECTION_STRING = "${auth_db_read_replica_connection_string}" - CLICKHOUSE_CONNECTION_STRING = "${clickhouse_connection_string}" - SUPABASE_JWT_SECRETS = "${supabase_jwt_secrets}" - OTEL_COLLECTOR_GRPC_ENDPOINT = "${otel_collector_grpc_endpoint}" - LOGS_COLLECTOR_ADDRESS = "${logs_collector_address}" - - %{ for key, val in env } - ${ key } = "${ val }" - %{ endfor } + %{ for key in sort(keys(env)) ~} + ${key} = "${env[key]}" + %{ endfor ~} } config { diff --git a/iac/modules/job-dashboard-api/main.tf b/iac/modules/job-dashboard-api/main.tf index aa577bfd2f..26dcc2a889 100644 --- a/iac/modules/job-dashboard-api/main.tf +++ b/iac/modules/job-dashboard-api/main.tf @@ -1,5 +1,29 @@ locals { - env = { for key, value in var.env : key => value if value != null && value != "" } + base_env = { + GIN_MODE = "release" + ENVIRONMENT = var.environment + NODE_ID = "$${node.unique.id}" + PORT = "$${NOMAD_PORT_api}" + POSTGRES_CONNECTION_STRING = var.postgres_connection_string + AUTH_DB_CONNECTION_STRING = var.auth_db_connection_string + AUTH_DB_READ_REPLICA_CONNECTION_STRING = var.auth_db_read_replica_connection_string + CLICKHOUSE_CONNECTION_STRING = var.clickhouse_connection_string + SUPABASE_JWT_SECRETS = var.supabase_jwt_secrets + OTEL_COLLECTOR_GRPC_ENDPOINT = "localhost:${var.otel_collector_grpc_port}" + LOGS_COLLECTOR_ADDRESS = "http://localhost:${var.logs_proxy_port.port}" + } + + extra_env = { + for key, value in var.extra_env : key => value + if value != null && trimspace(value) != "" + } + + conflicting_extra_env_keys = sort(tolist(setintersection( + toset(keys(local.base_env)), + toset(keys(local.extra_env)), + ))) + + env = merge(local.base_env, local.extra_env) } resource "nomad_job" "dashboard_api" { @@ -14,16 +38,15 @@ resource "nomad_job" "dashboard_api" { memory_mb = 512 cpu_count = 1 - postgres_connection_string = var.postgres_connection_string - auth_db_connection_string = var.auth_db_connection_string - auth_db_read_replica_connection_string = var.auth_db_read_replica_connection_string - clickhouse_connection_string = var.clickhouse_connection_string - supabase_jwt_secrets = var.supabase_jwt_secrets - env = local.env + env = local.env subdomain = "dashboard-api" - - otel_collector_grpc_endpoint = "localhost:${var.otel_collector_grpc_port}" - logs_collector_address = "http://localhost:${var.logs_proxy_port.port}" }) + + lifecycle { + precondition { + condition = length(local.conflicting_extra_env_keys) == 0 + error_message = "dashboard-api extra_env contains reserved keys: ${join(", ", local.conflicting_extra_env_keys)}" + } + } } diff --git a/iac/modules/job-dashboard-api/variables.tf b/iac/modules/job-dashboard-api/variables.tf index 625da7c82b..ab02574467 100644 --- a/iac/modules/job-dashboard-api/variables.tf +++ b/iac/modules/job-dashboard-api/variables.tf @@ -44,7 +44,7 @@ variable "supabase_jwt_secrets" { sensitive = true } -variable "env" { +variable "extra_env" { type = map(string) default = {} } diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index 87f3593b7d..21e17909b2 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -135,7 +135,7 @@ module "dashboard_api" { auth_db_read_replica_connection_string = trimspace(data.google_secret_manager_secret_version.postgres_read_replica_connection_string.secret_data) clickhouse_connection_string = local.clickhouse_connection_string supabase_jwt_secrets = trimspace(data.google_secret_manager_secret_version.supabase_jwt_secrets.secret_data) - env = var.dashboard_api_env_vars + extra_env = var.dashboard_api_env_vars otel_collector_grpc_port = var.otel_collector_grpc_port logs_proxy_port = var.logs_proxy_port From 6b5a5c8e33ad69348d3effab6db42926628bd788 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Wed, 1 Apr 2026 13:34:12 -0700 Subject: [PATCH 15/92] refactor: use river for queue worker --- packages/dashboard-api/Makefile | 10 +- packages/dashboard-api/go.mod | 14 +- packages/dashboard-api/go.sum | 24 + .../backgroundworker/auth_user_sync.go | 139 +++++ .../backgroundworker/auth_user_sync_test.go | 252 +++++++++ .../internal/backgroundworker/river.go | 38 ++ .../internal/supabaseauthusersync/config.go | 28 - .../internal/supabaseauthusersync/logging.go | 214 -------- .../supabaseauthusersync/processor.go | 170 ------ .../supabaseauthusersync/processor_test.go | 137 ----- .../internal/supabaseauthusersync/runner.go | 162 ------ .../supabaseauthusersync/runner_test.go | 494 ------------------ .../internal/supabaseauthusersync/store.go | 115 ---- .../supabaseauthusersync/supervisor.go | 117 ----- .../supabaseauthusersync/supervisor_test.go | 77 --- packages/dashboard-api/main.go | 49 +- packages/db/pkg/auth/client.go | 4 + ...1000003_river_auth_user_sync_triggers.sql} | 104 ++-- packages/db/pkg/auth/queries/models.go | 13 - .../supabase_auth_user_sync/ack_batch.sql | 3 - .../supabase_auth_user_sync/claim_batch.sql | 17 - .../supabase_auth_user_sync/dead_letter.sql | 8 - .../supabase_auth_user_sync/get_auth_user.sql | 4 - .../supabase_auth_user_sync/retry.sql | 8 - packages/db/pkg/testutils/db.go | 40 +- 25 files changed, 592 insertions(+), 1649 deletions(-) create mode 100644 packages/dashboard-api/internal/backgroundworker/auth_user_sync.go create mode 100644 packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go create mode 100644 packages/dashboard-api/internal/backgroundworker/river.go delete mode 100644 packages/dashboard-api/internal/supabaseauthusersync/config.go delete mode 100644 packages/dashboard-api/internal/supabaseauthusersync/logging.go delete mode 100644 packages/dashboard-api/internal/supabaseauthusersync/processor.go delete mode 100644 packages/dashboard-api/internal/supabaseauthusersync/processor_test.go delete mode 100644 packages/dashboard-api/internal/supabaseauthusersync/runner.go delete mode 100644 packages/dashboard-api/internal/supabaseauthusersync/runner_test.go delete mode 100644 packages/dashboard-api/internal/supabaseauthusersync/store.go delete mode 100644 packages/dashboard-api/internal/supabaseauthusersync/supervisor.go delete mode 100644 packages/dashboard-api/internal/supabaseauthusersync/supervisor_test.go rename packages/db/pkg/auth/migrations/{20260328000001_dashboard_supabase_auth_user_sync_queue.sql => 20260401000003_river_auth_user_sync_triggers.sql} (66%) delete mode 100644 packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/ack_batch.sql delete mode 100644 packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/claim_batch.sql delete mode 100644 packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/dead_letter.sql delete mode 100644 packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/get_auth_user.sql delete mode 100644 packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/retry.sql diff --git a/packages/dashboard-api/Makefile b/packages/dashboard-api/Makefile index 5ce6efda2a..1dfef7c5fe 100644 --- a/packages/dashboard-api/Makefile +++ b/packages/dashboard-api/Makefile @@ -13,6 +13,10 @@ endif HOSTNAME := $(shell hostname 2> /dev/null || hostnamectl hostname 2> /dev/null) $(if $(HOSTNAME),,$(error Failed to determine hostname: both 'hostname' and 'hostnamectl' failed)) +define DASHBOARD_API_EXTRA_ENV +$$(printf '%s' "$${DASHBOARD_API_ENV_VARS:-}" | jq -r '(if .=="" then empty elif type=="string" then (fromjson? // empty) else . end) | to_entries? // [] | map("\(.key)=\(.value|tostring|@sh)") | join(" ")') +endef + .PHONY: generate generate: go generate ./... @@ -33,12 +37,14 @@ build-and-upload: .PHONY: run run: make build - ./bin/dashboard-api + @EXTRA_ENV=$(DASHBOARD_API_EXTRA_ENV); \ + eval "env $$EXTRA_ENV ./bin/dashboard-api" .PHONY: run-local run-local: make build - NODE_ID=$(HOSTNAME) ./bin/dashboard-api + @EXTRA_ENV=$(DASHBOARD_API_EXTRA_ENV); \ + eval "env NODE_ID=$(HOSTNAME) $$EXTRA_ENV ./bin/dashboard-api" .PHONY: test test: diff --git a/packages/dashboard-api/go.mod b/packages/dashboard-api/go.mod index c92076b4f9..79457e0125 100644 --- a/packages/dashboard-api/go.mod +++ b/packages/dashboard-api/go.mod @@ -20,7 +20,7 @@ require ( github.com/gin-contrib/cors v1.7.6 github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.6.0 - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.9.1 github.com/oapi-codegen/gin-middleware v1.0.2 github.com/oapi-codegen/runtime v1.1.1 github.com/stretchr/testify v1.11.1 @@ -113,6 +113,11 @@ require ( github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/pressly/goose/v3 v3.26.0 // indirect github.com/redis/go-redis/v9 v9.17.3 // indirect + github.com/riverqueue/river v0.32.0 // indirect + github.com/riverqueue/river/riverdriver v0.32.0 // indirect + github.com/riverqueue/river/riverdriver/riverpgxv5 v0.32.0 // indirect + github.com/riverqueue/river/rivershared v0.32.0 // indirect + github.com/riverqueue/river/rivertype v0.32.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/shirou/gopsutil/v4 v4.25.9 // indirect @@ -120,6 +125,10 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/testcontainers/testcontainers-go v0.40.0 // indirect github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect github.com/tklauser/go-sysconf v0.3.15 // indirect github.com/tklauser/numcpus v0.10.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect @@ -142,10 +151,11 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.41.0 // indirect go.opentelemetry.io/otel/trace v1.41.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect + go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.18.0 // indirect golang.org/x/crypto v0.48.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.50.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.41.0 // indirect diff --git a/packages/dashboard-api/go.sum b/packages/dashboard-api/go.sum index 8bc4994cc9..d724d1c16c 100644 --- a/packages/dashboard-api/go.sum +++ b/packages/dashboard-api/go.sum @@ -141,6 +141,8 @@ github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7Ulw github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= @@ -245,6 +247,16 @@ github.com/redis/go-redis/v9 v9.17.3 h1:fN29NdNrE17KttK5Ndf20buqfDZwGNgoUr9qjl1D github.com/redis/go-redis/v9 v9.17.3/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/riverqueue/river v0.32.0 h1:j15EoFZ4oQWXcCq8NyzWwoi3fdaO8mECTB100NSv9Qw= +github.com/riverqueue/river v0.32.0/go.mod h1:zABAdLze3HI7K02N+veikXyK5FjiLzjimnQpZ1Duyng= +github.com/riverqueue/river/riverdriver v0.32.0 h1:AG6a2hNVOIGLx/+3IRtbwofJRYEI7xqnVVxULe9s4Lg= +github.com/riverqueue/river/riverdriver v0.32.0/go.mod h1:FRDMuqnLOsakeJOHlozKK+VH7W7NLp+6EToxQ2JAjBE= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.32.0 h1:CqrRxxcdA/0sHkxLNldsQff9DIG5qxn2EJO09Pau3w0= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.32.0/go.mod h1:j45UPpbMpcI10m+huTeNUaOwzoLJcEg0K6ihWXWeOec= +github.com/riverqueue/river/rivershared v0.32.0 h1:7DwdrppMU9uoU2iU9aGQiv91nBezjlcI85NV4PmnLHw= +github.com/riverqueue/river/rivershared v0.32.0/go.mod h1:UE7GEj3zaTV3cKw7Q3angCozlNEGsL50xZBKJQ9m6zU= +github.com/riverqueue/river/rivertype v0.32.0 h1:RW7uodfl86gYkjwDponTAPNnUqM+X6BjlsNHxbt6Ztg= +github.com/riverqueue/river/rivertype v0.32.0/go.mod h1:D1Ad+EaZiaXbQbJcJcfeicXJMBKno0n6UcfKI5Q7DIQ= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= @@ -275,7 +287,18 @@ github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+ github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY= github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 h1:REJz+XwNpGC/dCgTfYvM4SKqobNqDBfvhq74s2oHTUM= github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0/go.mod h1:4K2OhtHEeT+JSIFX4V8DkGKsyLa96Y2vLdd3xsxD5HE= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= +github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= @@ -355,6 +378,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go new file mode 100644 index 0000000000..f999926ae2 --- /dev/null +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go @@ -0,0 +1,139 @@ +package backgroundworker + +import ( + "context" + "fmt" + + "github.com/google/uuid" + "github.com/riverqueue/river" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.uber.org/zap" + + sqlcdb "github.com/e2b-dev/infra/packages/db/client" + "github.com/e2b-dev/infra/packages/db/queries" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" + "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" +) + +const AuthUserSyncKind = "auth_user_sync" + +type AuthUserSyncArgs struct { + UserID string `json:"user_id"` + Operation string `json:"operation"` + Email string `json:"email,omitempty"` +} + +func (AuthUserSyncArgs) Kind() string { return AuthUserSyncKind } + +type AuthUserSyncWorker struct { + river.WorkerDefaults[AuthUserSyncArgs] + + mainDB *sqlcdb.Client + l logger.Logger + jobsCounter metric.Int64Counter +} + +func NewAuthUserSyncWorker(mainDB *sqlcdb.Client, l logger.Logger) *AuthUserSyncWorker { + jobsCounter, err := otel.Meter("dashboard-api.backgroundworker.auth_user_sync").Int64Counter( + "dashboard_api.auth_user_sync.jobs_total", + metric.WithDescription("Total auth user sync jobs by operation and result."), + metric.WithUnit("{job}"), + ) + if err != nil { + l.Warn(context.Background(), "failed to initialize auth user sync metric", zap.Error(err)) + } + + return &AuthUserSyncWorker{ + mainDB: mainDB, + l: l, + jobsCounter: jobsCounter, + } +} + +func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSyncArgs]) error { + attrs := []attribute.KeyValue{ + attribute.String("job.kind", AuthUserSyncKind), + attribute.String("job.operation", job.Args.Operation), + attribute.Int64("job.id", job.ID), + telemetry.WithUserID(job.Args.UserID), + } + telemetry.ReportEvent(ctx, "auth_user_sync.job.started", attrs...) + + userID, err := uuid.Parse(job.Args.UserID) + if err != nil { + telemetry.ReportError(ctx, "auth user sync parse user_id", err, attrs...) + w.observeJob(ctx, job.Args.Operation, "error") + + return fmt.Errorf("parse user_id %q: %w", job.Args.UserID, err) + } + + w.l.Info(ctx, "processing auth user sync job", + zap.String("job.kind", AuthUserSyncKind), + zap.Int64("job.id", job.ID), + zap.String("job.operation", job.Args.Operation), + logger.WithUserID(job.Args.UserID), + zap.Int("job.attempt", job.Attempt), + ) + + switch job.Args.Operation { + case "delete": + if err := w.mainDB.DeletePublicUser(ctx, userID); err != nil { + telemetry.ReportError(ctx, "auth user sync delete public user", err, attrs...) + w.observeJob(ctx, job.Args.Operation, "error") + + return fmt.Errorf("delete public.users %s: %w", userID, err) + } + + case "upsert": + if job.Args.Email == "" { + err := fmt.Errorf("missing email in job args") + telemetry.ReportError(ctx, "auth user sync missing email", err, attrs...) + w.observeJob(ctx, job.Args.Operation, "error") + + return fmt.Errorf("upsert public.users %s: missing email in job args", userID) + } + + if err := w.mainDB.UpsertPublicUser(ctx, queries.UpsertPublicUserParams{ + ID: userID, + Email: job.Args.Email, + }); err != nil { + telemetry.ReportError(ctx, "auth user sync upsert public user", err, attrs...) + w.observeJob(ctx, job.Args.Operation, "error") + + return fmt.Errorf("upsert public.users %s: %w", userID, err) + } + + default: + err := fmt.Errorf("unknown operation %q", job.Args.Operation) + telemetry.ReportError(ctx, "auth user sync unknown operation", err, attrs...) + w.observeJob(ctx, job.Args.Operation, "error") + + return fmt.Errorf("unknown operation %q for user %s", job.Args.Operation, userID) + } + + w.l.Info(ctx, "completed auth user sync job", + zap.String("job.kind", AuthUserSyncKind), + zap.Int64("job.id", job.ID), + zap.String("job.operation", job.Args.Operation), + logger.WithUserID(job.Args.UserID), + ) + telemetry.ReportEvent(ctx, "auth_user_sync.job.completed", attrs...) + w.observeJob(ctx, job.Args.Operation, "success") + + return nil +} + +func (w *AuthUserSyncWorker) observeJob(ctx context.Context, operation, result string) { + if w.jobsCounter == nil { + return + } + + w.jobsCounter.Add(ctx, 1, metric.WithAttributes( + attribute.String("worker", "supabase_auth_user_sync"), + attribute.String("job.kind", AuthUserSyncKind), + attribute.String("job.operation", operation), + attribute.String("result", result), + )) +} diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go new file mode 100644 index 0000000000..7d0e928b66 --- /dev/null +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go @@ -0,0 +1,252 @@ +package backgroundworker + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/riverqueue/river" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/e2b-dev/infra/packages/db/pkg/testutils" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" +) + +const ( + testEventuallyTimeout = 10 * time.Second + testEventuallyTick = 50 * time.Millisecond + testStopTimeout = 5 * time.Second +) + +type riverProcess struct { + cancel context.CancelFunc + done chan struct{} + stopOnce sync.Once +} + +func TestAuthUserSync_EndToEnd(t *testing.T) { + t.Parallel() + + db := testutils.SetupDatabase(t) + + authMigrationsDir := "packages/db/pkg/auth/migrations" + + db.ApplyMigrationsUpTo(t, 20260401000001, authMigrationsDir) + + authPool := db.AuthDb.WritePool() + require.NoError(t, RunRiverMigrations(t.Context(), authPool)) + + db.ApplyMigrations(t, authMigrationsDir) + + runUpsertProjection(t, db) + runDeleteProjection(t, db) + runBurstBacklog(t, db) +} + +func runUpsertProjection(t *testing.T, db *testutils.Database) { + t.Helper() + + ctx := t.Context() + userID := uuid.New() + email := fmt.Sprintf("river-sync-%s@example.com", userID.String()[:8]) + + proc := startRiverWorker(t, db) + t.Cleanup(func() { proc.Stop(t) }) + + insertAuthUser(t, ctx, db, userID, email) + + waitForPublicUser(t, ctx, db, userID, email) + + updatedEmail := fmt.Sprintf("river-sync-%s-updated@example.com", userID.String()[:8]) + updateAuthUserEmail(t, ctx, db, userID, updatedEmail) + + waitForPublicUser(t, ctx, db, userID, updatedEmail) + + proc.Stop(t) +} + +func runDeleteProjection(t *testing.T, db *testutils.Database) { + t.Helper() + + ctx := t.Context() + userID := uuid.New() + email := fmt.Sprintf("river-del-%s@example.com", userID.String()[:8]) + + proc := startRiverWorker(t, db) + t.Cleanup(func() { proc.Stop(t) }) + + insertAuthUser(t, ctx, db, userID, email) + waitForPublicUser(t, ctx, db, userID, email) + + deleteAuthUser(t, ctx, db, userID) + waitForPublicUserGone(t, ctx, db, userID) + + proc.Stop(t) +} + +func runBurstBacklog(t *testing.T, db *testutils.Database) { + t.Helper() + + ctx := t.Context() + const userCount = 40 + + type testUser struct { + id uuid.UUID + email string + shouldDel bool + } + + users := make([]testUser, 0, userCount) + for i := range userCount { + u := testUser{ + id: uuid.New(), + email: fmt.Sprintf("river-burst-%02d@example.com", i), + shouldDel: i%3 == 0, + } + users = append(users, u) + insertAuthUser(t, ctx, db, u.id, u.email) + } + + proc := startRiverWorker(t, db) + t.Cleanup(func() { proc.Stop(t) }) + + for _, u := range users { + waitForPublicUser(t, ctx, db, u.id, u.email) + } + + for _, u := range users { + if u.shouldDel { + deleteAuthUser(t, ctx, db, u.id) + } + } + + for _, u := range users { + if u.shouldDel { + waitForPublicUserGone(t, ctx, db, u.id) + } else { + waitForPublicUser(t, ctx, db, u.id, u.email) + } + } + + proc.Stop(t) +} + +func startRiverWorker(t *testing.T, db *testutils.Database) *riverProcess { + t.Helper() + + authPool := db.AuthDb.WritePool() + l := logger.NewNopLogger() + + workers := river.NewWorkers() + river.AddWorker(workers, NewAuthUserSyncWorker(db.SqlcClient, l)) + + client, err := NewRiverClient(authPool, workers) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, client.Start(ctx)) + + done := make(chan struct{}) + + go func() { + <-ctx.Done() + stopCtx, stopCancel := context.WithTimeout(context.WithoutCancel(ctx), testStopTimeout) + defer stopCancel() + + _ = client.Stop(stopCtx) + close(done) + }() + + return &riverProcess{cancel: cancel, done: done} +} + +func (p *riverProcess) Stop(t *testing.T) { + t.Helper() + + p.stopOnce.Do(func() { + p.cancel() + + select { + case <-p.done: + case <-time.After(testStopTimeout): + t.Fatal("river client did not stop in time") + } + }) +} + +func insertAuthUser(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, email string) { + t.Helper() + + err := db.AuthDb.TestsRawSQL(ctx, + "INSERT INTO auth.users (id, email) VALUES ($1, $2)", userID, email) + require.NoError(t, err) +} + +func updateAuthUserEmail(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, email string) { + t.Helper() + + err := db.AuthDb.TestsRawSQL(ctx, + "UPDATE auth.users SET email = $1 WHERE id = $2", email, userID) + require.NoError(t, err) +} + +func deleteAuthUser(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID) { + t.Helper() + + err := db.AuthDb.TestsRawSQL(ctx, + "DELETE FROM auth.users WHERE id = $1", userID) + require.NoError(t, err) +} + +func waitForPublicUser(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, expectedEmail string) { + t.Helper() + + require.EventuallyWithT(t, func(c *assert.CollectT) { + var email string + + err := db.AuthDb.TestsRawSQLQuery(ctx, + "SELECT email FROM public.users WHERE id = $1", + func(rows pgx.Rows) error { + if !rows.Next() { + return fmt.Errorf("user %s not found in public.users", userID) + } + + return rows.Scan(&email) + }, userID) + + if !assert.NoError(c, err) { + return + } + + assert.Equal(c, expectedEmail, email) + }, testEventuallyTimeout, testEventuallyTick) +} + +func waitForPublicUserGone(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID) { + t.Helper() + + require.EventuallyWithT(t, func(c *assert.CollectT) { + var count int + + err := db.AuthDb.TestsRawSQLQuery(ctx, + "SELECT count(*) FROM public.users WHERE id = $1", + func(rows pgx.Rows) error { + if !rows.Next() { + return nil + } + + return rows.Scan(&count) + }, userID) + + if !assert.NoError(c, err) { + return + } + + assert.Equal(c, 0, count) + }, testEventuallyTimeout, testEventuallyTick) +} diff --git a/packages/dashboard-api/internal/backgroundworker/river.go b/packages/dashboard-api/internal/backgroundworker/river.go new file mode 100644 index 0000000000..a248833067 --- /dev/null +++ b/packages/dashboard-api/internal/backgroundworker/river.go @@ -0,0 +1,38 @@ +package backgroundworker + +import ( + "context" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/riverqueue/river" + "github.com/riverqueue/river/riverdriver/riverpgxv5" + "github.com/riverqueue/river/rivermigrate" +) + +const AuthCustomSchema = "auth_custom" + +func RunRiverMigrations(ctx context.Context, pool *pgxpool.Pool) error { + driver := riverpgxv5.New(pool) + + migrator, err := rivermigrate.New(driver, &rivermigrate.Config{ + Schema: AuthCustomSchema, + }) + if err != nil { + return err + } + + _, err = migrator.Migrate(ctx, rivermigrate.DirectionUp, nil) + + return err +} + +func NewRiverClient(pool *pgxpool.Pool, workers *river.Workers) (*river.Client[pgx.Tx], error) { + return river.NewClient(riverpgxv5.New(pool), &river.Config{ + Schema: AuthCustomSchema, + Queues: map[string]river.QueueConfig{ + "auth_sync": {MaxWorkers: 10}, + }, + Workers: workers, + }) +} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/config.go b/packages/dashboard-api/internal/supabaseauthusersync/config.go deleted file mode 100644 index 778be6d592..0000000000 --- a/packages/dashboard-api/internal/supabaseauthusersync/config.go +++ /dev/null @@ -1,28 +0,0 @@ -package supabaseauthusersync - -import "time" - -const ( - defaultBatchSize int32 = 50 - defaultPollInterval time.Duration = 2 * time.Second - defaultLockTimeout time.Duration = 2 * time.Minute - defaultMaxAttempts int32 = 20 -) - -type Config struct { - Enabled bool - BatchSize int32 - PollInterval time.Duration - LockTimeout time.Duration - MaxAttempts int32 -} - -func DefaultConfig() Config { - return Config{ - Enabled: false, - BatchSize: defaultBatchSize, - PollInterval: defaultPollInterval, - LockTimeout: defaultLockTimeout, - MaxAttempts: defaultMaxAttempts, - } -} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/logging.go b/packages/dashboard-api/internal/supabaseauthusersync/logging.go deleted file mode 100644 index 06759556e6..0000000000 --- a/packages/dashboard-api/internal/supabaseauthusersync/logging.go +++ /dev/null @@ -1,214 +0,0 @@ -package supabaseauthusersync - -import ( - "sort" - "time" - - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - - "github.com/e2b-dev/infra/packages/shared/pkg/logger" -) - -type processOutcome string - -const ( - processOutcomeReadyToAck processOutcome = "ready_to_ack" - processOutcomeAcked processOutcome = "acked" - processOutcomeAckFailed processOutcome = "ack_failed" - processOutcomeRetried processOutcome = "retried" - processOutcomeRetryFailed processOutcome = "retry_failed" - processOutcomeDeadLettered processOutcome = "dead_lettered" - processOutcomeDeadLetterFailed processOutcome = "dead_letter_failed" -) - -type reconcileAction string - -const ( - reconcileActionUpsertPublicUser reconcileAction = "upsert_public_user" - reconcileActionDeletePublicUser reconcileAction = "delete_public_user" -) - -type processResult struct { - Outcome processOutcome - Action reconcileAction - Duration time.Duration - Backoff time.Duration -} - -type batchSummary struct { - ClaimedCount int - AckedCount int - AckFailedCount int - RetriedCount int - RetryFailedCount int - DeadLetteredCount int - DeadLetterFailedCount int - MaxAttemptCount int32 - OldestCreatedAt time.Time - NewestCreatedAt time.Time - OldestItemAge time.Duration - NewestItemAge time.Duration - OperationCounts map[string]int - ActionCounts map[string]int -} - -func newBatchSummary(items []QueueItem, now time.Time) batchSummary { - summary := batchSummary{ - ClaimedCount: len(items), - OperationCounts: make(map[string]int), - ActionCounts: make(map[string]int), - } - - for i, item := range items { - if i == 0 || item.AttemptCount > summary.MaxAttemptCount { - summary.MaxAttemptCount = item.AttemptCount - } - - summary.OperationCounts[item.Operation]++ - - if item.CreatedAt.IsZero() { - continue - } - - if summary.OldestCreatedAt.IsZero() || item.CreatedAt.Before(summary.OldestCreatedAt) { - summary.OldestCreatedAt = item.CreatedAt - } - if summary.NewestCreatedAt.IsZero() || item.CreatedAt.After(summary.NewestCreatedAt) { - summary.NewestCreatedAt = item.CreatedAt - } - } - - if !summary.OldestCreatedAt.IsZero() { - summary.OldestItemAge = ageSince(summary.OldestCreatedAt, now) - summary.NewestItemAge = ageSince(summary.NewestCreatedAt, now) - } - - return summary -} - -func (s *batchSummary) Add(result processResult) { - switch result.Outcome { - case processOutcomeAcked: - s.AckedCount++ - case processOutcomeAckFailed: - s.AckFailedCount++ - case processOutcomeRetried: - s.RetriedCount++ - case processOutcomeRetryFailed: - s.RetryFailedCount++ - case processOutcomeDeadLettered: - s.DeadLetteredCount++ - case processOutcomeDeadLetterFailed: - s.DeadLetterFailedCount++ - } - - if result.Action != "" { - s.ActionCounts[string(result.Action)]++ - } -} - -func (s *batchSummary) Fields(totalDuration time.Duration) []zap.Field { - fields := []zap.Field{ - zap.Int("queue_batch.claimed_count", s.ClaimedCount), - zap.Int("queue_batch.acked_count", s.AckedCount), - zap.Int("queue_batch.ack_failed_count", s.AckFailedCount), - zap.Int("queue_batch.retried_count", s.RetriedCount), - zap.Int("queue_batch.retry_failed_count", s.RetryFailedCount), - zap.Int("queue_batch.dead_lettered_count", s.DeadLetteredCount), - zap.Int("queue_batch.dead_letter_failed_count", s.DeadLetterFailedCount), - zap.Int32("queue_batch.max_attempt", s.MaxAttemptCount), - zap.Duration("queue_batch.duration", totalDuration), - } - - if !s.OldestCreatedAt.IsZero() { - fields = append(fields, - logger.Time("queue_batch.oldest_item_created_at", s.OldestCreatedAt), - logger.Time("queue_batch.newest_item_created_at", s.NewestCreatedAt), - zap.Duration("queue_batch.oldest_item_age", s.OldestItemAge), - zap.Duration("queue_batch.newest_item_age", s.NewestItemAge), - ) - } - - if len(s.OperationCounts) > 0 { - fields = append(fields, zap.Object("queue_batch.operation_counts", countsField(s.OperationCounts))) - } - if len(s.ActionCounts) > 0 { - fields = append(fields, zap.Object("queue_batch.action_counts", countsField(s.ActionCounts))) - } - - return fields -} - -func (s *batchSummary) Level() zapcore.Level { - if s.AckFailedCount > 0 || s.RetryFailedCount > 0 || s.DeadLetteredCount > 0 || s.DeadLetterFailedCount > 0 { - return zap.ErrorLevel - } - if s.RetriedCount > 0 { - return zap.WarnLevel - } - - return zap.InfoLevel -} - -func processResultFields(item QueueItem, result processResult, now time.Time) []zap.Field { - fields := queueItemFields(item, now) - fields = append(fields, - zap.String("queue_item.outcome", string(result.Outcome)), - zap.Duration("queue_item.duration", result.Duration), - ) - - if result.Action != "" { - fields = append(fields, zap.String("queue_item.action", string(result.Action))) - } - if result.Backoff > 0 { - fields = append(fields, - zap.Duration("queue_item.retry_backoff", result.Backoff), - zap.Int32("queue_item.next_attempt", item.AttemptCount+1), - ) - } - - return fields -} - -func queueItemFields(item QueueItem, now time.Time) []zap.Field { - fields := []zap.Field{ - zap.Int64("queue_item.id", item.ID), - logger.WithUserID(item.UserID.String()), - zap.String("queue_item.operation", item.Operation), - zap.Int32("queue_item.attempt", item.AttemptCount), - } - - if !item.CreatedAt.IsZero() { - fields = append(fields, - logger.Time("queue_item.created_at", item.CreatedAt), - zap.Duration("queue_item.age", ageSince(item.CreatedAt, now)), - ) - } - - return fields -} - -func ageSince(createdAt time.Time, now time.Time) time.Duration { - if createdAt.IsZero() || now.Before(createdAt) { - return 0 - } - - return now.Sub(createdAt) -} - -type countsField map[string]int - -func (f countsField) MarshalLogObject(enc zapcore.ObjectEncoder) error { - keys := make([]string, 0, len(f)) - for key := range f { - keys = append(keys, key) - } - sort.Strings(keys) - - for _, key := range keys { - enc.AddInt(key, f[key]) - } - - return nil -} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/processor.go b/packages/dashboard-api/internal/supabaseauthusersync/processor.go deleted file mode 100644 index 0a6e9ab369..0000000000 --- a/packages/dashboard-api/internal/supabaseauthusersync/processor.go +++ /dev/null @@ -1,170 +0,0 @@ -package supabaseauthusersync - -import ( - "context" - "errors" - "fmt" - "runtime/debug" - "time" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5" - "go.uber.org/zap" - - "github.com/e2b-dev/infra/packages/shared/pkg/logger" -) - -type processorStore interface { - Retry(ctx context.Context, id int64, backoff time.Duration, lastError string) error - DeadLetter(ctx context.Context, id int64, lastError string) error - GetAuthUser(ctx context.Context, userID uuid.UUID) (*AuthUser, error) - UpsertPublicUser(ctx context.Context, id uuid.UUID, email string) error - DeletePublicUser(ctx context.Context, id uuid.UUID) error -} - -type Processor struct { - store processorStore - maxAttempts int32 - l logger.Logger -} - -func NewProcessor(store processorStore, maxAttempts int32, l logger.Logger) *Processor { - return &Processor{ - store: store, - maxAttempts: maxAttempts, - l: l, - } -} - -func (p *Processor) process(ctx context.Context, item QueueItem) processResult { - startedAt := time.Now() - action, err := p.processOnce(ctx, item) - result := processResult{ - Action: action, - Duration: time.Since(startedAt), - } - - if err == nil { - result.Outcome = processOutcomeReadyToAck - - return result - } - - if item.AttemptCount >= p.maxAttempts { - if dlErr := p.store.DeadLetter(ctx, item.ID, err.Error()); dlErr != nil { - result.Outcome = processOutcomeDeadLetterFailed - - p.l.Error(ctx, "failed to dead-letter supabase auth sync queue item", - append( - processResultFields(item, result, time.Now()), - zap.Int32("queue_item.max_attempts", p.maxAttempts), - zap.NamedError("processing_error", err), - zap.NamedError("dead_letter_error", dlErr), - )..., - ) - - return result - } - - result.Outcome = processOutcomeDeadLettered - p.l.Error(ctx, "dead-lettered supabase auth sync queue item after max attempts", - append( - processResultFields(item, result, time.Now()), - zap.Int32("queue_item.max_attempts", p.maxAttempts), - zap.NamedError("processing_error", err), - )..., - ) - - return result - } - - backoff := retryBackoff(item.AttemptCount) - result.Outcome = processOutcomeRetried - result.Backoff = backoff - - if retryErr := p.store.Retry(ctx, item.ID, backoff, err.Error()); retryErr != nil { - result.Outcome = processOutcomeRetryFailed - - p.l.Error(ctx, "failed to schedule supabase auth sync queue item retry", - append( - processResultFields(item, result, time.Now()), - zap.NamedError("processing_error", err), - zap.NamedError("retry_error", retryErr), - )..., - ) - - return result - } - - p.l.Warn(ctx, "retrying supabase auth sync queue item after processing error", - append( - processResultFields(item, result, time.Now()), - zap.NamedError("processing_error", err), - )..., - ) - - return result -} - -func (p *Processor) processOnce(ctx context.Context, item QueueItem) (action reconcileAction, err error) { - defer func() { - if recovered := recover(); recovered != nil { - p.l.Error(ctx, "panic while processing supabase auth sync queue item", - append( - queueItemFields(item, time.Now()), - zap.String("worker.panic", fmt.Sprint(recovered)), - zap.String("worker.stack", string(debug.Stack())), - )..., - ) - - err = fmt.Errorf("panic while processing queue item: %v", recovered) - } - }() - - return p.reconcile(ctx, item) -} - -func (p *Processor) reconcile(ctx context.Context, item QueueItem) (reconcileAction, error) { - if item.Operation == "delete" { - if err := p.store.DeletePublicUser(ctx, item.UserID); err != nil { - return "", fmt.Errorf("delete public.users %s: %w", item.UserID, err) - } - - return reconcileActionDeletePublicUser, nil - } - - authUser, err := p.store.GetAuthUser(ctx, item.UserID) - - if errors.Is(err, pgx.ErrNoRows) { - if delErr := p.store.DeletePublicUser(ctx, item.UserID); delErr != nil { - return "", fmt.Errorf("delete public.users %s: %w", item.UserID, delErr) - } - - return reconcileActionDeletePublicUser, nil - } - - if err != nil { - return "", fmt.Errorf("get auth.users %s: %w", item.UserID, err) - } - - if err = p.store.UpsertPublicUser(ctx, authUser.ID, authUser.Email); err != nil { - return "", fmt.Errorf("upsert public.users %s: %w", authUser.ID, err) - } - - return reconcileActionUpsertPublicUser, nil -} - -func retryBackoff(attempt int32) time.Duration { - switch { - case attempt <= 1: - return 5 * time.Second - case attempt <= 3: - return 30 * time.Second - case attempt <= 6: - return 2 * time.Minute - case attempt <= 10: - return 5 * time.Minute - default: - return 15 * time.Minute - } -} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/processor_test.go b/packages/dashboard-api/internal/supabaseauthusersync/processor_test.go deleted file mode 100644 index 446653a3d0..0000000000 --- a/packages/dashboard-api/internal/supabaseauthusersync/processor_test.go +++ /dev/null @@ -1,137 +0,0 @@ -package supabaseauthusersync - -import ( - "context" - "testing" - "time" - - "github.com/google/uuid" - "github.com/stretchr/testify/require" - - "github.com/e2b-dev/infra/packages/shared/pkg/logger" -) - -type retryCall struct { - id int64 - backoff time.Duration - lastError string -} - -type deadLetterCall struct { - id int64 - lastError string -} - -type fakeProcessorStore struct { - getAuthUserFn func(context.Context, uuid.UUID) (*AuthUser, error) - - deletePublicUserCalls int - retryCalls []retryCall - deadLetterCalls []deadLetterCall -} - -func (s *fakeProcessorStore) Retry(_ context.Context, id int64, backoff time.Duration, lastError string) error { - s.retryCalls = append(s.retryCalls, retryCall{ - id: id, - backoff: backoff, - lastError: lastError, - }) - - return nil -} - -func (s *fakeProcessorStore) DeadLetter(_ context.Context, id int64, lastError string) error { - s.deadLetterCalls = append(s.deadLetterCalls, deadLetterCall{ - id: id, - lastError: lastError, - }) - - return nil -} - -func (s *fakeProcessorStore) GetAuthUser(ctx context.Context, userID uuid.UUID) (*AuthUser, error) { - return s.getAuthUserFn(ctx, userID) -} - -func (s *fakeProcessorStore) UpsertPublicUser(_ context.Context, _ uuid.UUID, _ string) error { - return nil -} - -func (s *fakeProcessorStore) DeletePublicUser(_ context.Context, _ uuid.UUID) error { - s.deletePublicUserCalls++ - - return nil -} - -func TestProcessorProcessRetriesRecoveredPanic(t *testing.T) { - t.Parallel() - - store := &fakeProcessorStore{ - getAuthUserFn: func(context.Context, uuid.UUID) (*AuthUser, error) { - panic("boom") - }, - } - processor := NewProcessor(store, 3, logger.NewNopLogger()) - item := QueueItem{ - ID: 1, - UserID: uuid.New(), - AttemptCount: 1, - } - - require.NotPanics(t, func() { - processor.process(context.Background(), item) - }) - require.Len(t, store.retryCalls, 1) - require.Contains(t, store.retryCalls[0].lastError, "panic while processing queue item") - require.Empty(t, store.deadLetterCalls) -} - -func TestProcessorProcessDeadLettersRecoveredPanicAtMaxAttempts(t *testing.T) { - t.Parallel() - - store := &fakeProcessorStore{ - getAuthUserFn: func(context.Context, uuid.UUID) (*AuthUser, error) { - panic("boom") - }, - } - processor := NewProcessor(store, 3, logger.NewNopLogger()) - item := QueueItem{ - ID: 1, - UserID: uuid.New(), - AttemptCount: 3, - } - - require.NotPanics(t, func() { - processor.process(context.Background(), item) - }) - require.Empty(t, store.retryCalls) - require.Len(t, store.deadLetterCalls, 1) - require.Contains(t, store.deadLetterCalls[0].lastError, "panic while processing queue item") -} - -func TestProcessorProcessDeleteSkipsAuthLookup(t *testing.T) { - t.Parallel() - - getAuthUserCalled := false - store := &fakeProcessorStore{ - getAuthUserFn: func(context.Context, uuid.UUID) (*AuthUser, error) { - getAuthUserCalled = true - - return nil, nil - }, - } - processor := NewProcessor(store, 3, logger.NewNopLogger()) - item := QueueItem{ - ID: 1, - UserID: uuid.New(), - Operation: "delete", - AttemptCount: 1, - } - - result := processor.process(context.Background(), item) - - require.False(t, getAuthUserCalled) - require.Equal(t, 1, store.deletePublicUserCalls) - require.Equal(t, processOutcomeReadyToAck, result.Outcome) - require.Equal(t, reconcileActionDeletePublicUser, result.Action) -} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/runner.go b/packages/dashboard-api/internal/supabaseauthusersync/runner.go deleted file mode 100644 index 37d014bcc6..0000000000 --- a/packages/dashboard-api/internal/supabaseauthusersync/runner.go +++ /dev/null @@ -1,162 +0,0 @@ -package supabaseauthusersync - -import ( - "context" - "time" - - "go.uber.org/zap" - - sqlcdb "github.com/e2b-dev/infra/packages/db/client" - authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" - "github.com/e2b-dev/infra/packages/shared/pkg/logger" -) - -type runnerStore interface { - ClaimBatch(ctx context.Context, lockOwner string, lockTimeout time.Duration, batchSize int32) ([]QueueItem, error) - AckBatch(ctx context.Context, ids []int64) error -} - -type workerStore interface { - runnerStore - processorStore -} - -type Runner struct { - cfg Config - store runnerStore - processor *Processor - lockOwner string - l logger.Logger -} - -type ackCandidate struct { - item QueueItem - result processResult -} - -func NewRunner(cfg Config, authDB *authdb.Client, mainDB *sqlcdb.Client, lockOwner string, l logger.Logger) *Runner { - workerLogger := l.With(logger.WithServiceInstanceID(lockOwner)) - store := NewStore(authDB, mainDB) - - return &Runner{ - cfg: cfg, - store: store, - processor: NewProcessor(store, cfg.MaxAttempts, workerLogger), - lockOwner: lockOwner, - l: workerLogger, - } -} - -func (r *Runner) Run(ctx context.Context) error { - r.l.Info(ctx, "starting supabase auth user sync worker", - zap.String("worker.lock_owner", r.lockOwner), - zap.Duration("worker.poll_interval", r.cfg.PollInterval), - zap.Int32("worker.batch_size", r.cfg.BatchSize), - zap.Duration("worker.lock_timeout", r.cfg.LockTimeout), - zap.Int32("worker.max_attempts", r.cfg.MaxAttempts), - ) - - for { - r.drain(ctx) - if ctx.Err() != nil { - r.l.Info(ctx, "stopping supabase auth user sync worker", zap.Error(ctx.Err())) - - return ctx.Err() - } - - timer := time.NewTimer(r.cfg.PollInterval) - select { - case <-ctx.Done(): - if !timer.Stop() { - <-timer.C - } - - r.l.Info(ctx, "stopping supabase auth user sync worker", zap.Error(ctx.Err())) - - return ctx.Err() - case <-timer.C: - } - } -} - -func (r *Runner) drain(ctx context.Context) { - for { - processed := r.pollOnce(ctx) - if processed == 0 { - return - } - } -} - -func (r *Runner) pollOnce(ctx context.Context) int { - claimedAt := time.Now() - items, err := r.store.ClaimBatch(ctx, r.lockOwner, r.cfg.LockTimeout, r.cfg.BatchSize) - if err != nil { - r.l.Error(ctx, "failed to claim supabase auth sync queue batch", - zap.String("worker.lock_owner", r.lockOwner), - zap.Duration("worker.lock_timeout", r.cfg.LockTimeout), - zap.Int32("worker.batch_size", r.cfg.BatchSize), - zap.Error(err), - ) - - return 0 - } - - if len(items) == 0 { - return 0 - } - - summary := newBatchSummary(items, claimedAt) - ackCandidates := make([]ackCandidate, 0, len(items)) - - for _, item := range items { - result := r.processor.process(ctx, item) - if result.Outcome == processOutcomeReadyToAck { - ackCandidates = append(ackCandidates, ackCandidate{ - item: item, - result: result, - }) - - continue - } - - summary.Add(result) - } - - if len(ackCandidates) > 0 { - r.finalizeAcks(ctx, ackCandidates, &summary) - } - - r.l.Log(ctx, summary.Level(), "processed supabase auth sync queue batch", summary.Fields(time.Since(claimedAt))...) - - return len(items) -} - -func (r *Runner) finalizeAcks(ctx context.Context, candidates []ackCandidate, summary *batchSummary) { - ids := make([]int64, 0, len(candidates)) - for _, candidate := range candidates { - ids = append(ids, candidate.item.ID) - } - - if err := r.store.AckBatch(ctx, ids); err != nil { - for _, candidate := range candidates { - candidate.result.Outcome = processOutcomeAckFailed - summary.Add(candidate.result) - - r.l.Error(ctx, "processed supabase auth sync queue item but failed to ack", - append( - processResultFields(candidate.item, candidate.result, time.Now()), - zap.NamedError("ack_error", err), - )..., - ) - } - - return - } - - for _, candidate := range candidates { - candidate.result.Outcome = processOutcomeAcked - summary.Add(candidate.result) - r.l.Info(ctx, "processed supabase auth sync queue item", processResultFields(candidate.item, candidate.result, time.Now())...) - } -} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go b/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go deleted file mode 100644 index 25d2327af1..0000000000 --- a/packages/dashboard-api/internal/supabaseauthusersync/runner_test.go +++ /dev/null @@ -1,494 +0,0 @@ -package supabaseauthusersync - -import ( - "context" - "fmt" - "sync" - "testing" - "time" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/e2b-dev/infra/packages/db/pkg/testutils" - "github.com/e2b-dev/infra/packages/shared/pkg/logger" -) - -const ( - testRunnerPollInterval = 20 * time.Millisecond - testRunnerLockTimeout = 150 * time.Millisecond - testEventuallyTimeout = 8 * time.Second - testEventuallyTick = 25 * time.Millisecond - testRunnerStopTimeout = 2 * time.Second -) - -type runnerProcess struct { - cancel context.CancelFunc - done chan error - stopOnce sync.Once -} - -type userExpectation struct { - Email string - Exists bool -} - -type queueSnapshot struct { - Total int - DeadLettered int -} - -func TestSupabaseAuthUserSyncRunner_EndToEnd(t *testing.T) { - t.Parallel() - - db := testutils.SetupDatabase(t) - db.ApplyMigrations(t, "packages/db/pkg/auth/migrations") - - runRepairsInsertUpdateDeleteDrift(t, db) - runReclaimsStaleQueueLocks(t, db) - runDrainsBurstBacklogWithMultipleRunners(t, db) -} - -func runRepairsInsertUpdateDeleteDrift(t *testing.T, db *testutils.Database) { - t.Helper() - - ctx := t.Context() - userID := uuid.New() - initialEmail := fmt.Sprintf("auth-sync-%s-initial@example.com", userID.String()[:8]) - updatedEmail := fmt.Sprintf("auth-sync-%s-updated@example.com", userID.String()[:8]) - - insertAuthUser(t, ctx, db, userID, initialEmail) - deletePublicUser(t, ctx, db, userID) - assertQueueBacklog(t, ctx, db, 1) - - insertRunner := startRunnerProcess(t, db, newTestRunnerConfig(4), "repair-insert") - t.Cleanup(func() { - insertRunner.Stop(t) - }) - waitForPublicUsers(t, ctx, db, map[uuid.UUID]userExpectation{ - userID: { - Email: initialEmail, - Exists: true, - }, - }) - waitForQueueDrain(t, ctx, db) - insertRunner.Stop(t) - - updateAuthUserEmail(t, ctx, db, userID, updatedEmail) - setPublicUserEmail(t, ctx, db, userID, "stale@example.com") - assertQueueBacklog(t, ctx, db, 1) - - updateRunner := startRunnerProcess(t, db, newTestRunnerConfig(4), "repair-update") - t.Cleanup(func() { - updateRunner.Stop(t) - }) - waitForPublicUsers(t, ctx, db, map[uuid.UUID]userExpectation{ - userID: { - Email: updatedEmail, - Exists: true, - }, - }) - waitForQueueDrain(t, ctx, db) - updateRunner.Stop(t) - - deleteAuthUser(t, ctx, db, userID) - insertPublicUser(t, ctx, db, userID, "ghost@example.com") - assertQueueBacklog(t, ctx, db, 1) - - deleteRunner := startRunnerProcess(t, db, newTestRunnerConfig(4), "repair-delete") - t.Cleanup(func() { - deleteRunner.Stop(t) - }) - waitForPublicUsers(t, ctx, db, map[uuid.UUID]userExpectation{ - userID: { - Exists: false, - }, - }) - waitForQueueDrain(t, ctx, db) - deleteRunner.Stop(t) -} - -func runReclaimsStaleQueueLocks(t *testing.T, db *testutils.Database) { - t.Helper() - - ctx := t.Context() - userID := uuid.New() - email := fmt.Sprintf("auth-sync-%s-locked@example.com", userID.String()[:8]) - - insertAuthUser(t, ctx, db, userID, email) - deletePublicUser(t, ctx, db, userID) - lockQueueItems(t, ctx, db, userID, time.Now().Add(-time.Minute), "stale-worker") - assertQueueBacklog(t, ctx, db, 1) - - runner := startRunnerProcess(t, db, newTestRunnerConfig(2), "lock-reclaimer") - t.Cleanup(func() { - runner.Stop(t) - }) - - waitForPublicUsers(t, ctx, db, map[uuid.UUID]userExpectation{ - userID: { - Email: email, - Exists: true, - }, - }) - waitForQueueDrain(t, ctx, db) - runner.Stop(t) -} - -func runDrainsBurstBacklogWithMultipleRunners(t *testing.T, db *testutils.Database) { - t.Helper() - - ctx := t.Context() - const userCount = 60 - - userIDs := make([]uuid.UUID, 0, userCount) - - for i := range userCount { - userID := uuid.New() - userIDs = append(userIDs, userID) - - initialEmail := fmt.Sprintf("auth-sync-burst-%02d-initial@example.com", i) - insertAuthUser(t, ctx, db, userID, initialEmail) - - if i%2 == 0 { - updateAuthUserEmail(t, ctx, db, userID, fmt.Sprintf("auth-sync-burst-%02d-v2@example.com", i)) - } - if i%5 == 0 { - updateAuthUserEmail(t, ctx, db, userID, fmt.Sprintf("auth-sync-burst-%02d-v3@example.com", i)) - } - - if i%3 == 0 { - deleteAuthUser(t, ctx, db, userID) - enqueueUserSyncItem(t, ctx, db, userID, "delete") - if i%6 == 0 { - insertPublicUser(t, ctx, db, userID, fmt.Sprintf("ghost-%02d@example.com", i)) - } - - continue - } - - if i%8 == 0 { - deletePublicUser(t, ctx, db, userID) - } else if i%7 == 0 { - setPublicUserEmail(t, ctx, db, userID, fmt.Sprintf("stale-%02d@example.com", i)) - } - - if i%4 == 0 { - enqueueUserSyncItem(t, ctx, db, userID, "upsert") - } - if i%9 == 0 { - enqueueUserSyncItem(t, ctx, db, userID, "upsert") - } - } - - authUsers, err := loadAuthUsers(ctx, db) - require.NoError(t, err) - - want := expectedUsersForIDs(userIDs, authUsers) - assertQueueBacklog(t, ctx, db, userCount) - - runnerA := startRunnerProcess(t, db, newTestRunnerConfig(5), "burst-a") - runnerB := startRunnerProcess(t, db, newTestRunnerConfig(5), "burst-b") - t.Cleanup(func() { - runnerA.Stop(t) - runnerB.Stop(t) - }) - - waitForPublicUsers(t, ctx, db, want) - waitForQueueDrain(t, ctx, db) - - runnerA.Stop(t) - runnerB.Stop(t) -} - -func newTestRunnerConfig(batchSize int32) Config { - cfg := DefaultConfig() - cfg.Enabled = true - cfg.BatchSize = batchSize - cfg.PollInterval = testRunnerPollInterval - cfg.LockTimeout = testRunnerLockTimeout - cfg.MaxAttempts = 5 - - return cfg -} - -func startRunnerProcess(t *testing.T, db *testutils.Database, cfg Config, lockOwner string) *runnerProcess { - t.Helper() - - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan error, 1) - runner := NewRunner( - cfg, - db.AuthDb, - db.SqlcClient, - lockOwner, - logger.NewNopLogger(), - ) - - go func() { - done <- runner.Run(ctx) - }() - - return &runnerProcess{ - cancel: cancel, - done: done, - } -} - -func (p *runnerProcess) Stop(t *testing.T) { - t.Helper() - - p.stopOnce.Do(func() { - p.cancel() - - select { - case err := <-p.done: - require.ErrorIs(t, err, context.Canceled) - case <-time.After(testRunnerStopTimeout): - t.Fatalf("runner did not stop within %s", testRunnerStopTimeout) - } - }) -} - -func insertAuthUser(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, email string) { - t.Helper() - - err := db.AuthDb.TestsRawSQL(ctx, - "INSERT INTO auth.users (id, email) VALUES ($1, $2)", - userID, - email, - ) - require.NoError(t, err) -} - -func updateAuthUserEmail(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, email string) { - t.Helper() - - err := db.AuthDb.TestsRawSQL(ctx, - "UPDATE auth.users SET email = $1 WHERE id = $2", - email, - userID, - ) - require.NoError(t, err) -} - -func deleteAuthUser(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID) { - t.Helper() - - err := db.AuthDb.TestsRawSQL(ctx, - "DELETE FROM auth.users WHERE id = $1", - userID, - ) - require.NoError(t, err) -} - -func deletePublicUser(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID) { - t.Helper() - - err := db.AuthDb.TestsRawSQL(ctx, - "DELETE FROM public.users WHERE id = $1", - userID, - ) - require.NoError(t, err) -} - -func insertPublicUser(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, email string) { - t.Helper() - - err := db.AuthDb.TestsRawSQL(ctx, ` -INSERT INTO public.users (id, email) -VALUES ($1, $2) -ON CONFLICT (id) DO UPDATE -SET email = EXCLUDED.email, - updated_at = now() -`, - userID, - email, - ) - require.NoError(t, err) -} - -func setPublicUserEmail(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, email string) { - t.Helper() - - err := db.AuthDb.TestsRawSQL(ctx, - "UPDATE public.users SET email = $1, updated_at = now() WHERE id = $2", - email, - userID, - ) - require.NoError(t, err) -} - -func enqueueUserSyncItem(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, operation string) { - t.Helper() - - err := db.AuthDb.TestsRawSQL(ctx, - "INSERT INTO public.user_sync_queue (user_id, operation) VALUES ($1, $2)", - userID, - operation, - ) - require.NoError(t, err) -} - -func lockQueueItems(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, lockedAt time.Time, lockOwner string) { - t.Helper() - - err := db.AuthDb.TestsRawSQL(ctx, ` -UPDATE public.user_sync_queue -SET locked_at = $2, - lock_owner = $3 -WHERE user_id = $1 -`, - userID, - lockedAt, - lockOwner, - ) - require.NoError(t, err) -} - -func loadPublicUsers(ctx context.Context, db *testutils.Database) (map[uuid.UUID]string, error) { - users := make(map[uuid.UUID]string) - - err := db.AuthDb.TestsRawSQLQuery(ctx, - "SELECT id, email FROM public.users", - func(rows pgx.Rows) error { - for rows.Next() { - var userID uuid.UUID - var email string - if err := rows.Scan(&userID, &email); err != nil { - return err - } - - users[userID] = email - } - - return rows.Err() - }, - ) - if err != nil { - return nil, err - } - - return users, nil -} - -func loadAuthUsers(ctx context.Context, db *testutils.Database) (map[uuid.UUID]string, error) { - users := make(map[uuid.UUID]string) - - err := db.AuthDb.TestsRawSQLQuery(ctx, - "SELECT id, email FROM auth.users", - func(rows pgx.Rows) error { - for rows.Next() { - var userID uuid.UUID - var email string - if err := rows.Scan(&userID, &email); err != nil { - return err - } - - users[userID] = email - } - - return rows.Err() - }, - ) - if err != nil { - return nil, err - } - - return users, nil -} - -func loadQueueSnapshot(ctx context.Context, db *testutils.Database) (queueSnapshot, error) { - var snapshot queueSnapshot - - err := db.AuthDb.TestsRawSQLQuery(ctx, ` -SELECT - count(*)::int AS total, - count(*) FILTER (WHERE dead_lettered_at IS NOT NULL)::int AS dead_lettered -FROM public.user_sync_queue -`, - func(rows pgx.Rows) error { - if !rows.Next() { - return nil - } - - return rows.Scan(&snapshot.Total, &snapshot.DeadLettered) - }, - ) - if err != nil { - return queueSnapshot{}, err - } - - return snapshot, nil -} - -func expectedUsersForIDs(userIDs []uuid.UUID, authUsers map[uuid.UUID]string) map[uuid.UUID]userExpectation { - want := make(map[uuid.UUID]userExpectation, len(userIDs)) - - for _, userID := range userIDs { - email, ok := authUsers[userID] - want[userID] = userExpectation{ - Email: email, - Exists: ok, - } - } - - return want -} - -func assertQueueBacklog(t *testing.T, ctx context.Context, db *testutils.Database, minimum int) { - t.Helper() - - snapshot, err := loadQueueSnapshot(ctx, db) - require.NoError(t, err) - require.GreaterOrEqual(t, snapshot.Total, minimum) -} - -func waitForQueueDrain(t *testing.T, ctx context.Context, db *testutils.Database) { - t.Helper() - - require.EventuallyWithT(t, func(c *assert.CollectT) { - snapshot, err := loadQueueSnapshot(ctx, db) - if !assert.NoError(c, err) { - return - } - - assert.Equal(c, 0, snapshot.Total) - assert.Equal(c, 0, snapshot.DeadLettered) - }, testEventuallyTimeout, testEventuallyTick) -} - -func waitForPublicUsers(t *testing.T, ctx context.Context, db *testutils.Database, want map[uuid.UUID]userExpectation) { - t.Helper() - - require.EventuallyWithT(t, func(c *assert.CollectT) { - got, err := loadPublicUsers(ctx, db) - if !assert.NoError(c, err) { - return - } - - var gotExisting int - var wantExisting int - - for userID, expectation := range want { - email, ok := got[userID] - if ok { - gotExisting++ - } - if expectation.Exists { - wantExisting++ - } - - if !assert.Equalf(c, expectation.Exists, ok, "public.users presence for %s", userID) { - continue - } - if expectation.Exists { - assert.Equalf(c, expectation.Email, email, "public.users email for %s", userID) - } - } - - assert.Equal(c, wantExisting, gotExisting) - }, testEventuallyTimeout, testEventuallyTick) -} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/store.go b/packages/dashboard-api/internal/supabaseauthusersync/store.go deleted file mode 100644 index 63ebc988d0..0000000000 --- a/packages/dashboard-api/internal/supabaseauthusersync/store.go +++ /dev/null @@ -1,115 +0,0 @@ -package supabaseauthusersync - -import ( - "context" - "time" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" - - sqlcdb "github.com/e2b-dev/infra/packages/db/client" - authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" - authqueries "github.com/e2b-dev/infra/packages/db/pkg/auth/queries" - "github.com/e2b-dev/infra/packages/db/queries" -) - -type QueueItem struct { - ID int64 - UserID uuid.UUID - Operation string - CreatedAt time.Time - AttemptCount int32 -} - -type AuthUser struct { - ID uuid.UUID - Email string -} - -type Store struct { - authQueries *authqueries.Queries - mainQueries *queries.Queries -} - -var _ workerStore = (*Store)(nil) - -func NewStore(authDB *authdb.Client, mainDB *sqlcdb.Client) *Store { - return &Store{ - authQueries: authDB.Write, - mainQueries: mainDB.Queries, - } -} - -func (s *Store) ClaimBatch(ctx context.Context, lockOwner string, lockTimeout time.Duration, batchSize int32) ([]QueueItem, error) { - rows, err := s.authQueries.ClaimUserSyncQueueBatch(ctx, authqueries.ClaimUserSyncQueueBatchParams{ - LockOwner: lockOwner, - LockTimeout: durationToInterval(lockTimeout), - BatchSize: batchSize, - }) - if err != nil { - return nil, err - } - - items := make([]QueueItem, len(rows)) - for i, r := range rows { - items[i] = QueueItem{ - ID: r.ID, - UserID: r.UserID, - Operation: r.Operation, - CreatedAt: r.CreatedAt, - AttemptCount: r.AttemptCount, - } - } - - return items, nil -} - -func (s *Store) AckBatch(ctx context.Context, ids []int64) error { - if len(ids) == 0 { - return nil - } - - return s.authQueries.AckUserSyncQueueItems(ctx, ids) -} - -func (s *Store) Retry(ctx context.Context, id int64, backoff time.Duration, lastError string) error { - return s.authQueries.RetryUserSyncQueueItem(ctx, authqueries.RetryUserSyncQueueItemParams{ - ID: id, - Backoff: durationToInterval(backoff), - LastError: lastError, - }) -} - -func (s *Store) DeadLetter(ctx context.Context, id int64, lastError string) error { - return s.authQueries.DeadLetterUserSyncQueueItem(ctx, authqueries.DeadLetterUserSyncQueueItemParams{ - ID: id, - LastError: lastError, - }) -} - -func (s *Store) GetAuthUser(ctx context.Context, userID uuid.UUID) (*AuthUser, error) { - row, err := s.authQueries.GetAuthUserByID(ctx, userID) - if err != nil { - return nil, err - } - - return &AuthUser{ID: row.ID, Email: row.Email}, nil -} - -func (s *Store) UpsertPublicUser(ctx context.Context, id uuid.UUID, email string) error { - return s.mainQueries.UpsertPublicUser(ctx, queries.UpsertPublicUserParams{ - ID: id, - Email: email, - }) -} - -func (s *Store) DeletePublicUser(ctx context.Context, id uuid.UUID) error { - return s.mainQueries.DeletePublicUser(ctx, id) -} - -func durationToInterval(d time.Duration) pgtype.Interval { - return pgtype.Interval{ - Microseconds: d.Microseconds(), - Valid: true, - } -} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/supervisor.go b/packages/dashboard-api/internal/supabaseauthusersync/supervisor.go deleted file mode 100644 index de60dda43a..0000000000 --- a/packages/dashboard-api/internal/supabaseauthusersync/supervisor.go +++ /dev/null @@ -1,117 +0,0 @@ -package supabaseauthusersync - -import ( - "context" - "errors" - "fmt" - "runtime/debug" - "time" - - "go.uber.org/zap" - - "github.com/e2b-dev/infra/packages/shared/pkg/logger" -) - -const ( - defaultRestartDelay = time.Second - maxRestartDelay = 30 * time.Second - healthyRunResetThreshold = time.Minute -) - -type supervisorConfig struct { - RestartDelay time.Duration - MaxRestartDelay time.Duration - HealthyRunResetAfter time.Duration -} - -func defaultSupervisorConfig() supervisorConfig { - return supervisorConfig{ - RestartDelay: defaultRestartDelay, - MaxRestartDelay: maxRestartDelay, - HealthyRunResetAfter: healthyRunResetThreshold, - } -} - -func (r *Runner) RunWithRestart(ctx context.Context) error { - return supervise(ctx, r.l, defaultSupervisorConfig(), r.Run) -} - -func supervise(ctx context.Context, l logger.Logger, cfg supervisorConfig, run func(context.Context) error) error { - restartAttempt := 0 - - for { - startedAt := time.Now() - err := runRecovering(ctx, l, run) - runtime := time.Since(startedAt) - - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return err - } - if ctx.Err() != nil { - return ctx.Err() - } - - if runtime >= cfg.HealthyRunResetAfter { - restartAttempt = 0 - } - restartAttempt++ - - delay := restartBackoff(restartAttempt, cfg.RestartDelay, cfg.MaxRestartDelay) - l.Error(ctx, "supabase auth user sync worker exited unexpectedly; restarting", - zap.Error(err), - zap.Int("worker.restart_attempt", restartAttempt), - zap.Duration("worker.restart_in", delay), - zap.Duration("worker.runtime", runtime), - zap.Duration("worker.healthy_run_reset_after", cfg.HealthyRunResetAfter), - ) - - timer := time.NewTimer(delay) - select { - case <-ctx.Done(): - timer.Stop() - - return ctx.Err() - case <-timer.C: - } - } -} - -func runRecovering(ctx context.Context, l logger.Logger, run func(context.Context) error) (err error) { - defer func() { - if recovered := recover(); recovered != nil { - l.Error(ctx, "supabase auth user sync worker panicked", - zap.String("worker.panic", fmt.Sprint(recovered)), - zap.String("worker.stack", string(debug.Stack())), - ) - - err = fmt.Errorf("worker panic: %v", recovered) - } - }() - - err = run(ctx) - if err == nil && ctx.Err() == nil { - return errors.New("worker exited without error") - } - - return err -} - -func restartBackoff(attempt int, base time.Duration, maxDelay time.Duration) time.Duration { - if base <= 0 { - base = defaultRestartDelay - } - if maxDelay < base { - maxDelay = base - } - - delay := base - for i := 1; i < attempt; i++ { - if delay >= maxDelay/2 { - return maxDelay - } - - delay *= 2 - } - - return delay -} diff --git a/packages/dashboard-api/internal/supabaseauthusersync/supervisor_test.go b/packages/dashboard-api/internal/supabaseauthusersync/supervisor_test.go deleted file mode 100644 index fc53f77ba4..0000000000 --- a/packages/dashboard-api/internal/supabaseauthusersync/supervisor_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package supabaseauthusersync - -import ( - "context" - "errors" - "sync/atomic" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/e2b-dev/infra/packages/shared/pkg/logger" -) - -func TestSuperviseRestartsAfterUnexpectedError(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - var runs atomic.Int32 - errCh := make(chan error, 1) - - go func() { - errCh <- supervise(ctx, logger.NewNopLogger(), supervisorConfig{ - RestartDelay: time.Millisecond, - MaxRestartDelay: time.Millisecond, - HealthyRunResetAfter: time.Hour, - }, func(ctx context.Context) error { - attempt := runs.Add(1) - if attempt < 3 { - return errors.New("boom") - } - - cancel() - <-ctx.Done() - - return ctx.Err() - }) - }() - - err := <-errCh - require.ErrorIs(t, err, context.Canceled) - require.Equal(t, int32(3), runs.Load()) -} - -func TestSuperviseRestartsAfterPanic(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - var runs atomic.Int32 - errCh := make(chan error, 1) - - go func() { - errCh <- supervise(ctx, logger.NewNopLogger(), supervisorConfig{ - RestartDelay: time.Millisecond, - MaxRestartDelay: time.Millisecond, - HealthyRunResetAfter: time.Hour, - }, func(ctx context.Context) error { - attempt := runs.Add(1) - if attempt == 1 { - panic("boom") - } - - cancel() - <-ctx.Done() - - return ctx.Err() - }) - }() - - err := <-errCh - require.ErrorIs(t, err, context.Canceled) - require.Equal(t, int32(2), runs.Load()) -} diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index f1a75eae72..e6df2d9eb4 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -20,7 +20,9 @@ import ( "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" "github.com/google/uuid" + "github.com/jackc/pgx/v5" middleware "github.com/oapi-codegen/gin-middleware" + "github.com/riverqueue/river" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -28,9 +30,9 @@ import ( "github.com/e2b-dev/infra/packages/auth/pkg/types" clickhouse "github.com/e2b-dev/infra/packages/clickhouse/pkg" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" + "github.com/e2b-dev/infra/packages/dashboard-api/internal/backgroundworker" "github.com/e2b-dev/infra/packages/dashboard-api/internal/cfg" "github.com/e2b-dev/infra/packages/dashboard-api/internal/handlers" - "github.com/e2b-dev/infra/packages/dashboard-api/internal/supabaseauthusersync" sqlcdb "github.com/e2b-dev/infra/packages/db/client" authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" "github.com/e2b-dev/infra/packages/db/pkg/pool" @@ -230,24 +232,29 @@ func run() int { wg := sync.WaitGroup{} + var riverClient *river.Client[pgx.Tx] + if config.SupabaseAuthUserSyncEnabled { workerLogger := l.With(zap.String("worker", "supabase_auth_user_sync")) - syncConfig := supabaseauthusersync.DefaultConfig() - syncConfig.Enabled = true - syncRunner := supabaseauthusersync.NewRunner( - syncConfig, - authDB, - db, - serviceInstanceID, - workerLogger, - ) - - wg.Go(func() { - if err := syncRunner.RunWithRestart(signalCtx); err != nil && !errors.Is(err, context.Canceled) { - l.Error(ctx, "supabase auth user sync worker error", zap.Error(err)) - errorCode.Add(1) - } - }) + + authPool := authDB.WritePool() + if err := backgroundworker.RunRiverMigrations(ctx, authPool); err != nil { + l.Fatal(ctx, "failed to run River migrations on auth DB", zap.Error(err)) + } + + workers := river.NewWorkers() + river.AddWorker(workers, backgroundworker.NewAuthUserSyncWorker(db, workerLogger)) + + riverClient, err = backgroundworker.NewRiverClient(authPool, workers) + if err != nil { + l.Fatal(ctx, "failed to create River client", zap.Error(err)) + } + + if err := riverClient.Start(signalCtx); err != nil { + l.Fatal(ctx, "failed to start River client", zap.Error(err)) + } + + l.Info(ctx, "background worker started (River auth_custom)", zap.String("queue", "auth_sync")) } wg.Go(func() { @@ -257,6 +264,14 @@ func run() int { shutdownCtx, shutdownCancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) defer shutdownCancel() + if riverClient != nil { + if err := riverClient.Stop(shutdownCtx); err != nil { + l.Error(ctx, "River client shutdown error", zap.Error(err)) + + errorCode.Add(1) + } + } + if err := s.Shutdown(shutdownCtx); err != nil { l.Error(ctx, "HTTP server shutdown error", zap.Error(err)) diff --git a/packages/db/pkg/auth/client.go b/packages/db/pkg/auth/client.go index 02a1dcf2a0..fe9f33d666 100644 --- a/packages/db/pkg/auth/client.go +++ b/packages/db/pkg/auth/client.go @@ -60,6 +60,10 @@ func (db *Client) Close() error { return nil } +func (db *Client) WritePool() *pgxpool.Pool { + return db.writeConn +} + // WithTx runs the given function in a transaction. func (db *Client) WithTx(ctx context.Context) (*authqueries.Queries, pgx.Tx, error) { tx, err := db.writeConn.BeginTx(ctx, pgx.TxOptions{}) diff --git a/packages/db/pkg/auth/migrations/20260328000001_dashboard_supabase_auth_user_sync_queue.sql b/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql similarity index 66% rename from packages/db/pkg/auth/migrations/20260328000001_dashboard_supabase_auth_user_sync_queue.sql rename to packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql index e300948803..3d41c6d733 100644 --- a/packages/db/pkg/auth/migrations/20260328000001_dashboard_supabase_auth_user_sync_queue.sql +++ b/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql @@ -1,71 +1,66 @@ -- +goose Up -- +goose StatementBegin -CREATE TABLE public.user_sync_queue ( - id BIGSERIAL PRIMARY KEY, - user_id UUID NOT NULL, - operation TEXT NOT NULL CHECK (operation IN ('upsert', 'delete')), - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), - locked_at TIMESTAMPTZ NULL, - lock_owner TEXT NULL, - attempt_count INT NOT NULL DEFAULT 0, - last_error TEXT NULL, - dead_lettered_at TIMESTAMPTZ NULL -); - -ALTER TABLE public.user_sync_queue ENABLE ROW LEVEL SECURITY; - -CREATE INDEX auth_user_sync_queue_pending_idx - ON public.user_sync_queue (id) - WHERE dead_lettered_at IS NULL AND locked_at IS NULL; - -CREATE INDEX auth_user_sync_queue_user_idx - ON public.user_sync_queue (user_id); - -GRANT INSERT ON public.user_sync_queue TO trigger_user; -GRANT USAGE, SELECT ON SEQUENCE public.user_sync_queue_id_seq TO trigger_user; - -CREATE POLICY "Allow to create a user sync queue item" - ON public.user_sync_queue - AS PERMISSIVE - FOR INSERT - TO trigger_user - WITH CHECK (TRUE); + +CREATE SCHEMA IF NOT EXISTS auth_custom; CREATE OR REPLACE FUNCTION public.sync_insert_auth_users_to_public_users_trigger() RETURNS TRIGGER LANGUAGE plpgsql AS $func$ BEGIN - INSERT INTO public.user_sync_queue (user_id, operation) - VALUES (NEW.id, 'upsert'); + INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) + VALUES ( + jsonb_build_object('user_id', NEW.id, 'operation', 'upsert', 'email', NEW.email), + 'auth_user_sync', + 20, + 'auth_sync', + 'available' + ); + + PERFORM pg_notify('auth_custom.river_insert', '{"queue":"auth_sync"}'); RETURN NEW; END; -$func$ SECURITY DEFINER SET search_path = public; +$func$ SECURITY DEFINER SET search_path = public, auth_custom; CREATE OR REPLACE FUNCTION public.sync_update_auth_users_to_public_users_trigger() RETURNS TRIGGER LANGUAGE plpgsql AS $func$ BEGIN IF OLD.email IS DISTINCT FROM NEW.email THEN - INSERT INTO public.user_sync_queue (user_id, operation) - VALUES (NEW.id, 'upsert'); + INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) + VALUES ( + jsonb_build_object('user_id', NEW.id, 'operation', 'upsert', 'email', NEW.email), + 'auth_user_sync', + 20, + 'auth_sync', + 'available' + ); + + PERFORM pg_notify('auth_custom.river_insert', '{"queue":"auth_sync"}'); END IF; RETURN NEW; END; -$func$ SECURITY DEFINER SET search_path = public; +$func$ SECURITY DEFINER SET search_path = public, auth_custom; CREATE OR REPLACE FUNCTION public.sync_delete_auth_users_to_public_users_trigger() RETURNS TRIGGER LANGUAGE plpgsql AS $func$ BEGIN - INSERT INTO public.user_sync_queue (user_id, operation) - VALUES (OLD.id, 'delete'); + INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) + VALUES ( + jsonb_build_object('user_id', OLD.id, 'operation', 'delete'), + 'auth_user_sync', + 20, + 'auth_sync', + 'available' + ); + + PERFORM pg_notify('auth_custom.river_insert', '{"queue":"auth_sync"}'); RETURN OLD; END; -$func$ SECURITY DEFINER SET search_path = public; +$func$ SECURITY DEFINER SET search_path = public, auth_custom; ALTER FUNCTION public.sync_insert_auth_users_to_public_users_trigger() OWNER TO trigger_user; ALTER FUNCTION public.sync_update_auth_users_to_public_users_trigger() OWNER TO trigger_user; @@ -85,10 +80,16 @@ DROP TRIGGER IF EXISTS sync_deletes_to_public_users ON auth.users; CREATE TRIGGER sync_deletes_to_public_users AFTER DELETE ON auth.users FOR EACH ROW EXECUTE FUNCTION public.sync_delete_auth_users_to_public_users_trigger(); + +GRANT USAGE ON SCHEMA auth_custom TO trigger_user; +GRANT INSERT ON auth_custom.river_job TO trigger_user; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA auth_custom TO trigger_user; + -- +goose StatementEnd -- +goose Down -- +goose StatementBegin + DROP TRIGGER IF EXISTS sync_inserts_to_public_users ON auth.users; DROP TRIGGER IF EXISTS sync_updates_to_public_users ON auth.users; DROP TRIGGER IF EXISTS sync_deletes_to_public_users ON auth.users; @@ -97,8 +98,8 @@ CREATE OR REPLACE FUNCTION public.sync_insert_auth_users_to_public_users_trigger LANGUAGE plpgsql AS $func$ BEGIN - INSERT INTO public.users (id, email) - VALUES (NEW.id, NEW.email); + INSERT INTO public.user_sync_queue (user_id, operation) + VALUES (NEW.id, 'upsert'); RETURN NEW; END; @@ -108,13 +109,9 @@ CREATE OR REPLACE FUNCTION public.sync_update_auth_users_to_public_users_trigger LANGUAGE plpgsql AS $func$ BEGIN - UPDATE public.users - SET email = NEW.email, - updated_at = now() - WHERE id = NEW.id; - - IF NOT FOUND THEN - RAISE EXCEPTION 'User with id % does not exist in public.users', NEW.id; + IF OLD.email IS DISTINCT FROM NEW.email THEN + INSERT INTO public.user_sync_queue (user_id, operation) + VALUES (NEW.id, 'upsert'); END IF; RETURN NEW; @@ -125,7 +122,8 @@ CREATE OR REPLACE FUNCTION public.sync_delete_auth_users_to_public_users_trigger LANGUAGE plpgsql AS $func$ BEGIN - DELETE FROM public.users WHERE id = OLD.id; + INSERT INTO public.user_sync_queue (user_id, operation) + VALUES (OLD.id, 'delete'); RETURN OLD; END; @@ -147,10 +145,8 @@ CREATE TRIGGER sync_deletes_to_public_users AFTER DELETE ON auth.users FOR EACH ROW EXECUTE FUNCTION public.sync_delete_auth_users_to_public_users_trigger(); -REVOKE INSERT ON public.user_sync_queue FROM trigger_user; -REVOKE USAGE, SELECT ON SEQUENCE public.user_sync_queue_id_seq FROM trigger_user; +REVOKE ALL ON SCHEMA auth_custom FROM trigger_user; -DROP POLICY IF EXISTS "Allow to create a user sync queue item" ON public.user_sync_queue; +DROP SCHEMA IF EXISTS auth_custom CASCADE; -DROP TABLE public.user_sync_queue; -- +goose StatementEnd diff --git a/packages/db/pkg/auth/queries/models.go b/packages/db/pkg/auth/queries/models.go index e32050f8ad..de91154ef2 100644 --- a/packages/db/pkg/auth/queries/models.go +++ b/packages/db/pkg/auth/queries/models.go @@ -220,19 +220,6 @@ type User struct { Email string } -type UserSyncQueue struct { - ID int64 - UserID uuid.UUID - Operation string - CreatedAt time.Time - NextAttemptAt time.Time - LockedAt *time.Time - LockOwner *string - AttemptCount int32 - LastError *string - DeadLetteredAt *time.Time -} - type UsersTeam struct { ID int64 UserID uuid.UUID diff --git a/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/ack_batch.sql b/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/ack_batch.sql deleted file mode 100644 index 45dd7f6e49..0000000000 --- a/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/ack_batch.sql +++ /dev/null @@ -1,3 +0,0 @@ --- name: AckUserSyncQueueItems :exec -DELETE FROM public.user_sync_queue -WHERE id = ANY(sqlc.arg(ids)::bigint[]); diff --git a/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/claim_batch.sql b/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/claim_batch.sql deleted file mode 100644 index 08f35e662d..0000000000 --- a/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/claim_batch.sql +++ /dev/null @@ -1,17 +0,0 @@ --- name: ClaimUserSyncQueueBatch :many -UPDATE public.user_sync_queue -SET - locked_at = now(), - lock_owner = sqlc.arg(lock_owner)::text, - attempt_count = attempt_count + 1 -WHERE id IN ( - SELECT id - FROM public.user_sync_queue - WHERE dead_lettered_at IS NULL - AND next_attempt_at <= now() - AND (locked_at IS NULL OR locked_at < now() - sqlc.arg(lock_timeout)::interval) - ORDER BY id - FOR UPDATE SKIP LOCKED - LIMIT sqlc.arg(batch_size)::int -) -RETURNING id, user_id, operation, created_at, attempt_count; diff --git a/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/dead_letter.sql b/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/dead_letter.sql deleted file mode 100644 index a68d8cd363..0000000000 --- a/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/dead_letter.sql +++ /dev/null @@ -1,8 +0,0 @@ --- name: DeadLetterUserSyncQueueItem :exec -UPDATE public.user_sync_queue -SET - locked_at = NULL, - lock_owner = NULL, - dead_lettered_at = now(), - last_error = sqlc.arg(last_error)::text -WHERE id = sqlc.arg(id)::bigint; diff --git a/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/get_auth_user.sql b/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/get_auth_user.sql deleted file mode 100644 index 414b3fe3cf..0000000000 --- a/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/get_auth_user.sql +++ /dev/null @@ -1,4 +0,0 @@ --- name: GetAuthUserByID :one -SELECT id, email -FROM auth.users -WHERE id = sqlc.arg(user_id)::uuid; diff --git a/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/retry.sql b/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/retry.sql deleted file mode 100644 index 5386fdce5e..0000000000 --- a/packages/db/pkg/auth/sql_queries/supabase_auth_user_sync/retry.sql +++ /dev/null @@ -1,8 +0,0 @@ --- name: RetryUserSyncQueueItem :exec -UPDATE public.user_sync_queue -SET - locked_at = NULL, - lock_owner = NULL, - next_attempt_at = now() + sqlc.arg(backoff)::interval, - last_error = sqlc.arg(last_error)::text -WHERE id = sqlc.arg(id)::bigint; diff --git a/packages/db/pkg/testutils/db.go b/packages/db/pkg/testutils/db.go index 3b1f8accb5..a9fd66e4c4 100644 --- a/packages/db/pkg/testutils/db.go +++ b/packages/db/pkg/testutils/db.go @@ -118,6 +118,22 @@ func SetupDatabase(t *testing.T) *Database { func (db *Database) ApplyMigrations(t *testing.T, migrationDirs ...string) { t.Helper() + db.applyGooseMigrations(t, 0, migrationDirs...) +} + +func (db *Database) ApplyMigrationsUpTo(t *testing.T, version int64, migrationDirs ...string) { + t.Helper() + + db.applyGooseMigrations(t, version, migrationDirs...) +} + +func (db *Database) ConnStr() string { + return db.connStr +} + +func (db *Database) applyGooseMigrations(t *testing.T, upToVersion int64, migrationDirs ...string) { + t.Helper() + cmd := exec.CommandContext(t.Context(), "git", "rev-parse", "--show-toplevel") output, err := cmd.Output() require.NoError(t, err, "Failed to find git root") @@ -134,13 +150,23 @@ func (db *Database) ApplyMigrations(t *testing.T, migrationDirs ...string) { }) for _, migrationsDir := range migrationDirs { - err = goose.RunWithOptionsContext( - t.Context(), - "up", - sqlDB, - filepath.Join(repoRoot, migrationsDir), - nil, - ) + if upToVersion > 0 { + err = goose.UpToContext( + t.Context(), + sqlDB, + filepath.Join(repoRoot, migrationsDir), + upToVersion, + ) + } else { + err = goose.RunWithOptionsContext( + t.Context(), + "up", + sqlDB, + filepath.Join(repoRoot, migrationsDir), + nil, + ) + } + require.NoError(t, err) } } From 1113654a2f8ec475e6e459a825779adff6b33968 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 1 Apr 2026 20:38:50 +0000 Subject: [PATCH 16/92] chore: auto-commit generated changes --- packages/api/go.mod | 4 ++-- packages/api/go.sum | 6 ++---- packages/auth/go.mod | 2 +- packages/auth/go.sum | 3 +-- packages/clickhouse/go.mod | 4 ++-- packages/clickhouse/go.sum | 6 ++---- packages/client-proxy/go.mod | 2 +- packages/client-proxy/go.sum | 3 +-- packages/dashboard-api/go.mod | 8 ++++---- packages/dashboard-api/go.sum | 7 +++---- .../internal/backgroundworker/auth_user_sync.go | 2 +- packages/db/go.mod | 4 ++-- packages/db/go.sum | 6 ++---- packages/docker-reverse-proxy/go.mod | 4 ++-- packages/docker-reverse-proxy/go.sum | 6 ++---- packages/envd/go.mod | 2 +- packages/envd/go.sum | 3 +-- packages/local-dev/go.mod | 2 +- packages/local-dev/go.sum | 3 +-- packages/orchestrator/go.mod | 2 +- packages/orchestrator/go.sum | 3 +-- packages/shared/go.mod | 2 +- packages/shared/go.sum | 3 +-- tests/integration/go.mod | 4 ++-- tests/integration/go.sum | 6 ++---- 25 files changed, 40 insertions(+), 57 deletions(-) diff --git a/packages/api/go.mod b/packages/api/go.mod index 9d197cf26d..8a9763fe98 100644 --- a/packages/api/go.mod +++ b/packages/api/go.mod @@ -35,7 +35,7 @@ require ( github.com/google/uuid v1.6.0 github.com/grafana/loki/v3 v3.6.4 github.com/hashicorp/nomad/api v0.0.0-20251216171439-1dee0671280e - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.9.1 github.com/launchdarkly/go-sdk-common/v3 v3.3.0 github.com/launchdarkly/go-server-sdk/v7 v7.13.0 github.com/oapi-codegen/gin-middleware v1.0.2 @@ -381,7 +381,7 @@ require ( golang.org/x/crypto v0.48.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect golang.org/x/image v0.38.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.35.0 // indirect diff --git a/packages/api/go.sum b/packages/api/go.sum index 1d207bd808..5e33ec6518 100644 --- a/packages/api/go.sum +++ b/packages/api/go.sum @@ -557,8 +557,7 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jaegertracing/jaeger-idl v0.5.0 h1:zFXR5NL3Utu7MhPg8ZorxtCBjHrL3ReM1VoB65FOFGE= @@ -1159,8 +1158,7 @@ golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/auth/go.mod b/packages/auth/go.mod index 27782f8e62..a545b30053 100644 --- a/packages/auth/go.mod +++ b/packages/auth/go.mod @@ -61,7 +61,7 @@ require ( github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.7.5 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jellydator/ttlcache/v3 v3.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/packages/auth/go.sum b/packages/auth/go.sum index 4b463c263c..6d03e5cbdb 100644 --- a/packages/auth/go.sum +++ b/packages/auth/go.sum @@ -114,8 +114,7 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= diff --git a/packages/clickhouse/go.mod b/packages/clickhouse/go.mod index e7f4c03e67..2b151949d7 100644 --- a/packages/clickhouse/go.mod +++ b/packages/clickhouse/go.mod @@ -39,7 +39,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.7.5 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/joho/godotenv v1.5.1 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect @@ -93,7 +93,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.50.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.41.0 // indirect diff --git a/packages/clickhouse/go.sum b/packages/clickhouse/go.sum index af8e90ffb7..88ae0b3a51 100644 --- a/packages/clickhouse/go.sum +++ b/packages/clickhouse/go.sum @@ -128,8 +128,7 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -310,8 +309,7 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/client-proxy/go.mod b/packages/client-proxy/go.mod index 04bad15cce..0b041d6463 100644 --- a/packages/client-proxy/go.mod +++ b/packages/client-proxy/go.mod @@ -63,7 +63,7 @@ require ( go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.50.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.41.0 // indirect diff --git a/packages/client-proxy/go.sum b/packages/client-proxy/go.sum index dd487c5288..dde1f4e9af 100644 --- a/packages/client-proxy/go.sum +++ b/packages/client-proxy/go.sum @@ -207,8 +207,7 @@ golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= diff --git a/packages/dashboard-api/go.mod b/packages/dashboard-api/go.mod index 79457e0125..cb4fa8809d 100644 --- a/packages/dashboard-api/go.mod +++ b/packages/dashboard-api/go.mod @@ -23,7 +23,11 @@ require ( github.com/jackc/pgx/v5 v5.9.1 github.com/oapi-codegen/gin-middleware v1.0.2 github.com/oapi-codegen/runtime v1.1.1 + github.com/riverqueue/river v0.32.0 + github.com/riverqueue/river/riverdriver/riverpgxv5 v0.32.0 github.com/stretchr/testify v1.11.1 + go.opentelemetry.io/otel v1.41.0 + go.opentelemetry.io/otel/metric v1.41.0 go.uber.org/zap v1.27.1 ) @@ -113,9 +117,7 @@ require ( github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/pressly/goose/v3 v3.26.0 // indirect github.com/redis/go-redis/v9 v9.17.3 // indirect - github.com/riverqueue/river v0.32.0 // indirect github.com/riverqueue/river/riverdriver v0.32.0 // indirect - github.com/riverqueue/river/riverdriver/riverpgxv5 v0.32.0 // indirect github.com/riverqueue/river/rivershared v0.32.0 // indirect github.com/riverqueue/river/rivertype v0.32.0 // indirect github.com/segmentio/asm v1.2.0 // indirect @@ -139,13 +141,11 @@ require ( go.opentelemetry.io/contrib/bridges/otelzap v0.14.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect go.opentelemetry.io/contrib/instrumentation/runtime v0.66.0 // indirect - go.opentelemetry.io/otel v1.41.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.15.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.39.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0 // indirect go.opentelemetry.io/otel/log v0.15.0 // indirect - go.opentelemetry.io/otel/metric v1.41.0 // indirect go.opentelemetry.io/otel/sdk v1.41.0 // indirect go.opentelemetry.io/otel/sdk/log v0.15.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.41.0 // indirect diff --git a/packages/dashboard-api/go.sum b/packages/dashboard-api/go.sum index d724d1c16c..7a0dcff727 100644 --- a/packages/dashboard-api/go.sum +++ b/packages/dashboard-api/go.sum @@ -139,8 +139,6 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= @@ -257,6 +255,8 @@ github.com/riverqueue/river/rivershared v0.32.0 h1:7DwdrppMU9uoU2iU9aGQiv91nBezj github.com/riverqueue/river/rivershared v0.32.0/go.mod h1:UE7GEj3zaTV3cKw7Q3angCozlNEGsL50xZBKJQ9m6zU= github.com/riverqueue/river/rivertype v0.32.0 h1:RW7uodfl86gYkjwDponTAPNnUqM+X6BjlsNHxbt6Ztg= github.com/riverqueue/river/rivertype v0.32.0/go.mod h1:D1Ad+EaZiaXbQbJcJcfeicXJMBKno0n6UcfKI5Q7DIQ= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= @@ -376,8 +376,7 @@ golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05 golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go index f999926ae2..b962bec78b 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go @@ -36,7 +36,7 @@ type AuthUserSyncWorker struct { } func NewAuthUserSyncWorker(mainDB *sqlcdb.Client, l logger.Logger) *AuthUserSyncWorker { - jobsCounter, err := otel.Meter("dashboard-api.backgroundworker.auth_user_sync").Int64Counter( + jobsCounter, err := otel.Meter("github.com/e2b-dev/infra/packages/dashboard-api/internal/backgroundworker") "dashboard_api.auth_user_sync.jobs_total", metric.WithDescription("Total auth user sync jobs by operation and result."), metric.WithUnit("{job}"), diff --git a/packages/db/go.mod b/packages/db/go.mod index 22cbf8515a..f69bcb99b5 100644 --- a/packages/db/go.mod +++ b/packages/db/go.mod @@ -14,7 +14,7 @@ require ( github.com/exaring/otelpgx v0.9.3 github.com/google/uuid v1.6.0 github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.9.1 github.com/lib/pq v1.11.2 github.com/pressly/goose/v3 v3.26.0 github.com/stretchr/testify v1.11.1 @@ -140,7 +140,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.50.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.41.0 // indirect diff --git a/packages/db/go.sum b/packages/db/go.sum index b21cf6fa1b..3a807c79e1 100644 --- a/packages/db/go.sum +++ b/packages/db/go.sum @@ -175,8 +175,7 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -424,8 +423,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/docker-reverse-proxy/go.mod b/packages/docker-reverse-proxy/go.mod index 2f663be82b..c39a14622f 100644 --- a/packages/docker-reverse-proxy/go.mod +++ b/packages/docker-reverse-proxy/go.mod @@ -40,7 +40,7 @@ require ( github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.7.5 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/klauspost/compress v1.18.2 // indirect github.com/lib/pq v1.11.2 // indirect @@ -79,7 +79,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect golang.org/x/crypto v0.48.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.50.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.41.0 // indirect diff --git a/packages/docker-reverse-proxy/go.sum b/packages/docker-reverse-proxy/go.sum index 9d0c348df3..4a13a06795 100644 --- a/packages/docker-reverse-proxy/go.sum +++ b/packages/docker-reverse-proxy/go.sum @@ -69,8 +69,7 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= @@ -189,8 +188,7 @@ golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= diff --git a/packages/envd/go.mod b/packages/envd/go.mod index 6175013054..4d86a8140b 100644 --- a/packages/envd/go.mod +++ b/packages/envd/go.mod @@ -73,7 +73,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect golang.org/x/crypto v0.48.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.50.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/text v0.35.0 // indirect diff --git a/packages/envd/go.sum b/packages/envd/go.sum index 0834f4de21..737d5e4e69 100644 --- a/packages/envd/go.sum +++ b/packages/envd/go.sum @@ -213,8 +213,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/packages/local-dev/go.mod b/packages/local-dev/go.mod index bd14d7b6ba..8429d7768d 100644 --- a/packages/local-dev/go.mod +++ b/packages/local-dev/go.mod @@ -10,7 +10,7 @@ require ( github.com/e2b-dev/infra/packages/db v0.0.0 github.com/e2b-dev/infra/packages/shared v0.0.0 github.com/google/uuid v1.6.0 - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.9.1 github.com/pressly/goose/v3 v3.26.0 github.com/stretchr/testify v1.11.1 github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 diff --git a/packages/local-dev/go.sum b/packages/local-dev/go.sum index 5878ebb18b..4496ed8763 100644 --- a/packages/local-dev/go.sum +++ b/packages/local-dev/go.sum @@ -69,8 +69,7 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= diff --git a/packages/orchestrator/go.mod b/packages/orchestrator/go.mod index cd896e8d7a..0d5dde3fe9 100644 --- a/packages/orchestrator/go.mod +++ b/packages/orchestrator/go.mod @@ -309,7 +309,7 @@ require ( golang.org/x/arch v0.18.0 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.50.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/term v0.40.0 // indirect diff --git a/packages/orchestrator/go.sum b/packages/orchestrator/go.sum index f75731182b..4416ad76b9 100644 --- a/packages/orchestrator/go.sum +++ b/packages/orchestrator/go.sum @@ -1428,8 +1428,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/shared/go.mod b/packages/shared/go.mod index 093aa0ae83..e1d9394cc7 100644 --- a/packages/shared/go.mod +++ b/packages/shared/go.mod @@ -54,7 +54,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.41.0 go.opentelemetry.io/otel/trace v1.41.0 go.uber.org/zap v1.27.1 - golang.org/x/mod v0.33.0 + golang.org/x/mod v0.34.0 golang.org/x/oauth2 v0.34.0 golang.org/x/sync v0.20.0 google.golang.org/api v0.257.0 diff --git a/packages/shared/go.sum b/packages/shared/go.sum index a58712d7d6..c4e534ef81 100644 --- a/packages/shared/go.sum +++ b/packages/shared/go.sum @@ -1023,8 +1023,7 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/tests/integration/go.mod b/tests/integration/go.mod index a0e9a6bb59..7a4019b2de 100644 --- a/tests/integration/go.mod +++ b/tests/integration/go.mod @@ -25,7 +25,7 @@ require ( github.com/e2b-dev/infra/packages/envd v0.0.0-00010101000000-000000000000 github.com/e2b-dev/infra/packages/shared v0.0.0 github.com/google/uuid v1.6.0 - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.9.1 github.com/oapi-codegen/runtime v1.1.1 github.com/stretchr/testify v1.11.1 golang.org/x/sync v0.20.0 @@ -176,7 +176,7 @@ require ( go.uber.org/zap v1.27.1 // indirect golang.org/x/arch v0.18.0 // indirect golang.org/x/crypto v0.48.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.50.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/term v0.40.0 // indirect diff --git a/tests/integration/go.sum b/tests/integration/go.sum index 83d28a5089..a26fb9f43a 100644 --- a/tests/integration/go.sum +++ b/tests/integration/go.sum @@ -182,8 +182,7 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= @@ -447,8 +446,7 @@ golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkN golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= From fb0afd468e591857742206d17ea579de252462c6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 1 Apr 2026 20:44:00 +0000 Subject: [PATCH 17/92] chore: auto-commit generated changes --- packages/api/go.sum | 2 ++ packages/auth/go.sum | 1 + packages/clickhouse/go.sum | 2 ++ packages/client-proxy/go.sum | 1 + packages/db/go.sum | 2 ++ packages/docker-reverse-proxy/go.sum | 2 ++ packages/envd/go.sum | 1 + packages/local-dev/go.sum | 1 + packages/orchestrator/go.sum | 1 + packages/shared/go.sum | 1 + tests/integration/go.sum | 2 ++ 11 files changed, 16 insertions(+) diff --git a/packages/api/go.sum b/packages/api/go.sum index 5e33ec6518..f1c7dc11cf 100644 --- a/packages/api/go.sum +++ b/packages/api/go.sum @@ -558,6 +558,7 @@ github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5ey github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jaegertracing/jaeger-idl v0.5.0 h1:zFXR5NL3Utu7MhPg8ZorxtCBjHrL3ReM1VoB65FOFGE= @@ -1159,6 +1160,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/auth/go.sum b/packages/auth/go.sum index 6d03e5cbdb..d0e8192d66 100644 --- a/packages/auth/go.sum +++ b/packages/auth/go.sum @@ -115,6 +115,7 @@ github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5ey github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= diff --git a/packages/clickhouse/go.sum b/packages/clickhouse/go.sum index 88ae0b3a51..c290bd8497 100644 --- a/packages/clickhouse/go.sum +++ b/packages/clickhouse/go.sum @@ -129,6 +129,7 @@ github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5ey github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -310,6 +311,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/client-proxy/go.sum b/packages/client-proxy/go.sum index dde1f4e9af..a8aa57bff1 100644 --- a/packages/client-proxy/go.sum +++ b/packages/client-proxy/go.sum @@ -208,6 +208,7 @@ golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVo golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= diff --git a/packages/db/go.sum b/packages/db/go.sum index 3a807c79e1..b3908e0377 100644 --- a/packages/db/go.sum +++ b/packages/db/go.sum @@ -176,6 +176,7 @@ github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5ey github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -424,6 +425,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/docker-reverse-proxy/go.sum b/packages/docker-reverse-proxy/go.sum index 4a13a06795..fa1e0b24f5 100644 --- a/packages/docker-reverse-proxy/go.sum +++ b/packages/docker-reverse-proxy/go.sum @@ -70,6 +70,7 @@ github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5ey github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= @@ -189,6 +190,7 @@ golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVo golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= diff --git a/packages/envd/go.sum b/packages/envd/go.sum index 737d5e4e69..fa0212436b 100644 --- a/packages/envd/go.sum +++ b/packages/envd/go.sum @@ -214,6 +214,7 @@ golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/packages/local-dev/go.sum b/packages/local-dev/go.sum index 4496ed8763..def89b00d0 100644 --- a/packages/local-dev/go.sum +++ b/packages/local-dev/go.sum @@ -70,6 +70,7 @@ github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5ey github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= diff --git a/packages/orchestrator/go.sum b/packages/orchestrator/go.sum index 4416ad76b9..17645baedb 100644 --- a/packages/orchestrator/go.sum +++ b/packages/orchestrator/go.sum @@ -1429,6 +1429,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/shared/go.sum b/packages/shared/go.sum index c4e534ef81..69f3c15d7f 100644 --- a/packages/shared/go.sum +++ b/packages/shared/go.sum @@ -1024,6 +1024,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/tests/integration/go.sum b/tests/integration/go.sum index a26fb9f43a..4ef6ca9f30 100644 --- a/tests/integration/go.sum +++ b/tests/integration/go.sum @@ -183,6 +183,7 @@ github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5ey github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= @@ -447,6 +448,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= From bd4dd80223d12df4cdd969317ce6efc640e22a9c Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Wed, 1 Apr 2026 15:17:50 -0700 Subject: [PATCH 18/92] chore: update pgx dependency to v5.9.1 and golang.org/x/mod to v0.34.0 across multiple packages --- packages/api/go.mod | 4 +- packages/api/go.sum | 6 +- packages/auth/go.mod | 2 +- packages/auth/go.sum | 3 +- packages/clickhouse/go.mod | 4 +- packages/clickhouse/go.sum | 6 +- packages/client-proxy/go.mod | 2 +- packages/client-proxy/go.sum | 3 +- packages/dashboard-api/go.mod | 8 +- packages/dashboard-api/go.sum | 7 +- packages/db/go.mod | 4 +- packages/db/go.sum | 6 +- ...0260401000001_river_auth_custom_schema.sql | 13 +++ ...01000003_river_auth_user_sync_triggers.sql | 107 +++++------------- packages/docker-reverse-proxy/go.mod | 4 +- packages/docker-reverse-proxy/go.sum | 6 +- packages/envd/go.mod | 2 +- packages/envd/go.sum | 3 +- packages/local-dev/go.mod | 2 +- packages/local-dev/go.sum | 3 +- packages/orchestrator/go.mod | 2 +- packages/orchestrator/go.sum | 3 +- packages/shared/go.mod | 2 +- packages/shared/go.sum | 3 +- tests/integration/go.mod | 4 +- tests/integration/go.sum | 6 +- 26 files changed, 79 insertions(+), 136 deletions(-) create mode 100644 packages/db/pkg/auth/migrations/20260401000001_river_auth_custom_schema.sql diff --git a/packages/api/go.mod b/packages/api/go.mod index 9d197cf26d..8a9763fe98 100644 --- a/packages/api/go.mod +++ b/packages/api/go.mod @@ -35,7 +35,7 @@ require ( github.com/google/uuid v1.6.0 github.com/grafana/loki/v3 v3.6.4 github.com/hashicorp/nomad/api v0.0.0-20251216171439-1dee0671280e - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.9.1 github.com/launchdarkly/go-sdk-common/v3 v3.3.0 github.com/launchdarkly/go-server-sdk/v7 v7.13.0 github.com/oapi-codegen/gin-middleware v1.0.2 @@ -381,7 +381,7 @@ require ( golang.org/x/crypto v0.48.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect golang.org/x/image v0.38.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.35.0 // indirect diff --git a/packages/api/go.sum b/packages/api/go.sum index 1d207bd808..5e33ec6518 100644 --- a/packages/api/go.sum +++ b/packages/api/go.sum @@ -557,8 +557,7 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jaegertracing/jaeger-idl v0.5.0 h1:zFXR5NL3Utu7MhPg8ZorxtCBjHrL3ReM1VoB65FOFGE= @@ -1159,8 +1158,7 @@ golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/auth/go.mod b/packages/auth/go.mod index 27782f8e62..a545b30053 100644 --- a/packages/auth/go.mod +++ b/packages/auth/go.mod @@ -61,7 +61,7 @@ require ( github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.7.5 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jellydator/ttlcache/v3 v3.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/packages/auth/go.sum b/packages/auth/go.sum index 4b463c263c..6d03e5cbdb 100644 --- a/packages/auth/go.sum +++ b/packages/auth/go.sum @@ -114,8 +114,7 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= diff --git a/packages/clickhouse/go.mod b/packages/clickhouse/go.mod index e7f4c03e67..2b151949d7 100644 --- a/packages/clickhouse/go.mod +++ b/packages/clickhouse/go.mod @@ -39,7 +39,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.7.5 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/joho/godotenv v1.5.1 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect @@ -93,7 +93,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.50.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.41.0 // indirect diff --git a/packages/clickhouse/go.sum b/packages/clickhouse/go.sum index af8e90ffb7..88ae0b3a51 100644 --- a/packages/clickhouse/go.sum +++ b/packages/clickhouse/go.sum @@ -128,8 +128,7 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -310,8 +309,7 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/client-proxy/go.mod b/packages/client-proxy/go.mod index 04bad15cce..0b041d6463 100644 --- a/packages/client-proxy/go.mod +++ b/packages/client-proxy/go.mod @@ -63,7 +63,7 @@ require ( go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.50.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.41.0 // indirect diff --git a/packages/client-proxy/go.sum b/packages/client-proxy/go.sum index dd487c5288..dde1f4e9af 100644 --- a/packages/client-proxy/go.sum +++ b/packages/client-proxy/go.sum @@ -207,8 +207,7 @@ golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= diff --git a/packages/dashboard-api/go.mod b/packages/dashboard-api/go.mod index 79457e0125..cb4fa8809d 100644 --- a/packages/dashboard-api/go.mod +++ b/packages/dashboard-api/go.mod @@ -23,7 +23,11 @@ require ( github.com/jackc/pgx/v5 v5.9.1 github.com/oapi-codegen/gin-middleware v1.0.2 github.com/oapi-codegen/runtime v1.1.1 + github.com/riverqueue/river v0.32.0 + github.com/riverqueue/river/riverdriver/riverpgxv5 v0.32.0 github.com/stretchr/testify v1.11.1 + go.opentelemetry.io/otel v1.41.0 + go.opentelemetry.io/otel/metric v1.41.0 go.uber.org/zap v1.27.1 ) @@ -113,9 +117,7 @@ require ( github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/pressly/goose/v3 v3.26.0 // indirect github.com/redis/go-redis/v9 v9.17.3 // indirect - github.com/riverqueue/river v0.32.0 // indirect github.com/riverqueue/river/riverdriver v0.32.0 // indirect - github.com/riverqueue/river/riverdriver/riverpgxv5 v0.32.0 // indirect github.com/riverqueue/river/rivershared v0.32.0 // indirect github.com/riverqueue/river/rivertype v0.32.0 // indirect github.com/segmentio/asm v1.2.0 // indirect @@ -139,13 +141,11 @@ require ( go.opentelemetry.io/contrib/bridges/otelzap v0.14.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect go.opentelemetry.io/contrib/instrumentation/runtime v0.66.0 // indirect - go.opentelemetry.io/otel v1.41.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.15.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.39.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0 // indirect go.opentelemetry.io/otel/log v0.15.0 // indirect - go.opentelemetry.io/otel/metric v1.41.0 // indirect go.opentelemetry.io/otel/sdk v1.41.0 // indirect go.opentelemetry.io/otel/sdk/log v0.15.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.41.0 // indirect diff --git a/packages/dashboard-api/go.sum b/packages/dashboard-api/go.sum index d724d1c16c..7a0dcff727 100644 --- a/packages/dashboard-api/go.sum +++ b/packages/dashboard-api/go.sum @@ -139,8 +139,6 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= @@ -257,6 +255,8 @@ github.com/riverqueue/river/rivershared v0.32.0 h1:7DwdrppMU9uoU2iU9aGQiv91nBezj github.com/riverqueue/river/rivershared v0.32.0/go.mod h1:UE7GEj3zaTV3cKw7Q3angCozlNEGsL50xZBKJQ9m6zU= github.com/riverqueue/river/rivertype v0.32.0 h1:RW7uodfl86gYkjwDponTAPNnUqM+X6BjlsNHxbt6Ztg= github.com/riverqueue/river/rivertype v0.32.0/go.mod h1:D1Ad+EaZiaXbQbJcJcfeicXJMBKno0n6UcfKI5Q7DIQ= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= @@ -376,8 +376,7 @@ golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05 golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/packages/db/go.mod b/packages/db/go.mod index 22cbf8515a..f69bcb99b5 100644 --- a/packages/db/go.mod +++ b/packages/db/go.mod @@ -14,7 +14,7 @@ require ( github.com/exaring/otelpgx v0.9.3 github.com/google/uuid v1.6.0 github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.9.1 github.com/lib/pq v1.11.2 github.com/pressly/goose/v3 v3.26.0 github.com/stretchr/testify v1.11.1 @@ -140,7 +140,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.50.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.41.0 // indirect diff --git a/packages/db/go.sum b/packages/db/go.sum index b21cf6fa1b..3a807c79e1 100644 --- a/packages/db/go.sum +++ b/packages/db/go.sum @@ -175,8 +175,7 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -424,8 +423,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/db/pkg/auth/migrations/20260401000001_river_auth_custom_schema.sql b/packages/db/pkg/auth/migrations/20260401000001_river_auth_custom_schema.sql new file mode 100644 index 0000000000..73afedfbe3 --- /dev/null +++ b/packages/db/pkg/auth/migrations/20260401000001_river_auth_custom_schema.sql @@ -0,0 +1,13 @@ +-- +goose Up +-- +goose StatementBegin + +CREATE SCHEMA IF NOT EXISTS auth_custom; + +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin + +/* We don't want to drop the schema, as it is used by other services. */ + +-- +goose StatementEnd diff --git a/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql b/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql index 3d41c6d733..ba3ee8737c 100644 --- a/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql +++ b/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql @@ -1,11 +1,11 @@ -- +goose Up -- +goose StatementBegin -CREATE SCHEMA IF NOT EXISTS auth_custom; - -CREATE OR REPLACE FUNCTION public.sync_insert_auth_users_to_public_users_trigger() RETURNS TRIGGER +CREATE OR REPLACE FUNCTION auth_custom.enqueue_user_sync_on_insert() +RETURNS TRIGGER LANGUAGE plpgsql -AS $func$ +SECURITY DEFINER SET search_path = '' +AS $$ BEGIN INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) VALUES ( @@ -20,11 +20,13 @@ BEGIN RETURN NEW; END; -$func$ SECURITY DEFINER SET search_path = public, auth_custom; +$$; -CREATE OR REPLACE FUNCTION public.sync_update_auth_users_to_public_users_trigger() RETURNS TRIGGER +CREATE OR REPLACE FUNCTION auth_custom.enqueue_user_sync_on_update() +RETURNS TRIGGER LANGUAGE plpgsql -AS $func$ +SECURITY DEFINER SET search_path = '' +AS $$ BEGIN IF OLD.email IS DISTINCT FROM NEW.email THEN INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) @@ -41,11 +43,13 @@ BEGIN RETURN NEW; END; -$func$ SECURITY DEFINER SET search_path = public, auth_custom; +$$; -CREATE OR REPLACE FUNCTION public.sync_delete_auth_users_to_public_users_trigger() RETURNS TRIGGER +CREATE OR REPLACE FUNCTION auth_custom.enqueue_user_sync_on_delete() +RETURNS TRIGGER LANGUAGE plpgsql -AS $func$ +SECURITY DEFINER SET search_path = '' +AS $$ BEGIN INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) VALUES ( @@ -60,28 +64,20 @@ BEGIN RETURN OLD; END; -$func$ SECURITY DEFINER SET search_path = public, auth_custom; - -ALTER FUNCTION public.sync_insert_auth_users_to_public_users_trigger() OWNER TO trigger_user; -ALTER FUNCTION public.sync_update_auth_users_to_public_users_trigger() OWNER TO trigger_user; -ALTER FUNCTION public.sync_delete_auth_users_to_public_users_trigger() OWNER TO trigger_user; +$$; -DROP TRIGGER IF EXISTS sync_inserts_to_public_users ON auth.users; -CREATE TRIGGER sync_inserts_to_public_users +CREATE TRIGGER enqueue_user_sync_on_insert AFTER INSERT ON auth.users - FOR EACH ROW EXECUTE FUNCTION public.sync_insert_auth_users_to_public_users_trigger(); + FOR EACH ROW EXECUTE FUNCTION auth_custom.enqueue_user_sync_on_insert(); -DROP TRIGGER IF EXISTS sync_updates_to_public_users ON auth.users; -CREATE TRIGGER sync_updates_to_public_users +CREATE TRIGGER enqueue_user_sync_on_update AFTER UPDATE ON auth.users - FOR EACH ROW EXECUTE FUNCTION public.sync_update_auth_users_to_public_users_trigger(); + FOR EACH ROW EXECUTE FUNCTION auth_custom.enqueue_user_sync_on_update(); -DROP TRIGGER IF EXISTS sync_deletes_to_public_users ON auth.users; -CREATE TRIGGER sync_deletes_to_public_users +CREATE TRIGGER enqueue_user_sync_on_delete AFTER DELETE ON auth.users - FOR EACH ROW EXECUTE FUNCTION public.sync_delete_auth_users_to_public_users_trigger(); + FOR EACH ROW EXECUTE FUNCTION auth_custom.enqueue_user_sync_on_delete(); -GRANT USAGE ON SCHEMA auth_custom TO trigger_user; GRANT INSERT ON auth_custom.river_job TO trigger_user; GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA auth_custom TO trigger_user; @@ -90,63 +86,14 @@ GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA auth_custom TO trigger_user; -- +goose Down -- +goose StatementBegin -DROP TRIGGER IF EXISTS sync_inserts_to_public_users ON auth.users; -DROP TRIGGER IF EXISTS sync_updates_to_public_users ON auth.users; -DROP TRIGGER IF EXISTS sync_deletes_to_public_users ON auth.users; - -CREATE OR REPLACE FUNCTION public.sync_insert_auth_users_to_public_users_trigger() RETURNS TRIGGER -LANGUAGE plpgsql -AS $func$ -BEGIN - INSERT INTO public.user_sync_queue (user_id, operation) - VALUES (NEW.id, 'upsert'); - - RETURN NEW; -END; -$func$ SECURITY DEFINER SET search_path = public; +DROP TRIGGER IF EXISTS enqueue_user_sync_on_insert ON auth.users; +DROP TRIGGER IF EXISTS enqueue_user_sync_on_update ON auth.users; +DROP TRIGGER IF EXISTS enqueue_user_sync_on_delete ON auth.users; -CREATE OR REPLACE FUNCTION public.sync_update_auth_users_to_public_users_trigger() RETURNS TRIGGER -LANGUAGE plpgsql -AS $func$ -BEGIN - IF OLD.email IS DISTINCT FROM NEW.email THEN - INSERT INTO public.user_sync_queue (user_id, operation) - VALUES (NEW.id, 'upsert'); - END IF; - - RETURN NEW; -END; -$func$ SECURITY DEFINER SET search_path = public; - -CREATE OR REPLACE FUNCTION public.sync_delete_auth_users_to_public_users_trigger() RETURNS TRIGGER -LANGUAGE plpgsql -AS $func$ -BEGIN - INSERT INTO public.user_sync_queue (user_id, operation) - VALUES (OLD.id, 'delete'); - - RETURN OLD; -END; -$func$ SECURITY DEFINER SET search_path = public; - -ALTER FUNCTION public.sync_insert_auth_users_to_public_users_trigger() OWNER TO trigger_user; -ALTER FUNCTION public.sync_update_auth_users_to_public_users_trigger() OWNER TO trigger_user; -ALTER FUNCTION public.sync_delete_auth_users_to_public_users_trigger() OWNER TO trigger_user; - -CREATE TRIGGER sync_inserts_to_public_users - AFTER INSERT ON auth.users - FOR EACH ROW EXECUTE FUNCTION public.sync_insert_auth_users_to_public_users_trigger(); - -CREATE TRIGGER sync_updates_to_public_users - AFTER UPDATE ON auth.users - FOR EACH ROW EXECUTE FUNCTION public.sync_update_auth_users_to_public_users_trigger(); - -CREATE TRIGGER sync_deletes_to_public_users - AFTER DELETE ON auth.users - FOR EACH ROW EXECUTE FUNCTION public.sync_delete_auth_users_to_public_users_trigger(); +DROP FUNCTION IF EXISTS auth_custom.enqueue_user_sync_on_insert(); +DROP FUNCTION IF EXISTS auth_custom.enqueue_user_sync_on_update(); +DROP FUNCTION IF EXISTS auth_custom.enqueue_user_sync_on_delete(); REVOKE ALL ON SCHEMA auth_custom FROM trigger_user; -DROP SCHEMA IF EXISTS auth_custom CASCADE; - -- +goose StatementEnd diff --git a/packages/docker-reverse-proxy/go.mod b/packages/docker-reverse-proxy/go.mod index 2f663be82b..c39a14622f 100644 --- a/packages/docker-reverse-proxy/go.mod +++ b/packages/docker-reverse-proxy/go.mod @@ -40,7 +40,7 @@ require ( github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.7.5 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/klauspost/compress v1.18.2 // indirect github.com/lib/pq v1.11.2 // indirect @@ -79,7 +79,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect golang.org/x/crypto v0.48.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.50.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.41.0 // indirect diff --git a/packages/docker-reverse-proxy/go.sum b/packages/docker-reverse-proxy/go.sum index 9d0c348df3..4a13a06795 100644 --- a/packages/docker-reverse-proxy/go.sum +++ b/packages/docker-reverse-proxy/go.sum @@ -69,8 +69,7 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= @@ -189,8 +188,7 @@ golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= diff --git a/packages/envd/go.mod b/packages/envd/go.mod index 6175013054..4d86a8140b 100644 --- a/packages/envd/go.mod +++ b/packages/envd/go.mod @@ -73,7 +73,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect golang.org/x/crypto v0.48.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.50.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/text v0.35.0 // indirect diff --git a/packages/envd/go.sum b/packages/envd/go.sum index 0834f4de21..737d5e4e69 100644 --- a/packages/envd/go.sum +++ b/packages/envd/go.sum @@ -213,8 +213,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/packages/local-dev/go.mod b/packages/local-dev/go.mod index bd14d7b6ba..8429d7768d 100644 --- a/packages/local-dev/go.mod +++ b/packages/local-dev/go.mod @@ -10,7 +10,7 @@ require ( github.com/e2b-dev/infra/packages/db v0.0.0 github.com/e2b-dev/infra/packages/shared v0.0.0 github.com/google/uuid v1.6.0 - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.9.1 github.com/pressly/goose/v3 v3.26.0 github.com/stretchr/testify v1.11.1 github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 diff --git a/packages/local-dev/go.sum b/packages/local-dev/go.sum index 5878ebb18b..4496ed8763 100644 --- a/packages/local-dev/go.sum +++ b/packages/local-dev/go.sum @@ -69,8 +69,7 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= diff --git a/packages/orchestrator/go.mod b/packages/orchestrator/go.mod index cd896e8d7a..0d5dde3fe9 100644 --- a/packages/orchestrator/go.mod +++ b/packages/orchestrator/go.mod @@ -309,7 +309,7 @@ require ( golang.org/x/arch v0.18.0 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.50.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/term v0.40.0 // indirect diff --git a/packages/orchestrator/go.sum b/packages/orchestrator/go.sum index f75731182b..4416ad76b9 100644 --- a/packages/orchestrator/go.sum +++ b/packages/orchestrator/go.sum @@ -1428,8 +1428,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/shared/go.mod b/packages/shared/go.mod index 093aa0ae83..e1d9394cc7 100644 --- a/packages/shared/go.mod +++ b/packages/shared/go.mod @@ -54,7 +54,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.41.0 go.opentelemetry.io/otel/trace v1.41.0 go.uber.org/zap v1.27.1 - golang.org/x/mod v0.33.0 + golang.org/x/mod v0.34.0 golang.org/x/oauth2 v0.34.0 golang.org/x/sync v0.20.0 google.golang.org/api v0.257.0 diff --git a/packages/shared/go.sum b/packages/shared/go.sum index a58712d7d6..c4e534ef81 100644 --- a/packages/shared/go.sum +++ b/packages/shared/go.sum @@ -1023,8 +1023,7 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/tests/integration/go.mod b/tests/integration/go.mod index a0e9a6bb59..7a4019b2de 100644 --- a/tests/integration/go.mod +++ b/tests/integration/go.mod @@ -25,7 +25,7 @@ require ( github.com/e2b-dev/infra/packages/envd v0.0.0-00010101000000-000000000000 github.com/e2b-dev/infra/packages/shared v0.0.0 github.com/google/uuid v1.6.0 - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.9.1 github.com/oapi-codegen/runtime v1.1.1 github.com/stretchr/testify v1.11.1 golang.org/x/sync v0.20.0 @@ -176,7 +176,7 @@ require ( go.uber.org/zap v1.27.1 // indirect golang.org/x/arch v0.18.0 // indirect golang.org/x/crypto v0.48.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.50.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/term v0.40.0 // indirect diff --git a/tests/integration/go.sum b/tests/integration/go.sum index 83d28a5089..a26fb9f43a 100644 --- a/tests/integration/go.sum +++ b/tests/integration/go.sum @@ -182,8 +182,7 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= @@ -447,8 +446,7 @@ golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkN golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= From c98878f16605bd02d3f24851bcb8092885e51661 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Wed, 1 Apr 2026 19:17:49 -0700 Subject: [PATCH 19/92] chore: fix lint --- .../dashboard-api/internal/backgroundworker/auth_user_sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go index b962bec78b..f999926ae2 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go @@ -36,7 +36,7 @@ type AuthUserSyncWorker struct { } func NewAuthUserSyncWorker(mainDB *sqlcdb.Client, l logger.Logger) *AuthUserSyncWorker { - jobsCounter, err := otel.Meter("github.com/e2b-dev/infra/packages/dashboard-api/internal/backgroundworker") + jobsCounter, err := otel.Meter("dashboard-api.backgroundworker.auth_user_sync").Int64Counter( "dashboard_api.auth_user_sync.jobs_total", metric.WithDescription("Total auth user sync jobs by operation and result."), metric.WithUnit("{job}"), From e314b9604456f92b77b0003faf6b1fe4cbe4adb4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 2 Apr 2026 02:22:48 +0000 Subject: [PATCH 20/92] chore: auto-commit generated changes --- .../dashboard-api/internal/backgroundworker/auth_user_sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go index f999926ae2..b962bec78b 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go @@ -36,7 +36,7 @@ type AuthUserSyncWorker struct { } func NewAuthUserSyncWorker(mainDB *sqlcdb.Client, l logger.Logger) *AuthUserSyncWorker { - jobsCounter, err := otel.Meter("dashboard-api.backgroundworker.auth_user_sync").Int64Counter( + jobsCounter, err := otel.Meter("github.com/e2b-dev/infra/packages/dashboard-api/internal/backgroundworker") "dashboard_api.auth_user_sync.jobs_total", metric.WithDescription("Total auth user sync jobs by operation and result."), metric.WithUnit("{job}"), From db124a4452c4eb3924349f3ffdc37bedacb0e06e Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Wed, 1 Apr 2026 21:51:04 -0700 Subject: [PATCH 21/92] improve: retrying for river jobs --- iac/modules/job-dashboard-api/main.tf | 1 - .../internal/backgroundworker/auth_user_sync.go | 16 ++++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/iac/modules/job-dashboard-api/main.tf b/iac/modules/job-dashboard-api/main.tf index 26dcc2a889..442427d181 100644 --- a/iac/modules/job-dashboard-api/main.tf +++ b/iac/modules/job-dashboard-api/main.tf @@ -31,7 +31,6 @@ resource "nomad_job" "dashboard_api" { update_stanza = var.update_stanza node_pool = var.node_pool image_name = var.image - environment = var.environment count = var.count_instances diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go index f999926ae2..d22e67be5d 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go @@ -64,9 +64,9 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy userID, err := uuid.Parse(job.Args.UserID) if err != nil { telemetry.ReportError(ctx, "auth user sync parse user_id", err, attrs...) - w.observeJob(ctx, job.Args.Operation, "error") + w.observeJob(ctx, job.Args.Operation, "cancelled") - return fmt.Errorf("parse user_id %q: %w", job.Args.UserID, err) + return river.JobCancel(fmt.Errorf("parse user_id %q: %w", job.Args.UserID, err)) } w.l.Info(ctx, "processing auth user sync job", @@ -88,11 +88,11 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy case "upsert": if job.Args.Email == "" { - err := fmt.Errorf("missing email in job args") + err := fmt.Errorf("missing email in job args for user %s", userID) telemetry.ReportError(ctx, "auth user sync missing email", err, attrs...) - w.observeJob(ctx, job.Args.Operation, "error") + w.observeJob(ctx, job.Args.Operation, "cancelled") - return fmt.Errorf("upsert public.users %s: missing email in job args", userID) + return river.JobCancel(err) } if err := w.mainDB.UpsertPublicUser(ctx, queries.UpsertPublicUserParams{ @@ -106,11 +106,11 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy } default: - err := fmt.Errorf("unknown operation %q", job.Args.Operation) + err := fmt.Errorf("unknown operation %q for user %s", job.Args.Operation, userID) telemetry.ReportError(ctx, "auth user sync unknown operation", err, attrs...) - w.observeJob(ctx, job.Args.Operation, "error") + w.observeJob(ctx, job.Args.Operation, "cancelled") - return fmt.Errorf("unknown operation %q for user %s", job.Args.Operation, userID) + return river.JobCancel(err) } w.l.Info(ctx, "completed auth user sync job", From eed510b1dc8c41008a6ac77c822258f1a9c552db Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 2 Apr 2026 11:52:10 -0700 Subject: [PATCH 22/92] refactor: enhance auth user sync worker and streamline environment variable handling - Updated the AuthUserSyncWorker to utilize OpenTelemetry metrics for better monitoring. - Refactored the Makefile to improve environment variable exportation and streamline build/run commands. - Removed outdated SQL query files related to user sync queue operations to clean up the codebase. - Adjusted the database migration script to drop the auth_custom schema if it exists, ensuring a cleaner migration process. --- packages/dashboard-api/Makefile | 17 ++--- .../backgroundworker/auth_user_sync.go | 9 +-- packages/dashboard-api/main.go | 5 +- ...0260401000001_river_auth_custom_schema.sql | 2 +- packages/db/pkg/auth/queries/ack.sql.go | 20 ----- packages/db/pkg/auth/queries/ack_batch.sql.go | 20 ----- .../db/pkg/auth/queries/claim_batch.sql.go | 73 ------------------- .../db/pkg/auth/queries/dead_letter.sql.go | 30 -------- packages/db/pkg/auth/queries/retry.sql.go | 33 --------- 9 files changed, 15 insertions(+), 194 deletions(-) delete mode 100644 packages/db/pkg/auth/queries/ack.sql.go delete mode 100644 packages/db/pkg/auth/queries/ack_batch.sql.go delete mode 100644 packages/db/pkg/auth/queries/claim_batch.sql.go delete mode 100644 packages/db/pkg/auth/queries/dead_letter.sql.go delete mode 100644 packages/db/pkg/auth/queries/retry.sql.go diff --git a/packages/dashboard-api/Makefile b/packages/dashboard-api/Makefile index 1dfef7c5fe..fb8e5fe69e 100644 --- a/packages/dashboard-api/Makefile +++ b/packages/dashboard-api/Makefile @@ -13,8 +13,8 @@ endif HOSTNAME := $(shell hostname 2> /dev/null || hostnamectl hostname 2> /dev/null) $(if $(HOSTNAME),,$(error Failed to determine hostname: both 'hostname' and 'hostnamectl' failed)) -define DASHBOARD_API_EXTRA_ENV -$$(printf '%s' "$${DASHBOARD_API_ENV_VARS:-}" | jq -r '(if .=="" then empty elif type=="string" then (fromjson? // empty) else . end) | to_entries? // [] | map("\(.key)=\(.value|tostring|@sh)") | join(" ")') +define export_extra_env +export $$(printf '%s' "$$DASHBOARD_API_ENV_VARS" | jq -r '(if .=="" then empty elif type=="string" then (fromjson? // empty) else . end) | to_entries? // [] | map("\(.key)=\(.value|tostring)") | .[]' 2>/dev/null) 2>/dev/null; endef .PHONY: generate @@ -23,28 +23,25 @@ generate: .PHONY: build build: - # Allow for passing commit sha directly for docker builds $(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD)) $(eval EXPECTED_MIGRATION_TIMESTAMP ?= $(expectedMigration)) CGO_ENABLED=0 go build -o bin/dashboard-api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedMigrationTimestamp=$(EXPECTED_MIGRATION_TIMESTAMP)" . .PHONY: build-and-upload build-and-upload: - $(eval COMMIT_SHA := $(shell git rev-parse --short HEAD)) - $(eval EXPECTED_MIGRATION_TIMESTAMP := $(expectedMigration)) - @ docker build --platform linux/amd64 --tag $(IMAGE_REGISTRY) --push --build-arg COMMIT_SHA="$(COMMIT_SHA)" --build-arg EXPECTED_MIGRATION_TIMESTAMP="$(EXPECTED_MIGRATION_TIMESTAMP)" -f ./Dockerfile .. + $(eval COMMIT_SHA := $(shell git rev-parse --short HEAD)) + $(eval EXPECTED_MIGRATION_TIMESTAMP := $(expectedMigration)) + @docker build --platform linux/amd64 --tag $(IMAGE_REGISTRY) --push --build-arg COMMIT_SHA="$(COMMIT_SHA)" --build-arg EXPECTED_MIGRATION_TIMESTAMP="$(EXPECTED_MIGRATION_TIMESTAMP)" -f ./Dockerfile .. .PHONY: run run: make build - @EXTRA_ENV=$(DASHBOARD_API_EXTRA_ENV); \ - eval "env $$EXTRA_ENV ./bin/dashboard-api" + @$(export_extra_env) ./bin/dashboard-api .PHONY: run-local run-local: make build - @EXTRA_ENV=$(DASHBOARD_API_EXTRA_ENV); \ - eval "env NODE_ID=$(HOSTNAME) $$EXTRA_ENV ./bin/dashboard-api" + @$(export_extra_env) NODE_ID=$(HOSTNAME) ./bin/dashboard-api .PHONY: test test: diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go index 8a068c1f62..2f578675f9 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go @@ -6,7 +6,6 @@ import ( "github.com/google/uuid" "github.com/riverqueue/river" - "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" "go.uber.org/zap" @@ -35,14 +34,14 @@ type AuthUserSyncWorker struct { jobsCounter metric.Int64Counter } -func NewAuthUserSyncWorker(mainDB *sqlcdb.Client, l logger.Logger) *AuthUserSyncWorker { - jobsCounter, err := otel.Meter("github.com/e2b-dev/infra/packages/dashboard-api/internal/backgroundworker") - "dashboard_api.auth_user_sync.jobs_total", +func NewAuthUserSyncWorker(ctx context.Context, mainDB *sqlcdb.Client, meter metric.Meter, l logger.Logger) *AuthUserSyncWorker { + jobsCounter, err := meter.Int64Counter( + "jobs_total", metric.WithDescription("Total auth user sync jobs by operation and result."), metric.WithUnit("{job}"), ) if err != nil { - l.Warn(context.Background(), "failed to initialize auth user sync metric", zap.Error(err)) + l.Warn(ctx, "failed to initialize auth user sync metric", zap.Error(err)) } return &AuthUserSyncWorker{ diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index e6df2d9eb4..1f8f715f5c 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -236,14 +236,15 @@ func run() int { if config.SupabaseAuthUserSyncEnabled { workerLogger := l.With(zap.String("worker", "supabase_auth_user_sync")) - + workerMeter := tel.MeterProvider.Meter("dashboard-api.backgroundworker.auth_user_sync") + authPool := authDB.WritePool() if err := backgroundworker.RunRiverMigrations(ctx, authPool); err != nil { l.Fatal(ctx, "failed to run River migrations on auth DB", zap.Error(err)) } workers := river.NewWorkers() - river.AddWorker(workers, backgroundworker.NewAuthUserSyncWorker(db, workerLogger)) + river.AddWorker(workers, backgroundworker.NewAuthUserSyncWorker(ctx, db, workerMeter, workerLogger)) riverClient, err = backgroundworker.NewRiverClient(authPool, workers) if err != nil { diff --git a/packages/db/pkg/auth/migrations/20260401000001_river_auth_custom_schema.sql b/packages/db/pkg/auth/migrations/20260401000001_river_auth_custom_schema.sql index 73afedfbe3..0eba39b6c0 100644 --- a/packages/db/pkg/auth/migrations/20260401000001_river_auth_custom_schema.sql +++ b/packages/db/pkg/auth/migrations/20260401000001_river_auth_custom_schema.sql @@ -8,6 +8,6 @@ CREATE SCHEMA IF NOT EXISTS auth_custom; -- +goose Down -- +goose StatementBegin -/* We don't want to drop the schema, as it is used by other services. */ +DROP SCHEMA IF EXISTS auth_custom CASCADE; -- +goose StatementEnd diff --git a/packages/db/pkg/auth/queries/ack.sql.go b/packages/db/pkg/auth/queries/ack.sql.go deleted file mode 100644 index 2102f263db..0000000000 --- a/packages/db/pkg/auth/queries/ack.sql.go +++ /dev/null @@ -1,20 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.29.0 -// source: ack.sql - -package authqueries - -import ( - "context" -) - -const ackUserSyncQueueItem = `-- name: AckUserSyncQueueItem :exec -DELETE FROM public.user_sync_queue -WHERE id = $1::bigint -` - -func (q *Queries) AckUserSyncQueueItem(ctx context.Context, id int64) error { - _, err := q.db.Exec(ctx, ackUserSyncQueueItem, id) - return err -} diff --git a/packages/db/pkg/auth/queries/ack_batch.sql.go b/packages/db/pkg/auth/queries/ack_batch.sql.go deleted file mode 100644 index 1b0d49a0af..0000000000 --- a/packages/db/pkg/auth/queries/ack_batch.sql.go +++ /dev/null @@ -1,20 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.29.0 -// source: ack_batch.sql - -package authqueries - -import ( - "context" -) - -const ackUserSyncQueueItems = `-- name: AckUserSyncQueueItems :exec -DELETE FROM public.user_sync_queue -WHERE id = ANY($1::bigint[]) -` - -func (q *Queries) AckUserSyncQueueItems(ctx context.Context, ids []int64) error { - _, err := q.db.Exec(ctx, ackUserSyncQueueItems, ids) - return err -} diff --git a/packages/db/pkg/auth/queries/claim_batch.sql.go b/packages/db/pkg/auth/queries/claim_batch.sql.go deleted file mode 100644 index 6c1555549f..0000000000 --- a/packages/db/pkg/auth/queries/claim_batch.sql.go +++ /dev/null @@ -1,73 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.29.0 -// source: claim_batch.sql - -package authqueries - -import ( - "context" - "time" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" -) - -const claimUserSyncQueueBatch = `-- name: ClaimUserSyncQueueBatch :many -UPDATE public.user_sync_queue -SET - locked_at = now(), - lock_owner = $1::text, - attempt_count = attempt_count + 1 -WHERE id IN ( - SELECT id - FROM public.user_sync_queue - WHERE dead_lettered_at IS NULL - AND next_attempt_at <= now() - AND (locked_at IS NULL OR locked_at < now() - $2::interval) - ORDER BY id - FOR UPDATE SKIP LOCKED - LIMIT $3::int -) -RETURNING id, user_id, operation, created_at, attempt_count -` - -type ClaimUserSyncQueueBatchParams struct { - LockOwner string - LockTimeout pgtype.Interval - BatchSize int32 -} - -type ClaimUserSyncQueueBatchRow struct { - ID int64 - UserID uuid.UUID - Operation string - CreatedAt time.Time - AttemptCount int32 -} - -func (q *Queries) ClaimUserSyncQueueBatch(ctx context.Context, arg ClaimUserSyncQueueBatchParams) ([]ClaimUserSyncQueueBatchRow, error) { - rows, err := q.db.Query(ctx, claimUserSyncQueueBatch, arg.LockOwner, arg.LockTimeout, arg.BatchSize) - if err != nil { - return nil, err - } - defer rows.Close() - var items []ClaimUserSyncQueueBatchRow - for rows.Next() { - var i ClaimUserSyncQueueBatchRow - if err := rows.Scan( - &i.ID, - &i.UserID, - &i.Operation, - &i.CreatedAt, - &i.AttemptCount, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} diff --git a/packages/db/pkg/auth/queries/dead_letter.sql.go b/packages/db/pkg/auth/queries/dead_letter.sql.go deleted file mode 100644 index de2bcb3e80..0000000000 --- a/packages/db/pkg/auth/queries/dead_letter.sql.go +++ /dev/null @@ -1,30 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.29.0 -// source: dead_letter.sql - -package authqueries - -import ( - "context" -) - -const deadLetterUserSyncQueueItem = `-- name: DeadLetterUserSyncQueueItem :exec -UPDATE public.user_sync_queue -SET - locked_at = NULL, - lock_owner = NULL, - dead_lettered_at = now(), - last_error = $1::text -WHERE id = $2::bigint -` - -type DeadLetterUserSyncQueueItemParams struct { - LastError string - ID int64 -} - -func (q *Queries) DeadLetterUserSyncQueueItem(ctx context.Context, arg DeadLetterUserSyncQueueItemParams) error { - _, err := q.db.Exec(ctx, deadLetterUserSyncQueueItem, arg.LastError, arg.ID) - return err -} diff --git a/packages/db/pkg/auth/queries/retry.sql.go b/packages/db/pkg/auth/queries/retry.sql.go deleted file mode 100644 index 297fbd7c76..0000000000 --- a/packages/db/pkg/auth/queries/retry.sql.go +++ /dev/null @@ -1,33 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.29.0 -// source: retry.sql - -package authqueries - -import ( - "context" - - "github.com/jackc/pgx/v5/pgtype" -) - -const retryUserSyncQueueItem = `-- name: RetryUserSyncQueueItem :exec -UPDATE public.user_sync_queue -SET - locked_at = NULL, - lock_owner = NULL, - next_attempt_at = now() + $1::interval, - last_error = $2::text -WHERE id = $3::bigint -` - -type RetryUserSyncQueueItemParams struct { - Backoff pgtype.Interval - LastError string - ID int64 -} - -func (q *Queries) RetryUserSyncQueueItem(ctx context.Context, arg RetryUserSyncQueueItemParams) error { - _, err := q.db.Exec(ctx, retryUserSyncQueueItem, arg.Backoff, arg.LastError, arg.ID) - return err -} From 8226f67c5d90f82b36abf89e4639d74bb93ee738 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 2 Apr 2026 12:01:28 -0700 Subject: [PATCH 23/92] chore: rename config and fix test --- .../internal/backgroundworker/auth_user_sync.go | 2 +- .../internal/backgroundworker/auth_user_sync_test.go | 8 ++++++-- .../dashboard-api/internal/backgroundworker/river.go | 7 +++++-- packages/dashboard-api/internal/cfg/model.go | 2 +- packages/dashboard-api/main.go | 10 +++++----- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go index 2f578675f9..51159f4acf 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go @@ -34,7 +34,7 @@ type AuthUserSyncWorker struct { jobsCounter metric.Int64Counter } -func NewAuthUserSyncWorker(ctx context.Context, mainDB *sqlcdb.Client, meter metric.Meter, l logger.Logger) *AuthUserSyncWorker { +func NewAuthUserSyncWorker(ctx context.Context, mainDB *sqlcdb.Client, meter metric.Meter, l logger.Logger) *AuthUserSyncWorker { jobsCounter, err := meter.Int64Counter( "jobs_total", metric.WithDescription("Total auth user sync jobs by operation and result."), diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go index 7d0e928b66..59490d428d 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go @@ -15,6 +15,7 @@ import ( "github.com/e2b-dev/infra/packages/db/pkg/testutils" "github.com/e2b-dev/infra/packages/shared/pkg/logger" + "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) const ( @@ -138,17 +139,20 @@ func runBurstBacklog(t *testing.T, db *testutils.Database) { func startRiverWorker(t *testing.T, db *testutils.Database) *riverProcess { t.Helper() + ctx := t.Context() authPool := db.AuthDb.WritePool() l := logger.NewNopLogger() + tel := telemetry.NewNoopClient() + meter := tel.MeterProvider.Meter("dashboard-api.backgroundworker.auth_user_sync") workers := river.NewWorkers() - river.AddWorker(workers, NewAuthUserSyncWorker(db.SqlcClient, l)) + river.AddWorker(workers, NewAuthUserSyncWorker(ctx, db.SqlcClient, meter, l)) client, err := NewRiverClient(authPool, workers) require.NoError(t, err) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(ctx) require.NoError(t, client.Start(ctx)) done := make(chan struct{}) diff --git a/packages/dashboard-api/internal/backgroundworker/river.go b/packages/dashboard-api/internal/backgroundworker/river.go index a248833067..5f3b788cf1 100644 --- a/packages/dashboard-api/internal/backgroundworker/river.go +++ b/packages/dashboard-api/internal/backgroundworker/river.go @@ -10,7 +10,10 @@ import ( "github.com/riverqueue/river/rivermigrate" ) -const AuthCustomSchema = "auth_custom" +const ( + AuthCustomSchema = "auth_custom" + AuthUserSyncQueue = "auth_user_sync" +) func RunRiverMigrations(ctx context.Context, pool *pgxpool.Pool) error { driver := riverpgxv5.New(pool) @@ -31,7 +34,7 @@ func NewRiverClient(pool *pgxpool.Pool, workers *river.Workers) (*river.Client[p return river.NewClient(riverpgxv5.New(pool), &river.Config{ Schema: AuthCustomSchema, Queues: map[string]river.QueueConfig{ - "auth_sync": {MaxWorkers: 10}, + AuthUserSyncQueue: {MaxWorkers: 10}, }, Workers: workers, }) diff --git a/packages/dashboard-api/internal/cfg/model.go b/packages/dashboard-api/internal/cfg/model.go index bbd83194f6..7de479e753 100644 --- a/packages/dashboard-api/internal/cfg/model.go +++ b/packages/dashboard-api/internal/cfg/model.go @@ -11,7 +11,7 @@ type Config struct { AuthDBConnectionString string `env:"AUTH_DB_CONNECTION_STRING"` AuthDBReadReplicaConnectionString string `env:"AUTH_DB_READ_REPLICA_CONNECTION_STRING"` - SupabaseAuthUserSyncEnabled bool `env:"SUPABASE_AUTH_USER_SYNC_ENABLED" envDefault:"false"` + AuthUserSyncBackgroundWorkerEnabled bool `env:"AUTH_USER_SYNC_BACKGROUND_WORKER_ENABLED" envDefault:"false"` } func Parse() (Config, error) { diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 1f8f715f5c..deacb7b390 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -234,10 +234,10 @@ func run() int { var riverClient *river.Client[pgx.Tx] - if config.SupabaseAuthUserSyncEnabled { - workerLogger := l.With(zap.String("worker", "supabase_auth_user_sync")) - workerMeter := tel.MeterProvider.Meter("dashboard-api.backgroundworker.auth_user_sync") - + if config.AuthUserSyncBackgroundWorkerEnabled { + workerLogger := l.With(zap.String("worker", "auth_user_sync")) + workerMeter := tel.MeterProvider.Meter("github.com/e2b-dev/infra/packages/dashboard-api/internal/backgroundworker") + authPool := authDB.WritePool() if err := backgroundworker.RunRiverMigrations(ctx, authPool); err != nil { l.Fatal(ctx, "failed to run River migrations on auth DB", zap.Error(err)) @@ -255,7 +255,7 @@ func run() int { l.Fatal(ctx, "failed to start River client", zap.Error(err)) } - l.Info(ctx, "background worker started (River auth_custom)", zap.String("queue", "auth_sync")) + l.Info(ctx, "background worker started", zap.String("queue", backgroundworker.AuthUserSyncQueue), zap.String("schema", backgroundworker.AuthCustomSchema)) } wg.Go(func() { From 8485d7a6e8abf6628b45c895825bb9100c4600f7 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 2 Apr 2026 12:16:26 -0700 Subject: [PATCH 24/92] improve: config --- .../backgroundworker/auth_user_sync.go | 14 ++++++-------- .../internal/backgroundworker/config.go | 14 ++++++++++++++ .../internal/backgroundworker/river.go | 7 +------ packages/dashboard-api/main.go | 4 ++-- ...401000003_river_auth_user_sync_triggers.sql | 18 +++++++++--------- 5 files changed, 32 insertions(+), 25 deletions(-) create mode 100644 packages/dashboard-api/internal/backgroundworker/config.go diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go index 51159f4acf..4bd30c45d7 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go @@ -16,15 +16,13 @@ import ( "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) -const AuthUserSyncKind = "auth_user_sync" - type AuthUserSyncArgs struct { UserID string `json:"user_id"` Operation string `json:"operation"` Email string `json:"email,omitempty"` } -func (AuthUserSyncArgs) Kind() string { return AuthUserSyncKind } +func (AuthUserSyncArgs) Kind() string { return AuthUserProjectionKind } type AuthUserSyncWorker struct { river.WorkerDefaults[AuthUserSyncArgs] @@ -53,7 +51,7 @@ func NewAuthUserSyncWorker(ctx context.Context, mainDB *sqlcdb.Client, meter met func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSyncArgs]) error { attrs := []attribute.KeyValue{ - attribute.String("job.kind", AuthUserSyncKind), + attribute.String("job.kind", AuthUserProjectionKind), attribute.String("job.operation", job.Args.Operation), attribute.Int64("job.id", job.ID), telemetry.WithUserID(job.Args.UserID), @@ -69,7 +67,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy } w.l.Info(ctx, "processing auth user sync job", - zap.String("job.kind", AuthUserSyncKind), + zap.String("job.kind", AuthUserProjectionKind), zap.Int64("job.id", job.ID), zap.String("job.operation", job.Args.Operation), logger.WithUserID(job.Args.UserID), @@ -113,7 +111,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy } w.l.Info(ctx, "completed auth user sync job", - zap.String("job.kind", AuthUserSyncKind), + zap.String("job.kind", AuthUserProjectionKind), zap.Int64("job.id", job.ID), zap.String("job.operation", job.Args.Operation), logger.WithUserID(job.Args.UserID), @@ -130,8 +128,8 @@ func (w *AuthUserSyncWorker) observeJob(ctx context.Context, operation, result s } w.jobsCounter.Add(ctx, 1, metric.WithAttributes( - attribute.String("worker", "supabase_auth_user_sync"), - attribute.String("job.kind", AuthUserSyncKind), + attribute.String("worker", AuthUserProjectionKind), + attribute.String("job.kind", AuthUserProjectionKind), attribute.String("job.operation", operation), attribute.String("result", result), )) diff --git a/packages/dashboard-api/internal/backgroundworker/config.go b/packages/dashboard-api/internal/backgroundworker/config.go new file mode 100644 index 0000000000..d853dbb8b7 --- /dev/null +++ b/packages/dashboard-api/internal/backgroundworker/config.go @@ -0,0 +1,14 @@ +package backgroundworker + +const ( + // must match the schema used in packages/db/pkg/auth/migrations for River tables and triggers + AuthCustomSchema = "auth_custom" + + // must match the queue value in packages/db/pkg/auth/migrations trigger SQL inserts + AuthUserProjectionQueue = "auth_user_projection" + + // must match the kind value in packages/db/pkg/auth/migrations trigger SQL inserts + AuthUserProjectionKind = "auth_user_projection" + + AuthUserProjectionMaxWorkers = 10 +) diff --git a/packages/dashboard-api/internal/backgroundworker/river.go b/packages/dashboard-api/internal/backgroundworker/river.go index 5f3b788cf1..0dd603bde3 100644 --- a/packages/dashboard-api/internal/backgroundworker/river.go +++ b/packages/dashboard-api/internal/backgroundworker/river.go @@ -10,11 +10,6 @@ import ( "github.com/riverqueue/river/rivermigrate" ) -const ( - AuthCustomSchema = "auth_custom" - AuthUserSyncQueue = "auth_user_sync" -) - func RunRiverMigrations(ctx context.Context, pool *pgxpool.Pool) error { driver := riverpgxv5.New(pool) @@ -34,7 +29,7 @@ func NewRiverClient(pool *pgxpool.Pool, workers *river.Workers) (*river.Client[p return river.NewClient(riverpgxv5.New(pool), &river.Config{ Schema: AuthCustomSchema, Queues: map[string]river.QueueConfig{ - AuthUserSyncQueue: {MaxWorkers: 10}, + AuthUserProjectionQueue: {MaxWorkers: AuthUserProjectionMaxWorkers}, }, Workers: workers, }) diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index deacb7b390..99df76e429 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -235,7 +235,7 @@ func run() int { var riverClient *river.Client[pgx.Tx] if config.AuthUserSyncBackgroundWorkerEnabled { - workerLogger := l.With(zap.String("worker", "auth_user_sync")) + workerLogger := l.With(zap.String("worker", backgroundworker.AuthUserProjectionKind)) workerMeter := tel.MeterProvider.Meter("github.com/e2b-dev/infra/packages/dashboard-api/internal/backgroundworker") authPool := authDB.WritePool() @@ -255,7 +255,7 @@ func run() int { l.Fatal(ctx, "failed to start River client", zap.Error(err)) } - l.Info(ctx, "background worker started", zap.String("queue", backgroundworker.AuthUserSyncQueue), zap.String("schema", backgroundworker.AuthCustomSchema)) + l.Info(ctx, "background worker started", zap.String("queue", backgroundworker.AuthUserProjectionQueue), zap.String("schema", backgroundworker.AuthCustomSchema)) } wg.Go(func() { diff --git a/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql b/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql index ba3ee8737c..825e84640d 100644 --- a/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql +++ b/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql @@ -10,13 +10,13 @@ BEGIN INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) VALUES ( jsonb_build_object('user_id', NEW.id, 'operation', 'upsert', 'email', NEW.email), - 'auth_user_sync', + 'auth_user_projection', 20, - 'auth_sync', + 'auth_user_projection', 'available' ); - PERFORM pg_notify('auth_custom.river_insert', '{"queue":"auth_sync"}'); + PERFORM pg_notify('auth_custom.river_insert', '{"queue":"auth_user_projection"}'); RETURN NEW; END; @@ -32,13 +32,13 @@ BEGIN INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) VALUES ( jsonb_build_object('user_id', NEW.id, 'operation', 'upsert', 'email', NEW.email), - 'auth_user_sync', + 'auth_user_projection', 20, - 'auth_sync', + 'auth_user_projection', 'available' ); - PERFORM pg_notify('auth_custom.river_insert', '{"queue":"auth_sync"}'); + PERFORM pg_notify('auth_custom.river_insert', '{"queue":"auth_user_projection"}'); END IF; RETURN NEW; @@ -54,13 +54,13 @@ BEGIN INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) VALUES ( jsonb_build_object('user_id', OLD.id, 'operation', 'delete'), - 'auth_user_sync', + 'auth_user_projection', 20, - 'auth_sync', + 'auth_user_projection', 'available' ); - PERFORM pg_notify('auth_custom.river_insert', '{"queue":"auth_sync"}'); + PERFORM pg_notify('auth_custom.river_insert', '{"queue":"auth_user_projection"}'); RETURN OLD; END; From c0b11020cb16f39fa62a0250c71474607e57830c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 2 Apr 2026 20:53:12 +0000 Subject: [PATCH 25/92] chore: auto-commit generated changes --- .../internal/backgroundworker/auth_user_sync_test.go | 2 +- packages/dashboard-api/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go index 59490d428d..18cf96d151 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go @@ -144,7 +144,7 @@ func startRiverWorker(t *testing.T, db *testutils.Database) *riverProcess { authPool := db.AuthDb.WritePool() l := logger.NewNopLogger() tel := telemetry.NewNoopClient() - meter := tel.MeterProvider.Meter("dashboard-api.backgroundworker.auth_user_sync") + meter := tel.MeterProvider.Meter("github.com/e2b-dev/infra/packages/dashboard-api/internal/backgroundworker") workers := river.NewWorkers() river.AddWorker(workers, NewAuthUserSyncWorker(ctx, db.SqlcClient, meter, l)) diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 6337e5cab6..012980ab69 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -251,7 +251,7 @@ func run() int { if config.AuthUserSyncBackgroundWorkerEnabled { workerLogger := l.With(zap.String("worker", backgroundworker.AuthUserProjectionKind)) - workerMeter := tel.MeterProvider.Meter("github.com/e2b-dev/infra/packages/dashboard-api/internal/backgroundworker") + workerMeter := tel.MeterProvider.Meter("github.com/e2b-dev/infra/packages/dashboard-api") authPool := authDB.WritePool() if err := backgroundworker.RunRiverMigrations(ctx, authPool); err != nil { From 87ef8f0e7bab71745ff8bf149911694dbc99af18 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 2 Apr 2026 20:58:22 +0000 Subject: [PATCH 26/92] fix: correct env var name in template to match Go config Use AUTH_USER_SYNC_BACKGROUND_WORKER_ENABLED instead of SUPABASE_AUTH_USER_SYNC_ENABLED to match the actual config field in packages/dashboard-api/internal/cfg/model.go Co-authored-by: Ben Fornefeld --- .env.gcp.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.gcp.template b/.env.gcp.template index 5269692a3e..4bb2c5b353 100644 --- a/.env.gcp.template +++ b/.env.gcp.template @@ -79,7 +79,7 @@ CLICKHOUSE_CLUSTER_SIZE=1 DASHBOARD_API_COUNT= # Additional non-reserved dashboard-api env vars passed directly to the Nomad job (default: {}) # Reserved keys managed by the module cannot be overridden here. -# Example: '{"SUPABASE_AUTH_USER_SYNC_ENABLED":"true"}' +# Example: '{"AUTH_USER_SYNC_BACKGROUND_WORKER_ENABLED":"true"}' DASHBOARD_API_ENV_VARS= # Filestore cache for builds shared across cluster (default:false) From 45a7ccbfa73b90abd3d140d0a90896be57d49233 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 2 Apr 2026 19:32:49 -0700 Subject: [PATCH 27/92] fix: update permissions for trigger_user on river_job and sequences in auth_custom schema - Changed REVOKE statement to specify INSERT permission on river_job. - Added REVOKE for USAGE and SELECT on all sequences in the auth_custom schema for trigger_user. --- .../20260401000003_river_auth_user_sync_triggers.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql b/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql index 825e84640d..93ddb068e9 100644 --- a/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql +++ b/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql @@ -94,6 +94,7 @@ DROP FUNCTION IF EXISTS auth_custom.enqueue_user_sync_on_insert(); DROP FUNCTION IF EXISTS auth_custom.enqueue_user_sync_on_update(); DROP FUNCTION IF EXISTS auth_custom.enqueue_user_sync_on_delete(); -REVOKE ALL ON SCHEMA auth_custom FROM trigger_user; +REVOKE INSERT ON auth_custom.river_job FROM trigger_user; +REVOKE USAGE, SELECT ON ALL SEQUENCES IN SCHEMA auth_custom FROM trigger_user; -- +goose StatementEnd From aae228c01adebd9623164c85372b9295dd413154 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Fri, 3 Apr 2026 14:02:59 -0700 Subject: [PATCH 28/92] wip: team provisioning sink --- .../dashboard-api/internal/api/api.gen.go | 143 ++++++---- packages/dashboard-api/internal/cfg/model.go | 6 +- .../dashboard-api/internal/handlers/store.go | 25 +- .../internal/handlers/team_handlers_test.go | 160 +++++++++++ .../internal/handlers/team_provisioning.go | 267 ++++++++++++++++++ .../internal/teambilling/factory.go | 20 ++ .../internal/teambilling/http_sink.go | 84 ++++++ .../internal/teambilling/noop_sink.go | 17 ++ .../internal/teambilling/sink.go | 29 ++ .../internal/teamprovision/http_sink.go | 84 ++++++ .../internal/teamprovision/sink.go | 29 ++ packages/dashboard-api/main.go | 12 +- packages/db/queries/team_lifecycle.sql.go | 158 +++++++++++ packages/db/queries/teams/team_lifecycle.sql | 46 +++ packages/shared/pkg/teamprovision/request.go | 16 ++ spec/openapi-dashboard.yml | 52 ++++ 16 files changed, 1087 insertions(+), 61 deletions(-) create mode 100644 packages/dashboard-api/internal/handlers/team_provisioning.go create mode 100644 packages/dashboard-api/internal/teambilling/factory.go create mode 100644 packages/dashboard-api/internal/teambilling/http_sink.go create mode 100644 packages/dashboard-api/internal/teambilling/noop_sink.go create mode 100644 packages/dashboard-api/internal/teambilling/sink.go create mode 100644 packages/dashboard-api/internal/teamprovision/http_sink.go create mode 100644 packages/dashboard-api/internal/teamprovision/sink.go create mode 100644 packages/db/queries/team_lifecycle.sql.go create mode 100644 packages/db/queries/teams/team_lifecycle.sql create mode 100644 packages/shared/pkg/teamprovision/request.go diff --git a/packages/dashboard-api/internal/api/api.gen.go b/packages/dashboard-api/internal/api/api.gen.go index 5426f426e7..afaf2c5fd1 100644 --- a/packages/dashboard-api/internal/api/api.gen.go +++ b/packages/dashboard-api/internal/api/api.gen.go @@ -90,6 +90,11 @@ type BuildsStatusesResponse struct { // CPUCount CPU cores for the sandbox type CPUCount = int64 +// CreateTeamRequest defines model for CreateTeamRequest. +type CreateTeamRequest struct { + Name string `json:"name"` +} + // DefaultTemplate defines model for DefaultTemplate. type DefaultTemplate struct { Aliases []DefaultTemplateAlias `json:"aliases"` @@ -326,6 +331,9 @@ type GetTeamsResolveParams struct { Slug TeamSlug `form:"slug" json:"slug"` } +// PostTeamsJSONRequestBody defines body for PostTeams for application/json ContentType. +type PostTeamsJSONRequestBody = CreateTeamRequest + // PatchTeamsTeamIDJSONRequestBody defines body for PatchTeamsTeamID for application/json ContentType. type PatchTeamsTeamIDJSONRequestBody = UpdateTeamRequest @@ -352,6 +360,9 @@ type ServerInterface interface { // List user teams // (GET /teams) GetTeams(c *gin.Context) + // Create team + // (POST /teams) + PostTeams(c *gin.Context) // Resolve team identity // (GET /teams/resolve) GetTeamsResolve(c *gin.Context, params GetTeamsResolveParams) @@ -370,6 +381,9 @@ type ServerInterface interface { // List default templates // (GET /templates/defaults) GetTemplatesDefaults(c *gin.Context) + // Bootstrap user + // (POST /users/bootstrap) + PostUsersBootstrap(c *gin.Context) } // ServerInterfaceWrapper converts contexts to parameters. @@ -556,6 +570,21 @@ func (siw *ServerInterfaceWrapper) GetTeams(c *gin.Context) { siw.Handler.GetTeams(c) } +// PostTeams operation middleware +func (siw *ServerInterfaceWrapper) PostTeams(c *gin.Context) { + + c.Set(Supabase1TokenAuthScopes, []string{}) + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.PostTeams(c) +} + // GetTeamsResolve operation middleware func (siw *ServerInterfaceWrapper) GetTeamsResolve(c *gin.Context) { @@ -727,6 +756,21 @@ func (siw *ServerInterfaceWrapper) GetTemplatesDefaults(c *gin.Context) { siw.Handler.GetTemplatesDefaults(c) } +// PostUsersBootstrap operation middleware +func (siw *ServerInterfaceWrapper) PostUsersBootstrap(c *gin.Context) { + + c.Set(Supabase1TokenAuthScopes, []string{}) + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.PostUsersBootstrap(c) +} + // GinServerOptions provides options for the Gin server. type GinServerOptions struct { BaseURL string @@ -760,65 +804,68 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options router.GET(options.BaseURL+"/health", wrapper.GetHealth) router.GET(options.BaseURL+"/sandboxes/:sandboxID/record", wrapper.GetSandboxesSandboxIDRecord) router.GET(options.BaseURL+"/teams", wrapper.GetTeams) + router.POST(options.BaseURL+"/teams", wrapper.PostTeams) router.GET(options.BaseURL+"/teams/resolve", wrapper.GetTeamsResolve) router.PATCH(options.BaseURL+"/teams/:teamID", wrapper.PatchTeamsTeamID) router.GET(options.BaseURL+"/teams/:teamID/members", wrapper.GetTeamsTeamIDMembers) router.POST(options.BaseURL+"/teams/:teamID/members", wrapper.PostTeamsTeamIDMembers) router.DELETE(options.BaseURL+"/teams/:teamID/members/:userId", wrapper.DeleteTeamsTeamIDMembersUserId) router.GET(options.BaseURL+"/templates/defaults", wrapper.GetTemplatesDefaults) + router.POST(options.BaseURL+"/users/bootstrap", wrapper.PostUsersBootstrap) } // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xc62/cuBH/Vwi1QL8ou+tHitbf/Li7GI1bw3YOBQLD5kqzu7xIpI6k/Dh3//eCT0kr", - "6rGOnbp3+RSvRHJevxnODKk8RQnLC0aBShEdPEUF5jgHCVz/mpckS29Iqv5OQSScFJIwGh1EpylQSRYE", - "OGILJFeA9NhJFEdEvS+wXEVxRHEO0UG1Thxx+LUkHNLoQPIS4kgkK8ixIrBgPMcyOojKUo+Uj4WaKyQn", - "dBmt17Ff5obxGwl5kWEJbdb+pf/AGVqQTAJH80fDGyKe5xi56Y2HjFfPcUaw8OL8WgJ/bMvTYKQuSzfv", - "os3wMctz/E6A0r2EFGVESKVVw/XpiUCSoSVIJCSWpQCBFowr1uChyFgK0cECZwL6WRW9uicScjHCCHGU", - "44dTM3hnNvPvMedYES0p+bUEO0ARWceRkI+ZGqOWjrwmnCzbqsPrQDJEaJKVKYxVhScZlPzPHBbRQfSn", - "aeUQUzNMTI8U6Us9XUnQELpLQnGTlFwwHhBQP0ccZMkppAqgyoEKDneElcIIzEEUjApAhKLbhINSxQ2W", - "/3H2vEXGVF0QtcRHgFLcZCQnss3nGX4geZkjWuZz4+daWUrzhndUAEcFXkIXE2bhOg8pLHCZyejg/Syu", - "wEao3NuNNLgURYutnFD7y6ucUAlL4Jp5gWk6Zw+nJ2Oikx3cEZ+qpfqcpK0/CTgfR1+N7CBuF/m60KgW", - "uczKZZuXK8A5Elm5NHYTLLvrtJcatqUKSgH8dNQGoUZ2qMAu8jUqWKvJxmW0O+/PZuqfhFEJVIMbF0VG", - "Eqz4m/4iFJNPtfX73P8Hzhk3NJpCHuEUKZZBSOX3+7Od16d5WMqVUq1ZFYEZp4jvvT7xHxmfkzQFaiju", - "vz7FfzKJFqykqaL4/lsY9RL4HXCn2LUDoUbVYZoqfzoDFREvrOVV2sRZAVwSgz3IMckaoDVPQo5bIf6z", - "HXXth7H5L5BoZOkN6JQuWJuY3RsOAwFcz0J6gIKKJDkIifNC7SkXPx7v7e39vbaLeGZTLOGdGhza/xeE", - "ErHqpcfyIoMhijEiC+QW6yRPyyzDc7W5mnjQYkcFEBEKejaN0+8Rh0ynEpIhuSLCpBKaA3yHiaagI5PL", - "BdpkwnzUMgCdG2yXRphJZyAEXgby2B8xyUoOKDcD0P0KqE1/EBHodoFJBultjJhcAb8nAtCt4vN2Mqy4", - "DeBVGGoY2Mu1yWsnRC+9HkLIsMznuCggVThAKRarOcM8RUlGlK50LkfVpv/ZZCeK3Tgysio+yiQBIWoc", - "VEaqcaAy0LarvDHsbllXDabm/+cg1EJ5wAVgOIg+8ZEIeWGzgLb5UyzHp/xqKUj1sqGUn8KDPO7P7yVD", - "BRbCBB1AaoZL7fW+oetNoyyFJ6U/UDqlzIx1ifV2WtRCNvjrVtelLYi6VTavwBIKsx+DpVk9ko5EovbX", - "lpo3RGsyExLr+PzTMStpwL2Pzz+hhHFTO9crgqhZhvx1P+ovPOLoxJQwV7UGRFNpunVg/hylh40FD9X0", - "EOa0/F6+VvHU5lRPMMn5YPBopBHjUgGgd+nPwAUxadfIeNd6XJTzjCS1V3PGMsA6xeQ4P5tvSqtt1JZW", - "FPieBtXTMUEyibMTIr5ckt+gg0yHULVV7pKiHEUwFO4cVCpbOZntwm0u48ZubZXXAEdDFSE3CQIuDONw", - "NqSSqgInMMLsG1KbRUcw1ROUXMft2R42GGkqCkFOnTGO2nFGvUOC/AabcUZlEWfkqDfczEL4MnVKO+3X", - "3a5N8nowUu8mUTwmRORdG79Zyb6eDJYump1quZDaPgDO5KrbrJ2sfChzTN9xwKnCGVrpdVCyguQL4iDK", - "TA7z18dYfav/Xl59T1EbHHefM1w1jgoM2YKDACrrtPyJwunJJOohMK6J5kYPI94YoDqcqNHprOvirkow", - "5DZnkDP+GAqC5s1zIuDO7t9CUerSrHABCeNpz0610SnTdtlQXDD3KUqfN/Th0qeX6zhKG5tA7+ZTjVTz", - "WI4JDTg3FoDMSwUlDg3VSY4XC5IoPGNdABOF2RHwzWtG6mPSG/OZjfUOZ+cdofOK5NZR61LeY4HspNEB", - "U0hWFNsT0ZOeHRa9L51s47NowVmO7lckWSlD1pmybjfo1DXCcePUotJ1Dc418zcAG/Lmqq0Z8K80hfTo", - "MVRHDKpquK4YXMK3Uzu2p8Fdh4gTd+zULjJCYdO1a6uJdUH61Sf6Mhw9YHTaWrPJUMbqlu7i7cIc/HTz", - "NlKVwp4xjWnlqKEhfj4VyvyGK99Bzwk9rzG0E2/wZ06KnqIcP3wEupSr6GD3/Xu9dbjfOwF+C84WJINz", - "ksiSwyeejStZenn+ShU6SV6K15biNYGg4gVwJUKgz5Ox5AukF4DFyGL+BZzyCFMKabjwJ+LIsNT1usej", - "Y3PuPOheTh0fzeiXNk2ns8SRJCbMjjVm7E5m9cQqPrXZqmuupuN4w8LN0GbV1QeZj16jm2UoTUrOgUqb", - "o4EY2ZyqZrpE2jQlR05X21m7Z9NV5bqg8YGVXIxsD+X44SLUfuqm8XOgFRQcvRm8m+zFQa32aKwiXuPa", - "q6jPrL1dFpyP36p8aBlurahl2zwpd4Gk5EQ+Xqo1DROXZYHnWMDOFfsC9LBUYf7J3CBYAU61M9g7BP9+", - "5wa/04MrteOC/AN0B9WN2FWsjl5NidVaTDFM7IGsJFLf//lh9wid+BOtw/PTKI7uXIM0mk12JjPFBSuA", - "4oJEB9HeZDaZKT/GcqXlnc69DyxBBzdlEt1fUPVh9BNIb/P6Vb3PYetUQ6bBK2vreOQ839ofO8NdKho/", - "3l5YWl9vXOTYfcEz/8ApUegCgDljXJRZ9ljd0irwklB9imwYnpgrELMuml6IqRpU3Q4ZGrtTu8wxNHav", - "dimif6waVPcxDZmQd32+DrrJ52tlGFHmOeaP7uRH+bLVhnIQvBT+mEZE14qcNe60ftuvH9iX1SHS8wAu", - "vgWEWidno2G0cVT2R8bQTyDbJ4d9KHpyNl4P4+jIn6c8D0bfAEX6Ps+WwElBYpK54PM6YLD3uobG7r8B", - "4Fh1dOHGHBX0gcUcSkSvaOqNY4+AvT/UDzSEN75RmRe6Pkq/mgqXG06ffCtoPeW+Sdols88pL90s21jd", - "1leqBtSrOkuz+zvaYVxv7bvLWJdxCuHO2s5nPJCs2/jU3yKoqe0LrWCBcJbpDMB0MnF1LRVSfdcXzSFj", - "dCmQZDG6J3KFTJ2JMFWOq4tPtMjwchLFbZDq6uQ1/bJdAo1GlpZOi741qF7a+IGsrOKuZmJbdlXmndqr", - "4D1m1u8FwibPc1fI3W32vwj77Yx8jNEdzkiKJaFLf9Vbn1Ug05fstrClsnXo8ffdXzXyhDqnwyjR41N7", - "5f93GnSauLM6MkBxqOhF35P56mFtvjmTyaq9U52rxxokV+4Lie0x4vcm3Wg+YunjywWQVht73Wx22O+f", - "Xi+CtXvSQ+As9ZQGNv+gxYdRnlbEKKBOa4c3XYlVDaz2LOjrMPuKUW3zrGr03qdd3Opi8jtua+TegJvY", - "iKOCiQAAzpl4cQS8fNQKfsEyKnDttHOEBkT0KXFdeW8mwLz57PwwbShuq4A0fTJfz62NeTIwd5Wa2DzR", - "z9vo/OQ+vHseRoebu/bLvkA82x+AE4ec3b1RQP1PQHKhFTIKJ/b26tRWWcO1nEra3XfWrjTzy5jiTa6A", - "cKQfuO4LoQumqzl7jbkjzbfLnDhmXnFr67xEPHp/a0n/Fku8FpMNJPi7y1po+/yp8R8CmIOc5tfP0Hho", - "ANV44NZdX6//GwAA//86rG96N0IAAA==", + "H4sIAAAAAAAC/+xcWXPcuPH/Kij+/1V5oWdGlp1K9KZjd62Klah0bKXKpZIwZM8M1iTABUAdq8x3T+Ek", + "OQSPkTWOstkna0gAff260d0A/RwlLC8YBSpFdPAcFZjjHCRw/Wtekiy9Jan6OwWRcFJIwmh0EJ2mQCVZ", + "EOCILZBcAdJjJ1EcEfW+wHIVxRHFOUQH1TpxxOHXknBIowPJS4gjkawgx4rAgvEcy+ggKks9Uj4Vaq6Q", + "nNBltF7Hfplbxm8l5EWGJbRZ+4f+A2doQTIJHM2fDG+IeJ5j5KY3HjJePccZwcKL82sJ/KktT4ORuizd", + "vIs2w8csz/E7AUr3ElKUESGVVg3XpycCSYaWIJGQWJYCBFowrliDxyJjKUQHC5wJ6GdV9OqeSMjFCCPE", + "UY4fT83gvdnMv8ecY0W0pOTXEuwARWQdR0I+ZWqMWjrymnCybKsOrwPJEKFJVqYwVhWeZFDy/+ewiA6i", + "/5tWDjE1w8T0SJG+1NOVBA2huyQUt0nJBeMBAfVzxEGWnEKqAKocqOBwT1gpjMAcRMGoAEQouks4KFXc", + "YvkvZ887ZEzVBVFLfAQoxW1GciLbfJ7hR5KXOaJlPjd+rpWlNG94RwVwVOAldDFhFq7zkMICl5mMDj7O", + "4gpshMr995EGl6JosZUTan95lRMqYQlcMy8wTefs8fRkTHSygzviU7VUn5O09ScB5+Poq5EdxO0i3xYa", + "1SKXWbls83IFOEciK5fGboJl9532UsO2VEEpgJ+O2iDUyA4V2EW+RQVrNdm4jHbnD7OZ+idhVALV4MZF", + "kZEEK/6mvwjF5HNt/T73/4Fzxg2NppBHOEWKZRBS+f2H2d7uaR6WcqVUa1ZFYMYp4vu7J/4j43OSpkAN", + "xQ+7p/h3JtGClTRVFD9+D6NeAr8H7hS7diDUqDpMU+VPZ6Ai4oW1vEqbOCuAS2KwBzkmWQO05knIcSvE", + "f7GjbvwwNv8FEo0svQGd0gVrE7N7w2EggOtZSA9QUJEkByFxXqg95eLH4/39/b/WdhHPbIolvFODQ/v/", + "glAiVr30WF5kMEQxRmSB3GKd5GmZZXiuNlcTD1rsqAAiQkHPpnH6PeKQ6VRCMiRXRJhUQnOA7zHRFHRk", + "crlAm0yYj1oGoHOD7dIIM+kMhMDLQB77IyZZyQHlZgB6WAG16Q8iAt0tMMkgvYsRkyvgD0QAulN83k2G", + "FbcBvApDDQN7uTZ57YTopddDCBmW+RwXBaQKByjFYjVnmKcoyYjSlc7lqNr0v5jsRLEbR0ZWxUeZJCBE", + "jYPKSDUOVAbadpU3ht0t66rB1Py/HIRaKA+4AAwH0Sc+EyEvbBbQNn+K5fiUXy0FqV42lPJTeJTH/fm9", + "ZKjAQpigA0jNcKm93jd0vWmUpfCk9AdKp5SZsS6x3k6LWsgGf93qurQFUbfK5hVYQmH2c7A0q0fSkUjU", + "/tpS84ZoTWZCYh2fXx+zkgbc+/j8GiWMm9q5XhFEzTLkzx+i/sIjjo51sFRpQGcCYNLaZ1XPfAa6lKvo", + "4P3Hj3ph93tvyJB6jZCQJ6aEuqo1QJrUdevC/DnKDhsLHqrpIcxr/Xv9toq3tqb0BFMcDAavRhozLhUB", + "ep/+DFwQk/aNjLetx0U5z0hSezVnLAOsU1yO87P5prQaI21pRYEfaFA9HRMkkzg7IeLrJfkNOsh0CFVb", + "5T4pylEEQ+HWQaWylZPZLtzmMm5kC1Z5DXA0VDECwQZwYRiHszGV1BU4gRFm35DaLDqCqZ6g6Dp+L/aw", + "wUhXUQhy6oxx1I5z6h0S5DfYjHMqizkjR73hbhbCl6mT2mWH7rZtkteDkXo3ieIxISLvSjzMSvb1ZLB0", + "0uxUy4XU9glwJlfdZu1k5VOZY/qOA04VztBKr4OSFSRfEQdRZnKYvz7G6qnGH+XdHylyg+Puc46rxlGF", + "IVtwEEBlnZY/0Tg9mUQ9BMY18dzoYcQbA1SHIzU6nXVl3FWJhtzmDHLGn0JB0Lx5SQTce/+XUJS6NCtc", + "QMJ42rNTbXTqtF02FBfMfYrS5w19uPTp7TqO0sYm0Lv5VCPVPJZjQgPOjQUg81JBiUNDdZLjxYIkCs9Y", + "F+BEYXYEfPOakfqY9MZ8YWO/w9l5R+i8Irl11LqUD1ggO2l0wBSSFcX2RPSkF4dF70sn2/gsWnCWo4cV", + "SVbKkHWmrNsNOnWNcNw4Nal0XYNzzfwNwIa8uWqrBvwrTSE9egrVEYOqGq4rBpfw7dyO7Wlw1yHixB17", + "tYuMUNh07eJqYl2QfvWJvgxHDxidttZsMpSxuqW7eLswB0/dvI1UpbBnXGNaSWpoiJ/rIm0X8Dmh5zWG", + "9uLXKOn1IguSwTlJZMnhmmfjSpZenr9RhU6S1+K1pfjOzsW1AK5ECPSZMpZ8hfQCsBhZzL+CUx5hSiEN", + "F/5EHBmWul73eHRszr0H3cup47MZ/dqm6XSWOJLEhNmxxozdybCeWMWnNlt1zdV0HG9YuBnarLr6IPPZ", + "a3SzDKVJyTlQaXM0ECObU9VMl0ibpujI6Wo7a/dsuqpcFzQ+sZKLke2hHD9ehNpP3TR+DrSCgqM3g3eT", + "vTio1R6NVcRrXHsV9Zm1t8uC8/FblQ8tw60VtWybJ+UukJScyKdLtaZh4rIs8BwL2LtiX4EelirMP5sb", + "DCvAqXYGe4fhn+/c4Hd6cKV2XJC/ge6guhHvFaujV1NitRZTDBN7ICyJ1PePfnh/hE78idrh+WkUR/eu", + "QRrNJnuTmeKCFUBxQaKDaH8ym8yUH2O50vJO594HlqCDmzKJ7i+o+jD6CaS3ef2q4Jewdaoh0+CVuXU8", + "cp4/Whg7w11qGj/eXpha32xcJHn/incOAqdUoQsI5oxzUWbZU3VLrMBLQvUptmF4Yq5gzLpoeiGmalB1", + "O2Vo7F7tMsnQ2P3apYz+sWpQ3cc0ZELe9eUm6CZfbpRhRJnnmD+5kyfly1YbykHwUvhjIhHdKHLWuNP6", + "bcN+YF9Wh1gvA7j4HhBqndyNhtHGUd3/MoZ+Atk+uexD0bOz8XoYR0f+POVlMPoOKNL3ibYETgoSk8wF", + "n92Awd4rGxr74Q0Ax6qjCzfmqKAPLOZQItqhqTeOPQL2/lQ/0BDe+EZlXuj6KP1qKlxuOH32raD1lPsm", + "aZfMPqe8dLNsY3VbX6kaUDt1lmb3d7TDuN7aHy5jXcYphDtrO5/xQLJu41N/i6Cmti+0ggXCWaYzANPJ", + "xNW1WEj1XWM0h4zRpUCSxeiByBUydSbCVDmuLj7RIsPLSRS3Qaqrk136ZbsEGo0sLZ0WfWtQvbbxA1lZ", + "xV3NxLbsWsdRwUQgLJwzUVO57sodsfTp1bTdvrSzblaG9mOVnZk71AQdMrht+dqPB3aYqu0WFUb3WooA", + "IrzDT+3HCT2Or98LhE3m7z5qcN9X/EnYr7nkU4zucUZSLAld+o8P9OkVMp3qbp+3VLbejPwXGDvdi14C", + "I6vXBo5+f9tQE3NWRwYoDhW96Hs23+GszVeQMlkFgpR6rEFy5b7Z2R4jPlt5/SDXPtj4zkEucEoxBM5S", + "T/kOMe7Nl6NGecNh0gF1WjvO60q1a2C1p4PfhtkdRrXN08vR2ZB2cauLye+40ZV7A26dVL0iAl4/agW/", + "qRoVuPbaOUIDIvreQF15bybAvPl67TBtKG6rgDR9Nt9zro15MjC315rYPNHP2+i8dp+Cvgyjw+1++61p", + "IJ59GIATh5zdv1FA/UdAcqEVMgon9j7z1Nbdw9W9Strdl/+uWPfLmHJeroBwpB+4fhyhC6bre3uxvSPN", + "t8ucOGZ2uLV1Xisfvb+1pH+LRX+LyQYS/G12jQblgWI6Z0wKyXGhE+7O/UvFA3Hkx76x0soLUdi2zBuz", + "jVecZi7knWv/7LnxX4aYo9bm/48AjYdmicYDZ+f1zfrfAQAA//+hahcBWUYAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/packages/dashboard-api/internal/cfg/model.go b/packages/dashboard-api/internal/cfg/model.go index cf1b16bda7..012fd57d4b 100644 --- a/packages/dashboard-api/internal/cfg/model.go +++ b/packages/dashboard-api/internal/cfg/model.go @@ -2,6 +2,7 @@ package cfg import ( "fmt" + "time" "github.com/caarlos0/env/v11" ) @@ -19,7 +20,10 @@ type Config struct { RedisClusterURL string `env:"REDIS_CLUSTER_URL"` RedisTLSCABase64 string `env:"REDIS_TLS_CA_BASE64"` - AuthUserSyncBackgroundWorkerEnabled bool `env:"AUTH_USER_SYNC_BACKGROUND_WORKER_ENABLED" envDefault:"false"` + AuthUserSyncBackgroundWorkerEnabled bool `env:"AUTH_USER_SYNC_BACKGROUND_WORKER_ENABLED" envDefault:"false"` + BillingServerURL string `env:"BILLING_SERVER_URL"` + BillingServerAPIToken string `env:"BILLING_SERVER_API_TOKEN"` + BillingServerTimeout time.Duration `env:"BILLING_SERVER_TIMEOUT" envDefault:"15s"` } func Parse() (Config, error) { diff --git a/packages/dashboard-api/internal/handlers/store.go b/packages/dashboard-api/internal/handlers/store.go index 0cb45346a3..efcd10ee2b 100644 --- a/packages/dashboard-api/internal/handlers/store.go +++ b/packages/dashboard-api/internal/handlers/store.go @@ -12,6 +12,7 @@ import ( clickhouse "github.com/e2b-dev/infra/packages/clickhouse/pkg" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" "github.com/e2b-dev/infra/packages/dashboard-api/internal/cfg" + "github.com/e2b-dev/infra/packages/dashboard-api/internal/teambilling" sqlcdb "github.com/e2b-dev/infra/packages/db/client" authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" "github.com/e2b-dev/infra/packages/shared/pkg/apierrors" @@ -20,20 +21,22 @@ import ( var _ api.ServerInterface = (*APIStore)(nil) type APIStore struct { - config cfg.Config - db *sqlcdb.Client - authDB *authdb.Client - clickhouse clickhouse.Clickhouse - authService *sharedauth.AuthService[*types.Team] + config cfg.Config + db *sqlcdb.Client + authDB *authdb.Client + clickhouse clickhouse.Clickhouse + authService *sharedauth.AuthService[*types.Team] + teamProvisionSink teambilling.TeamProvisionSink } -func NewAPIStore(config cfg.Config, db *sqlcdb.Client, authDB *authdb.Client, ch clickhouse.Clickhouse, authService *sharedauth.AuthService[*types.Team]) *APIStore { +func NewAPIStore(config cfg.Config, db *sqlcdb.Client, authDB *authdb.Client, ch clickhouse.Clickhouse, authService *sharedauth.AuthService[*types.Team], teamProvisionSink teambilling.TeamProvisionSink) *APIStore { return &APIStore{ - config: config, - db: db, - authDB: authDB, - clickhouse: ch, - authService: authService, + config: config, + db: db, + authDB: authDB, + clickhouse: ch, + authService: authService, + teamProvisionSink: teamProvisionSink, } } diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index e4b4bd3366..b3f25d6ef6 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -1,6 +1,9 @@ package handlers import ( + "context" + "encoding/json" + "errors" "net/http" "net/http/httptest" "strings" @@ -12,9 +15,11 @@ import ( "github.com/e2b-dev/infra/packages/auth/pkg/auth" authtypes "github.com/e2b-dev/infra/packages/auth/pkg/types" + "github.com/e2b-dev/infra/packages/dashboard-api/internal/teambilling" authqueries "github.com/e2b-dev/infra/packages/db/pkg/auth/queries" "github.com/e2b-dev/infra/packages/db/pkg/testutils" "github.com/e2b-dev/infra/packages/db/queries" + "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" ) func TestParseUpdateTeamBody_ProfilePictureNullClearsValue(t *testing.T) { @@ -283,3 +288,158 @@ VALUES ($1, $2, $3) t.Fatalf("failed to create team member relation: %v", err) } } + +func TestPostUsersBootstrap_CreatesDefaultTeamAndCallsSink(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUser(t, testDB) + sink := &fakeTeamProvisionSink{} + + existingTeam, err := testDB.SqlcClient.GetDefaultTeamByUserID(ctx, userID) + if err != nil { + t.Fatalf("expected trigger-created default team: %v", err) + } + if err := testDB.SqlcClient.DeleteTeamByID(ctx, existingTeam.ID); err != nil { + t.Fatalf("failed to remove trigger-created default team: %v", err) + } + if err := testDB.SqlcClient.DeletePublicUser(ctx, userID); err != nil { + t.Fatalf("failed to remove trigger-created public user: %v", err) + } + + recorder := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(recorder) + ginCtx.Request = httptest.NewRequestWithContext(ctx, http.MethodPost, "/", nil) + auth.SetUserID(ginCtx, userID) + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDb, + teamProvisionSink: sink, + } + store.PostUsersBootstrap(ginCtx) + + if recorder.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d", recorder.Code) + } + + team, err := testDB.SqlcClient.GetDefaultTeamByUserID(ctx, userID) + if err != nil { + t.Fatalf("expected default team to be created: %v", err) + } + + if len(sink.requests) != 1 { + t.Fatalf("expected one billing provisioning call, got %d", len(sink.requests)) + } + + req := sink.requests[0] + if req.TeamID != team.ID { + t.Fatalf("expected sink team id %s, got %s", team.ID, req.TeamID) + } + if req.Reason != teamprovision.ReasonDefaultSignupTeam { + t.Fatalf("expected default signup reason, got %s", req.Reason) + } + + var responseBody map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &responseBody); err != nil { + t.Fatalf("failed to parse response body: %v", err) + } + if responseBody["slug"] != team.Slug { + t.Fatalf("expected slug %s, got %v", team.Slug, responseBody["slug"]) + } +} + +func TestCreateTeam_NoProvisionSinkLeavesCreatedTeam(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUser(t, testDB) + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDb, + teamProvisionSink: &fakeTeamProvisionSink{}, + } + + team, err := store.createTeam(ctx, userID, "Acme") + if err != nil { + t.Fatalf("expected team creation to succeed without external provisioning, got %v", err) + } + + rows, err := testDB.AuthDb.Read.GetTeamsWithUsersTeamsWithTier(ctx, userID) + if err != nil { + t.Fatalf("failed to query user teams: %v", err) + } + if len(rows) != 2 { + t.Fatalf("expected default team and created team, got %d rows", len(rows)) + } + + found := false + for _, row := range rows { + if row.Team.ID == team.ID { + found = true + if row.IsDefault { + t.Fatal("expected manually created team not to be default") + } + } + } + if !found { + t.Fatal("expected created team to remain in local state") + } +} + +func TestCreateTeam_BillingBadRequestCleansUpCreatedTeam(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUser(t, testDB) + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDb, + teamProvisionSink: &fakeTeamProvisionSink{ + err: &teambilling.ProvisionError{ + StatusCode: http.StatusBadRequest, + Message: "limit reached", + }, + }, + } + + _, err := store.createTeam(ctx, userID, "Acme") + if err == nil { + t.Fatal("expected billing error") + } + + var provisionErr *teambilling.ProvisionError + if !errors.As(err, &provisionErr) { + t.Fatalf("expected provisioning error, got %T: %v", err, err) + } + if provisionErr.StatusCode != http.StatusBadRequest { + t.Fatalf("expected status 400, got %d", provisionErr.StatusCode) + } + + rows, err := testDB.AuthDb.Read.GetTeamsWithUsersTeamsWithTier(ctx, userID) + if err != nil { + t.Fatalf("failed to query user teams: %v", err) + } + if len(rows) != 1 { + t.Fatalf("expected only the default team to remain, got %d rows", len(rows)) + } + if !rows[0].IsDefault { + t.Fatal("expected remaining team to be the default team") + } +} + +type fakeTeamProvisionSink struct { + requests []teamprovision.TeamBillingProvisionRequestedV1 + err error +} + +func (s *fakeTeamProvisionSink) ProvisionTeam(_ context.Context, req teamprovision.TeamBillingProvisionRequestedV1) error { + s.requests = append(s.requests, req) + + return s.err +} diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go new file mode 100644 index 0000000000..09dd427054 --- /dev/null +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -0,0 +1,267 @@ +package handlers + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/e2b-dev/infra/packages/auth/pkg/auth" + "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" + "github.com/e2b-dev/infra/packages/dashboard-api/internal/teambilling" + "github.com/e2b-dev/infra/packages/db/pkg/dberrors" + "github.com/e2b-dev/infra/packages/db/queries" + "github.com/e2b-dev/infra/packages/shared/pkg/ginutils" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" + "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" + "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "go.uber.org/zap" +) + +const baseTierID = "base_v1" + +type provisionedTeam struct { + ID uuid.UUID + Name string + Email string + Slug string +} + +func (s *APIStore) PostUsersBootstrap(c *gin.Context) { + ctx := c.Request.Context() + telemetry.ReportEvent(ctx, "bootstrap user") + + userID := auth.MustGetUserID(c) + team, err := s.bootstrapUser(ctx, userID) + if err != nil { + s.handleProvisioningError(ctx, c, "bootstrap user", err) + + return + } + + c.JSON(http.StatusOK, api.TeamResolveResponse{ + Id: team.ID, + Slug: team.Slug, + }) +} + +func (s *APIStore) PostTeams(c *gin.Context) { + ctx := c.Request.Context() + telemetry.ReportEvent(ctx, "create team") + + userID := auth.MustGetUserID(c) + body, err := ginutils.ParseBody[api.CreateTeamRequest](ctx, c) + if err != nil { + s.sendAPIStoreError(c, http.StatusBadRequest, "Invalid request body") + + return + } + + team, err := s.createTeam(ctx, userID, body.Name) + if err != nil { + s.handleProvisioningError(ctx, c, "create team", err) + + return + } + + c.JSON(http.StatusOK, api.TeamResolveResponse{ + Id: team.ID, + Slug: team.Slug, + }) +} + +func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisionedTeam, error) { + authUser, err := s.authDB.Write.GetUser(ctx, userID) + if err != nil { + return provisionedTeam{}, fmt.Errorf("get auth user: %w", err) + } + + txDB, tx, err := s.db.WithTx(ctx) + if err != nil { + return provisionedTeam{}, fmt.Errorf("start transaction: %w", err) + } + defer func() { + _ = tx.Rollback(ctx) + }() + + if err := txDB.UpsertPublicUser(ctx, queries.UpsertPublicUserParams{ + ID: authUser.ID, + Email: authUser.Email, + }); err != nil { + return provisionedTeam{}, fmt.Errorf("upsert public user: %w", err) + } + + existingTeam, err := txDB.GetDefaultTeamByUserID(ctx, userID) + if err == nil { + if err := tx.Commit(ctx); err != nil { + return provisionedTeam{}, fmt.Errorf("commit existing user bootstrap transaction: %w", err) + } + + err = s.teamProvisionSink.ProvisionTeam(ctx, teamprovision.TeamBillingProvisionRequestedV1{ + TeamID: existingTeam.ID, + TeamName: existingTeam.Name, + TeamEmail: existingTeam.Email, + OwnerUserID: userID, + Reason: teamprovision.ReasonDefaultSignupTeam, + }) + if err != nil { + return provisionedTeam{}, err + } + + return provisionedTeam{ + ID: existingTeam.ID, + Name: existingTeam.Name, + Email: existingTeam.Email, + Slug: existingTeam.Slug, + }, nil + } + if !dberrors.IsNotFoundError(err) { + return provisionedTeam{}, fmt.Errorf("get default team: %w", err) + } + + team, err := txDB.CreateTeam(ctx, queries.CreateTeamParams{ + Name: authUser.Email, + Tier: baseTierID, + Email: authUser.Email, + }) + if err != nil { + return provisionedTeam{}, fmt.Errorf("create default team: %w", err) + } + + if err := txDB.CreateTeamMembership(ctx, queries.CreateTeamMembershipParams{ + UserID: userID, + TeamID: team.ID, + IsDefault: true, + AddedBy: nil, + }); err != nil { + return provisionedTeam{}, fmt.Errorf("create default team membership: %w", err) + } + + if err := tx.Commit(ctx); err != nil { + return provisionedTeam{}, fmt.Errorf("commit user bootstrap transaction: %w", err) + } + + err = s.teamProvisionSink.ProvisionTeam(ctx, teamprovision.TeamBillingProvisionRequestedV1{ + TeamID: team.ID, + TeamName: team.Name, + TeamEmail: team.Email, + OwnerUserID: userID, + Reason: teamprovision.ReasonDefaultSignupTeam, + }) + if err != nil { + if cleanupErr := s.cleanupCreatedTeam(ctx, team.ID); cleanupErr != nil { + return provisionedTeam{}, fmt.Errorf("cleanup created default team: %w", cleanupErr) + } + + return provisionedTeam{}, err + } + + return provisionedTeam{ + ID: team.ID, + Name: team.Name, + Email: team.Email, + Slug: team.Slug, + }, nil +} + +func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string) (provisionedTeam, error) { + authUser, err := s.authDB.Read.GetUser(ctx, userID) + if err != nil { + return provisionedTeam{}, fmt.Errorf("get auth user: %w", err) + } + + txDB, tx, err := s.db.WithTx(ctx) + if err != nil { + return provisionedTeam{}, fmt.Errorf("start transaction: %w", err) + } + defer func() { + _ = tx.Rollback(ctx) + }() + + if err := txDB.UpsertPublicUser(ctx, queries.UpsertPublicUserParams{ + ID: authUser.ID, + Email: authUser.Email, + }); err != nil { + return provisionedTeam{}, fmt.Errorf("upsert public user: %w", err) + } + + team, err := txDB.CreateTeam(ctx, queries.CreateTeamParams{ + Name: name, + Tier: baseTierID, + Email: authUser.Email, + }) + if err != nil { + return provisionedTeam{}, fmt.Errorf("create team: %w", err) + } + + if err := txDB.CreateTeamMembership(ctx, queries.CreateTeamMembershipParams{ + UserID: userID, + TeamID: team.ID, + IsDefault: false, + AddedBy: &userID, + }); err != nil { + return provisionedTeam{}, fmt.Errorf("create team membership: %w", err) + } + + if err := tx.Commit(ctx); err != nil { + return provisionedTeam{}, fmt.Errorf("commit team creation transaction: %w", err) + } + + err = s.teamProvisionSink.ProvisionTeam(ctx, teamprovision.TeamBillingProvisionRequestedV1{ + TeamID: team.ID, + TeamName: team.Name, + TeamEmail: team.Email, + OwnerUserID: userID, + Reason: teamprovision.ReasonAdditionalTeam, + }) + if err != nil { + if cleanupErr := s.cleanupCreatedTeam(ctx, team.ID); cleanupErr != nil { + return provisionedTeam{}, fmt.Errorf("cleanup created team: %w", cleanupErr) + } + + return provisionedTeam{}, err + } + + return provisionedTeam{ + ID: team.ID, + Name: team.Name, + Email: team.Email, + Slug: team.Slug, + }, nil +} + +func (s *APIStore) cleanupCreatedTeam(ctx context.Context, teamID uuid.UUID) error { + if err := s.db.DeleteTeamByID(ctx, teamID); err != nil { + logger.L().Error(ctx, "failed to cleanup created team", + zap.String("teamID", teamID.String()), + zap.Error(err), + ) + + return err + } + + if s.authService != nil { + if err := s.authService.InvalidateTeamCache(ctx, teamID); err != nil { + logger.L().Warn(ctx, "failed to invalidate team cache after cleanup", + zap.String("teamID", teamID.String()), + zap.Error(err), + ) + } + } + + return nil +} + +func (s *APIStore) handleProvisioningError(ctx context.Context, c *gin.Context, operation string, err error) { + var provisionErr *teambilling.ProvisionError + if errors.As(err, &provisionErr) && provisionErr.IsBadRequest() { + s.sendAPIStoreError(c, http.StatusBadRequest, provisionErr.Error()) + + return + } + + logger.L().Error(ctx, operation+" failed", zap.Error(err)) + s.sendAPIStoreError(c, http.StatusInternalServerError, "Failed to "+operation) +} diff --git a/packages/dashboard-api/internal/teambilling/factory.go b/packages/dashboard-api/internal/teambilling/factory.go new file mode 100644 index 0000000000..aca7025ef3 --- /dev/null +++ b/packages/dashboard-api/internal/teambilling/factory.go @@ -0,0 +1,20 @@ +package teambilling + +import ( + "errors" + "time" +) + +var ErrMissingAPIToken = errors.New("billing server api token is required when billing server url is configured") + +func NewProvisionSink(baseURL, apiToken string, timeout time.Duration) (TeamProvisionSink, error) { + if baseURL == "" { + return NewNoopProvisionSink(), nil + } + + if apiToken == "" { + return nil, ErrMissingAPIToken + } + + return NewHTTPProvisionSink(baseURL, apiToken, timeout), nil +} diff --git a/packages/dashboard-api/internal/teambilling/http_sink.go b/packages/dashboard-api/internal/teambilling/http_sink.go new file mode 100644 index 0000000000..46f2173874 --- /dev/null +++ b/packages/dashboard-api/internal/teambilling/http_sink.go @@ -0,0 +1,84 @@ +package teambilling + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" +) + +const billingServerAPIKeyHeader = "X-Billing-Server-API-Key" + +type HTTPProvisionSink struct { + baseURL string + apiToken string + client *http.Client +} + +type errorResponse struct { + Message string `json:"message"` +} + +func NewHTTPProvisionSink(baseURL, apiToken string, timeout time.Duration) *HTTPProvisionSink { + if timeout <= 0 { + timeout = 15 * time.Second + } + + return &HTTPProvisionSink{ + baseURL: strings.TrimRight(baseURL, "/"), + apiToken: apiToken, + client: &http.Client{ + Timeout: timeout, + }, + } +} + +func (s *HTTPProvisionSink) ProvisionTeam(ctx context.Context, req teamprovision.TeamBillingProvisionRequestedV1) error { + if s.baseURL == "" || s.apiToken == "" { + return &ProvisionError{ + StatusCode: http.StatusServiceUnavailable, + Message: "billing provisioning sink is not configured", + } + } + + body, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("marshal billing provisioning request: %w", err) + } + + httpReq, err := http.NewRequestWithContext( + ctx, + http.MethodPost, + s.baseURL+"/internal/team-billing-provision", + bytes.NewReader(body), + ) + if err != nil { + return fmt.Errorf("create billing provisioning request: %w", err) + } + + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set(billingServerAPIKeyHeader, s.apiToken) + + resp, err := s.client.Do(httpReq) + if err != nil { + return fmt.Errorf("call billing provisioning endpoint: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + return nil + } + + var apiErr errorResponse + _ = json.NewDecoder(resp.Body).Decode(&apiErr) + + return &ProvisionError{ + StatusCode: resp.StatusCode, + Message: apiErr.Message, + } +} diff --git a/packages/dashboard-api/internal/teambilling/noop_sink.go b/packages/dashboard-api/internal/teambilling/noop_sink.go new file mode 100644 index 0000000000..e675b40576 --- /dev/null +++ b/packages/dashboard-api/internal/teambilling/noop_sink.go @@ -0,0 +1,17 @@ +package teambilling + +import ( + "context" + + "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" +) + +type NoopProvisionSink struct{} + +func NewNoopProvisionSink() *NoopProvisionSink { + return &NoopProvisionSink{} +} + +func (s *NoopProvisionSink) ProvisionTeam(context.Context, teamprovision.TeamBillingProvisionRequestedV1) error { + return nil +} diff --git a/packages/dashboard-api/internal/teambilling/sink.go b/packages/dashboard-api/internal/teambilling/sink.go new file mode 100644 index 0000000000..a23c9aa5ca --- /dev/null +++ b/packages/dashboard-api/internal/teambilling/sink.go @@ -0,0 +1,29 @@ +package teambilling + +import ( + "context" + "fmt" + + "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" +) + +type TeamProvisionSink interface { + ProvisionTeam(ctx context.Context, req teamprovision.TeamBillingProvisionRequestedV1) error +} + +type ProvisionError struct { + StatusCode int + Message string +} + +func (e *ProvisionError) Error() string { + if e.Message != "" { + return e.Message + } + + return fmt.Sprintf("billing provisioning failed with status %d", e.StatusCode) +} + +func (e *ProvisionError) IsBadRequest() bool { + return e.StatusCode == 400 +} diff --git a/packages/dashboard-api/internal/teamprovision/http_sink.go b/packages/dashboard-api/internal/teamprovision/http_sink.go new file mode 100644 index 0000000000..428b330e75 --- /dev/null +++ b/packages/dashboard-api/internal/teamprovision/http_sink.go @@ -0,0 +1,84 @@ +package teamprovision + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" +) + +const billingServerAPIKeyHeader = "X-Billing-Server-API-Key" + +type HTTPProvisionSink struct { + baseURL string + apiToken string + client *http.Client +} + +type errorResponse struct { + Message string `json:"message"` +} + +func NewHTTPProvisionSink(baseURL, apiToken string, timeout time.Duration) *HTTPProvisionSink { + if timeout <= 0 { + timeout = 15 * time.Second + } + + return &HTTPProvisionSink{ + baseURL: strings.TrimRight(baseURL, "/"), + apiToken: apiToken, + client: &http.Client{ + Timeout: timeout, + }, + } +} + +func (s *HTTPProvisionSink) ProvisionTeam(ctx context.Context, req teamprovision.TeamBillingProvisionRequestedV1) error { + if s.baseURL == "" || s.apiToken == "" { + return &ProvisionError{ + StatusCode: http.StatusServiceUnavailable, + Message: "billing provisioning sink is not configured", + } + } + + body, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("marshal billing provisioning request: %w", err) + } + + httpReq, err := http.NewRequestWithContext( + ctx, + http.MethodPost, + s.baseURL+"/internal/team-billing-provision", + bytes.NewReader(body), + ) + if err != nil { + return fmt.Errorf("create billing provisioning request: %w", err) + } + + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set(billingServerAPIKeyHeader, s.apiToken) + + resp, err := s.client.Do(httpReq) + if err != nil { + return fmt.Errorf("call billing provisioning endpoint: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + return nil + } + + var apiErr errorResponse + _ = json.NewDecoder(resp.Body).Decode(&apiErr) + + return &ProvisionError{ + StatusCode: resp.StatusCode, + Message: apiErr.Message, + } +} diff --git a/packages/dashboard-api/internal/teamprovision/sink.go b/packages/dashboard-api/internal/teamprovision/sink.go new file mode 100644 index 0000000000..45c0da78c0 --- /dev/null +++ b/packages/dashboard-api/internal/teamprovision/sink.go @@ -0,0 +1,29 @@ +package teamprovision + +import ( + "context" + "fmt" + + "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" +) + +type TeamProvisionSink interface { + ProvisionTeam(ctx context.Context, req teamprovision.TeamBillingProvisionRequestedV1) error +} + +type ProvisionError struct { + StatusCode int + Message string +} + +func (e *ProvisionError) Error() string { + if e.Message != "" { + return e.Message + } + + return fmt.Sprintf("billing provisioning failed with status %d", e.StatusCode) +} + +func (e *ProvisionError) IsBadRequest() bool { + return e.StatusCode >= 400 && e.StatusCode < 500 +} diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 012980ab69..29a47490d8 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -33,6 +33,7 @@ import ( "github.com/e2b-dev/infra/packages/dashboard-api/internal/backgroundworker" "github.com/e2b-dev/infra/packages/dashboard-api/internal/cfg" "github.com/e2b-dev/infra/packages/dashboard-api/internal/handlers" + "github.com/e2b-dev/infra/packages/dashboard-api/internal/teambilling" sqlcdb "github.com/e2b-dev/infra/packages/db/client" authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" "github.com/e2b-dev/infra/packages/db/pkg/pool" @@ -163,7 +164,16 @@ func run() int { authService := sharedauth.NewAuthService[*types.Team](authStore, authCache, config.SupabaseJWTSecrets) defer authService.Close(ctx) - apiStore := handlers.NewAPIStore(config, db, authDB, clickhouseClient, authService) + teamProvisionSink, err := teambilling.NewProvisionSink( + config.BillingServerURL, + config.BillingServerAPIToken, + config.BillingServerTimeout, + ) + if err != nil { + l.Fatal(ctx, "initializing team provision sink", zap.Error(err)) + } + + apiStore := handlers.NewAPIStore(config, db, authDB, clickhouseClient, authService, teamProvisionSink) swagger, err := api.GetSwagger() if err != nil { diff --git a/packages/db/queries/team_lifecycle.sql.go b/packages/db/queries/team_lifecycle.sql.go new file mode 100644 index 0000000000..7202f8b9d9 --- /dev/null +++ b/packages/db/queries/team_lifecycle.sql.go @@ -0,0 +1,158 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: team_lifecycle.sql + +package queries + +import ( + "context" + "time" + + "github.com/google/uuid" +) + +const createTeam = `-- name: CreateTeam :one +INSERT INTO public.teams (name, tier, email) +VALUES ($1::text, $2::text, $3::text) +RETURNING + id, + created_at, + is_blocked, + name, + tier, + email, + is_banned, + blocked_reason, + cluster_id, + sandbox_scheduling_labels, + slug +` + +type CreateTeamParams struct { + Name string + Tier string + Email string +} + +type CreateTeamRow struct { + ID uuid.UUID + CreatedAt time.Time + IsBlocked bool + Name string + Tier string + Email string + IsBanned bool + BlockedReason *string + ClusterID *uuid.UUID + SandboxSchedulingLabels []string + Slug string +} + +func (q *Queries) CreateTeam(ctx context.Context, arg CreateTeamParams) (CreateTeamRow, error) { + row := q.db.QueryRow(ctx, createTeam, arg.Name, arg.Tier, arg.Email) + var i CreateTeamRow + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.IsBlocked, + &i.Name, + &i.Tier, + &i.Email, + &i.IsBanned, + &i.BlockedReason, + &i.ClusterID, + &i.SandboxSchedulingLabels, + &i.Slug, + ) + return i, err +} + +const createTeamMembership = `-- name: CreateTeamMembership :exec +INSERT INTO public.users_teams (user_id, team_id, is_default, added_by) +VALUES ( + $1::uuid, + $2::uuid, + $3::boolean, + $4::uuid +) +` + +type CreateTeamMembershipParams struct { + UserID uuid.UUID + TeamID uuid.UUID + IsDefault bool + AddedBy *uuid.UUID +} + +func (q *Queries) CreateTeamMembership(ctx context.Context, arg CreateTeamMembershipParams) error { + _, err := q.db.Exec(ctx, createTeamMembership, + arg.UserID, + arg.TeamID, + arg.IsDefault, + arg.AddedBy, + ) + return err +} + +const deleteTeamByID = `-- name: DeleteTeamByID :exec +DELETE FROM public.teams +WHERE id = $1::uuid +` + +func (q *Queries) DeleteTeamByID(ctx context.Context, id uuid.UUID) error { + _, err := q.db.Exec(ctx, deleteTeamByID, id) + return err +} + +const getDefaultTeamByUserID = `-- name: GetDefaultTeamByUserID :one +SELECT + t.id, + t.created_at, + t.is_blocked, + t.name, + t.tier, + t.email, + t.is_banned, + t.blocked_reason, + t.cluster_id, + t.sandbox_scheduling_labels, + t.slug +FROM public.teams t +JOIN public.users_teams ut ON ut.team_id = t.id +WHERE ut.user_id = $1::uuid + AND ut.is_default = true +` + +type GetDefaultTeamByUserIDRow struct { + ID uuid.UUID + CreatedAt time.Time + IsBlocked bool + Name string + Tier string + Email string + IsBanned bool + BlockedReason *string + ClusterID *uuid.UUID + SandboxSchedulingLabels []string + Slug string +} + +func (q *Queries) GetDefaultTeamByUserID(ctx context.Context, userID uuid.UUID) (GetDefaultTeamByUserIDRow, error) { + row := q.db.QueryRow(ctx, getDefaultTeamByUserID, userID) + var i GetDefaultTeamByUserIDRow + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.IsBlocked, + &i.Name, + &i.Tier, + &i.Email, + &i.IsBanned, + &i.BlockedReason, + &i.ClusterID, + &i.SandboxSchedulingLabels, + &i.Slug, + ) + return i, err +} diff --git a/packages/db/queries/teams/team_lifecycle.sql b/packages/db/queries/teams/team_lifecycle.sql new file mode 100644 index 0000000000..f5b8706552 --- /dev/null +++ b/packages/db/queries/teams/team_lifecycle.sql @@ -0,0 +1,46 @@ +-- name: CreateTeam :one +INSERT INTO public.teams (name, tier, email) +VALUES (sqlc.arg(name)::text, sqlc.arg(tier)::text, sqlc.arg(email)::text) +RETURNING + id, + created_at, + is_blocked, + name, + tier, + email, + is_banned, + blocked_reason, + cluster_id, + sandbox_scheduling_labels, + slug; + +-- name: CreateTeamMembership :exec +INSERT INTO public.users_teams (user_id, team_id, is_default, added_by) +VALUES ( + sqlc.arg(user_id)::uuid, + sqlc.arg(team_id)::uuid, + sqlc.arg(is_default)::boolean, + sqlc.narg(added_by)::uuid +); + +-- name: GetDefaultTeamByUserID :one +SELECT + t.id, + t.created_at, + t.is_blocked, + t.name, + t.tier, + t.email, + t.is_banned, + t.blocked_reason, + t.cluster_id, + t.sandbox_scheduling_labels, + t.slug +FROM public.teams t +JOIN public.users_teams ut ON ut.team_id = t.id +WHERE ut.user_id = sqlc.arg(user_id)::uuid + AND ut.is_default = true; + +-- name: DeleteTeamByID :exec +DELETE FROM public.teams +WHERE id = sqlc.arg(id)::uuid; diff --git a/packages/shared/pkg/teamprovision/request.go b/packages/shared/pkg/teamprovision/request.go new file mode 100644 index 0000000000..0dc183b28a --- /dev/null +++ b/packages/shared/pkg/teamprovision/request.go @@ -0,0 +1,16 @@ +package teamprovision + +import "github.com/google/uuid" + +const ( + ReasonDefaultSignupTeam = "default_signup_team" + ReasonAdditionalTeam = "additional_team" +) + +type TeamBillingProvisionRequestedV1 struct { + TeamID uuid.UUID `json:"team_id"` + TeamName string `json:"team_name"` + TeamEmail string `json:"team_email"` + OwnerUserID uuid.UUID `json:"owner_user_id"` + Reason string `json:"reason"` +} diff --git a/spec/openapi-dashboard.yml b/spec/openapi-dashboard.yml index 9d0a4fbac1..b6962d65b8 100644 --- a/spec/openapi-dashboard.yml +++ b/spec/openapi-dashboard.yml @@ -493,6 +493,16 @@ components: type: string format: email + CreateTeamRequest: + type: object + required: + - name + properties: + name: + type: string + minLength: 1 + maxLength: 255 + DefaultTemplateAlias: type: object required: @@ -714,6 +724,48 @@ paths: $ref: "#/components/responses/401" "500": $ref: "#/components/responses/500" + post: + summary: Create team + tags: [teams] + security: + - Supabase1TokenAuth: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateTeamRequest" + responses: + "200": + description: Successfully created team. + content: + application/json: + schema: + $ref: "#/components/schemas/TeamResolveResponse" + "400": + $ref: "#/components/responses/400" + "401": + $ref: "#/components/responses/401" + "500": + $ref: "#/components/responses/500" + + /users/bootstrap: + post: + summary: Bootstrap user + tags: [teams] + security: + - Supabase1TokenAuth: [] + responses: + "200": + description: Successfully bootstrapped user. + content: + application/json: + schema: + $ref: "#/components/schemas/TeamResolveResponse" + "401": + $ref: "#/components/responses/401" + "500": + $ref: "#/components/responses/500" /teams/resolve: get: From 07d4c25408d71ab662567aa334bc4525517e0797 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Mon, 6 Apr 2026 11:24:30 -0700 Subject: [PATCH 29/92] refactor: update environment variable handling and improve auth user sync worker - Renamed environment variable from AUTH_USER_SYNC_BACKGROUND_WORKER_ENABLED to ENABLE_AUTH_USER_SYNC_BACKGROUND_WORKER for clarity and consistency. - Streamlined the AuthUserSyncWorker to enhance monitoring and error handling. - Adjusted the handling of environment variables in the Nomad job configuration to allow for better integration with module defaults. - Updated test cases to reflect changes in the database client naming convention from AuthDb to AuthDB for consistency across the codebase. --- .env.gcp.template | 6 +- .../job-dashboard-api/jobs/dashboard-api.hcl | 4 +- iac/modules/job-dashboard-api/main.tf | 19 +- packages/api/internal/db/snapshots_test.go | 2 +- .../internal/handlers/template_alias_test.go | 6 +- .../backgroundworker/auth_user_sync.go | 24 +-- .../backgroundworker/auth_user_sync_test.go | 165 +++++++++--------- .../internal/backgroundworker/config.go | 19 +- .../internal/backgroundworker/river.go | 40 ++++- packages/dashboard-api/internal/cfg/model.go | 2 +- .../internal/handlers/team_handlers_test.go | 4 +- packages/dashboard-api/main.go | 30 ++-- packages/db/pkg/testutils/db.go | 10 +- packages/db/pkg/testutils/queries.go | 4 +- .../internal/auth/validate_test.go | 8 +- tests/integration/internal/setup/db_client.go | 4 +- tests/integration/internal/tests/team_test.go | 4 +- tests/integration/internal/utils/team.go | 8 +- tests/integration/internal/utils/user.go | 6 +- 19 files changed, 183 insertions(+), 182 deletions(-) diff --git a/.env.gcp.template b/.env.gcp.template index 4bb2c5b353..f70a6bfa72 100644 --- a/.env.gcp.template +++ b/.env.gcp.template @@ -77,9 +77,9 @@ CLICKHOUSE_CLUSTER_SIZE=1 # Dashboard API instance count (default: 0) DASHBOARD_API_COUNT= -# Additional non-reserved dashboard-api env vars passed directly to the Nomad job (default: {}) -# Reserved keys managed by the module cannot be overridden here. -# Example: '{"AUTH_USER_SYNC_BACKGROUND_WORKER_ENABLED":"true"}' +# Additional dashboard-api env vars passed directly to the Nomad job (default: {}) +# Values here are merged into the job env and can override module defaults. +# Example: '{"ENABLE_AUTH_USER_SYNC_BACKGROUND_WORKER":"true"}' DASHBOARD_API_ENV_VARS= # Filestore cache for builds shared across cluster (default:false) diff --git a/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl b/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl index f05aadcbb5..885b59a7c5 100644 --- a/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl +++ b/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl @@ -71,8 +71,8 @@ job "dashboard-api" { } env { - %{ for key in sort(keys(env)) ~} - ${key} = "${env[key]}" + %{ for key, value in env ~} + ${key} = "${value}" %{ endfor ~} } diff --git a/iac/modules/job-dashboard-api/main.tf b/iac/modules/job-dashboard-api/main.tf index e2ad9f9c08..68476d109a 100644 --- a/iac/modules/job-dashboard-api/main.tf +++ b/iac/modules/job-dashboard-api/main.tf @@ -16,17 +16,7 @@ locals { LOGS_COLLECTOR_ADDRESS = "http://localhost:${var.logs_proxy_port.port}" } - extra_env = { - for key, value in var.extra_env : key => value - if value != null && trimspace(value) != "" - } - - conflicting_extra_env_keys = sort(tolist(setintersection( - toset(keys(local.base_env)), - toset(keys(local.extra_env)), - ))) - - env = merge(local.base_env, local.extra_env) + env = merge(local.base_env, var.extra_env) } resource "nomad_job" "dashboard_api" { @@ -44,11 +34,4 @@ resource "nomad_job" "dashboard_api" { subdomain = "dashboard-api" }) - - lifecycle { - precondition { - condition = length(local.conflicting_extra_env_keys) == 0 - error_message = "dashboard-api extra_env contains reserved keys: ${join(", ", local.conflicting_extra_env_keys)}" - } - } } diff --git a/packages/api/internal/db/snapshots_test.go b/packages/api/internal/db/snapshots_test.go index 83f7d2a872..6550885042 100644 --- a/packages/api/internal/db/snapshots_test.go +++ b/packages/api/internal/db/snapshots_test.go @@ -23,7 +23,7 @@ func createTestTeam(t *testing.T, db *testutils.Database) uuid.UUID { // Insert a team directly into the database using raw SQL // Using the default tier 'base_v1' that is created in migrations - err := db.AuthDb.TestsRawSQL(t.Context(), + err := db.AuthDB.TestsRawSQL(t.Context(), "INSERT INTO public.teams (id, name, tier, email, slug) VALUES ($1, $2, $3, $4, $5)", teamID, "Test Team "+teamID.String(), "base_v1", "test-"+teamID.String()+"@example.com", slug, ) diff --git a/packages/api/internal/handlers/template_alias_test.go b/packages/api/internal/handlers/template_alias_test.go index e89a660078..724952804b 100644 --- a/packages/api/internal/handlers/template_alias_test.go +++ b/packages/api/internal/handlers/template_alias_test.go @@ -28,7 +28,7 @@ func TestQueryNotExistingTemplateAlias(t *testing.T) { store := &APIStore{ sqlcDB: testDB.SqlcClient, - authDB: testDB.AuthDb, + authDB: testDB.AuthDB, templateCache: templatecache.NewTemplateCache(testDB.SqlcClient, redis), } @@ -69,7 +69,7 @@ func TestQueryExistingTemplateAlias(t *testing.T) { store := &APIStore{ sqlcDB: testDB.SqlcClient, - authDB: testDB.AuthDb, + authDB: testDB.AuthDB, templateCache: templatecache.NewTemplateCache(testDB.SqlcClient, redis), } @@ -114,7 +114,7 @@ func TestQueryExistingTemplateAliasAsNotOwnerTeam(t *testing.T) { store := &APIStore{ sqlcDB: testDB.SqlcClient, - authDB: testDB.AuthDb, + authDB: testDB.AuthDB, templateCache: templatecache.NewTemplateCache(testDB.SqlcClient, redis), } diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go index 4bd30c45d7..29e8b54216 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go @@ -22,7 +22,7 @@ type AuthUserSyncArgs struct { Email string `json:"email,omitempty"` } -func (AuthUserSyncArgs) Kind() string { return AuthUserProjectionKind } +func (AuthUserSyncArgs) Kind() string { return authUserProjectionKind } type AuthUserSyncWorker struct { river.WorkerDefaults[AuthUserSyncArgs] @@ -51,7 +51,7 @@ func NewAuthUserSyncWorker(ctx context.Context, mainDB *sqlcdb.Client, meter met func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSyncArgs]) error { attrs := []attribute.KeyValue{ - attribute.String("job.kind", AuthUserProjectionKind), + attribute.String("job.kind", authUserProjectionKind), attribute.String("job.operation", job.Args.Operation), attribute.Int64("job.id", job.ID), telemetry.WithUserID(job.Args.UserID), @@ -61,13 +61,13 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy userID, err := uuid.Parse(job.Args.UserID) if err != nil { telemetry.ReportError(ctx, "auth user sync parse user_id", err, attrs...) - w.observeJob(ctx, job.Args.Operation, "cancelled") + w.observeJob(ctx, job.Args.Operation, jobResultInvalidArgument) return river.JobCancel(fmt.Errorf("parse user_id %q: %w", job.Args.UserID, err)) } w.l.Info(ctx, "processing auth user sync job", - zap.String("job.kind", AuthUserProjectionKind), + zap.String("job.kind", authUserProjectionKind), zap.Int64("job.id", job.ID), zap.String("job.operation", job.Args.Operation), logger.WithUserID(job.Args.UserID), @@ -78,7 +78,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy case "delete": if err := w.mainDB.DeletePublicUser(ctx, userID); err != nil { telemetry.ReportError(ctx, "auth user sync delete public user", err, attrs...) - w.observeJob(ctx, job.Args.Operation, "error") + w.observeJob(ctx, job.Args.Operation, jobResultError) return fmt.Errorf("delete public.users %s: %w", userID, err) } @@ -87,7 +87,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy if job.Args.Email == "" { err := fmt.Errorf("missing email in job args for user %s", userID) telemetry.ReportError(ctx, "auth user sync missing email", err, attrs...) - w.observeJob(ctx, job.Args.Operation, "cancelled") + w.observeJob(ctx, job.Args.Operation, jobResultInvalidArgument) return river.JobCancel(err) } @@ -97,7 +97,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy Email: job.Args.Email, }); err != nil { telemetry.ReportError(ctx, "auth user sync upsert public user", err, attrs...) - w.observeJob(ctx, job.Args.Operation, "error") + w.observeJob(ctx, job.Args.Operation, jobResultError) return fmt.Errorf("upsert public.users %s: %w", userID, err) } @@ -105,19 +105,19 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy default: err := fmt.Errorf("unknown operation %q for user %s", job.Args.Operation, userID) telemetry.ReportError(ctx, "auth user sync unknown operation", err, attrs...) - w.observeJob(ctx, job.Args.Operation, "cancelled") + w.observeJob(ctx, job.Args.Operation, jobResultInvalidArgument) return river.JobCancel(err) } w.l.Info(ctx, "completed auth user sync job", - zap.String("job.kind", AuthUserProjectionKind), + zap.String("job.kind", authUserProjectionKind), zap.Int64("job.id", job.ID), zap.String("job.operation", job.Args.Operation), logger.WithUserID(job.Args.UserID), ) telemetry.ReportEvent(ctx, "auth_user_sync.job.completed", attrs...) - w.observeJob(ctx, job.Args.Operation, "success") + w.observeJob(ctx, job.Args.Operation, jobResultSuccess) return nil } @@ -128,8 +128,8 @@ func (w *AuthUserSyncWorker) observeJob(ctx context.Context, operation, result s } w.jobsCounter.Add(ctx, 1, metric.WithAttributes( - attribute.String("worker", AuthUserProjectionKind), - attribute.String("job.kind", AuthUserProjectionKind), + attribute.String("worker", authUserProjectionKind), + attribute.String("job.kind", authUserProjectionKind), attribute.String("job.operation", operation), attribute.String("result", result), )) diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go index 18cf96d151..01dda26d4f 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go @@ -22,6 +22,9 @@ const ( testEventuallyTimeout = 10 * time.Second testEventuallyTick = 50 * time.Millisecond testStopTimeout = 5 * time.Second + + authMigrationsDir = "packages/db/pkg/auth/migrations" + authCustomSchemaVersion int64 = 20260401000001 ) type riverProcess struct { @@ -34,117 +37,107 @@ func TestAuthUserSync_EndToEnd(t *testing.T) { t.Parallel() db := testutils.SetupDatabase(t) + applyAuthUserSyncMigrations(t, db) - authMigrationsDir := "packages/db/pkg/auth/migrations" - - db.ApplyMigrationsUpTo(t, 20260401000001, authMigrationsDir) - - authPool := db.AuthDb.WritePool() - require.NoError(t, RunRiverMigrations(t.Context(), authPool)) - - db.ApplyMigrations(t, authMigrationsDir) + t.Run("upsert projects auth users into public users", func(t *testing.T) { + ctx := t.Context() + userID := uuid.New() + email := fmt.Sprintf("river-sync-%s@example.com", userID.String()[:8]) - runUpsertProjection(t, db) - runDeleteProjection(t, db) - runBurstBacklog(t, db) -} + proc := startRiverWorker(t, db) + t.Cleanup(func() { proc.Stop(t) }) -func runUpsertProjection(t *testing.T, db *testutils.Database) { - t.Helper() + insertAuthUser(t, ctx, db, userID, email) + waitForPublicUser(t, ctx, db, userID, email) - ctx := t.Context() - userID := uuid.New() - email := fmt.Sprintf("river-sync-%s@example.com", userID.String()[:8]) + updatedEmail := fmt.Sprintf("river-sync-%s-updated@example.com", userID.String()[:8]) + updateAuthUserEmail(t, ctx, db, userID, updatedEmail) + waitForPublicUser(t, ctx, db, userID, updatedEmail) + }) - proc := startRiverWorker(t, db) - t.Cleanup(func() { proc.Stop(t) }) + t.Run("delete removes projected public users", func(t *testing.T) { + ctx := t.Context() + userID := uuid.New() + email := fmt.Sprintf("river-del-%s@example.com", userID.String()[:8]) - insertAuthUser(t, ctx, db, userID, email) + proc := startRiverWorker(t, db) + t.Cleanup(func() { proc.Stop(t) }) - waitForPublicUser(t, ctx, db, userID, email) + insertAuthUser(t, ctx, db, userID, email) + waitForPublicUser(t, ctx, db, userID, email) - updatedEmail := fmt.Sprintf("river-sync-%s-updated@example.com", userID.String()[:8]) - updateAuthUserEmail(t, ctx, db, userID, updatedEmail) + deleteAuthUser(t, ctx, db, userID) + waitForPublicUserGone(t, ctx, db, userID) + }) - waitForPublicUser(t, ctx, db, userID, updatedEmail) + t.Run("burst backlog drains mixed upsert and delete work", func(t *testing.T) { + ctx := t.Context() + const userCount = 40 - proc.Stop(t) -} + type testUser struct { + id uuid.UUID + email string + shouldDel bool + } -func runDeleteProjection(t *testing.T, db *testutils.Database) { - t.Helper() + users := make([]testUser, 0, userCount) + for i := range userCount { + u := testUser{ + id: uuid.New(), + email: fmt.Sprintf("river-burst-%02d@example.com", i), + shouldDel: i%3 == 0, + } + users = append(users, u) + insertAuthUser(t, ctx, db, u.id, u.email) + } - ctx := t.Context() - userID := uuid.New() - email := fmt.Sprintf("river-del-%s@example.com", userID.String()[:8]) + proc := startRiverWorker(t, db) + t.Cleanup(func() { proc.Stop(t) }) - proc := startRiverWorker(t, db) - t.Cleanup(func() { proc.Stop(t) }) + for _, u := range users { + waitForPublicUser(t, ctx, db, u.id, u.email) + } - insertAuthUser(t, ctx, db, userID, email) - waitForPublicUser(t, ctx, db, userID, email) + for _, u := range users { + if u.shouldDel { + deleteAuthUser(t, ctx, db, u.id) + } + } - deleteAuthUser(t, ctx, db, userID) - waitForPublicUserGone(t, ctx, db, userID) + for _, u := range users { + if u.shouldDel { + waitForPublicUserGone(t, ctx, db, u.id) + continue + } - proc.Stop(t) + waitForPublicUser(t, ctx, db, u.id, u.email) + } + }) } -func runBurstBacklog(t *testing.T, db *testutils.Database) { +func applyAuthUserSyncMigrations(t *testing.T, db *testutils.Database) { t.Helper() - ctx := t.Context() - const userCount = 40 - - type testUser struct { - id uuid.UUID - email string - shouldDel bool - } - - users := make([]testUser, 0, userCount) - for i := range userCount { - u := testUser{ - id: uuid.New(), - email: fmt.Sprintf("river-burst-%02d@example.com", i), - shouldDel: i%3 == 0, - } - users = append(users, u) - insertAuthUser(t, ctx, db, u.id, u.email) - } - - proc := startRiverWorker(t, db) - t.Cleanup(func() { proc.Stop(t) }) - - for _, u := range users { - waitForPublicUser(t, ctx, db, u.id, u.email) - } - - for _, u := range users { - if u.shouldDel { - deleteAuthUser(t, ctx, db, u.id) - } - } + // The auth user sync bootstraps `auth_custom` in three steps: + // 1. goose migration 20260401000001 creates the shared schema + // 2. River library migrations create River tables inside that schema + // 3. the remaining auth migrations add triggers that enqueue into River + db.ApplyMigrationsUpTo(t, authCustomSchemaVersion, authMigrationsDir) - for _, u := range users { - if u.shouldDel { - waitForPublicUserGone(t, ctx, db, u.id) - } else { - waitForPublicUser(t, ctx, db, u.id, u.email) - } - } + authPool := db.AuthDB.WritePool() + require.NoError(t, RunRiverMigrations(t.Context(), authPool)) - proc.Stop(t) + db.ApplyMigrations(t, authMigrationsDir) } func startRiverWorker(t *testing.T, db *testutils.Database) *riverProcess { t.Helper() ctx := t.Context() - authPool := db.AuthDb.WritePool() + authPool := db.AuthDB.WritePool() l := logger.NewNopLogger() tel := telemetry.NewNoopClient() - meter := tel.MeterProvider.Meter("github.com/e2b-dev/infra/packages/dashboard-api/internal/backgroundworker") + meter := tel.MeterProvider.Meter(workerMeterName) workers := river.NewWorkers() river.AddWorker(workers, NewAuthUserSyncWorker(ctx, db.SqlcClient, meter, l)) @@ -186,7 +179,7 @@ func (p *riverProcess) Stop(t *testing.T) { func insertAuthUser(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, email string) { t.Helper() - err := db.AuthDb.TestsRawSQL(ctx, + err := db.AuthDB.TestsRawSQL(ctx, "INSERT INTO auth.users (id, email) VALUES ($1, $2)", userID, email) require.NoError(t, err) } @@ -194,7 +187,7 @@ func insertAuthUser(t *testing.T, ctx context.Context, db *testutils.Database, u func updateAuthUserEmail(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, email string) { t.Helper() - err := db.AuthDb.TestsRawSQL(ctx, + err := db.AuthDB.TestsRawSQL(ctx, "UPDATE auth.users SET email = $1 WHERE id = $2", email, userID) require.NoError(t, err) } @@ -202,7 +195,7 @@ func updateAuthUserEmail(t *testing.T, ctx context.Context, db *testutils.Databa func deleteAuthUser(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID) { t.Helper() - err := db.AuthDb.TestsRawSQL(ctx, + err := db.AuthDB.TestsRawSQL(ctx, "DELETE FROM auth.users WHERE id = $1", userID) require.NoError(t, err) } @@ -213,7 +206,7 @@ func waitForPublicUser(t *testing.T, ctx context.Context, db *testutils.Database require.EventuallyWithT(t, func(c *assert.CollectT) { var email string - err := db.AuthDb.TestsRawSQLQuery(ctx, + err := db.AuthDB.TestsRawSQLQuery(ctx, "SELECT email FROM public.users WHERE id = $1", func(rows pgx.Rows) error { if !rows.Next() { @@ -237,7 +230,7 @@ func waitForPublicUserGone(t *testing.T, ctx context.Context, db *testutils.Data require.EventuallyWithT(t, func(c *assert.CollectT) { var count int - err := db.AuthDb.TestsRawSQLQuery(ctx, + err := db.AuthDB.TestsRawSQLQuery(ctx, "SELECT count(*) FROM public.users WHERE id = $1", func(rows pgx.Rows) error { if !rows.Next() { diff --git a/packages/dashboard-api/internal/backgroundworker/config.go b/packages/dashboard-api/internal/backgroundworker/config.go index d853dbb8b7..1bbbfbb1ed 100644 --- a/packages/dashboard-api/internal/backgroundworker/config.go +++ b/packages/dashboard-api/internal/backgroundworker/config.go @@ -1,14 +1,13 @@ package backgroundworker const ( - // must match the schema used in packages/db/pkg/auth/migrations for River tables and triggers - AuthCustomSchema = "auth_custom" - - // must match the queue value in packages/db/pkg/auth/migrations trigger SQL inserts - AuthUserProjectionQueue = "auth_user_projection" - - // must match the kind value in packages/db/pkg/auth/migrations trigger SQL inserts - AuthUserProjectionKind = "auth_user_projection" - - AuthUserProjectionMaxWorkers = 10 + authCustomSchema = "auth_custom" + authUserProjectionQueue = "auth_user_projection" + authUserProjectionKind = "auth_user_projection" + authUserProjectionMaxWorkers = 10 + + workerMeterName = "github.com/e2b-dev/infra/packages/dashboard-api/internal/backgroundworker" + jobResultError = "error" + jobResultInvalidArgument = "invalid_argument" + jobResultSuccess = "success" ) diff --git a/packages/dashboard-api/internal/backgroundworker/river.go b/packages/dashboard-api/internal/backgroundworker/river.go index 0dd603bde3..291a15a66a 100644 --- a/packages/dashboard-api/internal/backgroundworker/river.go +++ b/packages/dashboard-api/internal/backgroundworker/river.go @@ -2,19 +2,25 @@ package backgroundworker import ( "context" + "fmt" + sqlcdb "github.com/e2b-dev/infra/packages/db/client" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/riverqueue/river" "github.com/riverqueue/river/riverdriver/riverpgxv5" "github.com/riverqueue/river/rivermigrate" + "go.opentelemetry.io/otel/metric" + "go.uber.org/zap" + + "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) func RunRiverMigrations(ctx context.Context, pool *pgxpool.Pool) error { driver := riverpgxv5.New(pool) migrator, err := rivermigrate.New(driver, &rivermigrate.Config{ - Schema: AuthCustomSchema, + Schema: authCustomSchema, }) if err != nil { return err @@ -27,10 +33,38 @@ func RunRiverMigrations(ctx context.Context, pool *pgxpool.Pool) error { func NewRiverClient(pool *pgxpool.Pool, workers *river.Workers) (*river.Client[pgx.Tx], error) { return river.NewClient(riverpgxv5.New(pool), &river.Config{ - Schema: AuthCustomSchema, + Schema: authCustomSchema, Queues: map[string]river.QueueConfig{ - AuthUserProjectionQueue: {MaxWorkers: AuthUserProjectionMaxWorkers}, + authUserProjectionQueue: {MaxWorkers: authUserProjectionMaxWorkers}, }, Workers: workers, }) } + +func StartAuthUserSyncWorker(setupCtx, runCtx context.Context, authPool *pgxpool.Pool, mainDB *sqlcdb.Client, meterProvider metric.MeterProvider, l logger.Logger) (*river.Client[pgx.Tx], error) { + if err := RunRiverMigrations(setupCtx, authPool); err != nil { + return nil, fmt.Errorf("run River migrations on auth DB: %w", err) + } + + workerLogger := l.With(zap.String("worker", authUserProjectionKind)) + workerMeter := meterProvider.Meter(workerMeterName) + + workers := river.NewWorkers() + river.AddWorker(workers, NewAuthUserSyncWorker(setupCtx, mainDB, workerMeter, workerLogger)) + + riverClient, err := NewRiverClient(authPool, workers) + if err != nil { + return nil, fmt.Errorf("create River client: %w", err) + } + + if err := riverClient.Start(runCtx); err != nil { + return nil, fmt.Errorf("start River client: %w", err) + } + + l.Info(setupCtx, "background worker started", + zap.String("queue", authUserProjectionQueue), + zap.String("schema", authCustomSchema), + ) + + return riverClient, nil +} diff --git a/packages/dashboard-api/internal/cfg/model.go b/packages/dashboard-api/internal/cfg/model.go index cf1b16bda7..94365f06c9 100644 --- a/packages/dashboard-api/internal/cfg/model.go +++ b/packages/dashboard-api/internal/cfg/model.go @@ -19,7 +19,7 @@ type Config struct { RedisClusterURL string `env:"REDIS_CLUSTER_URL"` RedisTLSCABase64 string `env:"REDIS_TLS_CA_BASE64"` - AuthUserSyncBackgroundWorkerEnabled bool `env:"AUTH_USER_SYNC_BACKGROUND_WORKER_ENABLED" envDefault:"false"` + EnableAuthUserSyncBackgroundWorker bool `env:"ENABLE_AUTH_USER_SYNC_BACKGROUND_WORKER" envDefault:"false"` } func Parse() (Config, error) { diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index e4b4bd3366..c499113784 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -257,7 +257,7 @@ func createHandlerTestUser(t *testing.T, db *testutils.Database) uuid.UUID { userID := uuid.New() email := handlerTestUserEmail(userID) - err := db.AuthDb.TestsRawSQL(t.Context(), ` + err := db.AuthDB.TestsRawSQL(t.Context(), ` INSERT INTO auth.users (id, email) VALUES ($1, $2) `, userID, email) @@ -275,7 +275,7 @@ func handlerTestUserEmail(userID uuid.UUID) string { func insertHandlerTestTeamMember(t *testing.T, db *testutils.Database, userID, teamID uuid.UUID, isDefault bool) { t.Helper() - err := db.AuthDb.TestsRawSQL(t.Context(), ` + err := db.AuthDB.TestsRawSQL(t.Context(), ` INSERT INTO public.users_teams (user_id, team_id, is_default) VALUES ($1, $2, $3) `, userID, teamID, isDefault) diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 012980ab69..1fb91fe75e 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -249,28 +249,18 @@ func run() int { var riverClient *river.Client[pgx.Tx] - if config.AuthUserSyncBackgroundWorkerEnabled { - workerLogger := l.With(zap.String("worker", backgroundworker.AuthUserProjectionKind)) - workerMeter := tel.MeterProvider.Meter("github.com/e2b-dev/infra/packages/dashboard-api") - - authPool := authDB.WritePool() - if err := backgroundworker.RunRiverMigrations(ctx, authPool); err != nil { - l.Fatal(ctx, "failed to run River migrations on auth DB", zap.Error(err)) - } - - workers := river.NewWorkers() - river.AddWorker(workers, backgroundworker.NewAuthUserSyncWorker(ctx, db, workerMeter, workerLogger)) - - riverClient, err = backgroundworker.NewRiverClient(authPool, workers) + if config.EnableAuthUserSyncBackgroundWorker { + riverClient, err = backgroundworker.StartAuthUserSyncWorker( + ctx, + signalCtx, + authDB.WritePool(), + db, + tel.MeterProvider, + l, + ) if err != nil { - l.Fatal(ctx, "failed to create River client", zap.Error(err)) + l.Fatal(ctx, "failed to start auth user sync worker", zap.Error(err)) } - - if err := riverClient.Start(signalCtx); err != nil { - l.Fatal(ctx, "failed to start River client", zap.Error(err)) - } - - l.Info(ctx, "background worker started", zap.String("queue", backgroundworker.AuthUserProjectionQueue), zap.String("schema", backgroundworker.AuthCustomSchema)) } wg.Go(func() { diff --git a/packages/db/pkg/testutils/db.go b/packages/db/pkg/testutils/db.go index a9fd66e4c4..aa5199f4ac 100644 --- a/packages/db/pkg/testutils/db.go +++ b/packages/db/pkg/testutils/db.go @@ -37,7 +37,7 @@ func init() { // Database encapsulates the test database container and clients type Database struct { SqlcClient *db.Client - AuthDb *authdb.Client + AuthDB *authdb.Client TestQueries *queries.Queries connStr string } @@ -100,16 +100,16 @@ func SetupDatabase(t *testing.T) *Database { }) // Create the auth db client - authDb, err := authdb.NewClient(t.Context(), connStr, connStr) + authDB, err := authdb.NewClient(t.Context(), connStr, connStr) require.NoError(t, err, "Failed to create auth db client") t.Cleanup(func() { - err := authDb.Close() + err := authDB.Close() assert.NoError(t, err) }) return &Database{ SqlcClient: sqlcClient, - AuthDb: authDb, + AuthDB: authDB, TestQueries: testQueries, connStr: connStr, } @@ -124,6 +124,8 @@ func (db *Database) ApplyMigrations(t *testing.T, migrationDirs ...string) { func (db *Database) ApplyMigrationsUpTo(t *testing.T, version int64, migrationDirs ...string) { t.Helper() + // This is only used for staged bootstrap flows that must interleave + // third-party migrations with goose-managed SQL migrations. db.applyGooseMigrations(t, version, migrationDirs...) } diff --git a/packages/db/pkg/testutils/queries.go b/packages/db/pkg/testutils/queries.go index c264ef69d8..ab3c5529f7 100644 --- a/packages/db/pkg/testutils/queries.go +++ b/packages/db/pkg/testutils/queries.go @@ -23,7 +23,7 @@ func CreateTestTeam(t *testing.T, db *Database) uuid.UUID { // Insert a team directly into the database using raw SQL // Using the default tier 'base_v1' that is created in migrations - err := db.AuthDb.TestsRawSQL(t.Context(), + err := db.AuthDB.TestsRawSQL(t.Context(), "INSERT INTO public.teams (id, name, tier, email, slug) VALUES ($1, $2, $3, $4, $5)", teamID, "Test Team "+teamID.String(), "base_v1", "test-"+teamID.String()+"@example.com", slug, ) @@ -92,7 +92,7 @@ func GetTeamSlug(t *testing.T, ctx context.Context, db *Database, teamID uuid.UU t.Helper() var slug string - err := db.AuthDb.TestsRawSQLQuery(ctx, + err := db.AuthDB.TestsRawSQLQuery(ctx, "SELECT slug FROM public.teams WHERE id = $1", func(rows pgx.Rows) error { if rows.Next() { diff --git a/packages/docker-reverse-proxy/internal/auth/validate_test.go b/packages/docker-reverse-proxy/internal/auth/validate_test.go index 29ff34d215..fa7e62276f 100644 --- a/packages/docker-reverse-proxy/internal/auth/validate_test.go +++ b/packages/docker-reverse-proxy/internal/auth/validate_test.go @@ -111,27 +111,27 @@ func setupValidateTest(tb testing.TB, db *testutils.Database, userID, teamID uui tb.Helper() // Create team - err := db.AuthDb.TestsRawSQL(tb.Context(), ` + err := db.AuthDB.TestsRawSQL(tb.Context(), ` INSERT INTO "auth"."users" (id, email) VALUES ($1, 'test@e2b.dev') `, userID) require.NoError(tb, err) - err = db.AuthDb.TestsRawSQL(tb.Context(), ` + err = db.AuthDB.TestsRawSQL(tb.Context(), ` INSERT INTO teams (id, name, email, tier, slug) VALUES ($1, 'test-team', 'test@e2b.dev', 'base_v1', 'test-team-slug') `, teamID) require.NoError(tb, err) // Link user to team - err = db.AuthDb.TestsRawSQL(tb.Context(), ` + err = db.AuthDB.TestsRawSQL(tb.Context(), ` INSERT INTO users_teams (user_id, team_id, is_default) VALUES ($1, $2, true) `, userID, teamID) require.NoError(tb, err) // Create access token - _, err = db.AuthDb.Write.CreateAccessToken(tb.Context(), authqueries.CreateAccessTokenParams{ + _, err = db.AuthDB.Write.CreateAccessToken(tb.Context(), authqueries.CreateAccessTokenParams{ ID: uuid.New(), UserID: userID, AccessTokenHash: accessToken.HashedValue, diff --git a/tests/integration/internal/setup/db_client.go b/tests/integration/internal/setup/db_client.go index 97cbf5fe26..2092e28bd8 100644 --- a/tests/integration/internal/setup/db_client.go +++ b/tests/integration/internal/setup/db_client.go @@ -12,7 +12,7 @@ import ( type Database struct { Db *client.Client - AuthDb *authdb.Client + AuthDB *authdb.Client } func GetTestDBClient(tb testing.TB) *Database { @@ -33,6 +33,6 @@ func GetTestDBClient(tb testing.TB) *Database { return &Database{ Db: db, - AuthDb: authDb, + AuthDB: authDb, } } diff --git a/tests/integration/internal/tests/team_test.go b/tests/integration/internal/tests/team_test.go index 63104897f4..93d8871d3c 100644 --- a/tests/integration/internal/tests/team_test.go +++ b/tests/integration/internal/tests/team_test.go @@ -27,7 +27,7 @@ func TestBannedTeam(t *testing.T) { teamID := utils.CreateTeamWithUser(t, db, teamName, setup.UserID) apiKey := utils.CreateAPIKey(t, ctx, c, setup.UserID, teamID) - err := db.AuthDb.TestsRawSQL(ctx, ` + err := db.AuthDB.TestsRawSQL(ctx, ` UPDATE teams SET is_banned = $1 WHERE id = $2 `, true, teamID) require.NoError(t, err) @@ -56,7 +56,7 @@ func TestBlockedTeam(t *testing.T) { teamID := utils.CreateTeamWithUser(t, db, teamName, setup.UserID) apiKey := utils.CreateAPIKey(t, ctx, c, setup.UserID, teamID) - err := db.AuthDb.TestsRawSQL(ctx, ` + err := db.AuthDB.TestsRawSQL(ctx, ` UPDATE teams SET is_blocked = $1, blocked_reason = $2 WHERE id = $3 `, true, blockReason, teamID) require.NoError(t, err) diff --git a/tests/integration/internal/utils/team.go b/tests/integration/internal/utils/team.go index 7628247fd1..1f21fa94ad 100644 --- a/tests/integration/internal/utils/team.go +++ b/tests/integration/internal/utils/team.go @@ -28,7 +28,7 @@ func CreateTeamWithUser( teamID := uuid.New() slug := fmt.Sprintf("test-%s", teamID.String()[:8]) - err := db.AuthDb.TestsRawSQL(t.Context(), ` + err := db.AuthDB.TestsRawSQL(t.Context(), ` INSERT INTO teams (id, email, name, tier, is_blocked, slug) VALUES ($1, $2, $3, $4, $5, $6) `, teamID, fmt.Sprintf("test-integration-%s@e2b.dev", teamID), teamName, "base_v1", false, slug) @@ -39,7 +39,7 @@ VALUES ($1, $2, $3, $4, $5, $6) } t.Cleanup(func() { - db.AuthDb.TestsRawSQL(t.Context(), ` + db.AuthDB.TestsRawSQL(t.Context(), ` DELETE FROM teams WHERE id = $1 `, teamID) }) @@ -53,14 +53,14 @@ func AddUserToTeam(t *testing.T, db *setup.Database, teamID uuid.UUID, userID st userUUID, err := uuid.Parse(userID) require.NoError(t, err) - err = db.AuthDb.TestsRawSQL(t.Context(), ` + err = db.AuthDB.TestsRawSQL(t.Context(), ` INSERT INTO users_teams (user_id, team_id, is_default) VALUES ($1, $2, $3) `, userUUID, teamID, false) require.NoError(t, err) t.Cleanup(func() { - db.AuthDb.TestsRawSQL(t.Context(), ` + db.AuthDB.TestsRawSQL(t.Context(), ` DELETE FROM users_teams WHERE user_id = $1 and team_id = $2 `, userUUID, teamID) }) diff --git a/tests/integration/internal/utils/user.go b/tests/integration/internal/utils/user.go index d766adceb9..4073959ba1 100644 --- a/tests/integration/internal/utils/user.go +++ b/tests/integration/internal/utils/user.go @@ -19,14 +19,14 @@ func CreateUser(t *testing.T, db *setup.Database) uuid.UUID { userID := uuid.New() - err := db.AuthDb.TestsRawSQL(t.Context(), ` + err := db.AuthDB.TestsRawSQL(t.Context(), ` INSERT INTO auth.users (id, email) VALUES ($1, $2) `, userID, fmt.Sprintf("user-test-integration-%s@e2b.dev", userID)) require.NoError(t, err) t.Cleanup(func() { - db.AuthDb.TestsRawSQL(t.Context(), ` + db.AuthDB.TestsRawSQL(t.Context(), ` DELETE FROM auth.users WHERE id = $1 `, userID) }) @@ -49,7 +49,7 @@ func CreateAccessToken(t *testing.T, db *setup.Database, userID uuid.UUID) strin accessTokenMask, err := keys.MaskKey(keys.AccessTokenPrefix, tokenWithoutPrefix) require.NoError(t, err) - _, err = db.AuthDb.Write.CreateAccessToken(t.Context(), authqueries.CreateAccessTokenParams{ + _, err = db.AuthDB.Write.CreateAccessToken(t.Context(), authqueries.CreateAccessTokenParams{ ID: uuid.New(), UserID: userID, AccessTokenHash: accessTokenHash, From bd1ebb160d21b3045d58934ca24326ce712da28a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 6 Apr 2026 18:29:14 +0000 Subject: [PATCH 30/92] chore: auto-commit generated changes --- packages/dashboard-api/internal/backgroundworker/river.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dashboard-api/internal/backgroundworker/river.go b/packages/dashboard-api/internal/backgroundworker/river.go index 291a15a66a..319143e06c 100644 --- a/packages/dashboard-api/internal/backgroundworker/river.go +++ b/packages/dashboard-api/internal/backgroundworker/river.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - sqlcdb "github.com/e2b-dev/infra/packages/db/client" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/riverqueue/river" @@ -13,6 +12,7 @@ import ( "go.opentelemetry.io/otel/metric" "go.uber.org/zap" + sqlcdb "github.com/e2b-dev/infra/packages/db/client" "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) From 126d984a89b068833dfc3690a84cdce521cc84e0 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Mon, 6 Apr 2026 13:48:19 -0700 Subject: [PATCH 31/92] fix: lint --- .../internal/handlers/template_alias_test.go | 6 +++--- .../placement/placement_benchmark_test.go | 20 +++++++++---------- .../sandbox/storage/memory/operations_test.go | 18 ++++++++--------- .../backgroundworker/auth_user_sync_test.go | 7 +++++++ .../internal/backgroundworker/river.go | 2 +- tests/integration/internal/utils/process.go | 15 +++++++------- 6 files changed, 38 insertions(+), 30 deletions(-) diff --git a/packages/api/internal/handlers/template_alias_test.go b/packages/api/internal/handlers/template_alias_test.go index 724952804b..52d39d296b 100644 --- a/packages/api/internal/handlers/template_alias_test.go +++ b/packages/api/internal/handlers/template_alias_test.go @@ -38,7 +38,7 @@ func TestQueryNotExistingTemplateAlias(t *testing.T) { w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) - c.Request = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/templates/aliases/%s", alias), nil) + c.Request = httptest.NewRequestWithContext(t.Context(), http.MethodGet, fmt.Sprintf("/templates/aliases/%s", alias), nil) auth.SetTeamInfo(c, &types.Team{ Team: &authqueries.Team{ ID: teamID, @@ -75,7 +75,7 @@ func TestQueryExistingTemplateAlias(t *testing.T) { w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) - c.Request = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/templates/aliases/%s", alias), nil) + c.Request = httptest.NewRequestWithContext(t.Context(), http.MethodGet, fmt.Sprintf("/templates/aliases/%s", alias), nil) auth.SetTeamInfo(c, &types.Team{ Team: &authqueries.Team{ ID: teamID, @@ -120,7 +120,7 @@ func TestQueryExistingTemplateAliasAsNotOwnerTeam(t *testing.T) { w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) - c.Request = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/templates/aliases/%s", alias), nil) + c.Request = httptest.NewRequestWithContext(t.Context(), http.MethodGet, fmt.Sprintf("/templates/aliases/%s", alias), nil) auth.SetTeamInfo(c, &types.Team{ Team: &authqueries.Team{ diff --git a/packages/api/internal/orchestrator/placement/placement_benchmark_test.go b/packages/api/internal/orchestrator/placement/placement_benchmark_test.go index 7b6ad043c7..b941800713 100644 --- a/packages/api/internal/orchestrator/placement/placement_benchmark_test.go +++ b/packages/api/internal/orchestrator/placement/placement_benchmark_test.go @@ -55,8 +55,8 @@ type SimulatedNode struct { mu sync.RWMutex sandboxes map[string]*LiveSandbox - totalPlacements int64 - rejectedPlacements int64 + totalPlacements atomic.Int64 + rejectedPlacements atomic.Int64 lastUpdateTime time.Time } @@ -119,7 +119,7 @@ func (n *SimulatedNode) placeSandbox(sbx *LiveSandbox) bool { metrics := n.Metrics() // Check capacity with overcommit if metrics.CpuAllocated+uint32(sbx.RequestedCPU) > metrics.CpuCount*4 { // 4x overcommit - atomic.AddInt64(&n.rejectedPlacements, 1) + n.rejectedPlacements.Add(1) return false } @@ -137,7 +137,7 @@ func (n *SimulatedNode) placeSandbox(sbx *LiveSandbox) bool { MetricMemoryAllocatedBytes: metrics.MemoryAllocatedBytes + uint64(sbx.RequestedMemory)*1024*1024, }) n.sandboxes[sbx.ID] = sbx - atomic.AddInt64(&n.totalPlacements, 1) + n.totalPlacements.Add(1) return true } @@ -210,12 +210,12 @@ func runBenchmark(b *testing.B, algorithm Algorithm, config BenchmarkConfig) *Be mu sync.Mutex placementTimes []time.Duration activeSandboxes sync.Map // sandboxID -> *LiveSandbox - sandboxIDCounter int64 + sandboxIDCounter atomic.Int64 // Metrics for time series recentPlacements []time.Duration - recentSuccesses int64 - recentFailures int64 + recentSuccesses atomic.Int64 + recentFailures atomic.Int64 ) // Start sandbox cleanup goroutine @@ -260,7 +260,7 @@ func runBenchmark(b *testing.B, algorithm Algorithm, config BenchmarkConfig) *Be select { case <-ticker.C: // Generate sandbox with variance - sandboxID := atomic.AddInt64(&sandboxIDCounter, 1) + sandboxID := sandboxIDCounter.Add(1) cpuVariance := (rand.Float64()*2 - 1) * config.CPUVariance requestedCPU := max(int64(float64(config.AvgSandboxCPU)*(1+cpuVariance)), 1) @@ -323,7 +323,7 @@ func runBenchmark(b *testing.B, algorithm Algorithm, config BenchmarkConfig) *Be if simNode.placeSandbox(sbx) { activeSandboxes.Store(sbx.ID, sbx) metrics.SuccessfulPlacements++ - atomic.AddInt64(&recentSuccesses, 1) + recentSuccesses.Add(1) success = true } } @@ -331,7 +331,7 @@ func runBenchmark(b *testing.B, algorithm Algorithm, config BenchmarkConfig) *Be if !success { metrics.FailedPlacements++ - atomic.AddInt64(&recentFailures, 1) + recentFailures.Add(1) } mu.Unlock() } diff --git a/packages/api/internal/sandbox/storage/memory/operations_test.go b/packages/api/internal/sandbox/storage/memory/operations_test.go index 7d995e8ce4..e4e364ae61 100644 --- a/packages/api/internal/sandbox/storage/memory/operations_test.go +++ b/packages/api/internal/sandbox/storage/memory/operations_test.go @@ -832,8 +832,8 @@ func TestConcurrency_StressTest(t *testing.T) { stopCh := make(chan struct{}) // Metrics - var opsCompleted uint64 - var errorCount uint64 + var opsCompleted atomic.Uint64 + var errorCount atomic.Uint64 // Launch workers that continuously perform random operations for i := range 200 { @@ -857,21 +857,21 @@ func TestConcurrency_StressTest(t *testing.T) { if finish != nil { finish(t.Context(), nil) } - atomic.AddUint64(&opsCompleted, 1) + opsCompleted.Add(1) } else if err != nil { - atomic.AddUint64(&errorCount, 1) + errorCount.Add(1) } case 1: // Read state _ = sbx.State() - atomic.AddUint64(&opsCompleted, 1) + opsCompleted.Add(1) case 2: // Wait with timeout waitCtx, cancel := context.WithTimeout(t.Context(), time.Microsecond*10) _ = waitForStateChange(waitCtx, sbx) cancel() - atomic.AddUint64(&opsCompleted, 1) + opsCompleted.Add(1) case 3: // Read _data _ = sbx.Data() - atomic.AddUint64(&opsCompleted, 1) + opsCompleted.Add(1) } } @@ -887,8 +887,8 @@ func TestConcurrency_StressTest(t *testing.T) { close(stopCh) wg.Wait() - finalOps := atomic.LoadUint64(&opsCompleted) - finalErrors := atomic.LoadUint64(&errorCount) + finalOps := opsCompleted.Load() + finalErrors := errorCount.Load() t.Logf("Stress test completed: %d operations, %d errors", finalOps, finalErrors) // Should have completed many operations without panic diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go index 01dda26d4f..1d82d57276 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go @@ -40,6 +40,8 @@ func TestAuthUserSync_EndToEnd(t *testing.T) { applyAuthUserSyncMigrations(t, db) t.Run("upsert projects auth users into public users", func(t *testing.T) { + t.Parallel() + ctx := t.Context() userID := uuid.New() email := fmt.Sprintf("river-sync-%s@example.com", userID.String()[:8]) @@ -56,6 +58,8 @@ func TestAuthUserSync_EndToEnd(t *testing.T) { }) t.Run("delete removes projected public users", func(t *testing.T) { + t.Parallel() + ctx := t.Context() userID := uuid.New() email := fmt.Sprintf("river-del-%s@example.com", userID.String()[:8]) @@ -71,6 +75,8 @@ func TestAuthUserSync_EndToEnd(t *testing.T) { }) t.Run("burst backlog drains mixed upsert and delete work", func(t *testing.T) { + t.Parallel() + ctx := t.Context() const userCount = 40 @@ -107,6 +113,7 @@ func TestAuthUserSync_EndToEnd(t *testing.T) { for _, u := range users { if u.shouldDel { waitForPublicUserGone(t, ctx, db, u.id) + continue } diff --git a/packages/dashboard-api/internal/backgroundworker/river.go b/packages/dashboard-api/internal/backgroundworker/river.go index 291a15a66a..319143e06c 100644 --- a/packages/dashboard-api/internal/backgroundworker/river.go +++ b/packages/dashboard-api/internal/backgroundworker/river.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - sqlcdb "github.com/e2b-dev/infra/packages/db/client" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/riverqueue/river" @@ -13,6 +12,7 @@ import ( "go.opentelemetry.io/otel/metric" "go.uber.org/zap" + sqlcdb "github.com/e2b-dev/infra/packages/db/client" "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) diff --git a/tests/integration/internal/utils/process.go b/tests/integration/internal/utils/process.go index 765a095539..69a04b54f7 100644 --- a/tests/integration/internal/utils/process.go +++ b/tests/integration/internal/utils/process.go @@ -3,6 +3,7 @@ package utils import ( "context" "fmt" + "strings" "testing" "connectrpc.com/connect" @@ -82,7 +83,7 @@ func ExecCommandWithOutput(tb testing.TB, ctx context.Context, sbx *api.Sandbox, } }() - var output string + var output strings.Builder for stream.Receive() { select { case <-ctx.Done(): @@ -95,28 +96,28 @@ func ExecCommandWithOutput(tb testing.TB, ctx context.Context, sbx *api.Sandbox, // Capture stdout if msg.GetEvent().GetData() != nil { if stdout := msg.GetEvent().GetData().GetStdout(); stdout != nil { - output += string(stdout) + output.Write(stdout) } if stderr := msg.GetEvent().GetData().GetStderr(); stderr != nil { - output += string(stderr) + output.Write(stderr) } } if msg.GetEvent().GetEnd() != nil { if msg.GetEvent().GetEnd().GetExitCode() != 0 { - return output, fmt.Errorf("command %s in sandbox %s failed with exit code %d", command, sbx.SandboxID, msg.GetEvent().GetEnd().GetExitCode()) + return output.String(), fmt.Errorf("command %s in sandbox %s failed with exit code %d", command, sbx.SandboxID, msg.GetEvent().GetEnd().GetExitCode()) } tb.Logf("Command [%s] completed successfully in sandbox %s", command, sbx.SandboxID) - return output, nil + return output.String(), nil } } } if err := stream.Err(); err != nil { - return output, fmt.Errorf("failed to execute command %s in sandbox %s: %w", command, sbx.SandboxID, err) + return output.String(), fmt.Errorf("failed to execute command %s in sandbox %s: %w", command, sbx.SandboxID, err) } - return output, nil + return output.String(), nil } From 24a425c09be4c76870d6e3629bb3ab356a0b7ca2 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Mon, 6 Apr 2026 15:03:40 -0700 Subject: [PATCH 32/92] chore: use write auth db for user check in team creation --- packages/dashboard-api/internal/handlers/team_provisioning.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index 09dd427054..b58a37db83 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -167,7 +167,7 @@ func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisi } func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string) (provisionedTeam, error) { - authUser, err := s.authDB.Read.GetUser(ctx, userID) + authUser, err := s.authDB.Write.GetUser(ctx, userID) if err != nil { return provisionedTeam{}, fmt.Errorf("get auth user: %w", err) } From 4155a9600d334d868f9a4297f3c0a3e4cf0a56b8 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Wed, 8 Apr 2026 13:41:13 -0700 Subject: [PATCH 33/92] refactor(dashboard-api): localize team creation policy Move create-team checks into dashboard-api transactions, keep billing provisioning post-commit, and rename the internal sink package to teamprovision. Made-with: Cursor --- packages/dashboard-api/go.mod | 2 + packages/dashboard-api/go.sum | 4 + .../dashboard-api/internal/api/api.gen.go | 102 +++++---- .../dashboard-api/internal/handlers/store.go | 6 +- .../internal/handlers/team_handlers_test.go | 209 +++++++++++++++++- .../internal/handlers/team_provisioning.go | 84 ++++++- .../internal/teambilling/http_sink.go | 84 ------- .../internal/teambilling/noop_sink.go | 17 -- .../internal/teambilling/sink.go | 29 --- .../{teambilling => teamprovision}/factory.go | 2 +- .../internal/teamprovision/http_sink.go | 6 +- .../internal/teamprovision/noop_sink.go | 17 ++ .../internal/teamprovision/sink.go | 6 +- packages/dashboard-api/main.go | 4 +- .../db/queries/team_creation_guard.sql.go | 130 +++++++++++ .../db/queries/teams/team_creation_guard.sql | 32 +++ spec/openapi-dashboard.yml | 8 + 17 files changed, 546 insertions(+), 196 deletions(-) delete mode 100644 packages/dashboard-api/internal/teambilling/http_sink.go delete mode 100644 packages/dashboard-api/internal/teambilling/noop_sink.go delete mode 100644 packages/dashboard-api/internal/teambilling/sink.go rename packages/dashboard-api/internal/{teambilling => teamprovision}/factory.go (95%) create mode 100644 packages/dashboard-api/internal/teamprovision/noop_sink.go create mode 100644 packages/db/queries/team_creation_guard.sql.go create mode 100644 packages/db/queries/teams/team_creation_guard.sql diff --git a/packages/dashboard-api/go.mod b/packages/dashboard-api/go.mod index 9ca06440bd..b10c432561 100644 --- a/packages/dashboard-api/go.mod +++ b/packages/dashboard-api/go.mod @@ -37,6 +37,7 @@ require ( github.com/ClickHouse/ch-go v0.67.0 // indirect github.com/ClickHouse/clickhouse-go/v2 v2.40.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/alicebob/miniredis/v2 v2.37.0 // indirect github.com/andybalholm/brotli v1.2.0 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/bsm/redislock v0.9.4 // indirect @@ -138,6 +139,7 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.0 // indirect github.com/woodsbury/decimal128 v1.3.0 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/bridges/otelzap v0.14.0 // indirect diff --git a/packages/dashboard-api/go.sum b/packages/dashboard-api/go.sum index 1a5405cacc..b8391c802f 100644 --- a/packages/dashboard-api/go.sum +++ b/packages/dashboard-api/go.sum @@ -11,6 +11,8 @@ github.com/ClickHouse/clickhouse-go/v2 v2.40.1/go.mod h1:GDzSBLVhladVm8V01aEB36I github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/alicebob/miniredis/v2 v2.37.0 h1:RheObYW32G1aiJIj81XVt78ZHJpHonHLHW7OLIshq68= +github.com/alicebob/miniredis/v2 v2.37.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= @@ -321,6 +323,8 @@ github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3i github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= diff --git a/packages/dashboard-api/internal/api/api.gen.go b/packages/dashboard-api/internal/api/api.gen.go index afaf2c5fd1..6904148416 100644 --- a/packages/dashboard-api/internal/api/api.gen.go +++ b/packages/dashboard-api/internal/api/api.gen.go @@ -301,6 +301,9 @@ type N403 = Error // N404 defines model for 404. type N404 = Error +// N429 defines model for 429. +type N429 = Error + // N500 defines model for 500. type N500 = Error @@ -817,55 +820,56 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xcWXPcuPH/Kij+/1V5oWdGlp1K9KZjd62Klah0bKXKpZIwZM8M1iTABUAdq8x3T+Ek", - "OQSPkTWOstkna0gAff260d0A/RwlLC8YBSpFdPAcFZjjHCRw/Wtekiy9Jan6OwWRcFJIwmh0EJ2mQCVZ", - "EOCILZBcAdJjJ1EcEfW+wHIVxRHFOUQH1TpxxOHXknBIowPJS4gjkawgx4rAgvEcy+ggKks9Uj4Vaq6Q", - "nNBltF7Hfplbxm8l5EWGJbRZ+4f+A2doQTIJHM2fDG+IeJ5j5KY3HjJePccZwcKL82sJ/KktT4ORuizd", - "vIs2w8csz/E7AUr3ElKUESGVVg3XpycCSYaWIJGQWJYCBFowrliDxyJjKUQHC5wJ6GdV9OqeSMjFCCPE", - "UY4fT83gvdnMv8ecY0W0pOTXEuwARWQdR0I+ZWqMWjrymnCybKsOrwPJEKFJVqYwVhWeZFDy/+ewiA6i", - "/5tWDjE1w8T0SJG+1NOVBA2huyQUt0nJBeMBAfVzxEGWnEKqAKocqOBwT1gpjMAcRMGoAEQouks4KFXc", - "YvkvZ887ZEzVBVFLfAQoxW1GciLbfJ7hR5KXOaJlPjd+rpWlNG94RwVwVOAldDFhFq7zkMICl5mMDj7O", - "4gpshMr995EGl6JosZUTan95lRMqYQlcMy8wTefs8fRkTHSygzviU7VUn5O09ScB5+Poq5EdxO0i3xYa", - "1SKXWbls83IFOEciK5fGboJl9532UsO2VEEpgJ+O2iDUyA4V2EW+RQVrNdm4jHbnD7OZ+idhVALV4MZF", - "kZEEK/6mvwjF5HNt/T73/4Fzxg2NppBHOEWKZRBS+f2H2d7uaR6WcqVUa1ZFYMYp4vu7J/4j43OSpkAN", - "xQ+7p/h3JtGClTRVFD9+D6NeAr8H7hS7diDUqDpMU+VPZ6Ai4oW1vEqbOCuAS2KwBzkmWQO05knIcSvE", - "f7GjbvwwNv8FEo0svQGd0gVrE7N7w2EggOtZSA9QUJEkByFxXqg95eLH4/39/b/WdhHPbIolvFODQ/v/", - "glAiVr30WF5kMEQxRmSB3GKd5GmZZXiuNlcTD1rsqAAiQkHPpnH6PeKQ6VRCMiRXRJhUQnOA7zHRFHRk", - "crlAm0yYj1oGoHOD7dIIM+kMhMDLQB77IyZZyQHlZgB6WAG16Q8iAt0tMMkgvYsRkyvgD0QAulN83k2G", - "FbcBvApDDQN7uTZ57YTopddDCBmW+RwXBaQKByjFYjVnmKcoyYjSlc7lqNr0v5jsRLEbR0ZWxUeZJCBE", - "jYPKSDUOVAbadpU3ht0t66rB1Py/HIRaKA+4AAwH0Sc+EyEvbBbQNn+K5fiUXy0FqV42lPJTeJTH/fm9", - "ZKjAQpigA0jNcKm93jd0vWmUpfCk9AdKp5SZsS6x3k6LWsgGf93qurQFUbfK5hVYQmH2c7A0q0fSkUjU", - "/tpS84ZoTWZCYh2fXx+zkgbc+/j8GiWMm9q5XhFEzTLkzx+i/sIjjo51sFRpQGcCYNLaZ1XPfAa6lKvo", - "4P3Hj3ph93tvyJB6jZCQJ6aEuqo1QJrUdevC/DnKDhsLHqrpIcxr/Xv9toq3tqb0BFMcDAavRhozLhUB", - "ep/+DFwQk/aNjLetx0U5z0hSezVnLAOsU1yO87P5prQaI21pRYEfaFA9HRMkkzg7IeLrJfkNOsh0CFVb", - "5T4pylEEQ+HWQaWylZPZLtzmMm5kC1Z5DXA0VDECwQZwYRiHszGV1BU4gRFm35DaLDqCqZ6g6Dp+L/aw", - "wUhXUQhy6oxx1I5z6h0S5DfYjHMqizkjR73hbhbCl6mT2mWH7rZtkteDkXo3ieIxISLvSjzMSvb1ZLB0", - "0uxUy4XU9glwJlfdZu1k5VOZY/qOA04VztBKr4OSFSRfEQdRZnKYvz7G6qnGH+XdHylyg+Puc46rxlGF", - "IVtwEEBlnZY/0Tg9mUQ9BMY18dzoYcQbA1SHIzU6nXVl3FWJhtzmDHLGn0JB0Lx5SQTce/+XUJS6NCtc", - "QMJ42rNTbXTqtF02FBfMfYrS5w19uPTp7TqO0sYm0Lv5VCPVPJZjQgPOjQUg81JBiUNDdZLjxYIkCs9Y", - "F+BEYXYEfPOakfqY9MZ8YWO/w9l5R+i8Irl11LqUD1ggO2l0wBSSFcX2RPSkF4dF70sn2/gsWnCWo4cV", - "SVbKkHWmrNsNOnWNcNw4Nal0XYNzzfwNwIa8uWqrBvwrTSE9egrVEYOqGq4rBpfw7dyO7Wlw1yHixB17", - "tYuMUNh07eJqYl2QfvWJvgxHDxidttZsMpSxuqW7eLswB0/dvI1UpbBnXGNaSWpoiJ/rIm0X8Dmh5zWG", - "9uLXKOn1IguSwTlJZMnhmmfjSpZenr9RhU6S1+K1pfjOzsW1AK5ECPSZMpZ8hfQCsBhZzL+CUx5hSiEN", - "F/5EHBmWul73eHRszr0H3cup47MZ/dqm6XSWOJLEhNmxxozdybCeWMWnNlt1zdV0HG9YuBnarLr6IPPZ", - "a3SzDKVJyTlQaXM0ECObU9VMl0ibpujI6Wo7a/dsuqpcFzQ+sZKLke2hHD9ehNpP3TR+DrSCgqM3g3eT", - "vTio1R6NVcRrXHsV9Zm1t8uC8/FblQ8tw60VtWybJ+UukJScyKdLtaZh4rIs8BwL2LtiX4EelirMP5sb", - "DCvAqXYGe4fhn+/c4Hd6cKV2XJC/ge6guhHvFaujV1NitRZTDBN7ICyJ1PePfnh/hE78idrh+WkUR/eu", - "QRrNJnuTmeKCFUBxQaKDaH8ym8yUH2O50vJO594HlqCDmzKJ7i+o+jD6CaS3ef2q4Jewdaoh0+CVuXU8", - "cp4/Whg7w11qGj/eXpha32xcJHn/incOAqdUoQsI5oxzUWbZU3VLrMBLQvUptmF4Yq5gzLpoeiGmalB1", - "O2Vo7F7tMsnQ2P3apYz+sWpQ3cc0ZELe9eUm6CZfbpRhRJnnmD+5kyfly1YbykHwUvhjIhHdKHLWuNP6", - "bcN+YF9Wh1gvA7j4HhBqndyNhtHGUd3/MoZ+Atk+uexD0bOz8XoYR0f+POVlMPoOKNL3ibYETgoSk8wF", - "n92Awd4rGxr74Q0Ax6qjCzfmqKAPLOZQItqhqTeOPQL2/lQ/0BDe+EZlXuj6KP1qKlxuOH32raD1lPsm", - "aZfMPqe8dLNsY3VbX6kaUDt1lmb3d7TDuN7aHy5jXcYphDtrO5/xQLJu41N/i6Cmti+0ggXCWaYzANPJ", - "xNW1WEj1XWM0h4zRpUCSxeiByBUydSbCVDmuLj7RIsPLSRS3Qaqrk136ZbsEGo0sLZ0WfWtQvbbxA1lZ", - "xV3NxLbsWsdRwUQgLJwzUVO57sodsfTp1bTdvrSzblaG9mOVnZk71AQdMrht+dqPB3aYqu0WFUb3WooA", - "IrzDT+3HCT2Or98LhE3m7z5qcN9X/EnYr7nkU4zucUZSLAld+o8P9OkVMp3qbp+3VLbejPwXGDvdi14C", - "I6vXBo5+f9tQE3NWRwYoDhW96Hs23+GszVeQMlkFgpR6rEFy5b7Z2R4jPlt5/SDXPtj4zkEucEoxBM5S", - "T/kOMe7Nl6NGecNh0gF1WjvO60q1a2C1p4PfhtkdRrXN08vR2ZB2cauLye+40ZV7A26dVL0iAl4/agW/", - "qRoVuPbaOUIDIvreQF15bybAvPl67TBtKG6rgDR9Nt9zro15MjC315rYPNHP2+i8dp+Cvgyjw+1++61p", - "IJ59GIATh5zdv1FA/UdAcqEVMgon9j7z1Nbdw9W9Strdl/+uWPfLmHJeroBwpB+4fhyhC6bre3uxvSPN", - "t8ucOGZ2uLV1Xisfvb+1pH+LRX+LyQYS/G12jQblgWI6Z0wKyXGhE+7O/UvFA3Hkx76x0soLUdi2zBuz", - "jVecZi7knWv/7LnxX4aYo9bm/48AjYdmicYDZ+f1zfrfAQAA//+hahcBWUYAAA==", + "H4sIAAAAAAAC/+xc3XPbuBH/VzBsZ/rCSLKddHp+88fdxdOkzdjOTWcyGRsiVxIuJMADQNs6V/97B58k", + "RfBDjpW61zzZJhfYxe5vF7sL0I9RwvKCUaBSRMePUYE5zkEC13/NS5KlNyRVv6cgEk4KSRiNjqOLFKgk", + "CwIcsQWSK0CadhLFEVHvCyxXURxRnEN0XM0TRxx+KwmHNDqWvIQ4EskKcqwYLBjPsYyOo7LUlHJdqLFC", + "ckKX0WYT+2luGL+RkBcZltAW7Z/6F5yhBckkcDRfG9kQ8TLHyA1vPGS8eo4zgoVfzm8l8HV7PQ1B6mvp", + "ll20BT5jeY5fCVC6l5CijAiptGqkvjgXSDK0BImExLIUINCCcSUaPBQZSyE6XuBMQL+oolf3REIuRhgh", + "jnL8cGGID2Yz/x5zjhXTkpLfSrAEiskmjoRcZ4pGTR15Tbi17KoOrwPJEKFJVqYwVhWeZXDlf+awiI6j", + "P00rh5gaMjE9Vayv9HC1gsaiu1YobpKSC8YDC9TPEQdZcgqpAqhyoILDHWGlMAvmIApGBSBC0W3CQani", + "Bst/O3veImOqLoha5iNAKW4ykhPZlvM9fiB5mSNa5nPj51pZSvNGdlQARwVeQpcQZuK6DCkscJnJ6PjN", + "LK7ARqg8Oow0uBRHi62cUPuXVzmhEpbAtfAC03TOHi7Ox0QnS9wRn6qp+pykrT8JOB/HX1F2MLeTfF1o", + "VJNcZeWyLcs14ByJrFwauwmW3XXaS5HtqIJSAL8YtUEoyg4V2Em+RgUbNdi4jHbn17OZ+pEwKoFqcOOi", + "yEiClXzTX4US8rE2f5/7/8g544ZHc5GnOEVKZBBS+f3r2cH+eZ6UcqVUa2ZFYOgU86P9M/+J8TlJU6CG", + "4+v9c/wHk2jBSppqjoc/7J/jNWMox3TtLKtD/ptvAacr4HfAnUk3Dv4azydpqjz5PahYfGkxpxI2zgrg", + "khjUQ45J1nAX8yQUMipf+2SpPnsyNv8VEo1pvfVd0AVrM7O70klg69CjkCZQIJUkByFxXqjd7PKns6Oj", + "ox9q+5cXNsUSXiniUOaxIJSIVS8/lhcZDHGMEVkgN1kne1pmGZ6rbd1EopY4KnSJULi1CaR+jzhkOomR", + "DMkVESaJ0RLgO0w0Bx0TXRbSZhOWo5Z76KxktwTGDHoPQuBlIIP+CZOs5IByQ4DuV0Bt4oWIQLcLTDJI", + "b2PE5Ar4PRGAbpWct5NhxW0Br8JQw8B+XduydkL0yushhAwrfI6LAlKFA5RisZozzFOUZETpSmeRVKUb", + "n0xepMSNI7NWJUeZJCBETYLKSDUJVO7bdpUXht0dK7rBouB/HIR6UR5wARgOok+8I0Je2vyjbf4Uy/HF", + "hpoKUj1tqNig8CDP+isLyVCBhTBBB5Aa4YoKvW/oStcoS+FJ6Q+UTikztC6l302LepEN+brVdWVLsW6V", + "zSuwhMLsu2BRWI+kI5Go/bWl5q2lNYUJLevsw8czVtKAe599+IgSxk3VXq9FomYB9NfXUX/JE0dnOliq", + "NKAzATAJ9aOqpN4BXcpVdHz45o2e2P19MGRIPUdokeemeLuutV6a3HXTxPw6yg5bE56o4SHMa/17/bbK", + "xram9ABTlgwGr0YaMy4VAXqX/gJcEJP2jYy3rcdFOc9IUns1ZywDrJNrjvP38+3Vaoy0VysKfE+D6ukY", + "IJnE2TkRX67I79DBpmNRtVnukqIcxTAUbh1UKlu5NduJ21LGjWzBKq8BjoYqRiDYAC4M43A2ppK6Aicw", + "wuxbqzaTjhCqJyi6XuOTPWww0lUcgpI6Y5y245x6hwT5HbbjnMpi3pPT3nA3C+HL1EntskP3+bbZa2Kk", + "3k2ieEyIyLsSDzOTfT0ZLJ20ONV0IbW9BZzJVbdZO0V5W+aYvuKAU4UztNLzoGQFyRfEQZSZHJavT7B6", + "qvG9vPueIjck7j5huW4ckhi2BQcBVNZ5+bOUi/NJ1MNgXPvQUQ8j3higOpap8emsK+OuSjTkNu8hZ3wd", + "CoLmzVMi4MHh30JR6srMcAkJ42nPTrXVI9R22VJcMPcpSp839OHSp7ebOEobm0Dv5lNRqnEsx4QGnBsL", + "QOalghKHhuokx4sFSRSesS7AicLsCPjmNSP1CemN+cQjhQ5n5x2h85rk1lHrq7zHAtlBowOmkKwodmei", + "Bz05LHpfOt/FZ9GCsxzdr0iyUoasC2XdbtCpa4zjxnlNpesanGvmbwA25M1VWzXgX2kK6ek6VEcMqmq4", + "rhicwrdzO7anwV2HiHN34NYuMkJh07WLq4H1hfSrT/RlOJpgdNpas8lQxuqm7pLt0hx5dcs2UpXCnq6N", + "aSUp0pA8H4u0XcDnhH6oCXQQP0dJrydZkAw+kESWHD7ybFzJ0ivzV6rQreS5ZG0pvrNz8VEAV0sI9Jky", + "lnyB9BKwGFnMP4NTnmJKIQ0X/kScGpG6Xvd4dGxO3Afdy6njnaF+btN0OkscSWLC7Fhjxu5MWg+s4lNb", + "rLrmajqOtyzcDG1WXX2Qeec1ul2G0qTkHKi0ORqIkc2paqRLpE1TdORwtZ21ezZdVa4LGm9ZycXI9lCO", + "Hy5D7aduHr8EWkFB6u3g3RQvDmq1R2MV85rUXkV9Zu3tsuB8/FblQ8twa0VN25ZJuQskJSdyfaXmNEJc", + "lQWeYwEH1+wL0JNShflHc3diBTjVzmBvT/zrlSN+pYkrteOC/B10B9VRHCpRR8+mltWaTAlM7IGwJFLf", + "fPrx8BSd+xO1kw8XURzduQZpNJscTGZKClYAxQWJjqOjyWwyU36M5Uqvdzr3PrAEHdyUSXR/QdWH0c8g", + "vc3rlxQ/ha1TkUyDl/U28chx/mhh7Ah3nWo8vb2qtfm8dYXl8BnvHAROqUIXEMwZ56LMsnV1P63AS0L1", + "KbYReGIuf8y6ePpFTBVRdS9miPagdo1liPaodimjn1YR1X1MQybkXZ8+B93k02dlGFHmOeZrd/KkfNlq", + "QzkIXgp/TCSiz4qdNe60fs+xH9hX1SHW0wAuvgWEWid3o2G0dVT3/4yhn0G2Ty77UPTobLwZxtGpP095", + "Goy+AYr0faIdgZOCxCRzwWc/YLA32oZoX78A4Fh1dOHGHBX0gcUcSkR7NPXWsUfA3m/rBxrCG9+ozC+6", + "TqVfTYXLDaePvhW0mXLfJO1as88pr9wo21jd1VeqBtRenaXZ/R3tMK639t1lrMs4hXBnbeczHkjWbXzq", + "bxHU1PalVrBAOMt0BmA6mbi6kAupvuWM5pAxuhRIshjdE7lCps5EmCrH1cUnWmR4OYniNkh1dbJPv2yX", + "QKORpVenl74zqJ7b+IGsrJKuZmJbdm3iqGAiEBY+MFFTue7KnbJ0/Wzabl/a2TQrQ/uZzN7MHWqCDhnc", + "tnztZwv7TNXMje4B2sMf9o8gYye94gB6fHCY2k8oeoKEfi8QNlWC+/TCfQXyF2G/OZPrGN3hjKRYErr0", + "n0joky5kutrd8cFy2Xnj8t+J7HXfegrkrF4bmPvjbVlNzFkdGaA4VPSi79F8LbQx32rKZBUIaOqxBsm1", + "+7Jod4z4zOb5A2L7EOQbB8TAicYQOEs95FvEw5deuhrlDYdJB9Rp7eivKy2vgdWeJH4dZvcY1bZPOkdn", + "TtrFrS4mf+CmWO4NuHMC9owIeP6oFfz+alTgOmjnCA2I6DsGdeW9mADz4mu7k7ShuJ0C0vTRfHW6MebJ", + "wNx0a2LzXD9vo/Oj+2D1aRgdPhqwX8QG4tnrAThxyNndCwXUfwUkl1oho3Bi7z5PbY0+3AlQSbv7/wSu", + "sPfTmNJfroBwpB+43h2hC6Z7AfYSfEeab6c5d8LscWvrvII+en9rrf4lNghaQjaQ4G++azQoDxTTOWNS", + "SI4LnXB37l8qHohTT/vCSiu/iMK2cF6YbbzitHAh79z4Z4+Nf2xijmWb/8UBGg/NFI0Hzs6bz5v/BAAA", + "///XFttq/0YAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/packages/dashboard-api/internal/handlers/store.go b/packages/dashboard-api/internal/handlers/store.go index efcd10ee2b..c1c92f0018 100644 --- a/packages/dashboard-api/internal/handlers/store.go +++ b/packages/dashboard-api/internal/handlers/store.go @@ -12,7 +12,7 @@ import ( clickhouse "github.com/e2b-dev/infra/packages/clickhouse/pkg" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" "github.com/e2b-dev/infra/packages/dashboard-api/internal/cfg" - "github.com/e2b-dev/infra/packages/dashboard-api/internal/teambilling" + internalteamprovision "github.com/e2b-dev/infra/packages/dashboard-api/internal/teamprovision" sqlcdb "github.com/e2b-dev/infra/packages/db/client" authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" "github.com/e2b-dev/infra/packages/shared/pkg/apierrors" @@ -26,10 +26,10 @@ type APIStore struct { authDB *authdb.Client clickhouse clickhouse.Clickhouse authService *sharedauth.AuthService[*types.Team] - teamProvisionSink teambilling.TeamProvisionSink + teamProvisionSink internalteamprovision.TeamProvisionSink } -func NewAPIStore(config cfg.Config, db *sqlcdb.Client, authDB *authdb.Client, ch clickhouse.Clickhouse, authService *sharedauth.AuthService[*types.Team], teamProvisionSink teambilling.TeamProvisionSink) *APIStore { +func NewAPIStore(config cfg.Config, db *sqlcdb.Client, authDB *authdb.Client, ch clickhouse.Clickhouse, authService *sharedauth.AuthService[*types.Team], teamProvisionSink internalteamprovision.TeamProvisionSink) *APIStore { return &APIStore{ config: config, db: db, diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index b3f25d6ef6..d02fcf5395 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "testing" "time" @@ -15,7 +16,7 @@ import ( "github.com/e2b-dev/infra/packages/auth/pkg/auth" authtypes "github.com/e2b-dev/infra/packages/auth/pkg/types" - "github.com/e2b-dev/infra/packages/dashboard-api/internal/teambilling" + internalteamprovision "github.com/e2b-dev/infra/packages/dashboard-api/internal/teamprovision" authqueries "github.com/e2b-dev/infra/packages/db/pkg/auth/queries" "github.com/e2b-dev/infra/packages/db/pkg/testutils" "github.com/e2b-dev/infra/packages/db/queries" @@ -401,7 +402,7 @@ func TestCreateTeam_BillingBadRequestCleansUpCreatedTeam(t *testing.T) { db: testDB.SqlcClient, authDB: testDB.AuthDb, teamProvisionSink: &fakeTeamProvisionSink{ - err: &teambilling.ProvisionError{ + err: &internalteamprovision.ProvisionError{ StatusCode: http.StatusBadRequest, Message: "limit reached", }, @@ -413,7 +414,7 @@ func TestCreateTeam_BillingBadRequestCleansUpCreatedTeam(t *testing.T) { t.Fatal("expected billing error") } - var provisionErr *teambilling.ProvisionError + var provisionErr *internalteamprovision.ProvisionError if !errors.As(err, &provisionErr) { t.Fatalf("expected provisioning error, got %T: %v", err, err) } @@ -433,6 +434,208 @@ func TestCreateTeam_BillingBadRequestCleansUpCreatedTeam(t *testing.T) { } } +func TestPostTeams_LocalPolicyDeniedReturnsBadRequestWithoutCreatingTeam(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUser(t, testDB) + sink := &fakeTeamProvisionSink{} + + for range 2 { + team, err := testDB.SqlcClient.CreateTeam(ctx, queries.CreateTeamParams{ + Name: "extra", + Tier: baseTierID, + Email: handlerTestUserEmail(userID), + }) + if err != nil { + t.Fatalf("failed to create extra team: %v", err) + } + if err := testDB.SqlcClient.CreateTeamMembership(ctx, queries.CreateTeamMembershipParams{ + UserID: userID, + TeamID: team.ID, + IsDefault: false, + AddedBy: &userID, + }); err != nil { + t.Fatalf("failed to attach extra team membership: %v", err) + } + } + + recorder := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(recorder) + ginCtx.Request = httptest.NewRequestWithContext(ctx, http.MethodPost, "/", strings.NewReader(`{"name":"Acme"}`)) + ginCtx.Request.Header.Set("Content-Type", "application/json") + auth.SetUserID(ginCtx, userID) + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDb, + teamProvisionSink: sink, + } + store.PostTeams(ginCtx) + + if recorder.Code != http.StatusBadRequest { + t.Fatalf("expected status 400, got %d", recorder.Code) + } + if len(sink.requests) != 0 { + t.Fatalf("expected no provisioning call, got %d", len(sink.requests)) + } + + rows, err := testDB.AuthDb.Read.GetTeamsWithUsersTeamsWithTier(ctx, userID) + if err != nil { + t.Fatalf("failed to query user teams: %v", err) + } + if len(rows) != 3 { + t.Fatalf("expected existing teams to remain unchanged, got %d rows", len(rows)) + } +} + +func TestPostTeams_ProvisioningSuccessReturnsTeam(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUser(t, testDB) + sink := &fakeTeamProvisionSink{} + + recorder := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(recorder) + ginCtx.Request = httptest.NewRequestWithContext(ctx, http.MethodPost, "/", strings.NewReader(`{"name":"Acme"}`)) + ginCtx.Request.Header.Set("Content-Type", "application/json") + auth.SetUserID(ginCtx, userID) + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDb, + teamProvisionSink: sink, + } + store.PostTeams(ginCtx) + + if recorder.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d", recorder.Code) + } + if len(sink.requests) != 1 { + t.Fatalf("expected one provisioning call, got %d", len(sink.requests)) + } +} + +func TestPostTeams_ProvisioningFailureCleansUpTeam(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUser(t, testDB) + sink := &fakeTeamProvisionSink{ + err: &internalteamprovision.ProvisionError{ + StatusCode: http.StatusInternalServerError, + Message: "boom", + }, + } + + recorder := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(recorder) + ginCtx.Request = httptest.NewRequestWithContext(ctx, http.MethodPost, "/", strings.NewReader(`{"name":"Acme"}`)) + ginCtx.Request.Header.Set("Content-Type", "application/json") + auth.SetUserID(ginCtx, userID) + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDb, + teamProvisionSink: sink, + } + store.PostTeams(ginCtx) + + if recorder.Code != http.StatusInternalServerError { + t.Fatalf("expected status 500, got %d", recorder.Code) + } + if len(sink.requests) != 1 { + t.Fatalf("expected one provisioning call, got %d", len(sink.requests)) + } + + rows, err := testDB.AuthDb.Read.GetTeamsWithUsersTeamsWithTier(ctx, userID) + if err != nil { + t.Fatalf("failed to query user teams: %v", err) + } + if len(rows) != 1 { + t.Fatalf("expected only default team to remain, got %d rows", len(rows)) + } +} + +func TestCreateTeam_ConcurrentRequestsRespectLocalPolicy(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUser(t, testDB) + + for range 1 { + team, err := testDB.SqlcClient.CreateTeam(ctx, queries.CreateTeamParams{ + Name: "extra", + Tier: baseTierID, + Email: handlerTestUserEmail(userID), + }) + if err != nil { + t.Fatalf("failed to create extra team: %v", err) + } + if err := testDB.SqlcClient.CreateTeamMembership(ctx, queries.CreateTeamMembershipParams{ + UserID: userID, + TeamID: team.ID, + IsDefault: false, + AddedBy: &userID, + }); err != nil { + t.Fatalf("failed to attach extra team membership: %v", err) + } + } + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDb, + teamProvisionSink: &fakeTeamProvisionSink{}, + } + + var wg sync.WaitGroup + results := make(chan error, 2) + + for _, name := range []string{"Acme-1", "Acme-2"} { + wg.Add(1) + go func(teamName string) { + defer wg.Done() + _, err := store.createTeam(ctx, userID, teamName) + results <- err + }(name) + } + + wg.Wait() + close(results) + + var successCount int + var badRequestCount int + for err := range results { + if err == nil { + successCount++ + continue + } + + var provisionErr *internalteamprovision.ProvisionError + if !errors.As(err, &provisionErr) { + t.Fatalf("expected provisioning error, got %T: %v", err, err) + } + if provisionErr.StatusCode == http.StatusBadRequest { + badRequestCount++ + continue + } + + t.Fatalf("expected bad request or success, got %d", provisionErr.StatusCode) + } + + if successCount != 1 { + t.Fatalf("expected one success, got %d", successCount) + } + if badRequestCount != 1 { + t.Fatalf("expected one bad request, got %d", badRequestCount) + } +} + type fakeTeamProvisionSink struct { requests []teamprovision.TeamBillingProvisionRequestedV1 err error diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index b58a37db83..3153119961 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -8,7 +8,8 @@ import ( "github.com/e2b-dev/infra/packages/auth/pkg/auth" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" - "github.com/e2b-dev/infra/packages/dashboard-api/internal/teambilling" + internalteamprovision "github.com/e2b-dev/infra/packages/dashboard-api/internal/teamprovision" + sqlcdb "github.com/e2b-dev/infra/packages/db/client" "github.com/e2b-dev/infra/packages/db/pkg/dberrors" "github.com/e2b-dev/infra/packages/db/queries" "github.com/e2b-dev/infra/packages/shared/pkg/ginutils" @@ -21,6 +22,8 @@ import ( ) const baseTierID = "base_v1" +const maxTeamsPerUser = 3 +const maxTeamsPerUserWithProTier = 10 type provisionedTeam struct { ID uuid.UUID @@ -187,6 +190,14 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string return provisionedTeam{}, fmt.Errorf("upsert public user: %w", err) } + if _, err := txDB.LockUserTeamMembershipsForUpdate(ctx, userID); err != nil { + return provisionedTeam{}, fmt.Errorf("lock user team memberships: %w", err) + } + + if err := validateTeamCreationAllowed(ctx, txDB, userID); err != nil { + return provisionedTeam{}, err + } + team, err := txDB.CreateTeam(ctx, queries.CreateTeamParams{ Name: name, Tier: baseTierID, @@ -209,6 +220,13 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string return provisionedTeam{}, fmt.Errorf("commit team creation transaction: %w", err) } + logger.L().Info(ctx, "team created locally", + zap.String("user_id", userID.String()), + zap.String("team_id", team.ID.String()), + zap.String("reason", teamprovision.ReasonAdditionalTeam), + zap.String("result", "created"), + ) + err = s.teamProvisionSink.ProvisionTeam(ctx, teamprovision.TeamBillingProvisionRequestedV1{ TeamID: team.ID, TeamName: team.Name, @@ -217,6 +235,13 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string Reason: teamprovision.ReasonAdditionalTeam, }) if err != nil { + logger.L().Error(ctx, "team billing provisioning failed", + zap.String("user_id", userID.String()), + zap.String("team_id", team.ID.String()), + zap.String("reason", teamprovision.ReasonAdditionalTeam), + zap.String("result", "failed"), + zap.Error(err), + ) if cleanupErr := s.cleanupCreatedTeam(ctx, team.ID); cleanupErr != nil { return provisionedTeam{}, fmt.Errorf("cleanup created team: %w", cleanupErr) } @@ -224,6 +249,13 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string return provisionedTeam{}, err } + logger.L().Info(ctx, "team billing provisioning succeeded", + zap.String("user_id", userID.String()), + zap.String("team_id", team.ID.String()), + zap.String("reason", teamprovision.ReasonAdditionalTeam), + zap.String("result", "provisioned"), + ) + return provisionedTeam{ ID: team.ID, Name: team.Name, @@ -236,6 +268,7 @@ func (s *APIStore) cleanupCreatedTeam(ctx context.Context, teamID uuid.UUID) err if err := s.db.DeleteTeamByID(ctx, teamID); err != nil { logger.L().Error(ctx, "failed to cleanup created team", zap.String("teamID", teamID.String()), + zap.String("result", "cleanup_failed"), zap.Error(err), ) @@ -251,11 +284,58 @@ func (s *APIStore) cleanupCreatedTeam(ctx context.Context, teamID uuid.UUID) err } } + logger.L().Info(ctx, "cleaned up created team", + zap.String("teamID", teamID.String()), + zap.String("result", "cleanup_succeeded"), + ) + + return nil +} + +func validateTeamCreationAllowed(ctx context.Context, txDB *sqlcdb.Client, ownerUserID uuid.UUID) error { + teams, err := txDB.GetTeamsWithUsersTeamsWithTierForUpdate(ctx, ownerUserID) + if err != nil { + return fmt.Errorf("query user teams for limit check: %w", err) + } + + hasProTier := false + for _, row := range teams { + if row.Tier != baseTierID { + hasProTier = true + } + if row.IsBanned { + return &internalteamprovision.ProvisionError{ + StatusCode: http.StatusBadRequest, + Message: "You're unable to create a team right now. Please contact support if this persists.", + } + } + } + + if hasProTier { + if len(teams) >= maxTeamsPerUserWithProTier { + return &internalteamprovision.ProvisionError{ + StatusCode: http.StatusBadRequest, + Message: fmt.Sprintf("You can't create more than %d teams", maxTeamsPerUserWithProTier), + } + } + } else { + if len(teams) >= maxTeamsPerUser { + return &internalteamprovision.ProvisionError{ + StatusCode: http.StatusBadRequest, + Message: fmt.Sprintf( + "You can't create more than %d teams, you can upgrade to Pro tier to create up to %d teams", + maxTeamsPerUser, + maxTeamsPerUserWithProTier, + ), + } + } + } + return nil } func (s *APIStore) handleProvisioningError(ctx context.Context, c *gin.Context, operation string, err error) { - var provisionErr *teambilling.ProvisionError + var provisionErr *internalteamprovision.ProvisionError if errors.As(err, &provisionErr) && provisionErr.IsBadRequest() { s.sendAPIStoreError(c, http.StatusBadRequest, provisionErr.Error()) diff --git a/packages/dashboard-api/internal/teambilling/http_sink.go b/packages/dashboard-api/internal/teambilling/http_sink.go deleted file mode 100644 index 46f2173874..0000000000 --- a/packages/dashboard-api/internal/teambilling/http_sink.go +++ /dev/null @@ -1,84 +0,0 @@ -package teambilling - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "net/http" - "strings" - "time" - - "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" -) - -const billingServerAPIKeyHeader = "X-Billing-Server-API-Key" - -type HTTPProvisionSink struct { - baseURL string - apiToken string - client *http.Client -} - -type errorResponse struct { - Message string `json:"message"` -} - -func NewHTTPProvisionSink(baseURL, apiToken string, timeout time.Duration) *HTTPProvisionSink { - if timeout <= 0 { - timeout = 15 * time.Second - } - - return &HTTPProvisionSink{ - baseURL: strings.TrimRight(baseURL, "/"), - apiToken: apiToken, - client: &http.Client{ - Timeout: timeout, - }, - } -} - -func (s *HTTPProvisionSink) ProvisionTeam(ctx context.Context, req teamprovision.TeamBillingProvisionRequestedV1) error { - if s.baseURL == "" || s.apiToken == "" { - return &ProvisionError{ - StatusCode: http.StatusServiceUnavailable, - Message: "billing provisioning sink is not configured", - } - } - - body, err := json.Marshal(req) - if err != nil { - return fmt.Errorf("marshal billing provisioning request: %w", err) - } - - httpReq, err := http.NewRequestWithContext( - ctx, - http.MethodPost, - s.baseURL+"/internal/team-billing-provision", - bytes.NewReader(body), - ) - if err != nil { - return fmt.Errorf("create billing provisioning request: %w", err) - } - - httpReq.Header.Set("Content-Type", "application/json") - httpReq.Header.Set(billingServerAPIKeyHeader, s.apiToken) - - resp, err := s.client.Do(httpReq) - if err != nil { - return fmt.Errorf("call billing provisioning endpoint: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode == http.StatusOK { - return nil - } - - var apiErr errorResponse - _ = json.NewDecoder(resp.Body).Decode(&apiErr) - - return &ProvisionError{ - StatusCode: resp.StatusCode, - Message: apiErr.Message, - } -} diff --git a/packages/dashboard-api/internal/teambilling/noop_sink.go b/packages/dashboard-api/internal/teambilling/noop_sink.go deleted file mode 100644 index e675b40576..0000000000 --- a/packages/dashboard-api/internal/teambilling/noop_sink.go +++ /dev/null @@ -1,17 +0,0 @@ -package teambilling - -import ( - "context" - - "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" -) - -type NoopProvisionSink struct{} - -func NewNoopProvisionSink() *NoopProvisionSink { - return &NoopProvisionSink{} -} - -func (s *NoopProvisionSink) ProvisionTeam(context.Context, teamprovision.TeamBillingProvisionRequestedV1) error { - return nil -} diff --git a/packages/dashboard-api/internal/teambilling/sink.go b/packages/dashboard-api/internal/teambilling/sink.go deleted file mode 100644 index a23c9aa5ca..0000000000 --- a/packages/dashboard-api/internal/teambilling/sink.go +++ /dev/null @@ -1,29 +0,0 @@ -package teambilling - -import ( - "context" - "fmt" - - "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" -) - -type TeamProvisionSink interface { - ProvisionTeam(ctx context.Context, req teamprovision.TeamBillingProvisionRequestedV1) error -} - -type ProvisionError struct { - StatusCode int - Message string -} - -func (e *ProvisionError) Error() string { - if e.Message != "" { - return e.Message - } - - return fmt.Sprintf("billing provisioning failed with status %d", e.StatusCode) -} - -func (e *ProvisionError) IsBadRequest() bool { - return e.StatusCode == 400 -} diff --git a/packages/dashboard-api/internal/teambilling/factory.go b/packages/dashboard-api/internal/teamprovision/factory.go similarity index 95% rename from packages/dashboard-api/internal/teambilling/factory.go rename to packages/dashboard-api/internal/teamprovision/factory.go index aca7025ef3..94a5a7339e 100644 --- a/packages/dashboard-api/internal/teambilling/factory.go +++ b/packages/dashboard-api/internal/teamprovision/factory.go @@ -1,4 +1,4 @@ -package teambilling +package teamprovision import ( "errors" diff --git a/packages/dashboard-api/internal/teamprovision/http_sink.go b/packages/dashboard-api/internal/teamprovision/http_sink.go index 428b330e75..8eb88eb95b 100644 --- a/packages/dashboard-api/internal/teamprovision/http_sink.go +++ b/packages/dashboard-api/internal/teamprovision/http_sink.go @@ -9,7 +9,7 @@ import ( "strings" "time" - "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" + sharedteamprovision "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" ) const billingServerAPIKeyHeader = "X-Billing-Server-API-Key" @@ -38,7 +38,7 @@ func NewHTTPProvisionSink(baseURL, apiToken string, timeout time.Duration) *HTTP } } -func (s *HTTPProvisionSink) ProvisionTeam(ctx context.Context, req teamprovision.TeamBillingProvisionRequestedV1) error { +func (s *HTTPProvisionSink) ProvisionTeam(ctx context.Context, req sharedteamprovision.TeamBillingProvisionRequestedV1) error { if s.baseURL == "" || s.apiToken == "" { return &ProvisionError{ StatusCode: http.StatusServiceUnavailable, @@ -54,7 +54,7 @@ func (s *HTTPProvisionSink) ProvisionTeam(ctx context.Context, req teamprovision httpReq, err := http.NewRequestWithContext( ctx, http.MethodPost, - s.baseURL+"/internal/team-billing-provision", + s.baseURL+"/internal/teams/provision", bytes.NewReader(body), ) if err != nil { diff --git a/packages/dashboard-api/internal/teamprovision/noop_sink.go b/packages/dashboard-api/internal/teamprovision/noop_sink.go new file mode 100644 index 0000000000..ad91ad5fb4 --- /dev/null +++ b/packages/dashboard-api/internal/teamprovision/noop_sink.go @@ -0,0 +1,17 @@ +package teamprovision + +import ( + "context" + + sharedteamprovision "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" +) + +type NoopProvisionSink struct{} + +func NewNoopProvisionSink() *NoopProvisionSink { + return &NoopProvisionSink{} +} + +func (s *NoopProvisionSink) ProvisionTeam(context.Context, sharedteamprovision.TeamBillingProvisionRequestedV1) error { + return nil +} diff --git a/packages/dashboard-api/internal/teamprovision/sink.go b/packages/dashboard-api/internal/teamprovision/sink.go index 45c0da78c0..a61d732f65 100644 --- a/packages/dashboard-api/internal/teamprovision/sink.go +++ b/packages/dashboard-api/internal/teamprovision/sink.go @@ -4,11 +4,11 @@ import ( "context" "fmt" - "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" + sharedteamprovision "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" ) type TeamProvisionSink interface { - ProvisionTeam(ctx context.Context, req teamprovision.TeamBillingProvisionRequestedV1) error + ProvisionTeam(ctx context.Context, req sharedteamprovision.TeamBillingProvisionRequestedV1) error } type ProvisionError struct { @@ -25,5 +25,5 @@ func (e *ProvisionError) Error() string { } func (e *ProvisionError) IsBadRequest() bool { - return e.StatusCode >= 400 && e.StatusCode < 500 + return e.StatusCode == 400 } diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 29a47490d8..a424614b68 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -33,7 +33,7 @@ import ( "github.com/e2b-dev/infra/packages/dashboard-api/internal/backgroundworker" "github.com/e2b-dev/infra/packages/dashboard-api/internal/cfg" "github.com/e2b-dev/infra/packages/dashboard-api/internal/handlers" - "github.com/e2b-dev/infra/packages/dashboard-api/internal/teambilling" + internalteamprovision "github.com/e2b-dev/infra/packages/dashboard-api/internal/teamprovision" sqlcdb "github.com/e2b-dev/infra/packages/db/client" authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" "github.com/e2b-dev/infra/packages/db/pkg/pool" @@ -164,7 +164,7 @@ func run() int { authService := sharedauth.NewAuthService[*types.Team](authStore, authCache, config.SupabaseJWTSecrets) defer authService.Close(ctx) - teamProvisionSink, err := teambilling.NewProvisionSink( + teamProvisionSink, err := internalteamprovision.NewProvisionSink( config.BillingServerURL, config.BillingServerAPIToken, config.BillingServerTimeout, diff --git a/packages/db/queries/team_creation_guard.sql.go b/packages/db/queries/team_creation_guard.sql.go new file mode 100644 index 0000000000..830dc3b2c3 --- /dev/null +++ b/packages/db/queries/team_creation_guard.sql.go @@ -0,0 +1,130 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: team_creation_guard.sql + +package queries + +import ( + "context" + "time" + + "github.com/google/uuid" +) + +const getTeamsWithUsersTeamsWithTierForUpdate = `-- name: GetTeamsWithUsersTeamsWithTierForUpdate :many +SELECT + t.id, + t.created_at, + t.is_blocked, + t.name, + t.tier, + t.email, + t.is_banned, + t.blocked_reason, + t.cluster_id, + t.sandbox_scheduling_labels, + t.slug, + ut.is_default, + tl.id, + tl.max_length_hours, + tl.concurrent_sandboxes, + tl.concurrent_template_builds, + tl.max_vcpu, + tl.max_ram_mb, + tl.disk_mb +FROM public.teams t +JOIN public.users_teams ut ON ut.team_id = t.id +JOIN public.team_limits tl ON tl.id = t.id +WHERE ut.user_id = $1::uuid +FOR UPDATE OF ut +` + +type GetTeamsWithUsersTeamsWithTierForUpdateRow struct { + ID uuid.UUID + CreatedAt time.Time + IsBlocked bool + Name string + Tier string + Email string + IsBanned bool + BlockedReason *string + ClusterID *uuid.UUID + SandboxSchedulingLabels []string + Slug string + IsDefault bool + ID_2 uuid.UUID + MaxLengthHours int64 + ConcurrentSandboxes int32 + ConcurrentTemplateBuilds int32 + MaxVcpu int32 + MaxRamMb int32 + DiskMb int32 +} + +func (q *Queries) GetTeamsWithUsersTeamsWithTierForUpdate(ctx context.Context, userID uuid.UUID) ([]GetTeamsWithUsersTeamsWithTierForUpdateRow, error) { + rows, err := q.db.Query(ctx, getTeamsWithUsersTeamsWithTierForUpdate, userID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetTeamsWithUsersTeamsWithTierForUpdateRow + for rows.Next() { + var i GetTeamsWithUsersTeamsWithTierForUpdateRow + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.IsBlocked, + &i.Name, + &i.Tier, + &i.Email, + &i.IsBanned, + &i.BlockedReason, + &i.ClusterID, + &i.SandboxSchedulingLabels, + &i.Slug, + &i.IsDefault, + &i.ID_2, + &i.MaxLengthHours, + &i.ConcurrentSandboxes, + &i.ConcurrentTemplateBuilds, + &i.MaxVcpu, + &i.MaxRamMb, + &i.DiskMb, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const lockUserTeamMembershipsForUpdate = `-- name: LockUserTeamMembershipsForUpdate :many +SELECT team_id +FROM public.users_teams +WHERE user_id = $1::uuid +FOR UPDATE +` + +func (q *Queries) LockUserTeamMembershipsForUpdate(ctx context.Context, userID uuid.UUID) ([]uuid.UUID, error) { + rows, err := q.db.Query(ctx, lockUserTeamMembershipsForUpdate, userID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []uuid.UUID + for rows.Next() { + var team_id uuid.UUID + if err := rows.Scan(&team_id); err != nil { + return nil, err + } + items = append(items, team_id) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/packages/db/queries/teams/team_creation_guard.sql b/packages/db/queries/teams/team_creation_guard.sql new file mode 100644 index 0000000000..e4b80ebc80 --- /dev/null +++ b/packages/db/queries/teams/team_creation_guard.sql @@ -0,0 +1,32 @@ +-- name: LockUserTeamMembershipsForUpdate :many +SELECT team_id +FROM public.users_teams +WHERE user_id = sqlc.arg(user_id)::uuid +FOR UPDATE; + +-- name: GetTeamsWithUsersTeamsWithTierForUpdate :many +SELECT + t.id, + t.created_at, + t.is_blocked, + t.name, + t.tier, + t.email, + t.is_banned, + t.blocked_reason, + t.cluster_id, + t.sandbox_scheduling_labels, + t.slug, + ut.is_default, + tl.id, + tl.max_length_hours, + tl.concurrent_sandboxes, + tl.concurrent_template_builds, + tl.max_vcpu, + tl.max_ram_mb, + tl.disk_mb +FROM public.teams t +JOIN public.users_teams ut ON ut.team_id = t.id +JOIN public.team_limits tl ON tl.id = t.id +WHERE ut.user_id = sqlc.arg(user_id)::uuid +FOR UPDATE OF ut; diff --git a/spec/openapi-dashboard.yml b/spec/openapi-dashboard.yml index b6962d65b8..c9587c43f0 100644 --- a/spec/openapi-dashboard.yml +++ b/spec/openapi-dashboard.yml @@ -130,6 +130,12 @@ components: application/json: schema: $ref: "#/components/schemas/Error" + "429": + description: Too many requests + content: + application/json: + schema: + $ref: "#/components/schemas/Error" "500": description: Server error content: @@ -746,6 +752,8 @@ paths: $ref: "#/components/responses/400" "401": $ref: "#/components/responses/401" + "429": + $ref: "#/components/responses/429" "500": $ref: "#/components/responses/500" From 4187a1f01a161ac8c249240ab8f0a980e162e72c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 8 Apr 2026 20:58:03 +0000 Subject: [PATCH 34/92] chore: auto-commit generated changes --- packages/clickhouse/go.mod | 4 ++-- packages/clickhouse/go.sum | 1 - packages/client-proxy/go.mod | 2 +- packages/client-proxy/go.sum | 4 ++-- .../internal/handlers/team_provisioning.go | 15 +++++++++------ packages/db/go.mod | 4 ++-- packages/db/go.sum | 1 - packages/docker-reverse-proxy/go.mod | 4 ++-- packages/docker-reverse-proxy/go.sum | 7 +++---- packages/envd/go.mod | 2 +- packages/orchestrator/go.mod | 2 +- tests/integration/go.mod | 4 ++-- 12 files changed, 25 insertions(+), 25 deletions(-) diff --git a/packages/clickhouse/go.mod b/packages/clickhouse/go.mod index 932709551a..e7d399d034 100644 --- a/packages/clickhouse/go.mod +++ b/packages/clickhouse/go.mod @@ -42,7 +42,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.7.5 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/joho/godotenv v1.5.1 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect @@ -93,7 +93,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect diff --git a/packages/clickhouse/go.sum b/packages/clickhouse/go.sum index c5f05d320c..1c485d02b4 100644 --- a/packages/clickhouse/go.sum +++ b/packages/clickhouse/go.sum @@ -129,7 +129,6 @@ github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5ey github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= -github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= diff --git a/packages/client-proxy/go.mod b/packages/client-proxy/go.mod index 5ac838ffdf..517bb8ba97 100644 --- a/packages/client-proxy/go.mod +++ b/packages/client-proxy/go.mod @@ -65,7 +65,7 @@ require ( go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect diff --git a/packages/client-proxy/go.sum b/packages/client-proxy/go.sum index 2402ce9653..2987c46eb7 100644 --- a/packages/client-proxy/go.sum +++ b/packages/client-proxy/go.sum @@ -131,8 +131,8 @@ go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index 3153119961..ff9aeb25a7 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -6,6 +6,10 @@ import ( "fmt" "net/http" + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "go.uber.org/zap" + "github.com/e2b-dev/infra/packages/auth/pkg/auth" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" internalteamprovision "github.com/e2b-dev/infra/packages/dashboard-api/internal/teamprovision" @@ -16,14 +20,13 @@ import ( "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" - "github.com/gin-gonic/gin" - "github.com/google/uuid" - "go.uber.org/zap" ) -const baseTierID = "base_v1" -const maxTeamsPerUser = 3 -const maxTeamsPerUserWithProTier = 10 +const ( + baseTierID = "base_v1" + maxTeamsPerUser = 3 + maxTeamsPerUserWithProTier = 10 +) type provisionedTeam struct { ID uuid.UUID diff --git a/packages/db/go.mod b/packages/db/go.mod index 189933634d..e9bc205fbe 100644 --- a/packages/db/go.mod +++ b/packages/db/go.mod @@ -14,7 +14,7 @@ require ( github.com/exaring/otelpgx v0.9.3 github.com/google/uuid v1.6.0 github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.9.1 github.com/lib/pq v1.11.2 github.com/pressly/goose/v3 v3.26.0 github.com/stretchr/testify v1.11.1 @@ -140,7 +140,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect diff --git a/packages/db/go.sum b/packages/db/go.sum index 08348e1ce0..4758da5a5a 100644 --- a/packages/db/go.sum +++ b/packages/db/go.sum @@ -176,7 +176,6 @@ github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5ey github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= -github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= diff --git a/packages/docker-reverse-proxy/go.mod b/packages/docker-reverse-proxy/go.mod index b57f92bc28..4b07e1c717 100644 --- a/packages/docker-reverse-proxy/go.mod +++ b/packages/docker-reverse-proxy/go.mod @@ -40,7 +40,7 @@ require ( github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.7.5 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/klauspost/compress v1.18.2 // indirect github.com/lib/pq v1.11.2 // indirect @@ -79,7 +79,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect golang.org/x/crypto v0.49.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect diff --git a/packages/docker-reverse-proxy/go.sum b/packages/docker-reverse-proxy/go.sum index b17ae0b6e7..c633017b48 100644 --- a/packages/docker-reverse-proxy/go.sum +++ b/packages/docker-reverse-proxy/go.sum @@ -69,8 +69,7 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= @@ -189,8 +188,8 @@ golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= diff --git a/packages/envd/go.mod b/packages/envd/go.mod index 42241adf20..bcae5ece5e 100644 --- a/packages/envd/go.mod +++ b/packages/envd/go.mod @@ -73,7 +73,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect golang.org/x/crypto v0.49.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/text v0.35.0 // indirect diff --git a/packages/orchestrator/go.mod b/packages/orchestrator/go.mod index 7c719201b3..34e58ce46b 100644 --- a/packages/orchestrator/go.mod +++ b/packages/orchestrator/go.mod @@ -314,7 +314,7 @@ require ( golang.org/x/arch v0.25.0 // indirect golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/term v0.41.0 // indirect diff --git a/tests/integration/go.mod b/tests/integration/go.mod index e81b2fe7ef..4bc14e49c4 100644 --- a/tests/integration/go.mod +++ b/tests/integration/go.mod @@ -25,7 +25,7 @@ require ( github.com/e2b-dev/infra/packages/envd v0.0.0-00010101000000-000000000000 github.com/e2b-dev/infra/packages/shared v0.0.0 github.com/google/uuid v1.6.0 - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.9.1 github.com/oapi-codegen/runtime v1.1.1 github.com/stretchr/testify v1.11.1 golang.org/x/sync v0.20.0 @@ -181,7 +181,7 @@ require ( go.uber.org/zap v1.27.1 // indirect golang.org/x/arch v0.25.0 // indirect golang.org/x/crypto v0.49.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/term v0.41.0 // indirect From 407530ac723ed5c303d14107ce619f00ba1ecada Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 8 Apr 2026 21:03:29 +0000 Subject: [PATCH 35/92] chore: auto-commit generated changes --- packages/clickhouse/go.sum | 1 + packages/db/go.sum | 1 + packages/docker-reverse-proxy/go.sum | 1 + 3 files changed, 3 insertions(+) diff --git a/packages/clickhouse/go.sum b/packages/clickhouse/go.sum index 1c485d02b4..c5f05d320c 100644 --- a/packages/clickhouse/go.sum +++ b/packages/clickhouse/go.sum @@ -129,6 +129,7 @@ github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5ey github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= diff --git a/packages/db/go.sum b/packages/db/go.sum index 4758da5a5a..08348e1ce0 100644 --- a/packages/db/go.sum +++ b/packages/db/go.sum @@ -176,6 +176,7 @@ github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5ey github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= diff --git a/packages/docker-reverse-proxy/go.sum b/packages/docker-reverse-proxy/go.sum index c633017b48..ca2293ebd5 100644 --- a/packages/docker-reverse-proxy/go.sum +++ b/packages/docker-reverse-proxy/go.sum @@ -70,6 +70,7 @@ github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5ey github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= From 7760e3f3cc2ff86bda681ce939f628facaa1f389 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Wed, 8 Apr 2026 14:43:29 -0700 Subject: [PATCH 36/92] refactor(dashboard-api): block new teams for recent users --- .../internal/handlers/team_handlers_test.go | 43 ++++++++++- .../internal/handlers/team_provisioning.go | 77 +++++++++++++------ .../db/migrations/20000101000000_auth.sql | 1 + packages/db/pkg/auth/queries/get_user.sql.go | 4 +- packages/db/pkg/auth/queries/models.go | 5 +- packages/db/queries/models.go | 5 +- packages/db/queries/team_lifecycle.sql.go | 26 +++++-- packages/db/queries/teams/team_lifecycle.sql | 10 ++- 8 files changed, 129 insertions(+), 42 deletions(-) diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index 32ededff37..02ddc9e76b 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -258,13 +258,19 @@ func TestDeleteTeamsTeamIDMembersUserId_RechecksDefaultAfterLock(t *testing.T) { func createHandlerTestUser(t *testing.T, db *testutils.Database) uuid.UUID { t.Helper() + return createHandlerTestUserAt(t, db, time.Now().Add(-newUserNewTeamRequireBillingMethodThreshold-time.Hour)) +} + +func createHandlerTestUserAt(t *testing.T, db *testutils.Database, createdAt time.Time) uuid.UUID { + t.Helper() + userID := uuid.New() email := handlerTestUserEmail(userID) err := db.AuthDb.TestsRawSQL(t.Context(), ` -INSERT INTO auth.users (id, email) -VALUES ($1, $2) -`, userID, email) +INSERT INTO auth.users (id, email, created_at) +VALUES ($1, $2, $3) +`, userID, email, createdAt) if err != nil { t.Fatalf("failed to create test user: %v", err) } @@ -387,6 +393,37 @@ func TestCreateTeam_NoProvisionSinkLeavesCreatedTeam(t *testing.T) { if !found { t.Fatal("expected created team to remain in local state") } + if team.IsBlocked { + t.Fatal("expected older user team to remain unblocked") + } + if team.BlockedReason != nil { + t.Fatalf("expected no blocked reason, got %v", *team.BlockedReason) + } +} + +func TestCreateTeam_RecentUserCreatesBlockedTeam(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUserAt(t, testDB, time.Now().Add(-time.Hour)) + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDb, + teamProvisionSink: &fakeTeamProvisionSink{}, + } + + team, err := store.createTeam(ctx, userID, "Acme") + if err != nil { + t.Fatalf("expected team creation to succeed for recent user, got %v", err) + } + if !team.IsBlocked { + t.Fatal("expected recent user team to be blocked") + } + if team.BlockedReason == nil || *team.BlockedReason != blockedReasonMissingPayment { + t.Fatalf("expected blocked reason %q, got %v", blockedReasonMissingPayment, team.BlockedReason) + } } func TestCreateTeam_BillingBadRequestCleansUpCreatedTeam(t *testing.T) { diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index ff9aeb25a7..7391b6dd1f 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "time" "github.com/gin-gonic/gin" "github.com/google/uuid" @@ -23,16 +24,20 @@ import ( ) const ( - baseTierID = "base_v1" - maxTeamsPerUser = 3 - maxTeamsPerUserWithProTier = 10 + baseTierID = "base_v1" + maxTeamsPerUser = 3 + maxTeamsPerUserWithProTier = 10 + newUserNewTeamRequireBillingMethodThreshold = 3 * 24 * time.Hour + blockedReasonMissingPayment = "missing_payment" ) type provisionedTeam struct { - ID uuid.UUID - Name string - Email string - Slug string + ID uuid.UUID + Name string + Email string + Slug string + IsBlocked bool + BlockedReason *string } func (s *APIStore) PostUsersBootstrap(c *gin.Context) { @@ -117,10 +122,12 @@ func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisi } return provisionedTeam{ - ID: existingTeam.ID, - Name: existingTeam.Name, - Email: existingTeam.Email, - Slug: existingTeam.Slug, + ID: existingTeam.ID, + Name: existingTeam.Name, + Email: existingTeam.Email, + Slug: existingTeam.Slug, + IsBlocked: existingTeam.IsBlocked, + BlockedReason: existingTeam.BlockedReason, }, nil } if !dberrors.IsNotFoundError(err) { @@ -128,9 +135,11 @@ func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisi } team, err := txDB.CreateTeam(ctx, queries.CreateTeamParams{ - Name: authUser.Email, - Tier: baseTierID, - Email: authUser.Email, + Name: authUser.Email, + Tier: baseTierID, + Email: authUser.Email, + IsBlocked: false, + BlockedReason: nil, }) if err != nil { return provisionedTeam{}, fmt.Errorf("create default team: %w", err) @@ -165,10 +174,12 @@ func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisi } return provisionedTeam{ - ID: team.ID, - Name: team.Name, - Email: team.Email, - Slug: team.Slug, + ID: team.ID, + Name: team.Name, + Email: team.Email, + Slug: team.Slug, + IsBlocked: team.IsBlocked, + BlockedReason: team.BlockedReason, }, nil } @@ -201,10 +212,14 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string return provisionedTeam{}, err } + isBlocked, blockedReason := teamBlockPolicy(authUser.CreatedAt, time.Now()) + team, err := txDB.CreateTeam(ctx, queries.CreateTeamParams{ - Name: name, - Tier: baseTierID, - Email: authUser.Email, + Name: name, + Tier: baseTierID, + Email: authUser.Email, + IsBlocked: isBlocked, + BlockedReason: blockedReason, }) if err != nil { return provisionedTeam{}, fmt.Errorf("create team: %w", err) @@ -260,10 +275,12 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string ) return provisionedTeam{ - ID: team.ID, - Name: team.Name, - Email: team.Email, - Slug: team.Slug, + ID: team.ID, + Name: team.Name, + Email: team.Email, + Slug: team.Slug, + IsBlocked: team.IsBlocked, + BlockedReason: team.BlockedReason, }, nil } @@ -337,6 +354,16 @@ func validateTeamCreationAllowed(ctx context.Context, txDB *sqlcdb.Client, owner return nil } +func teamBlockPolicy(userCreatedAt, now time.Time) (bool, *string) { + if userCreatedAt.After(now.Add(-newUserNewTeamRequireBillingMethodThreshold)) { + reason := blockedReasonMissingPayment + + return true, &reason + } + + return false, nil +} + func (s *APIStore) handleProvisioningError(ctx context.Context, c *gin.Context, operation string, err error) { var provisionErr *internalteamprovision.ProvisionError if errors.As(err, &provisionErr) && provisionErr.IsBadRequest() { diff --git a/packages/db/migrations/20000101000000_auth.sql b/packages/db/migrations/20000101000000_auth.sql index 1a7d0d90b0..e420affef4 100644 --- a/packages/db/migrations/20000101000000_auth.sql +++ b/packages/db/migrations/20000101000000_auth.sql @@ -16,6 +16,7 @@ GRANT EXECUTE ON FUNCTION auth.uid() TO postgres; CREATE TABLE auth.users ( id uuid NOT NULL DEFAULT gen_random_uuid(), email text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (id) ); -- +goose StatementEnd diff --git a/packages/db/pkg/auth/queries/get_user.sql.go b/packages/db/pkg/auth/queries/get_user.sql.go index 5d7ee53330..8a842acc2f 100644 --- a/packages/db/pkg/auth/queries/get_user.sql.go +++ b/packages/db/pkg/auth/queries/get_user.sql.go @@ -12,12 +12,12 @@ import ( ) const getUser = `-- name: GetUser :one -SELECT id, email FROM "auth"."users" where id = $1 +SELECT id, email, created_at FROM "auth"."users" where id = $1 ` func (q *Queries) GetUser(ctx context.Context, userID uuid.UUID) (AuthUser, error) { row := q.db.QueryRow(ctx, getUser, userID) var i AuthUser - err := row.Scan(&i.ID, &i.Email) + err := row.Scan(&i.ID, &i.Email, &i.CreatedAt) return i, err } diff --git a/packages/db/pkg/auth/queries/models.go b/packages/db/pkg/auth/queries/models.go index de91154ef2..e6a3a542f3 100644 --- a/packages/db/pkg/auth/queries/models.go +++ b/packages/db/pkg/auth/queries/models.go @@ -50,8 +50,9 @@ type Addon struct { } type AuthUser struct { - ID uuid.UUID - Email string + ID uuid.UUID + Email string + CreatedAt time.Time } type BillingSandboxLog struct { diff --git a/packages/db/queries/models.go b/packages/db/queries/models.go index 6c960a7a59..3f6cb7f4cd 100644 --- a/packages/db/queries/models.go +++ b/packages/db/queries/models.go @@ -50,8 +50,9 @@ type Addon struct { } type AuthUser struct { - ID uuid.UUID - Email string + ID uuid.UUID + Email string + CreatedAt time.Time } type BillingSandboxLog struct { diff --git a/packages/db/queries/team_lifecycle.sql.go b/packages/db/queries/team_lifecycle.sql.go index 7202f8b9d9..815d5f3dd5 100644 --- a/packages/db/queries/team_lifecycle.sql.go +++ b/packages/db/queries/team_lifecycle.sql.go @@ -13,8 +13,14 @@ import ( ) const createTeam = `-- name: CreateTeam :one -INSERT INTO public.teams (name, tier, email) -VALUES ($1::text, $2::text, $3::text) +INSERT INTO public.teams (name, tier, email, is_blocked, blocked_reason) +VALUES ( + $1::text, + $2::text, + $3::text, + $4::boolean, + $5::text +) RETURNING id, created_at, @@ -30,9 +36,11 @@ RETURNING ` type CreateTeamParams struct { - Name string - Tier string - Email string + Name string + Tier string + Email string + IsBlocked bool + BlockedReason *string } type CreateTeamRow struct { @@ -50,7 +58,13 @@ type CreateTeamRow struct { } func (q *Queries) CreateTeam(ctx context.Context, arg CreateTeamParams) (CreateTeamRow, error) { - row := q.db.QueryRow(ctx, createTeam, arg.Name, arg.Tier, arg.Email) + row := q.db.QueryRow(ctx, createTeam, + arg.Name, + arg.Tier, + arg.Email, + arg.IsBlocked, + arg.BlockedReason, + ) var i CreateTeamRow err := row.Scan( &i.ID, diff --git a/packages/db/queries/teams/team_lifecycle.sql b/packages/db/queries/teams/team_lifecycle.sql index f5b8706552..c385a5c66a 100644 --- a/packages/db/queries/teams/team_lifecycle.sql +++ b/packages/db/queries/teams/team_lifecycle.sql @@ -1,6 +1,12 @@ -- name: CreateTeam :one -INSERT INTO public.teams (name, tier, email) -VALUES (sqlc.arg(name)::text, sqlc.arg(tier)::text, sqlc.arg(email)::text) +INSERT INTO public.teams (name, tier, email, is_blocked, blocked_reason) +VALUES ( + sqlc.arg(name)::text, + sqlc.arg(tier)::text, + sqlc.arg(email)::text, + sqlc.arg(is_blocked)::boolean, + sqlc.narg(blocked_reason)::text +) RETURNING id, created_at, From 5842767b8177fc0d4f21bc932b1171ecb04319ab Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Wed, 8 Apr 2026 15:00:22 -0700 Subject: [PATCH 37/92] fix(dashboard-api): serialize user bootstrap team creation --- .../internal/handlers/team_handlers_test.go | 90 +++++++++++++++++++ .../internal/handlers/team_provisioning.go | 5 ++ .../internal/teamprovision/http_sink.go | 2 + .../internal/teamprovision/noop_sink.go | 2 + .../db/queries/team_creation_guard.sql.go | 13 +++ .../db/queries/teams/team_creation_guard.sql | 6 ++ 6 files changed, 118 insertions(+) diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index 02ddc9e76b..d0a3dd5178 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -13,6 +13,7 @@ import ( "github.com/gin-gonic/gin" "github.com/google/uuid" + "github.com/jackc/pgx/v5" "github.com/e2b-dev/infra/packages/auth/pkg/auth" authtypes "github.com/e2b-dev/infra/packages/auth/pkg/types" @@ -355,6 +356,91 @@ func TestPostUsersBootstrap_CreatesDefaultTeamAndCallsSink(t *testing.T) { } } +func TestBootstrapUser_ConcurrentRequestsCreateSingleDefaultTeam(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUser(t, testDB) + sink := &fakeTeamProvisionSink{} + + existingTeam, err := testDB.SqlcClient.GetDefaultTeamByUserID(ctx, userID) + if err != nil { + t.Fatalf("expected trigger-created default team: %v", err) + } + if err := testDB.SqlcClient.DeleteTeamByID(ctx, existingTeam.ID); err != nil { + t.Fatalf("failed to remove trigger-created default team: %v", err) + } + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDb, + teamProvisionSink: sink, + } + + var wg sync.WaitGroup + results := make(chan provisionedTeam, 2) + errs := make(chan error, 2) + + for range 2 { + wg.Add(1) + go func() { + defer wg.Done() + + team, err := store.bootstrapUser(ctx, userID) + if err != nil { + errs <- err + + return + } + + results <- team + }() + } + + wg.Wait() + close(results) + close(errs) + + for err := range errs { + if err != nil { + t.Fatalf("expected bootstrap to succeed, got %v", err) + } + } + + var teamIDs []uuid.UUID + for team := range results { + teamIDs = append(teamIDs, team.ID) + } + if len(teamIDs) != 2 { + t.Fatalf("expected two bootstrap results, got %d", len(teamIDs)) + } + if teamIDs[0] != teamIDs[1] { + t.Fatalf("expected both bootstrap requests to resolve to the same team, got %s and %s", teamIDs[0], teamIDs[1]) + } + + var defaultTeamCount int + err = testDB.AuthDb.TestsRawSQLQuery(ctx, + `SELECT count(*) + FROM public.users_teams + WHERE user_id = $1 AND is_default = true`, + func(rows pgx.Rows) error { + if !rows.Next() { + return errors.New("missing default team count row") + } + + return rows.Scan(&defaultTeamCount) + }, + userID, + ) + if err != nil { + t.Fatalf("failed to count default team memberships: %v", err) + } + if defaultTeamCount != 1 { + t.Fatalf("expected exactly one default team membership, got %d", defaultTeamCount) + } +} + func TestCreateTeam_NoProvisionSinkLeavesCreatedTeam(t *testing.T) { t.Parallel() @@ -672,11 +758,15 @@ func TestCreateTeam_ConcurrentRequestsRespectLocalPolicy(t *testing.T) { } type fakeTeamProvisionSink struct { + mu sync.Mutex requests []teamprovision.TeamBillingProvisionRequestedV1 err error } func (s *fakeTeamProvisionSink) ProvisionTeam(_ context.Context, req teamprovision.TeamBillingProvisionRequestedV1) error { + s.mu.Lock() + defer s.mu.Unlock() + s.requests = append(s.requests, req) return s.err diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index 7391b6dd1f..cbee25d209 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -104,6 +104,11 @@ func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisi return provisionedTeam{}, fmt.Errorf("upsert public user: %w", err) } + // Serialize bootstrap for a user even when they have no team memberships yet. + if _, err := txDB.LockPublicUserForUpdate(ctx, authUser.ID); err != nil { + return provisionedTeam{}, fmt.Errorf("lock public user: %w", err) + } + existingTeam, err := txDB.GetDefaultTeamByUserID(ctx, userID) if err == nil { if err := tx.Commit(ctx); err != nil { diff --git a/packages/dashboard-api/internal/teamprovision/http_sink.go b/packages/dashboard-api/internal/teamprovision/http_sink.go index 8eb88eb95b..024a78e535 100644 --- a/packages/dashboard-api/internal/teamprovision/http_sink.go +++ b/packages/dashboard-api/internal/teamprovision/http_sink.go @@ -20,6 +20,8 @@ type HTTPProvisionSink struct { client *http.Client } +var _ TeamProvisionSink = (*HTTPProvisionSink)(nil) + type errorResponse struct { Message string `json:"message"` } diff --git a/packages/dashboard-api/internal/teamprovision/noop_sink.go b/packages/dashboard-api/internal/teamprovision/noop_sink.go index ad91ad5fb4..21fbf13a74 100644 --- a/packages/dashboard-api/internal/teamprovision/noop_sink.go +++ b/packages/dashboard-api/internal/teamprovision/noop_sink.go @@ -8,6 +8,8 @@ import ( type NoopProvisionSink struct{} +var _ TeamProvisionSink = (*NoopProvisionSink)(nil) + func NewNoopProvisionSink() *NoopProvisionSink { return &NoopProvisionSink{} } diff --git a/packages/db/queries/team_creation_guard.sql.go b/packages/db/queries/team_creation_guard.sql.go index 830dc3b2c3..37226ba423 100644 --- a/packages/db/queries/team_creation_guard.sql.go +++ b/packages/db/queries/team_creation_guard.sql.go @@ -102,6 +102,19 @@ func (q *Queries) GetTeamsWithUsersTeamsWithTierForUpdate(ctx context.Context, u return items, nil } +const lockPublicUserForUpdate = `-- name: LockPublicUserForUpdate :one +SELECT id +FROM public.users +WHERE id = $1::uuid +FOR UPDATE +` + +func (q *Queries) LockPublicUserForUpdate(ctx context.Context, id uuid.UUID) (uuid.UUID, error) { + row := q.db.QueryRow(ctx, lockPublicUserForUpdate, id) + err := row.Scan(&id) + return id, err +} + const lockUserTeamMembershipsForUpdate = `-- name: LockUserTeamMembershipsForUpdate :many SELECT team_id FROM public.users_teams diff --git a/packages/db/queries/teams/team_creation_guard.sql b/packages/db/queries/teams/team_creation_guard.sql index e4b80ebc80..b64aaaa0fe 100644 --- a/packages/db/queries/teams/team_creation_guard.sql +++ b/packages/db/queries/teams/team_creation_guard.sql @@ -4,6 +4,12 @@ FROM public.users_teams WHERE user_id = sqlc.arg(user_id)::uuid FOR UPDATE; +-- name: LockPublicUserForUpdate :one +SELECT id +FROM public.users +WHERE id = sqlc.arg(id)::uuid +FOR UPDATE; + -- name: GetTeamsWithUsersTeamsWithTierForUpdate :many SELECT t.id, From acb137482eccacfc433f8c25deca8759d12a57f9 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Wed, 8 Apr 2026 17:30:34 -0700 Subject: [PATCH 38/92] test(dashboard-api): add tests for provisioning failures and ensure teams remain intact - Introduced tests to verify that provisioning failures do not delete existing default teams during user bootstrap and team creation. - Updated existing tests to reflect changes in expected behavior when provisioning fails, ensuring teams are retained. - Refactored team provisioning logic to log failures without cleanup, enhancing error handling and observability. --- .../internal/handlers/team_handlers_test.go | 152 +++++++++++++++--- .../internal/handlers/team_provisioning.go | 77 +++------ 2 files changed, 154 insertions(+), 75 deletions(-) diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index d0a3dd5178..05b5145eee 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -356,6 +356,113 @@ func TestPostUsersBootstrap_CreatesDefaultTeamAndCallsSink(t *testing.T) { } } +func TestPostUsersBootstrap_ProvisioningFailureKeepsCreatedDefaultTeam(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUser(t, testDB) + sink := &fakeTeamProvisionSink{ + err: &internalteamprovision.ProvisionError{ + StatusCode: http.StatusInternalServerError, + Message: "boom", + }, + } + + existingTeam, err := testDB.SqlcClient.GetDefaultTeamByUserID(ctx, userID) + if err != nil { + t.Fatalf("expected trigger-created default team: %v", err) + } + if err := testDB.SqlcClient.DeleteTeamByID(ctx, existingTeam.ID); err != nil { + t.Fatalf("failed to remove trigger-created default team: %v", err) + } + if err := testDB.SqlcClient.DeletePublicUser(ctx, userID); err != nil { + t.Fatalf("failed to remove trigger-created public user: %v", err) + } + + recorder := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(recorder) + ginCtx.Request = httptest.NewRequestWithContext(ctx, http.MethodPost, "/", nil) + auth.SetUserID(ginCtx, userID) + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDb, + teamProvisionSink: sink, + } + store.PostUsersBootstrap(ginCtx) + + if recorder.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d", recorder.Code) + } + if len(sink.requests) != 1 { + t.Fatalf("expected one provisioning call, got %d", len(sink.requests)) + } + + team, err := testDB.SqlcClient.GetDefaultTeamByUserID(ctx, userID) + if err != nil { + t.Fatalf("expected default team to remain after provisioning failure: %v", err) + } + + rows, err := testDB.AuthDb.Read.GetTeamsWithUsersTeamsWithTier(ctx, userID) + if err != nil { + t.Fatalf("failed to query user teams: %v", err) + } + if len(rows) != 1 { + t.Fatalf("expected one default team to remain, got %d rows", len(rows)) + } + if rows[0].Team.ID != team.ID { + t.Fatalf("expected remaining team %s, got %s", team.ID, rows[0].Team.ID) + } + if !rows[0].IsDefault { + t.Fatal("expected remaining team to be the default team") + } +} + +func TestBootstrapUser_ProvisioningFailureWithExistingDefaultTeamStillSucceeds(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUser(t, testDB) + sink := &fakeTeamProvisionSink{ + err: &internalteamprovision.ProvisionError{ + StatusCode: http.StatusInternalServerError, + Message: "boom", + }, + } + + existingTeam, err := testDB.SqlcClient.GetDefaultTeamByUserID(ctx, userID) + if err != nil { + t.Fatalf("expected trigger-created default team: %v", err) + } + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDb, + teamProvisionSink: sink, + } + + team, err := store.bootstrapUser(ctx, userID) + if err != nil { + t.Fatalf("expected bootstrap to succeed, got %v", err) + } + if len(sink.requests) != 1 { + t.Fatalf("expected one provisioning call, got %d", len(sink.requests)) + } + if team.ID != existingTeam.ID { + t.Fatalf("expected existing default team %s, got %s", existingTeam.ID, team.ID) + } + + rows, err := testDB.AuthDb.Read.GetTeamsWithUsersTeamsWithTier(ctx, userID) + if err != nil { + t.Fatalf("failed to query user teams: %v", err) + } + if len(rows) != 1 { + t.Fatalf("expected existing team to remain unchanged, got %d rows", len(rows)) + } +} + func TestBootstrapUser_ConcurrentRequestsCreateSingleDefaultTeam(t *testing.T) { t.Parallel() @@ -512,7 +619,7 @@ func TestCreateTeam_RecentUserCreatesBlockedTeam(t *testing.T) { } } -func TestCreateTeam_BillingBadRequestCleansUpCreatedTeam(t *testing.T) { +func TestCreateTeam_BillingBadRequestStillReturnsCreatedTeam(t *testing.T) { t.Parallel() testDB := testutils.SetupDatabase(t) @@ -530,28 +637,33 @@ func TestCreateTeam_BillingBadRequestCleansUpCreatedTeam(t *testing.T) { }, } - _, err := store.createTeam(ctx, userID, "Acme") - if err == nil { - t.Fatal("expected billing error") - } - - var provisionErr *internalteamprovision.ProvisionError - if !errors.As(err, &provisionErr) { - t.Fatalf("expected provisioning error, got %T: %v", err, err) + team, err := store.createTeam(ctx, userID, "Acme") + if err != nil { + t.Fatalf("expected createTeam to succeed, got %v", err) } - if provisionErr.StatusCode != http.StatusBadRequest { - t.Fatalf("expected status 400, got %d", provisionErr.StatusCode) + if len(store.teamProvisionSink.(*fakeTeamProvisionSink).requests) != 1 { + t.Fatalf("expected one provisioning call, got %d", len(store.teamProvisionSink.(*fakeTeamProvisionSink).requests)) } rows, err := testDB.AuthDb.Read.GetTeamsWithUsersTeamsWithTier(ctx, userID) if err != nil { t.Fatalf("failed to query user teams: %v", err) } - if len(rows) != 1 { - t.Fatalf("expected only the default team to remain, got %d rows", len(rows)) + if len(rows) != 2 { + t.Fatalf("expected default team and created team to remain, got %d rows", len(rows)) } - if !rows[0].IsDefault { - t.Fatal("expected remaining team to be the default team") + + foundCreatedTeam := false + for _, row := range rows { + if row.Team.ID == team.ID { + foundCreatedTeam = true + if row.IsDefault { + t.Fatal("expected created team not to be default") + } + } + } + if !foundCreatedTeam { + t.Fatalf("expected created team %s to remain", team.ID) } } @@ -640,7 +752,7 @@ func TestPostTeams_ProvisioningSuccessReturnsTeam(t *testing.T) { } } -func TestPostTeams_ProvisioningFailureCleansUpTeam(t *testing.T) { +func TestPostTeams_ProvisioningFailureStillReturnsTeam(t *testing.T) { t.Parallel() testDB := testutils.SetupDatabase(t) @@ -666,8 +778,8 @@ func TestPostTeams_ProvisioningFailureCleansUpTeam(t *testing.T) { } store.PostTeams(ginCtx) - if recorder.Code != http.StatusInternalServerError { - t.Fatalf("expected status 500, got %d", recorder.Code) + if recorder.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d", recorder.Code) } if len(sink.requests) != 1 { t.Fatalf("expected one provisioning call, got %d", len(sink.requests)) @@ -677,8 +789,8 @@ func TestPostTeams_ProvisioningFailureCleansUpTeam(t *testing.T) { if err != nil { t.Fatalf("failed to query user teams: %v", err) } - if len(rows) != 1 { - t.Fatalf("expected only default team to remain, got %d rows", len(rows)) + if len(rows) != 2 { + t.Fatalf("expected default team and created team to remain, got %d rows", len(rows)) } } diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index cbee25d209..4807f97df2 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -115,15 +115,15 @@ func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisi return provisionedTeam{}, fmt.Errorf("commit existing user bootstrap transaction: %w", err) } - err = s.teamProvisionSink.ProvisionTeam(ctx, teamprovision.TeamBillingProvisionRequestedV1{ + req := teamprovision.TeamBillingProvisionRequestedV1{ TeamID: existingTeam.ID, TeamName: existingTeam.Name, TeamEmail: existingTeam.Email, OwnerUserID: userID, Reason: teamprovision.ReasonDefaultSignupTeam, - }) - if err != nil { - return provisionedTeam{}, err + } + if err := s.teamProvisionSink.ProvisionTeam(ctx, req); err != nil { + logTeamProvisioningFailure(ctx, req, err) } return provisionedTeam{ @@ -163,19 +163,15 @@ func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisi return provisionedTeam{}, fmt.Errorf("commit user bootstrap transaction: %w", err) } - err = s.teamProvisionSink.ProvisionTeam(ctx, teamprovision.TeamBillingProvisionRequestedV1{ + req := teamprovision.TeamBillingProvisionRequestedV1{ TeamID: team.ID, TeamName: team.Name, TeamEmail: team.Email, OwnerUserID: userID, Reason: teamprovision.ReasonDefaultSignupTeam, - }) - if err != nil { - if cleanupErr := s.cleanupCreatedTeam(ctx, team.ID); cleanupErr != nil { - return provisionedTeam{}, fmt.Errorf("cleanup created default team: %w", cleanupErr) - } - - return provisionedTeam{}, err + } + if err := s.teamProvisionSink.ProvisionTeam(ctx, req); err != nil { + logTeamProvisioningFailure(ctx, req, err) } return provisionedTeam{ @@ -250,35 +246,24 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string zap.String("result", "created"), ) - err = s.teamProvisionSink.ProvisionTeam(ctx, teamprovision.TeamBillingProvisionRequestedV1{ + req := teamprovision.TeamBillingProvisionRequestedV1{ TeamID: team.ID, TeamName: team.Name, TeamEmail: team.Email, OwnerUserID: userID, Reason: teamprovision.ReasonAdditionalTeam, - }) - if err != nil { - logger.L().Error(ctx, "team billing provisioning failed", + } + if err := s.teamProvisionSink.ProvisionTeam(ctx, req); err != nil { + logTeamProvisioningFailure(ctx, req, err) + } else { + logger.L().Info(ctx, "team billing provisioning succeeded", zap.String("user_id", userID.String()), zap.String("team_id", team.ID.String()), zap.String("reason", teamprovision.ReasonAdditionalTeam), - zap.String("result", "failed"), - zap.Error(err), + zap.String("result", "provisioned"), ) - if cleanupErr := s.cleanupCreatedTeam(ctx, team.ID); cleanupErr != nil { - return provisionedTeam{}, fmt.Errorf("cleanup created team: %w", cleanupErr) - } - - return provisionedTeam{}, err } - logger.L().Info(ctx, "team billing provisioning succeeded", - zap.String("user_id", userID.String()), - zap.String("team_id", team.ID.String()), - zap.String("reason", teamprovision.ReasonAdditionalTeam), - zap.String("result", "provisioned"), - ) - return provisionedTeam{ ID: team.ID, Name: team.Name, @@ -289,32 +274,14 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string }, nil } -func (s *APIStore) cleanupCreatedTeam(ctx context.Context, teamID uuid.UUID) error { - if err := s.db.DeleteTeamByID(ctx, teamID); err != nil { - logger.L().Error(ctx, "failed to cleanup created team", - zap.String("teamID", teamID.String()), - zap.String("result", "cleanup_failed"), - zap.Error(err), - ) - - return err - } - - if s.authService != nil { - if err := s.authService.InvalidateTeamCache(ctx, teamID); err != nil { - logger.L().Warn(ctx, "failed to invalidate team cache after cleanup", - zap.String("teamID", teamID.String()), - zap.Error(err), - ) - } - } - - logger.L().Info(ctx, "cleaned up created team", - zap.String("teamID", teamID.String()), - zap.String("result", "cleanup_succeeded"), +func logTeamProvisioningFailure(ctx context.Context, req teamprovision.TeamBillingProvisionRequestedV1, err error) { + logger.L().Error(ctx, "team billing provisioning failed", + zap.String("user_id", req.OwnerUserID.String()), + zap.String("team_id", req.TeamID.String()), + zap.String("reason", req.Reason), + zap.String("result", "failed"), + zap.Error(err), ) - - return nil } func validateTeamCreationAllowed(ctx context.Context, txDB *sqlcdb.Client, ownerUserID uuid.UUID) error { From b31e568e24873757d202fac9cad80211d2d6e6e9 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Wed, 8 Apr 2026 18:01:56 -0700 Subject: [PATCH 39/92] chore: cleanup --- .../internal/teamprovision/http_sink.go | 28 ++++++- .../internal/teamprovision/http_sink_test.go | 78 +++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 packages/dashboard-api/internal/teamprovision/http_sink_test.go diff --git a/packages/dashboard-api/internal/teamprovision/http_sink.go b/packages/dashboard-api/internal/teamprovision/http_sink.go index 024a78e535..501df9108e 100644 --- a/packages/dashboard-api/internal/teamprovision/http_sink.go +++ b/packages/dashboard-api/internal/teamprovision/http_sink.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "strings" "time" @@ -76,11 +77,32 @@ func (s *HTTPProvisionSink) ProvisionTeam(ctx context.Context, req sharedteampro return nil } - var apiErr errorResponse - _ = json.NewDecoder(resp.Body).Decode(&apiErr) + message, err := readProvisionErrorMessage(resp) + if err != nil { + return fmt.Errorf("read billing provisioning error response: %w", err) + } return &ProvisionError{ StatusCode: resp.StatusCode, - Message: apiErr.Message, + Message: message, + } +} + +func readProvisionErrorMessage(resp *http.Response) (string, error) { + body, err := io.ReadAll(io.LimitReader(resp.Body, 2048)) + if err != nil { + return "", err } + + var apiErr errorResponse + if err := json.Unmarshal(body, &apiErr); err == nil && apiErr.Message != "" { + return apiErr.Message, nil + } + + message := strings.TrimSpace(string(body)) + if message != "" { + return message, nil + } + + return http.StatusText(resp.StatusCode), nil } diff --git a/packages/dashboard-api/internal/teamprovision/http_sink_test.go b/packages/dashboard-api/internal/teamprovision/http_sink_test.go new file mode 100644 index 0000000000..c60e26bfdb --- /dev/null +++ b/packages/dashboard-api/internal/teamprovision/http_sink_test.go @@ -0,0 +1,78 @@ +package teamprovision + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + sharedteamprovision "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" +) + +func TestHTTPProvisionSink_ReturnsJSONErrorMessage(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"message":"invalid payload"}`)) + })) + defer server.Close() + + sink := NewHTTPProvisionSink(server.URL, "token", 0) + err := sink.ProvisionTeam(t.Context(), testProvisionRequest()) + require.Error(t, err) + + provisionErr, ok := err.(*ProvisionError) + require.True(t, ok) + require.Equal(t, http.StatusBadRequest, provisionErr.StatusCode) + require.Equal(t, "invalid payload", provisionErr.Message) +} + +func TestHTTPProvisionSink_FallsBackToPlainTextResponse(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadGateway) + _, _ = w.Write([]byte("upstream gateway exploded")) + })) + defer server.Close() + + sink := NewHTTPProvisionSink(server.URL, "token", 0) + err := sink.ProvisionTeam(t.Context(), testProvisionRequest()) + require.Error(t, err) + + provisionErr, ok := err.(*ProvisionError) + require.True(t, ok) + require.Equal(t, http.StatusBadGateway, provisionErr.StatusCode) + require.Equal(t, "upstream gateway exploded", provisionErr.Message) +} + +func TestHTTPProvisionSink_FallsBackToStatusTextForEmptyBody(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + + sink := NewHTTPProvisionSink(server.URL, "token", 0) + err := sink.ProvisionTeam(t.Context(), testProvisionRequest()) + require.Error(t, err) + + provisionErr, ok := err.(*ProvisionError) + require.True(t, ok) + require.Equal(t, http.StatusServiceUnavailable, provisionErr.StatusCode) + require.Equal(t, http.StatusText(http.StatusServiceUnavailable), provisionErr.Message) +} + +func testProvisionRequest() sharedteamprovision.TeamBillingProvisionRequestedV1 { + return sharedteamprovision.TeamBillingProvisionRequestedV1{ + TeamID: uuid.New(), + TeamName: "Acme", + TeamEmail: "acme@example.com", + OwnerUserID: uuid.New(), + Reason: sharedteamprovision.ReasonAdditionalTeam, + } +} From 7117c1381b1060a46c64bad0e2275aed4006c7cc Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 01:52:08 -0700 Subject: [PATCH 40/92] feat(dashboard-api): add billing server integration for team provisioning - Introduced configuration options for enabling billing HTTP team provision sink. - Updated dashboard API to utilize billing server URL and API token when the feature is enabled. - Enhanced error handling in team provisioning logic to ensure required parameters are validated. - Refactored related components to support the new billing integration. --- iac/provider-gcp/nomad/main.tf | 28 ++++++++++++- packages/dashboard-api/internal/cfg/model.go | 1 + .../internal/teamprovision/factory.go | 28 +++++++++++-- .../internal/teamprovision/factory_test.go | 40 +++++++++++++++++++ packages/dashboard-api/main.go | 1 + 5 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 packages/dashboard-api/internal/teamprovision/factory_test.go diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index 5f937179fe..0962f6357c 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -3,6 +3,17 @@ locals { redis_url = trimspace(data.google_secret_manager_secret_version.redis_cluster_url.secret_data) == "" ? "redis.service.consul:${var.redis_port.port}" : "" redis_cluster_url = trimspace(data.google_secret_manager_secret_version.redis_cluster_url.secret_data) loki_url = "http://loki.service.consul:${var.loki_service_port.port}" + enable_billing_http_team_provision_sink = ( + var.dashboard_api_count > 0 && + lower(trimspace(lookup(var.dashboard_api_env_vars, "ENABLE_BILLING_HTTP_TEAM_PROVISION_SINK", "false"))) == "true" + ) + dashboard_api_extra_env = merge( + var.dashboard_api_env_vars, + local.enable_billing_http_team_provision_sink ? { + BILLING_SERVER_URL = data.google_cloud_run_v2_service.billing_server[0].uri + BILLING_SERVER_API_TOKEN = data.google_secret_manager_secret_version.billing_server_api_token[0].secret_data + } : {}, + ) } # API @@ -35,6 +46,21 @@ data "google_secret_manager_secret_version" "launch_darkly_api_key" { secret = var.launch_darkly_api_key_secret_name } +data "google_secret_manager_secret_version" "billing_server_api_token" { + count = local.enable_billing_http_team_provision_sink ? 1 : 0 + + project = var.gcp_project_id + secret = "${var.prefix}billing-server-api-token" +} + +data "google_cloud_run_v2_service" "billing_server" { + count = local.enable_billing_http_team_provision_sink ? 1 : 0 + + project = var.gcp_project_id + location = var.gcp_region + name = "${var.prefix}billing-server" +} + provider "nomad" { address = "https://nomad.${var.domain_name}" secret_id = var.nomad_acl_token_secret @@ -138,7 +164,7 @@ module "dashboard_api" { redis_url = local.redis_url redis_cluster_url = local.redis_cluster_url redis_tls_ca_base64 = trimspace(data.google_secret_manager_secret_version.redis_tls_ca_base64.secret_data) - extra_env = var.dashboard_api_env_vars + extra_env = local.dashboard_api_extra_env otel_collector_grpc_port = var.otel_collector_grpc_port logs_proxy_port = var.logs_proxy_port diff --git a/packages/dashboard-api/internal/cfg/model.go b/packages/dashboard-api/internal/cfg/model.go index 4915c861ab..9b5e148c17 100644 --- a/packages/dashboard-api/internal/cfg/model.go +++ b/packages/dashboard-api/internal/cfg/model.go @@ -21,6 +21,7 @@ type Config struct { RedisTLSCABase64 string `env:"REDIS_TLS_CA_BASE64"` EnableAuthUserSyncBackgroundWorker bool `env:"ENABLE_AUTH_USER_SYNC_BACKGROUND_WORKER" envDefault:"false"` + EnableBillingHTTPTeamProvisionSink bool `env:"ENABLE_BILLING_HTTP_TEAM_PROVISION_SINK" envDefault:"false"` BillingServerURL string `env:"BILLING_SERVER_URL"` BillingServerAPIToken string `env:"BILLING_SERVER_API_TOKEN"` BillingServerTimeout time.Duration `env:"BILLING_SERVER_TIMEOUT" envDefault:"15s"` diff --git a/packages/dashboard-api/internal/teamprovision/factory.go b/packages/dashboard-api/internal/teamprovision/factory.go index 94a5a7339e..117c993e38 100644 --- a/packages/dashboard-api/internal/teamprovision/factory.go +++ b/packages/dashboard-api/internal/teamprovision/factory.go @@ -1,20 +1,42 @@ package teamprovision import ( + "context" "errors" "time" + + "github.com/e2b-dev/infra/packages/shared/pkg/logger" + "go.uber.org/zap" +) + +var ( + ErrMissingBaseURL = errors.New("billing server url is required when billing http team provision sink is enabled") + ErrMissingAPIToken = errors.New("billing server api token is required when billing http team provision sink is enabled") ) -var ErrMissingAPIToken = errors.New("billing server api token is required when billing server url is configured") +func NewProvisionSink(enabled bool, baseURL, apiToken string, timeout time.Duration) (TeamProvisionSink, error) { + if !enabled { + logger.L().Info(context.Background(), "team provision sink configured", + zap.String("sink", "noop"), + zap.String("result", "disabled"), + ) -func NewProvisionSink(baseURL, apiToken string, timeout time.Duration) (TeamProvisionSink, error) { - if baseURL == "" { return NewNoopProvisionSink(), nil } + if baseURL == "" { + return nil, ErrMissingBaseURL + } + if apiToken == "" { return nil, ErrMissingAPIToken } + logger.L().Info(context.Background(), "team provision sink configured", + zap.String("sink", "http"), + zap.String("result", "enabled"), + zap.String("base_url", baseURL), + ) + return NewHTTPProvisionSink(baseURL, apiToken, timeout), nil } diff --git a/packages/dashboard-api/internal/teamprovision/factory_test.go b/packages/dashboard-api/internal/teamprovision/factory_test.go new file mode 100644 index 0000000000..8bd786e094 --- /dev/null +++ b/packages/dashboard-api/internal/teamprovision/factory_test.go @@ -0,0 +1,40 @@ +package teamprovision + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestNewProvisionSink_DisabledReturnsNoop(t *testing.T) { + t.Parallel() + + sink, err := NewProvisionSink(false, "", "", 15*time.Second) + require.NoError(t, err) + require.IsType(t, &NoopProvisionSink{}, sink) +} + +func TestNewProvisionSink_EnabledRequiresBaseURL(t *testing.T) { + t.Parallel() + + sink, err := NewProvisionSink(true, "", "token", 15*time.Second) + require.Nil(t, sink) + require.ErrorIs(t, err, ErrMissingBaseURL) +} + +func TestNewProvisionSink_EnabledRequiresAPIToken(t *testing.T) { + t.Parallel() + + sink, err := NewProvisionSink(true, "https://billing.example.com", "", 15*time.Second) + require.Nil(t, sink) + require.ErrorIs(t, err, ErrMissingAPIToken) +} + +func TestNewProvisionSink_EnabledReturnsHTTPSink(t *testing.T) { + t.Parallel() + + sink, err := NewProvisionSink(true, "https://billing.example.com", "token", 15*time.Second) + require.NoError(t, err) + require.IsType(t, &HTTPProvisionSink{}, sink) +} diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 5bb169be0c..6ffe936874 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -164,6 +164,7 @@ func run() int { defer authService.Close(ctx) teamProvisionSink, err := internalteamprovision.NewProvisionSink( + config.EnableBillingHTTPTeamProvisionSink, config.BillingServerURL, config.BillingServerAPIToken, config.BillingServerTimeout, From 66efe05a023d8a3c89bfb3b48035d66a87b73947 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Apr 2026 08:56:45 +0000 Subject: [PATCH 41/92] chore: auto-commit generated changes --- packages/dashboard-api/internal/teamprovision/factory.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/dashboard-api/internal/teamprovision/factory.go b/packages/dashboard-api/internal/teamprovision/factory.go index 117c993e38..a704457076 100644 --- a/packages/dashboard-api/internal/teamprovision/factory.go +++ b/packages/dashboard-api/internal/teamprovision/factory.go @@ -5,8 +5,9 @@ import ( "errors" "time" - "github.com/e2b-dev/infra/packages/shared/pkg/logger" "go.uber.org/zap" + + "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) var ( From c21f0fdaa121d9b8ac8090efe7eb164ba8f84dab Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 02:00:52 -0700 Subject: [PATCH 42/92] chore: observability --- .../internal/handlers/team_provisioning.go | 31 +++-------- .../internal/teamprovision/http_sink.go | 52 ++++++++++++++++++- .../internal/teamprovision/noop_sink.go | 17 +++++- .../internal/teamprovision/sink.go | 29 +++++++++++ 4 files changed, 102 insertions(+), 27 deletions(-) diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index 4807f97df2..12f20a6fc6 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -122,9 +122,7 @@ func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisi OwnerUserID: userID, Reason: teamprovision.ReasonDefaultSignupTeam, } - if err := s.teamProvisionSink.ProvisionTeam(ctx, req); err != nil { - logTeamProvisioningFailure(ctx, req, err) - } + _ = s.teamProvisionSink.ProvisionTeam(ctx, req) return provisionedTeam{ ID: existingTeam.ID, @@ -170,9 +168,7 @@ func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisi OwnerUserID: userID, Reason: teamprovision.ReasonDefaultSignupTeam, } - if err := s.teamProvisionSink.ProvisionTeam(ctx, req); err != nil { - logTeamProvisioningFailure(ctx, req, err) - } + _ = s.teamProvisionSink.ProvisionTeam(ctx, req) return provisionedTeam{ ID: team.ID, @@ -254,14 +250,11 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string Reason: teamprovision.ReasonAdditionalTeam, } if err := s.teamProvisionSink.ProvisionTeam(ctx, req); err != nil { - logTeamProvisioningFailure(ctx, req, err) - } else { - logger.L().Info(ctx, "team billing provisioning succeeded", - zap.String("user_id", userID.String()), - zap.String("team_id", team.ID.String()), - zap.String("reason", teamprovision.ReasonAdditionalTeam), - zap.String("result", "provisioned"), - ) + if cleanupErr := s.cleanupCreatedTeam(ctx, team.ID); cleanupErr != nil { + return provisionedTeam{}, fmt.Errorf("cleanup created team: %w", cleanupErr) + } + + return provisionedTeam{}, err } return provisionedTeam{ @@ -274,16 +267,6 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string }, nil } -func logTeamProvisioningFailure(ctx context.Context, req teamprovision.TeamBillingProvisionRequestedV1, err error) { - logger.L().Error(ctx, "team billing provisioning failed", - zap.String("user_id", req.OwnerUserID.String()), - zap.String("team_id", req.TeamID.String()), - zap.String("reason", req.Reason), - zap.String("result", "failed"), - zap.Error(err), - ) -} - func validateTeamCreationAllowed(ctx context.Context, txDB *sqlcdb.Client, ownerUserID uuid.UUID) error { teams, err := txDB.GetTeamsWithUsersTeamsWithTierForUpdate(ctx, ownerUserID) if err != nil { diff --git a/packages/dashboard-api/internal/teamprovision/http_sink.go b/packages/dashboard-api/internal/teamprovision/http_sink.go index 501df9108e..48f21e9bad 100644 --- a/packages/dashboard-api/internal/teamprovision/http_sink.go +++ b/packages/dashboard-api/internal/teamprovision/http_sink.go @@ -10,7 +10,11 @@ import ( "strings" "time" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" sharedteamprovision "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" + "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" + "go.opentelemetry.io/otel/attribute" + "go.uber.org/zap" ) const billingServerAPIKeyHeader = "X-Billing-Server-API-Key" @@ -42,15 +46,28 @@ func NewHTTPProvisionSink(baseURL, apiToken string, timeout time.Duration) *HTTP } func (s *HTTPProvisionSink) ProvisionTeam(ctx context.Context, req sharedteamprovision.TeamBillingProvisionRequestedV1) error { + baseAttrs := provisionTelemetryAttrs(req, provisionSinkHTTP) + telemetry.SetAttributes(ctx, baseAttrs...) + telemetry.ReportEvent(ctx, "team_provision.started", baseAttrs...) + if s.baseURL == "" || s.apiToken == "" { - return &ProvisionError{ + err := &ProvisionError{ StatusCode: http.StatusServiceUnavailable, Message: "billing provisioning sink is not configured", } + failureAttrs := provisionTelemetryAttrs(req, provisionSinkHTTP, + attribute.String("team.provision.result", "failed"), + attribute.Int64("http.response.status_code", int64(err.StatusCode)), + ) + telemetry.ReportErrorByCode(ctx, err.StatusCode, "team provisioning failed", err, failureAttrs...) + + return err } body, err := json.Marshal(req) if err != nil { + telemetry.ReportCriticalError(ctx, "marshal billing provisioning request", err, baseAttrs...) + return fmt.Errorf("marshal billing provisioning request: %w", err) } @@ -61,31 +78,62 @@ func (s *HTTPProvisionSink) ProvisionTeam(ctx context.Context, req sharedteampro bytes.NewReader(body), ) if err != nil { + telemetry.ReportCriticalError(ctx, "create billing provisioning request", err, baseAttrs...) + return fmt.Errorf("create billing provisioning request: %w", err) } httpReq.Header.Set("Content-Type", "application/json") httpReq.Header.Set(billingServerAPIKeyHeader, s.apiToken) + startedAt := time.Now() resp, err := s.client.Do(httpReq) if err != nil { + telemetry.ReportCriticalError(ctx, "call billing provisioning endpoint", err, baseAttrs...) + return fmt.Errorf("call billing provisioning endpoint: %w", err) } defer resp.Body.Close() if resp.StatusCode == http.StatusOK { + duration := time.Since(startedAt) + successAttrs := provisionTelemetryAttrs(req, provisionSinkHTTP, + attribute.String("team.provision.result", "success"), + attribute.Int64("http.response.status_code", int64(resp.StatusCode)), + attribute.Int64("team.provision.duration_ms", duration.Milliseconds()), + ) + telemetry.ReportEvent(ctx, "team_provision.completed", successAttrs...) + + fields := append(provisionLogFields(req, provisionSinkHTTP), + zap.String("team.provision.result", "success"), + zap.Int("http.response.status_code", resp.StatusCode), + zap.Duration("team.provision.duration", duration), + ) + logger.L().Info(ctx, "team provisioning completed", fields...) + return nil } message, err := readProvisionErrorMessage(resp) if err != nil { + telemetry.ReportCriticalError(ctx, "read billing provisioning error response", err, baseAttrs...) + return fmt.Errorf("read billing provisioning error response: %w", err) } - return &ProvisionError{ + duration := time.Since(startedAt) + provisionErr := &ProvisionError{ StatusCode: resp.StatusCode, Message: message, } + failureAttrs := provisionTelemetryAttrs(req, provisionSinkHTTP, + attribute.String("team.provision.result", "failed"), + attribute.Int64("http.response.status_code", int64(resp.StatusCode)), + attribute.Int64("team.provision.duration_ms", duration.Milliseconds()), + ) + telemetry.ReportErrorByCode(ctx, resp.StatusCode, "team provisioning failed", provisionErr, failureAttrs...) + + return provisionErr } func readProvisionErrorMessage(resp *http.Response) (string, error) { diff --git a/packages/dashboard-api/internal/teamprovision/noop_sink.go b/packages/dashboard-api/internal/teamprovision/noop_sink.go index 21fbf13a74..7807711989 100644 --- a/packages/dashboard-api/internal/teamprovision/noop_sink.go +++ b/packages/dashboard-api/internal/teamprovision/noop_sink.go @@ -3,7 +3,11 @@ package teamprovision import ( "context" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" sharedteamprovision "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" + "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" + "go.opentelemetry.io/otel/attribute" + "go.uber.org/zap" ) type NoopProvisionSink struct{} @@ -14,6 +18,17 @@ func NewNoopProvisionSink() *NoopProvisionSink { return &NoopProvisionSink{} } -func (s *NoopProvisionSink) ProvisionTeam(context.Context, sharedteamprovision.TeamBillingProvisionRequestedV1) error { +func (s *NoopProvisionSink) ProvisionTeam(ctx context.Context, req sharedteamprovision.TeamBillingProvisionRequestedV1) error { + attrs := provisionTelemetryAttrs(req, provisionSinkNoop, + attribute.String("team.provision.result", "skipped"), + ) + telemetry.SetAttributes(ctx, attrs...) + telemetry.ReportEvent(ctx, "team_provision.skipped", attrs...) + + fields := append(provisionLogFields(req, provisionSinkNoop), + zap.String("team.provision.result", "skipped"), + ) + logger.L().Info(ctx, "team provisioning skipped", fields...) + return nil } diff --git a/packages/dashboard-api/internal/teamprovision/sink.go b/packages/dashboard-api/internal/teamprovision/sink.go index a61d732f65..8928e8b325 100644 --- a/packages/dashboard-api/internal/teamprovision/sink.go +++ b/packages/dashboard-api/internal/teamprovision/sink.go @@ -4,13 +4,22 @@ import ( "context" "fmt" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" sharedteamprovision "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" + "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" + "go.opentelemetry.io/otel/attribute" + "go.uber.org/zap" ) type TeamProvisionSink interface { ProvisionTeam(ctx context.Context, req sharedteamprovision.TeamBillingProvisionRequestedV1) error } +const ( + provisionSinkHTTP = "http" + provisionSinkNoop = "noop" +) + type ProvisionError struct { StatusCode int Message string @@ -27,3 +36,23 @@ func (e *ProvisionError) Error() string { func (e *ProvisionError) IsBadRequest() bool { return e.StatusCode == 400 } + +func provisionLogFields(req sharedteamprovision.TeamBillingProvisionRequestedV1, sink string) []zap.Field { + return []zap.Field{ + logger.WithTeamID(req.TeamID.String()), + logger.WithUserID(req.OwnerUserID.String()), + zap.String("team.provision.reason", req.Reason), + zap.String("team.provision.sink", sink), + } +} + +func provisionTelemetryAttrs(req sharedteamprovision.TeamBillingProvisionRequestedV1, sink string, attrs ...attribute.KeyValue) []attribute.KeyValue { + base := []attribute.KeyValue{ + telemetry.WithTeamID(req.TeamID.String()), + telemetry.WithUserID(req.OwnerUserID.String()), + attribute.String("team.provision.reason", req.Reason), + attribute.String("team.provision.sink", sink), + } + + return append(base, attrs...) +} From a78fd000106b4d20d0d4abf2295e47e286f831ce Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Apr 2026 09:06:22 +0000 Subject: [PATCH 43/92] chore: auto-commit generated changes --- packages/dashboard-api/internal/teamprovision/http_sink.go | 5 +++-- packages/dashboard-api/internal/teamprovision/noop_sink.go | 5 +++-- packages/dashboard-api/internal/teamprovision/sink.go | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/dashboard-api/internal/teamprovision/http_sink.go b/packages/dashboard-api/internal/teamprovision/http_sink.go index 48f21e9bad..f7e8f512ea 100644 --- a/packages/dashboard-api/internal/teamprovision/http_sink.go +++ b/packages/dashboard-api/internal/teamprovision/http_sink.go @@ -10,11 +10,12 @@ import ( "strings" "time" + "go.opentelemetry.io/otel/attribute" + "go.uber.org/zap" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" sharedteamprovision "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" - "go.opentelemetry.io/otel/attribute" - "go.uber.org/zap" ) const billingServerAPIKeyHeader = "X-Billing-Server-API-Key" diff --git a/packages/dashboard-api/internal/teamprovision/noop_sink.go b/packages/dashboard-api/internal/teamprovision/noop_sink.go index 7807711989..60df22408a 100644 --- a/packages/dashboard-api/internal/teamprovision/noop_sink.go +++ b/packages/dashboard-api/internal/teamprovision/noop_sink.go @@ -3,11 +3,12 @@ package teamprovision import ( "context" + "go.opentelemetry.io/otel/attribute" + "go.uber.org/zap" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" sharedteamprovision "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" - "go.opentelemetry.io/otel/attribute" - "go.uber.org/zap" ) type NoopProvisionSink struct{} diff --git a/packages/dashboard-api/internal/teamprovision/sink.go b/packages/dashboard-api/internal/teamprovision/sink.go index 8928e8b325..eba557abaa 100644 --- a/packages/dashboard-api/internal/teamprovision/sink.go +++ b/packages/dashboard-api/internal/teamprovision/sink.go @@ -4,11 +4,12 @@ import ( "context" "fmt" + "go.opentelemetry.io/otel/attribute" + "go.uber.org/zap" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" sharedteamprovision "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" - "go.opentelemetry.io/otel/attribute" - "go.uber.org/zap" ) type TeamProvisionSink interface { From 6ae0aa662fc4aa8fa9ab1d3c0096bc8fb512d7c6 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 02:10:57 -0700 Subject: [PATCH 44/92] chore: observability --- .../internal/handlers/team_provisioning.go | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index 12f20a6fc6..04071db501 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -235,13 +235,6 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string return provisionedTeam{}, fmt.Errorf("commit team creation transaction: %w", err) } - logger.L().Info(ctx, "team created locally", - zap.String("user_id", userID.String()), - zap.String("team_id", team.ID.String()), - zap.String("reason", teamprovision.ReasonAdditionalTeam), - zap.String("result", "created"), - ) - req := teamprovision.TeamBillingProvisionRequestedV1{ TeamID: team.ID, TeamName: team.Name, @@ -249,13 +242,7 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string OwnerUserID: userID, Reason: teamprovision.ReasonAdditionalTeam, } - if err := s.teamProvisionSink.ProvisionTeam(ctx, req); err != nil { - if cleanupErr := s.cleanupCreatedTeam(ctx, team.ID); cleanupErr != nil { - return provisionedTeam{}, fmt.Errorf("cleanup created team: %w", cleanupErr) - } - - return provisionedTeam{}, err - } + _ = s.teamProvisionSink.ProvisionTeam(ctx, req) return provisionedTeam{ ID: team.ID, From cc77e883e0fa16efd0242315db8b1a40c5e51137 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 02:17:13 -0700 Subject: [PATCH 45/92] chore: fix lint --- .../internal/handlers/team_handlers_test.go | 9 ++++----- .../internal/teamprovision/http_sink_test.go | 12 ++++++------ .../dashboard-api/internal/teamprovision/sink.go | 3 ++- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index c28a019397..dff67cba56 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -490,10 +490,7 @@ func TestBootstrapUser_ConcurrentRequestsCreateSingleDefaultTeam(t *testing.T) { errs := make(chan error, 2) for range 2 { - wg.Add(1) - go func() { - defer wg.Done() - + wg.Go(func() { team, err := store.bootstrapUser(ctx, userID) if err != nil { errs <- err @@ -502,7 +499,7 @@ func TestBootstrapUser_ConcurrentRequestsCreateSingleDefaultTeam(t *testing.T) { } results <- team - }() + }) } wg.Wait() @@ -846,6 +843,7 @@ func TestCreateTeam_ConcurrentRequestsRespectLocalPolicy(t *testing.T) { for err := range results { if err == nil { successCount++ + continue } @@ -855,6 +853,7 @@ func TestCreateTeam_ConcurrentRequestsRespectLocalPolicy(t *testing.T) { } if provisionErr.StatusCode == http.StatusBadRequest { badRequestCount++ + continue } diff --git a/packages/dashboard-api/internal/teamprovision/http_sink_test.go b/packages/dashboard-api/internal/teamprovision/http_sink_test.go index c60e26bfdb..4674e2eb5c 100644 --- a/packages/dashboard-api/internal/teamprovision/http_sink_test.go +++ b/packages/dashboard-api/internal/teamprovision/http_sink_test.go @@ -24,8 +24,8 @@ func TestHTTPProvisionSink_ReturnsJSONErrorMessage(t *testing.T) { err := sink.ProvisionTeam(t.Context(), testProvisionRequest()) require.Error(t, err) - provisionErr, ok := err.(*ProvisionError) - require.True(t, ok) + var provisionErr *ProvisionError + require.ErrorAs(t, err, &provisionErr) require.Equal(t, http.StatusBadRequest, provisionErr.StatusCode) require.Equal(t, "invalid payload", provisionErr.Message) } @@ -43,8 +43,8 @@ func TestHTTPProvisionSink_FallsBackToPlainTextResponse(t *testing.T) { err := sink.ProvisionTeam(t.Context(), testProvisionRequest()) require.Error(t, err) - provisionErr, ok := err.(*ProvisionError) - require.True(t, ok) + var provisionErr *ProvisionError + require.ErrorAs(t, err, &provisionErr) require.Equal(t, http.StatusBadGateway, provisionErr.StatusCode) require.Equal(t, "upstream gateway exploded", provisionErr.Message) } @@ -61,8 +61,8 @@ func TestHTTPProvisionSink_FallsBackToStatusTextForEmptyBody(t *testing.T) { err := sink.ProvisionTeam(t.Context(), testProvisionRequest()) require.Error(t, err) - provisionErr, ok := err.(*ProvisionError) - require.True(t, ok) + var provisionErr *ProvisionError + require.ErrorAs(t, err, &provisionErr) require.Equal(t, http.StatusServiceUnavailable, provisionErr.StatusCode) require.Equal(t, http.StatusText(http.StatusServiceUnavailable), provisionErr.Message) } diff --git a/packages/dashboard-api/internal/teamprovision/sink.go b/packages/dashboard-api/internal/teamprovision/sink.go index eba557abaa..c429290b25 100644 --- a/packages/dashboard-api/internal/teamprovision/sink.go +++ b/packages/dashboard-api/internal/teamprovision/sink.go @@ -3,6 +3,7 @@ package teamprovision import ( "context" "fmt" + "net/http" "go.opentelemetry.io/otel/attribute" "go.uber.org/zap" @@ -35,7 +36,7 @@ func (e *ProvisionError) Error() string { } func (e *ProvisionError) IsBadRequest() bool { - return e.StatusCode == 400 + return e.StatusCode == http.StatusBadRequest } func provisionLogFields(req sharedteamprovision.TeamBillingProvisionRequestedV1, sink string) []zap.Field { From d53308ca6c6383a58a5b40e2dde3855c1b5401ab Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 02:28:01 -0700 Subject: [PATCH 46/92] fix: lint --- packages/dashboard-api/internal/cfg/model.go | 2 +- packages/dashboard-api/internal/utils/builds.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/dashboard-api/internal/cfg/model.go b/packages/dashboard-api/internal/cfg/model.go index 9b5e148c17..555623f6b8 100644 --- a/packages/dashboard-api/internal/cfg/model.go +++ b/packages/dashboard-api/internal/cfg/model.go @@ -24,7 +24,7 @@ type Config struct { EnableBillingHTTPTeamProvisionSink bool `env:"ENABLE_BILLING_HTTP_TEAM_PROVISION_SINK" envDefault:"false"` BillingServerURL string `env:"BILLING_SERVER_URL"` BillingServerAPIToken string `env:"BILLING_SERVER_API_TOKEN"` - BillingServerTimeout time.Duration `env:"BILLING_SERVER_TIMEOUT" envDefault:"15s"` + BillingServerTimeout time.Duration `env:"BILLING_SERVER_TIMEOUT" envDefault:"30s"` } func Parse() (Config, error) { diff --git a/packages/dashboard-api/internal/utils/builds.go b/packages/dashboard-api/internal/utils/builds.go index 1ad18210fb..b3afd1c5b2 100644 --- a/packages/dashboard-api/internal/utils/builds.go +++ b/packages/dashboard-api/internal/utils/builds.go @@ -1,4 +1,4 @@ -package utils +package buildutils import ( "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" From 4544bf71fc1eb975e2727c0eecfac27b9e253b32 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 02:36:43 -0700 Subject: [PATCH 47/92] address comments --- iac/provider-gcp/nomad/main.tf | 2 +- .../backgroundworker/auth_user_sync_test.go | 22 +++++++++++-- .../internal/handlers/team_provisioning.go | 10 ++++-- ...01000003_river_auth_user_sync_triggers.sql | 32 ++++++++++++++++--- .../db/pkg/auth/queries/get_auth_user.sql.go | 25 --------------- 5 files changed, 56 insertions(+), 35 deletions(-) delete mode 100644 packages/db/pkg/auth/queries/get_auth_user.sql.go diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index 0962f6357c..0ac03f5d7b 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -8,11 +8,11 @@ locals { lower(trimspace(lookup(var.dashboard_api_env_vars, "ENABLE_BILLING_HTTP_TEAM_PROVISION_SINK", "false"))) == "true" ) dashboard_api_extra_env = merge( - var.dashboard_api_env_vars, local.enable_billing_http_team_provision_sink ? { BILLING_SERVER_URL = data.google_cloud_run_v2_service.billing_server[0].uri BILLING_SERVER_API_TOKEN = data.google_secret_manager_secret_version.billing_server_api_token[0].secret_data } : {}, + var.dashboard_api_env_vars, ) } diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go index 1d82d57276..5ce6eca0db 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go @@ -23,8 +23,9 @@ const ( testEventuallyTick = 50 * time.Millisecond testStopTimeout = 5 * time.Second - authMigrationsDir = "packages/db/pkg/auth/migrations" - authCustomSchemaVersion int64 = 20260401000001 + authMigrationsDir = "packages/db/pkg/auth/migrations" + authCustomSchemaVersion int64 = 20260401000001 + authTriggerMigrationVersion int64 = 20260401000003 ) type riverProcess struct { @@ -122,6 +123,23 @@ func TestAuthUserSync_EndToEnd(t *testing.T) { }) } +func TestAuthUserSyncMigrations_AuthWritesSucceedBeforeRiverTablesExist(t *testing.T) { + t.Parallel() + + db := testutils.SetupDatabase(t) + db.ApplyMigrationsUpTo(t, authTriggerMigrationVersion, authMigrationsDir) + + ctx := t.Context() + userID := uuid.New() + + require.NoError(t, db.AuthDB.TestsRawSQL(ctx, + "INSERT INTO auth.users (id, email) VALUES ($1, $2)", userID, "before-river@example.com")) + require.NoError(t, db.AuthDB.TestsRawSQL(ctx, + "UPDATE auth.users SET email = $1 WHERE id = $2", "before-river-updated@example.com", userID)) + require.NoError(t, db.AuthDB.TestsRawSQL(ctx, + "DELETE FROM auth.users WHERE id = $1", userID)) +} + func applyAuthUserSyncMigrations(t *testing.T, db *testutils.Database) { t.Helper() diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index 04071db501..41ff86ef51 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -9,7 +9,7 @@ import ( "github.com/gin-gonic/gin" "github.com/google/uuid" - "go.uber.org/zap" + "go.opentelemetry.io/otel/attribute" "github.com/e2b-dev/infra/packages/auth/pkg/auth" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" @@ -18,7 +18,6 @@ import ( "github.com/e2b-dev/infra/packages/db/pkg/dberrors" "github.com/e2b-dev/infra/packages/db/queries" "github.com/e2b-dev/infra/packages/shared/pkg/ginutils" - "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) @@ -307,13 +306,18 @@ func teamBlockPolicy(userCreatedAt, now time.Time) (bool, *string) { } func (s *APIStore) handleProvisioningError(ctx context.Context, c *gin.Context, operation string, err error) { + attrs := []attribute.KeyValue{ + attribute.String("team.provision.operation", operation), + } + var provisionErr *internalteamprovision.ProvisionError if errors.As(err, &provisionErr) && provisionErr.IsBadRequest() { + telemetry.ReportErrorByCode(ctx, http.StatusBadRequest, operation+" failed", err, attrs...) s.sendAPIStoreError(c, http.StatusBadRequest, provisionErr.Error()) return } - logger.L().Error(ctx, operation+" failed", zap.Error(err)) + telemetry.ReportErrorByCode(ctx, http.StatusInternalServerError, operation+" failed", err, attrs...) s.sendAPIStoreError(c, http.StatusInternalServerError, "Failed to "+operation) } diff --git a/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql b/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql index 93ddb068e9..5059b9e006 100644 --- a/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql +++ b/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql @@ -7,6 +7,10 @@ LANGUAGE plpgsql SECURITY DEFINER SET search_path = '' AS $$ BEGIN + IF to_regclass('auth_custom.river_job') IS NULL THEN + RETURN NEW; + END IF; + INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) VALUES ( jsonb_build_object('user_id', NEW.id, 'operation', 'upsert', 'email', NEW.email), @@ -28,6 +32,10 @@ LANGUAGE plpgsql SECURITY DEFINER SET search_path = '' AS $$ BEGIN + IF to_regclass('auth_custom.river_job') IS NULL THEN + RETURN NEW; + END IF; + IF OLD.email IS DISTINCT FROM NEW.email THEN INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) VALUES ( @@ -51,6 +59,10 @@ LANGUAGE plpgsql SECURITY DEFINER SET search_path = '' AS $$ BEGIN + IF to_regclass('auth_custom.river_job') IS NULL THEN + RETURN OLD; + END IF; + INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) VALUES ( jsonb_build_object('user_id', OLD.id, 'operation', 'delete'), @@ -78,8 +90,14 @@ CREATE TRIGGER enqueue_user_sync_on_delete AFTER DELETE ON auth.users FOR EACH ROW EXECUTE FUNCTION auth_custom.enqueue_user_sync_on_delete(); -GRANT INSERT ON auth_custom.river_job TO trigger_user; -GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA auth_custom TO trigger_user; +DO $grant$ +BEGIN + IF to_regclass('auth_custom.river_job') IS NOT NULL THEN + EXECUTE 'GRANT INSERT ON auth_custom.river_job TO trigger_user'; + EXECUTE 'GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA auth_custom TO trigger_user'; + END IF; +END; +$grant$; -- +goose StatementEnd @@ -94,7 +112,13 @@ DROP FUNCTION IF EXISTS auth_custom.enqueue_user_sync_on_insert(); DROP FUNCTION IF EXISTS auth_custom.enqueue_user_sync_on_update(); DROP FUNCTION IF EXISTS auth_custom.enqueue_user_sync_on_delete(); -REVOKE INSERT ON auth_custom.river_job FROM trigger_user; -REVOKE USAGE, SELECT ON ALL SEQUENCES IN SCHEMA auth_custom FROM trigger_user; +DO $revoke$ +BEGIN + IF to_regclass('auth_custom.river_job') IS NOT NULL THEN + EXECUTE 'REVOKE INSERT ON auth_custom.river_job FROM trigger_user'; + EXECUTE 'REVOKE USAGE, SELECT ON ALL SEQUENCES IN SCHEMA auth_custom FROM trigger_user'; + END IF; +END; +$revoke$; -- +goose StatementEnd diff --git a/packages/db/pkg/auth/queries/get_auth_user.sql.go b/packages/db/pkg/auth/queries/get_auth_user.sql.go deleted file mode 100644 index 4c34da8bdf..0000000000 --- a/packages/db/pkg/auth/queries/get_auth_user.sql.go +++ /dev/null @@ -1,25 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.29.0 -// source: get_auth_user.sql - -package authqueries - -import ( - "context" - - "github.com/google/uuid" -) - -const getAuthUserByID = `-- name: GetAuthUserByID :one -SELECT id, email -FROM auth.users -WHERE id = $1::uuid -` - -func (q *Queries) GetAuthUserByID(ctx context.Context, userID uuid.UUID) (AuthUser, error) { - row := q.db.QueryRow(ctx, getAuthUserByID, userID) - var i AuthUser - err := row.Scan(&i.ID, &i.Email) - return i, err -} From 04e18eaf9d434dceb29c0001a9e90f8881916e6b Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 03:15:01 -0700 Subject: [PATCH 48/92] refactor(dashboard-api): simplify team provisioning configuration + add retrying - Removed the billing server timeout configuration from the team provisioning setup. - Updated related functions and tests to reflect the removal of the timeout parameter. - Enhanced error handling and retry logic in the HTTP provision sink for improved reliability. --- packages/dashboard-api/internal/cfg/model.go | 10 +- .../internal/teamprovision/factory.go | 5 +- .../internal/teamprovision/factory_test.go | 9 +- .../internal/teamprovision/http_sink.go | 169 +++++++++++++----- .../internal/teamprovision/http_sink_test.go | 81 ++++++++- .../internal/teamprovision/sink.go | 9 + packages/dashboard-api/main.go | 1 - 7 files changed, 217 insertions(+), 67 deletions(-) diff --git a/packages/dashboard-api/internal/cfg/model.go b/packages/dashboard-api/internal/cfg/model.go index 555623f6b8..f77bc5ed57 100644 --- a/packages/dashboard-api/internal/cfg/model.go +++ b/packages/dashboard-api/internal/cfg/model.go @@ -2,7 +2,6 @@ package cfg import ( "fmt" - "time" "github.com/caarlos0/env/v11" ) @@ -20,11 +19,10 @@ type Config struct { RedisClusterURL string `env:"REDIS_CLUSTER_URL"` RedisTLSCABase64 string `env:"REDIS_TLS_CA_BASE64"` - EnableAuthUserSyncBackgroundWorker bool `env:"ENABLE_AUTH_USER_SYNC_BACKGROUND_WORKER" envDefault:"false"` - EnableBillingHTTPTeamProvisionSink bool `env:"ENABLE_BILLING_HTTP_TEAM_PROVISION_SINK" envDefault:"false"` - BillingServerURL string `env:"BILLING_SERVER_URL"` - BillingServerAPIToken string `env:"BILLING_SERVER_API_TOKEN"` - BillingServerTimeout time.Duration `env:"BILLING_SERVER_TIMEOUT" envDefault:"30s"` + EnableAuthUserSyncBackgroundWorker bool `env:"ENABLE_AUTH_USER_SYNC_BACKGROUND_WORKER" envDefault:"false"` + EnableBillingHTTPTeamProvisionSink bool `env:"ENABLE_BILLING_HTTP_TEAM_PROVISION_SINK" envDefault:"false"` + BillingServerURL string `env:"BILLING_SERVER_URL"` + BillingServerAPIToken string `env:"BILLING_SERVER_API_TOKEN"` } func Parse() (Config, error) { diff --git a/packages/dashboard-api/internal/teamprovision/factory.go b/packages/dashboard-api/internal/teamprovision/factory.go index a704457076..7c39b88047 100644 --- a/packages/dashboard-api/internal/teamprovision/factory.go +++ b/packages/dashboard-api/internal/teamprovision/factory.go @@ -3,7 +3,6 @@ package teamprovision import ( "context" "errors" - "time" "go.uber.org/zap" @@ -15,7 +14,7 @@ var ( ErrMissingAPIToken = errors.New("billing server api token is required when billing http team provision sink is enabled") ) -func NewProvisionSink(enabled bool, baseURL, apiToken string, timeout time.Duration) (TeamProvisionSink, error) { +func NewProvisionSink(enabled bool, baseURL, apiToken string) (TeamProvisionSink, error) { if !enabled { logger.L().Info(context.Background(), "team provision sink configured", zap.String("sink", "noop"), @@ -39,5 +38,5 @@ func NewProvisionSink(enabled bool, baseURL, apiToken string, timeout time.Durat zap.String("base_url", baseURL), ) - return NewHTTPProvisionSink(baseURL, apiToken, timeout), nil + return NewHTTPProvisionSink(baseURL, apiToken), nil } diff --git a/packages/dashboard-api/internal/teamprovision/factory_test.go b/packages/dashboard-api/internal/teamprovision/factory_test.go index 8bd786e094..407c608d4a 100644 --- a/packages/dashboard-api/internal/teamprovision/factory_test.go +++ b/packages/dashboard-api/internal/teamprovision/factory_test.go @@ -2,7 +2,6 @@ package teamprovision import ( "testing" - "time" "github.com/stretchr/testify/require" ) @@ -10,7 +9,7 @@ import ( func TestNewProvisionSink_DisabledReturnsNoop(t *testing.T) { t.Parallel() - sink, err := NewProvisionSink(false, "", "", 15*time.Second) + sink, err := NewProvisionSink(false, "", "") require.NoError(t, err) require.IsType(t, &NoopProvisionSink{}, sink) } @@ -18,7 +17,7 @@ func TestNewProvisionSink_DisabledReturnsNoop(t *testing.T) { func TestNewProvisionSink_EnabledRequiresBaseURL(t *testing.T) { t.Parallel() - sink, err := NewProvisionSink(true, "", "token", 15*time.Second) + sink, err := NewProvisionSink(true, "", "token") require.Nil(t, sink) require.ErrorIs(t, err, ErrMissingBaseURL) } @@ -26,7 +25,7 @@ func TestNewProvisionSink_EnabledRequiresBaseURL(t *testing.T) { func TestNewProvisionSink_EnabledRequiresAPIToken(t *testing.T) { t.Parallel() - sink, err := NewProvisionSink(true, "https://billing.example.com", "", 15*time.Second) + sink, err := NewProvisionSink(true, "https://billing.example.com", "") require.Nil(t, sink) require.ErrorIs(t, err, ErrMissingAPIToken) } @@ -34,7 +33,7 @@ func TestNewProvisionSink_EnabledRequiresAPIToken(t *testing.T) { func TestNewProvisionSink_EnabledReturnsHTTPSink(t *testing.T) { t.Parallel() - sink, err := NewProvisionSink(true, "https://billing.example.com", "token", 15*time.Second) + sink, err := NewProvisionSink(true, "https://billing.example.com", "token") require.NoError(t, err) require.IsType(t, &HTTPProvisionSink{}, sink) } diff --git a/packages/dashboard-api/internal/teamprovision/http_sink.go b/packages/dashboard-api/internal/teamprovision/http_sink.go index f7e8f512ea..78bccad2e1 100644 --- a/packages/dashboard-api/internal/teamprovision/http_sink.go +++ b/packages/dashboard-api/internal/teamprovision/http_sink.go @@ -1,15 +1,18 @@ package teamprovision import ( - "bytes" "context" "encoding/json" + "errors" "fmt" "io" + "math/rand" "net/http" "strings" "time" + "github.com/hashicorp/go-retryablehttp" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel/attribute" "go.uber.org/zap" @@ -20,10 +23,19 @@ import ( const billingServerAPIKeyHeader = "X-Billing-Server-API-Key" +const ( + defaultProvisionTimeout = 30 * time.Second + defaultProvisionRetryMaxAttempts = 3 + defaultProvisionRetryInitialWait = 100 * time.Millisecond + defaultProvisionRetryWaitCeiling = 2 * time.Second + provisionBackoffMultiplier = 2.0 +) + type HTTPProvisionSink struct { baseURL string apiToken string - client *http.Client + client *retryablehttp.Client + timeout time.Duration } var _ TeamProvisionSink = (*HTTPProvisionSink)(nil) @@ -32,17 +44,12 @@ type errorResponse struct { Message string `json:"message"` } -func NewHTTPProvisionSink(baseURL, apiToken string, timeout time.Duration) *HTTPProvisionSink { - if timeout <= 0 { - timeout = 15 * time.Second - } - +func NewHTTPProvisionSink(baseURL, apiToken string) *HTTPProvisionSink { return &HTTPProvisionSink{ baseURL: strings.TrimRight(baseURL, "/"), apiToken: apiToken, - client: &http.Client{ - Timeout: timeout, - }, + client: newRetryableProvisionClient(defaultProvisionTimeout), + timeout: defaultProvisionTimeout, } } @@ -72,42 +79,26 @@ func (s *HTTPProvisionSink) ProvisionTeam(ctx context.Context, req sharedteampro return fmt.Errorf("marshal billing provisioning request: %w", err) } - httpReq, err := http.NewRequestWithContext( - ctx, - http.MethodPost, - s.baseURL+"/internal/teams/provision", - bytes.NewReader(body), - ) - if err != nil { - telemetry.ReportCriticalError(ctx, "create billing provisioning request", err, baseAttrs...) - - return fmt.Errorf("create billing provisioning request: %w", err) - } - - httpReq.Header.Set("Content-Type", "application/json") - httpReq.Header.Set(billingServerAPIKeyHeader, s.apiToken) - + retryCtx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() startedAt := time.Now() - resp, err := s.client.Do(httpReq) - if err != nil { - telemetry.ReportCriticalError(ctx, "call billing provisioning endpoint", err, baseAttrs...) - - return fmt.Errorf("call billing provisioning endpoint: %w", err) + resp, err := s.provisionTeamOnce(retryCtx, body) + if resp != nil { + defer resp.Body.Close() } - defer resp.Body.Close() - if resp.StatusCode == http.StatusOK { - duration := time.Since(startedAt) + duration := time.Since(startedAt) + if err == nil && resp != nil && resp.StatusCode == http.StatusOK { successAttrs := provisionTelemetryAttrs(req, provisionSinkHTTP, attribute.String("team.provision.result", "success"), - attribute.Int64("http.response.status_code", int64(resp.StatusCode)), + attribute.Int64("http.response.status_code", int64(http.StatusOK)), attribute.Int64("team.provision.duration_ms", duration.Milliseconds()), ) telemetry.ReportEvent(ctx, "team_provision.completed", successAttrs...) fields := append(provisionLogFields(req, provisionSinkHTTP), zap.String("team.provision.result", "success"), - zap.Int("http.response.status_code", resp.StatusCode), + zap.Int("http.response.status_code", http.StatusOK), zap.Duration("team.provision.duration", duration), ) logger.L().Info(ctx, "team provisioning completed", fields...) @@ -115,28 +106,108 @@ func (s *HTTPProvisionSink) ProvisionTeam(ctx context.Context, req sharedteampro return nil } - message, err := readProvisionErrorMessage(resp) - if err != nil { - telemetry.ReportCriticalError(ctx, "read billing provisioning error response", err, baseAttrs...) - - return fmt.Errorf("read billing provisioning error response: %w", err) - } - - duration := time.Since(startedAt) - provisionErr := &ProvisionError{ - StatusCode: resp.StatusCode, - Message: message, - } + provisionErr := buildProvisionError(resp, err) failureAttrs := provisionTelemetryAttrs(req, provisionSinkHTTP, attribute.String("team.provision.result", "failed"), - attribute.Int64("http.response.status_code", int64(resp.StatusCode)), attribute.Int64("team.provision.duration_ms", duration.Milliseconds()), + attribute.Int64("http.response.status_code", int64(provisionErr.StatusCode)), ) - telemetry.ReportErrorByCode(ctx, resp.StatusCode, "team provisioning failed", provisionErr, failureAttrs...) + telemetry.ReportErrorByCode(ctx, provisionErr.StatusCode, "team provisioning failed", provisionErr, failureAttrs...) return provisionErr } +func (s *HTTPProvisionSink) provisionTeamOnce(ctx context.Context, body []byte) (*http.Response, error) { + httpReq, err := retryablehttp.NewRequestWithContext( + ctx, + http.MethodPost, + s.baseURL+"/internal/teams/provision", + body, + ) + if err != nil { + return nil, fmt.Errorf("create billing provisioning request: %w", err) + } + + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set(billingServerAPIKeyHeader, s.apiToken) + + return s.client.Do(httpReq) +} + +func buildProvisionError(resp *http.Response, err error) *ProvisionError { + if resp != nil { + message, readErr := readProvisionErrorMessage(resp) + if readErr != nil { + return &ProvisionError{ + StatusCode: http.StatusBadGateway, + Message: "billing provisioning response was unreadable", + Err: fmt.Errorf("read billing provisioning error response: %w", readErr), + } + } + + return &ProvisionError{ + StatusCode: resp.StatusCode, + Message: message, + Err: err, + } + } + + if errors.Is(err, context.DeadlineExceeded) { + return &ProvisionError{ + StatusCode: http.StatusGatewayTimeout, + Message: "billing provisioning request timed out", + Err: err, + } + } + if errors.Is(err, context.Canceled) { + return &ProvisionError{ + StatusCode: http.StatusServiceUnavailable, + Message: "billing provisioning request was canceled", + Err: err, + } + } + + return &ProvisionError{ + StatusCode: http.StatusServiceUnavailable, + Message: "billing provisioning request failed", + Err: err, + } +} + +func newRetryableProvisionClient(timeout time.Duration) *retryablehttp.Client { + client := retryablehttp.NewClient() + client.Logger = nil + client.RetryMax = defaultProvisionRetryMaxAttempts - 1 + client.RetryWaitMin = defaultProvisionRetryInitialWait + client.RetryWaitMax = defaultProvisionRetryWaitCeiling + client.ErrorHandler = retryablehttp.PassthroughErrorHandler + client.Backoff = func(minWait, maxWait time.Duration, attemptNum int, resp *http.Response) time.Duration { + if resp != nil && (resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable) { + return retryablehttp.DefaultBackoff(minWait, maxWait, attemptNum, resp) + } + + backoff := minWait + for range attemptNum { + backoff = time.Duration(float64(backoff) * provisionBackoffMultiplier) + if backoff > maxWait { + backoff = maxWait + + break + } + } + + if backoff > 0 { + return time.Duration(rand.Int63n(int64(backoff))) + } + + return backoff + } + client.HTTPClient.Timeout = timeout + client.HTTPClient.Transport = otelhttp.NewTransport(client.HTTPClient.Transport) + + return client +} + func readProvisionErrorMessage(resp *http.Response) (string, error) { body, err := io.ReadAll(io.LimitReader(resp.Body, 2048)) if err != nil { diff --git a/packages/dashboard-api/internal/teamprovision/http_sink_test.go b/packages/dashboard-api/internal/teamprovision/http_sink_test.go index 4674e2eb5c..6bd613ddeb 100644 --- a/packages/dashboard-api/internal/teamprovision/http_sink_test.go +++ b/packages/dashboard-api/internal/teamprovision/http_sink_test.go @@ -1,9 +1,14 @@ package teamprovision import ( + "errors" + "io" "net/http" "net/http/httptest" + "strings" + "sync/atomic" "testing" + "time" "github.com/google/uuid" "github.com/stretchr/testify/require" @@ -14,13 +19,15 @@ import ( func TestHTTPProvisionSink_ReturnsJSONErrorMessage(t *testing.T) { t.Parallel() + var requestCount atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requestCount.Add(1) w.WriteHeader(http.StatusBadRequest) _, _ = w.Write([]byte(`{"message":"invalid payload"}`)) })) defer server.Close() - sink := NewHTTPProvisionSink(server.URL, "token", 0) + sink := NewHTTPProvisionSink(server.URL, "token") err := sink.ProvisionTeam(t.Context(), testProvisionRequest()) require.Error(t, err) @@ -28,18 +35,23 @@ func TestHTTPProvisionSink_ReturnsJSONErrorMessage(t *testing.T) { require.ErrorAs(t, err, &provisionErr) require.Equal(t, http.StatusBadRequest, provisionErr.StatusCode) require.Equal(t, "invalid payload", provisionErr.Message) + require.EqualValues(t, 1, requestCount.Load()) } func TestHTTPProvisionSink_FallsBackToPlainTextResponse(t *testing.T) { t.Parallel() + var requestCount atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requestCount.Add(1) w.WriteHeader(http.StatusBadGateway) _, _ = w.Write([]byte("upstream gateway exploded")) })) defer server.Close() - sink := NewHTTPProvisionSink(server.URL, "token", 0) + sink := NewHTTPProvisionSink(server.URL, "token") + sink.client.RetryWaitMin = time.Millisecond + sink.client.RetryWaitMax = time.Millisecond err := sink.ProvisionTeam(t.Context(), testProvisionRequest()) require.Error(t, err) @@ -47,17 +59,22 @@ func TestHTTPProvisionSink_FallsBackToPlainTextResponse(t *testing.T) { require.ErrorAs(t, err, &provisionErr) require.Equal(t, http.StatusBadGateway, provisionErr.StatusCode) require.Equal(t, "upstream gateway exploded", provisionErr.Message) + require.EqualValues(t, sink.client.RetryMax+1, requestCount.Load()) } func TestHTTPProvisionSink_FallsBackToStatusTextForEmptyBody(t *testing.T) { t.Parallel() + var requestCount atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requestCount.Add(1) w.WriteHeader(http.StatusServiceUnavailable) })) defer server.Close() - sink := NewHTTPProvisionSink(server.URL, "token", 0) + sink := NewHTTPProvisionSink(server.URL, "token") + sink.client.RetryWaitMin = time.Millisecond + sink.client.RetryWaitMax = time.Millisecond err := sink.ProvisionTeam(t.Context(), testProvisionRequest()) require.Error(t, err) @@ -65,6 +82,58 @@ func TestHTTPProvisionSink_FallsBackToStatusTextForEmptyBody(t *testing.T) { require.ErrorAs(t, err, &provisionErr) require.Equal(t, http.StatusServiceUnavailable, provisionErr.StatusCode) require.Equal(t, http.StatusText(http.StatusServiceUnavailable), provisionErr.Message) + require.EqualValues(t, sink.client.RetryMax+1, requestCount.Load()) +} + +func TestHTTPProvisionSink_RetriesRetryableResponsesAndSucceeds(t *testing.T) { + t.Parallel() + + var requestCount atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + attempt := requestCount.Add(1) + if attempt == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte("temporary outage")) + + return + } + + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + sink := NewHTTPProvisionSink(server.URL, "token") + sink.client.RetryWaitMin = time.Millisecond + sink.client.RetryWaitMax = time.Millisecond + err := sink.ProvisionTeam(t.Context(), testProvisionRequest()) + require.NoError(t, err) + require.EqualValues(t, 2, requestCount.Load()) +} + +func TestHTTPProvisionSink_RetriesTransportErrorsAndSucceeds(t *testing.T) { + t.Parallel() + + sink := NewHTTPProvisionSink("http://billing.example", "token") + sink.client.RetryWaitMin = time.Millisecond + sink.client.RetryWaitMax = time.Millisecond + + var attemptCount atomic.Int32 + sink.client.HTTPClient.Transport = roundTripFunc(func(_ *http.Request) (*http.Response, error) { + attempt := attemptCount.Add(1) + if attempt == 1 { + return nil, errors.New("temporary dial failure") + } + + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader("")), + Header: make(http.Header), + }, nil + }) + + err := sink.ProvisionTeam(t.Context(), testProvisionRequest()) + require.NoError(t, err) + require.EqualValues(t, 2, attemptCount.Load()) } func testProvisionRequest() sharedteamprovision.TeamBillingProvisionRequestedV1 { @@ -76,3 +145,9 @@ func testProvisionRequest() sharedteamprovision.TeamBillingProvisionRequestedV1 Reason: sharedteamprovision.ReasonAdditionalTeam, } } + +type roundTripFunc func(req *http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} diff --git a/packages/dashboard-api/internal/teamprovision/sink.go b/packages/dashboard-api/internal/teamprovision/sink.go index c429290b25..1a75bcea63 100644 --- a/packages/dashboard-api/internal/teamprovision/sink.go +++ b/packages/dashboard-api/internal/teamprovision/sink.go @@ -25,6 +25,7 @@ const ( type ProvisionError struct { StatusCode int Message string + Err error } func (e *ProvisionError) Error() string { @@ -35,6 +36,14 @@ func (e *ProvisionError) Error() string { return fmt.Sprintf("billing provisioning failed with status %d", e.StatusCode) } +func (e *ProvisionError) Unwrap() error { + if e == nil { + return nil + } + + return e.Err +} + func (e *ProvisionError) IsBadRequest() bool { return e.StatusCode == http.StatusBadRequest } diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 6ffe936874..b2c97f2186 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -167,7 +167,6 @@ func run() int { config.EnableBillingHTTPTeamProvisionSink, config.BillingServerURL, config.BillingServerAPIToken, - config.BillingServerTimeout, ) if err != nil { l.Fatal(ctx, "initializing team provision sink", zap.Error(err)) From b695386845a40fc9003a5202977fc64101e35806 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 03:15:33 -0700 Subject: [PATCH 49/92] chore: work sync --- packages/dashboard-api/go.mod | 4 +++- packages/dashboard-api/go.sum | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/dashboard-api/go.mod b/packages/dashboard-api/go.mod index 56c9873431..be303ce9f2 100644 --- a/packages/dashboard-api/go.mod +++ b/packages/dashboard-api/go.mod @@ -20,12 +20,14 @@ require ( github.com/gin-contrib/cors v1.7.6 github.com/gin-gonic/gin v1.12.0 github.com/google/uuid v1.6.0 + github.com/hashicorp/go-retryablehttp v0.7.7 github.com/jackc/pgx/v5 v5.9.1 github.com/oapi-codegen/gin-middleware v1.0.2 github.com/oapi-codegen/runtime v1.1.1 github.com/riverqueue/river v0.33.0 github.com/riverqueue/river/riverdriver/riverpgxv5 v0.33.0 github.com/stretchr/testify v1.11.1 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/metric v1.43.0 go.uber.org/zap v1.27.1 @@ -79,6 +81,7 @@ require ( github.com/gorilla/mux v1.8.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect @@ -147,7 +150,6 @@ require ( go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/bridges/otelzap v0.14.0 // indirect go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.68.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect go.opentelemetry.io/contrib/instrumentation/runtime v0.66.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.15.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.39.0 // indirect diff --git a/packages/dashboard-api/go.sum b/packages/dashboard-api/go.sum index fe1f0c118f..9d3f88d8ee 100644 --- a/packages/dashboard-api/go.sum +++ b/packages/dashboard-api/go.sum @@ -70,6 +70,7 @@ github.com/ebitengine/purego v0.9.0 h1:mh0zpKBIXDceC63hpvPuGLiJ8ZAa3DfrFTudmfi8A github.com/ebitengine/purego v0.9.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/exaring/otelpgx v0.9.3 h1:4yO02tXC7ZJZ+hcqcUkfxblYNCIFGVhpUWI0iw1TzPU= github.com/exaring/otelpgx v0.9.3/go.mod h1:R5/M5LWsPPBZc1SrRE5e0DiU48bI78C1/GPTWs6I66U= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= @@ -135,6 +136,9 @@ github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2 h1:sGm2vDRFUrQJO/Veii4h4z github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2/go.mod h1:wd1YpapPLivG6nQgbf7ZkG1hhSOXDhhn4MLTknx2aAc= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 h1:D/V0gu4zQ3cL2WKeVNVM4r2gLxGGf6McLwgXzRTo2RQ= github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -176,6 +180,7 @@ github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8S github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= From 41b247eb3ea379660dc2f2e63bdab7d67a5496be Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Apr 2026 10:20:03 +0000 Subject: [PATCH 50/92] chore: auto-commit generated changes --- packages/dashboard-api/go.sum | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/dashboard-api/go.sum b/packages/dashboard-api/go.sum index 9d3f88d8ee..c86d1d6a68 100644 --- a/packages/dashboard-api/go.sum +++ b/packages/dashboard-api/go.sum @@ -71,6 +71,7 @@ github.com/ebitengine/purego v0.9.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI github.com/exaring/otelpgx v0.9.3 h1:4yO02tXC7ZJZ+hcqcUkfxblYNCIFGVhpUWI0iw1TzPU= github.com/exaring/otelpgx v0.9.3/go.mod h1:R5/M5LWsPPBZc1SrRE5e0DiU48bI78C1/GPTWs6I66U= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= @@ -137,8 +138,11 @@ github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2/go.mod h1:wd1YpapPLivG6nQ github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 h1:D/V0gu4zQ3cL2WKeVNVM4r2gLxGGf6McLwgXzRTo2RQ= github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -181,6 +185,7 @@ github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3 github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= From bf9b998e31af281a1acabfc7d23ddf537ab79e79 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 07:57:38 -0700 Subject: [PATCH 51/92] chore: bugbot comments --- .../internal/handlers/team_handlers_test.go | 66 +++++++++++++++++++ .../internal/handlers/team_provisioning.go | 5 +- .../internal/teamprovision/http_sink.go | 3 +- .../internal/teamprovision/http_sink_test.go | 25 +++++++ 4 files changed, 96 insertions(+), 3 deletions(-) diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index dff67cba56..8798cb234f 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -868,6 +868,72 @@ func TestCreateTeam_ConcurrentRequestsRespectLocalPolicy(t *testing.T) { } } +func TestCreateTeam_ConcurrentRequestsRespectLocalPolicyWithZeroMemberships(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUser(t, testDB) + + existingTeam, err := testDB.SqlcClient.GetDefaultTeamByUserID(ctx, userID) + if err != nil { + t.Fatalf("expected trigger-created default team: %v", err) + } + if err := testDB.SqlcClient.DeleteTeamByID(ctx, existingTeam.ID); err != nil { + t.Fatalf("failed to remove default team: %v", err) + } + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDB, + teamProvisionSink: &fakeTeamProvisionSink{}, + } + + var wg sync.WaitGroup + results := make(chan error, 4) + + for _, name := range []string{"Acme-1", "Acme-2", "Acme-3", "Acme-4"} { + wg.Add(1) + go func(teamName string) { + defer wg.Done() + _, err := store.createTeam(ctx, userID, teamName) + results <- err + }(name) + } + + wg.Wait() + close(results) + + var successCount int + var badRequestCount int + for err := range results { + if err == nil { + successCount++ + + continue + } + + var provisionErr *internalteamprovision.ProvisionError + if !errors.As(err, &provisionErr) { + t.Fatalf("expected provisioning error, got %T: %v", err, err) + } + if provisionErr.StatusCode == http.StatusBadRequest { + badRequestCount++ + + continue + } + + t.Fatalf("expected bad request or success, got %d", provisionErr.StatusCode) + } + + if successCount != maxTeamsPerUser { + t.Fatalf("expected %d successes, got %d", maxTeamsPerUser, successCount) + } + if badRequestCount != 1 { + t.Fatalf("expected one bad request, got %d", badRequestCount) + } +} + type fakeTeamProvisionSink struct { mu sync.Mutex requests []teamprovision.TeamBillingProvisionRequestedV1 diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index 41ff86ef51..9487d71546 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -200,8 +200,9 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string return provisionedTeam{}, fmt.Errorf("upsert public user: %w", err) } - if _, err := txDB.LockUserTeamMembershipsForUpdate(ctx, userID); err != nil { - return provisionedTeam{}, fmt.Errorf("lock user team memberships: %w", err) + // Serialize team creation even when the user currently has no team memberships. + if _, err := txDB.LockPublicUserForUpdate(ctx, authUser.ID); err != nil { + return provisionedTeam{}, fmt.Errorf("lock public user: %w", err) } if err := validateTeamCreationAllowed(ctx, txDB, userID); err != nil { diff --git a/packages/dashboard-api/internal/teamprovision/http_sink.go b/packages/dashboard-api/internal/teamprovision/http_sink.go index 78bccad2e1..96e0332ec3 100644 --- a/packages/dashboard-api/internal/teamprovision/http_sink.go +++ b/packages/dashboard-api/internal/teamprovision/http_sink.go @@ -28,6 +28,7 @@ const ( defaultProvisionRetryMaxAttempts = 3 defaultProvisionRetryInitialWait = 100 * time.Millisecond defaultProvisionRetryWaitCeiling = 2 * time.Second + defaultProvisionAttemptTimeout = defaultProvisionTimeout / defaultProvisionRetryMaxAttempts provisionBackoffMultiplier = 2.0 ) @@ -48,7 +49,7 @@ func NewHTTPProvisionSink(baseURL, apiToken string) *HTTPProvisionSink { return &HTTPProvisionSink{ baseURL: strings.TrimRight(baseURL, "/"), apiToken: apiToken, - client: newRetryableProvisionClient(defaultProvisionTimeout), + client: newRetryableProvisionClient(defaultProvisionAttemptTimeout), timeout: defaultProvisionTimeout, } } diff --git a/packages/dashboard-api/internal/teamprovision/http_sink_test.go b/packages/dashboard-api/internal/teamprovision/http_sink_test.go index 6bd613ddeb..eda7f843ce 100644 --- a/packages/dashboard-api/internal/teamprovision/http_sink_test.go +++ b/packages/dashboard-api/internal/teamprovision/http_sink_test.go @@ -136,6 +136,31 @@ func TestHTTPProvisionSink_RetriesTransportErrorsAndSucceeds(t *testing.T) { require.EqualValues(t, 2, attemptCount.Load()) } +func TestHTTPProvisionSink_RetriesRequestTimeoutWithinOverallBudget(t *testing.T) { + t.Parallel() + + var requestCount atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + attempt := requestCount.Add(1) + if attempt == 1 { + time.Sleep(40 * time.Millisecond) + } + + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + sink := NewHTTPProvisionSink(server.URL, "token") + sink.timeout = 80 * time.Millisecond + sink.client.HTTPClient.Timeout = 25 * time.Millisecond + sink.client.RetryWaitMin = time.Millisecond + sink.client.RetryWaitMax = time.Millisecond + + err := sink.ProvisionTeam(t.Context(), testProvisionRequest()) + require.NoError(t, err) + require.EqualValues(t, 2, requestCount.Load()) +} + func testProvisionRequest() sharedteamprovision.TeamBillingProvisionRequestedV1 { return sharedteamprovision.TeamBillingProvisionRequestedV1{ TeamID: uuid.New(), From da28e6a8e124b4638d1d6bf99dc16657b34148d8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Apr 2026 18:11:51 +0000 Subject: [PATCH 52/92] chore: auto-commit generated changes --- packages/clickhouse/go.mod | 4 ++-- packages/clickhouse/go.sum | 1 - packages/client-proxy/go.mod | 2 +- packages/client-proxy/go.sum | 4 ++-- packages/dashboard-api/go.sum | 20 ++++++++++---------- packages/db/go.mod | 2 +- packages/docker-reverse-proxy/go.mod | 4 ++-- packages/docker-reverse-proxy/go.sum | 4 ++-- packages/envd/go.mod | 2 +- packages/orchestrator/go.mod | 2 +- tests/integration/go.mod | 4 ++-- 11 files changed, 24 insertions(+), 25 deletions(-) diff --git a/packages/clickhouse/go.mod b/packages/clickhouse/go.mod index 932709551a..e7d399d034 100644 --- a/packages/clickhouse/go.mod +++ b/packages/clickhouse/go.mod @@ -42,7 +42,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.7.5 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/joho/godotenv v1.5.1 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect @@ -93,7 +93,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect diff --git a/packages/clickhouse/go.sum b/packages/clickhouse/go.sum index c5f05d320c..1c485d02b4 100644 --- a/packages/clickhouse/go.sum +++ b/packages/clickhouse/go.sum @@ -129,7 +129,6 @@ github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5ey github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= -github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= diff --git a/packages/client-proxy/go.mod b/packages/client-proxy/go.mod index 5ac838ffdf..517bb8ba97 100644 --- a/packages/client-proxy/go.mod +++ b/packages/client-proxy/go.mod @@ -65,7 +65,7 @@ require ( go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect diff --git a/packages/client-proxy/go.sum b/packages/client-proxy/go.sum index 2402ce9653..2987c46eb7 100644 --- a/packages/client-proxy/go.sum +++ b/packages/client-proxy/go.sum @@ -131,8 +131,8 @@ go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= diff --git a/packages/dashboard-api/go.sum b/packages/dashboard-api/go.sum index 56fe3ae394..fe1f0c118f 100644 --- a/packages/dashboard-api/go.sum +++ b/packages/dashboard-api/go.sum @@ -253,16 +253,16 @@ github.com/redis/go-redis/v9 v9.17.3 h1:fN29NdNrE17KttK5Ndf20buqfDZwGNgoUr9qjl1D github.com/redis/go-redis/v9 v9.17.3/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/riverqueue/river v0.32.0 h1:j15EoFZ4oQWXcCq8NyzWwoi3fdaO8mECTB100NSv9Qw= -github.com/riverqueue/river v0.32.0/go.mod h1:zABAdLze3HI7K02N+veikXyK5FjiLzjimnQpZ1Duyng= -github.com/riverqueue/river/riverdriver v0.32.0 h1:AG6a2hNVOIGLx/+3IRtbwofJRYEI7xqnVVxULe9s4Lg= -github.com/riverqueue/river/riverdriver v0.32.0/go.mod h1:FRDMuqnLOsakeJOHlozKK+VH7W7NLp+6EToxQ2JAjBE= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.32.0 h1:CqrRxxcdA/0sHkxLNldsQff9DIG5qxn2EJO09Pau3w0= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.32.0/go.mod h1:j45UPpbMpcI10m+huTeNUaOwzoLJcEg0K6ihWXWeOec= -github.com/riverqueue/river/rivershared v0.32.0 h1:7DwdrppMU9uoU2iU9aGQiv91nBezjlcI85NV4PmnLHw= -github.com/riverqueue/river/rivershared v0.32.0/go.mod h1:UE7GEj3zaTV3cKw7Q3angCozlNEGsL50xZBKJQ9m6zU= -github.com/riverqueue/river/rivertype v0.32.0 h1:RW7uodfl86gYkjwDponTAPNnUqM+X6BjlsNHxbt6Ztg= -github.com/riverqueue/river/rivertype v0.32.0/go.mod h1:D1Ad+EaZiaXbQbJcJcfeicXJMBKno0n6UcfKI5Q7DIQ= +github.com/riverqueue/river v0.33.0 h1:dVB1p91HKAFrOOPIndvsNtCiq5smW6Ii/XYCqZupmvM= +github.com/riverqueue/river v0.33.0/go.mod h1:OMDbi/nfD2uQ9v5kQo53LIypbJbR/bEqx2KyuXeHpdU= +github.com/riverqueue/river/riverdriver v0.33.0 h1:omnVHLRcq6Gy2F59HRI2NMdOAFGf2/iWnJ252nSALy0= +github.com/riverqueue/river/riverdriver v0.33.0/go.mod h1:TZVIUtKC9kaiOGmYTjNffu4IkqBB+i2iEepWLKm2emM= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.33.0 h1:Q5oVOHI3KPFkkH6WLXwrNkqAeRWiBqPI5YjVy/QNyd0= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.33.0/go.mod h1:RjltKK9O9vMbdvlzh/oZoyV62oTNACWuzm3vkchuBFA= +github.com/riverqueue/river/rivershared v0.33.0 h1:gE19JWgu0RgO78PTb5C1OqrI6T2x65mCDtlXHk+hl3E= +github.com/riverqueue/river/rivershared v0.33.0/go.mod h1:/wv6gmMJ4yC23Y9FFZaN/3GgctGmzxsGJ1/moYv1AnE= +github.com/riverqueue/river/rivertype v0.33.0 h1:GIFeAX+JMUL6CaWj7iAUNG86eGSp2KP1NTbTfipsJOo= +github.com/riverqueue/river/rivertype v0.33.0/go.mod h1:D1Ad+EaZiaXbQbJcJcfeicXJMBKno0n6UcfKI5Q7DIQ= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= diff --git a/packages/db/go.mod b/packages/db/go.mod index c02c539552..e9bc205fbe 100644 --- a/packages/db/go.mod +++ b/packages/db/go.mod @@ -140,7 +140,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect diff --git a/packages/docker-reverse-proxy/go.mod b/packages/docker-reverse-proxy/go.mod index b57f92bc28..4b07e1c717 100644 --- a/packages/docker-reverse-proxy/go.mod +++ b/packages/docker-reverse-proxy/go.mod @@ -40,7 +40,7 @@ require ( github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.7.5 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/klauspost/compress v1.18.2 // indirect github.com/lib/pq v1.11.2 // indirect @@ -79,7 +79,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect golang.org/x/crypto v0.49.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect diff --git a/packages/docker-reverse-proxy/go.sum b/packages/docker-reverse-proxy/go.sum index c3f12aa731..ca2293ebd5 100644 --- a/packages/docker-reverse-proxy/go.sum +++ b/packages/docker-reverse-proxy/go.sum @@ -189,8 +189,8 @@ golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= diff --git a/packages/envd/go.mod b/packages/envd/go.mod index 42241adf20..bcae5ece5e 100644 --- a/packages/envd/go.mod +++ b/packages/envd/go.mod @@ -73,7 +73,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect golang.org/x/crypto v0.49.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/text v0.35.0 // indirect diff --git a/packages/orchestrator/go.mod b/packages/orchestrator/go.mod index 7c719201b3..34e58ce46b 100644 --- a/packages/orchestrator/go.mod +++ b/packages/orchestrator/go.mod @@ -314,7 +314,7 @@ require ( golang.org/x/arch v0.25.0 // indirect golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/term v0.41.0 // indirect diff --git a/tests/integration/go.mod b/tests/integration/go.mod index e81b2fe7ef..4bc14e49c4 100644 --- a/tests/integration/go.mod +++ b/tests/integration/go.mod @@ -25,7 +25,7 @@ require ( github.com/e2b-dev/infra/packages/envd v0.0.0-00010101000000-000000000000 github.com/e2b-dev/infra/packages/shared v0.0.0 github.com/google/uuid v1.6.0 - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.9.1 github.com/oapi-codegen/runtime v1.1.1 github.com/stretchr/testify v1.11.1 golang.org/x/sync v0.20.0 @@ -181,7 +181,7 @@ require ( go.uber.org/zap v1.27.1 // indirect golang.org/x/arch v0.25.0 // indirect golang.org/x/crypto v0.49.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/term v0.41.0 // indirect From 1512edea7e4e50f51541f4e70788ddde26158893 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Apr 2026 18:17:10 +0000 Subject: [PATCH 53/92] chore: auto-commit generated changes --- packages/clickhouse/go.sum | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/clickhouse/go.sum b/packages/clickhouse/go.sum index 1c485d02b4..c5f05d320c 100644 --- a/packages/clickhouse/go.sum +++ b/packages/clickhouse/go.sum @@ -129,6 +129,7 @@ github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5ey github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= From d25d5313ebf29d033c1d4a78c24aec30489b52cd Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 12:33:18 -0700 Subject: [PATCH 54/92] refactor(dashboard-api): align auth user sync worker with review feedback --- .../backgroundworker/auth_user_sync.go | 20 +++++++++++++++---- .../backgroundworker/auth_user_sync_test.go | 5 +---- .../internal/backgroundworker/river.go | 6 ++---- packages/dashboard-api/main.go | 1 - 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go index 29e8b54216..e4e3df9796 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go @@ -6,8 +6,10 @@ import ( "github.com/google/uuid" "github.com/riverqueue/river" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" sqlcdb "github.com/e2b-dev/infra/packages/db/client" @@ -16,6 +18,11 @@ import ( "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) +var ( + workerMeter = otel.Meter(workerMeterName) + workerTracer = otel.Tracer(workerMeterName) +) + type AuthUserSyncArgs struct { UserID string `json:"user_id"` Operation string `json:"operation"` @@ -24,6 +31,8 @@ type AuthUserSyncArgs struct { func (AuthUserSyncArgs) Kind() string { return authUserProjectionKind } +var _ river.Worker[AuthUserSyncArgs] = (*AuthUserSyncWorker)(nil) + type AuthUserSyncWorker struct { river.WorkerDefaults[AuthUserSyncArgs] @@ -32,8 +41,8 @@ type AuthUserSyncWorker struct { jobsCounter metric.Int64Counter } -func NewAuthUserSyncWorker(ctx context.Context, mainDB *sqlcdb.Client, meter metric.Meter, l logger.Logger) *AuthUserSyncWorker { - jobsCounter, err := meter.Int64Counter( +func NewAuthUserSyncWorker(ctx context.Context, mainDB *sqlcdb.Client, l logger.Logger) *AuthUserSyncWorker { + jobsCounter, err := workerMeter.Int64Counter( "jobs_total", metric.WithDescription("Total auth user sync jobs by operation and result."), metric.WithUnit("{job}"), @@ -56,7 +65,10 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy attribute.Int64("job.id", job.ID), telemetry.WithUserID(job.Args.UserID), } - telemetry.ReportEvent(ctx, "auth_user_sync.job.started", attrs...) + ctx, span := workerTracer.Start(ctx, "auth_user_sync.work", trace.WithAttributes(attrs...)) + defer span.End() + + telemetry.ReportEvent(ctx, "auth_user_sync.job.started") userID, err := uuid.Parse(job.Args.UserID) if err != nil { @@ -116,7 +128,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy zap.String("job.operation", job.Args.Operation), logger.WithUserID(job.Args.UserID), ) - telemetry.ReportEvent(ctx, "auth_user_sync.job.completed", attrs...) + telemetry.ReportEvent(ctx, "auth_user_sync.job.completed") w.observeJob(ctx, job.Args.Operation, jobResultSuccess) return nil diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go index 1d82d57276..64f2931429 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go @@ -15,7 +15,6 @@ import ( "github.com/e2b-dev/infra/packages/db/pkg/testutils" "github.com/e2b-dev/infra/packages/shared/pkg/logger" - "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) const ( @@ -143,11 +142,9 @@ func startRiverWorker(t *testing.T, db *testutils.Database) *riverProcess { authPool := db.AuthDB.WritePool() l := logger.NewNopLogger() - tel := telemetry.NewNoopClient() - meter := tel.MeterProvider.Meter(workerMeterName) workers := river.NewWorkers() - river.AddWorker(workers, NewAuthUserSyncWorker(ctx, db.SqlcClient, meter, l)) + river.AddWorker(workers, NewAuthUserSyncWorker(ctx, db.SqlcClient, l)) client, err := NewRiverClient(authPool, workers) require.NoError(t, err) diff --git a/packages/dashboard-api/internal/backgroundworker/river.go b/packages/dashboard-api/internal/backgroundworker/river.go index 319143e06c..39f1bf0df3 100644 --- a/packages/dashboard-api/internal/backgroundworker/river.go +++ b/packages/dashboard-api/internal/backgroundworker/river.go @@ -9,7 +9,6 @@ import ( "github.com/riverqueue/river" "github.com/riverqueue/river/riverdriver/riverpgxv5" "github.com/riverqueue/river/rivermigrate" - "go.opentelemetry.io/otel/metric" "go.uber.org/zap" sqlcdb "github.com/e2b-dev/infra/packages/db/client" @@ -41,16 +40,15 @@ func NewRiverClient(pool *pgxpool.Pool, workers *river.Workers) (*river.Client[p }) } -func StartAuthUserSyncWorker(setupCtx, runCtx context.Context, authPool *pgxpool.Pool, mainDB *sqlcdb.Client, meterProvider metric.MeterProvider, l logger.Logger) (*river.Client[pgx.Tx], error) { +func StartAuthUserSyncWorker(setupCtx, runCtx context.Context, authPool *pgxpool.Pool, mainDB *sqlcdb.Client, l logger.Logger) (*river.Client[pgx.Tx], error) { if err := RunRiverMigrations(setupCtx, authPool); err != nil { return nil, fmt.Errorf("run River migrations on auth DB: %w", err) } workerLogger := l.With(zap.String("worker", authUserProjectionKind)) - workerMeter := meterProvider.Meter(workerMeterName) workers := river.NewWorkers() - river.AddWorker(workers, NewAuthUserSyncWorker(setupCtx, mainDB, workerMeter, workerLogger)) + river.AddWorker(workers, NewAuthUserSyncWorker(setupCtx, mainDB, workerLogger)) riverClient, err := NewRiverClient(authPool, workers) if err != nil { diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index d918bc15e5..205f8f1a48 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -268,7 +268,6 @@ func run() int { signalCtx, authDB.WritePool(), db, - tel.MeterProvider, l, ) if err != nil { From fe217ea8f0b256546051aa45aaeb80f3fc9fe244 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Apr 2026 19:35:00 +0000 Subject: [PATCH 55/92] chore: auto-commit generated changes --- packages/dashboard-api/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dashboard-api/go.mod b/packages/dashboard-api/go.mod index 56c9873431..7d35bc5877 100644 --- a/packages/dashboard-api/go.mod +++ b/packages/dashboard-api/go.mod @@ -28,6 +28,7 @@ require ( github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/metric v1.43.0 + go.opentelemetry.io/otel/trace v1.43.0 go.uber.org/zap v1.27.1 ) @@ -157,7 +158,6 @@ require ( go.opentelemetry.io/otel/sdk v1.43.0 // indirect go.opentelemetry.io/otel/sdk/log v0.15.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect - go.opentelemetry.io/otel/trace v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect From 40a5cd0f386b735b2ca1f81a2dba1ade09cd75d1 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 13:20:56 -0700 Subject: [PATCH 56/92] refactor(db): split supabase auth sync source client --- .../backgroundworker/auth_user_sync.go | 45 +++- .../backgroundworker/auth_user_sync_test.go | 20 +- .../internal/backgroundworker/river.go | 11 +- packages/dashboard-api/main.go | 14 +- ...0260401000001_river_auth_custom_schema.sql | 13 - packages/db/pkg/supabase/client.go | 90 +++++++ ...0260401000001_river_auth_custom_schema.sql | 31 +++ ...01000003_river_auth_user_sync_triggers.sql | 4 +- packages/db/pkg/supabase/queries/db.go | 32 +++ .../queries/get_auth_user.sql.go | 8 +- packages/db/pkg/supabase/queries/models.go | 238 ++++++++++++++++++ .../sql_queries/users/get_auth_user.sql | 4 + packages/db/pkg/testutils/db.go | 10 + packages/db/sqlc.yaml | 33 ++- 14 files changed, 506 insertions(+), 47 deletions(-) delete mode 100644 packages/db/pkg/auth/migrations/20260401000001_river_auth_custom_schema.sql create mode 100644 packages/db/pkg/supabase/client.go create mode 100644 packages/db/pkg/supabase/migrations/20260401000001_river_auth_custom_schema.sql rename packages/db/pkg/{auth => supabase}/migrations/20260401000003_river_auth_user_sync_triggers.sql (98%) create mode 100644 packages/db/pkg/supabase/queries/db.go rename packages/db/pkg/{auth => supabase}/queries/get_auth_user.sql.go (59%) create mode 100644 packages/db/pkg/supabase/queries/models.go create mode 100644 packages/db/pkg/supabase/sql_queries/users/get_auth_user.sql diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go index e4e3df9796..b72755eeb5 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go @@ -2,9 +2,11 @@ package backgroundworker import ( "context" + "errors" "fmt" "github.com/google/uuid" + "github.com/jackc/pgx/v5" "github.com/riverqueue/river" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -13,6 +15,7 @@ import ( "go.uber.org/zap" sqlcdb "github.com/e2b-dev/infra/packages/db/client" + supabasedb "github.com/e2b-dev/infra/packages/db/pkg/supabase" "github.com/e2b-dev/infra/packages/db/queries" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -26,7 +29,6 @@ var ( type AuthUserSyncArgs struct { UserID string `json:"user_id"` Operation string `json:"operation"` - Email string `json:"email,omitempty"` } func (AuthUserSyncArgs) Kind() string { return authUserProjectionKind } @@ -36,12 +38,13 @@ var _ river.Worker[AuthUserSyncArgs] = (*AuthUserSyncWorker)(nil) type AuthUserSyncWorker struct { river.WorkerDefaults[AuthUserSyncArgs] - mainDB *sqlcdb.Client + supabaseDB *supabasedb.Client + authDB *sqlcdb.Client l logger.Logger jobsCounter metric.Int64Counter } -func NewAuthUserSyncWorker(ctx context.Context, mainDB *sqlcdb.Client, l logger.Logger) *AuthUserSyncWorker { +func NewAuthUserSyncWorker(ctx context.Context, supabaseDB *supabasedb.Client, authDB *sqlcdb.Client, l logger.Logger) *AuthUserSyncWorker { jobsCounter, err := workerMeter.Int64Counter( "jobs_total", metric.WithDescription("Total auth user sync jobs by operation and result."), @@ -52,7 +55,8 @@ func NewAuthUserSyncWorker(ctx context.Context, mainDB *sqlcdb.Client, l logger. } return &AuthUserSyncWorker{ - mainDB: mainDB, + supabaseDB: supabaseDB, + authDB: authDB, l: l, jobsCounter: jobsCounter, } @@ -88,7 +92,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy switch job.Args.Operation { case "delete": - if err := w.mainDB.DeletePublicUser(ctx, userID); err != nil { + if err := w.authDB.DeletePublicUser(ctx, userID); err != nil { telemetry.ReportError(ctx, "auth user sync delete public user", err, attrs...) w.observeJob(ctx, job.Args.Operation, jobResultError) @@ -96,17 +100,38 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy } case "upsert": - if job.Args.Email == "" { - err := fmt.Errorf("missing email in job args for user %s", userID) - telemetry.ReportError(ctx, "auth user sync missing email", err, attrs...) + supabaseUser, err := w.supabaseDB.Read.GetAuthUserByID(ctx, userID) + if errors.Is(err, pgx.ErrNoRows) { + if err := w.authDB.DeletePublicUser(ctx, userID); err != nil { + telemetry.ReportError(ctx, "auth user sync delete stale public user", err, attrs...) + w.observeJob(ctx, job.Args.Operation, jobResultError) + + return fmt.Errorf("delete stale public.users %s: %w", userID, err) + } + + telemetry.ReportEvent(ctx, "auth_user_sync.job.source_user_missing") + w.observeJob(ctx, job.Args.Operation, jobResultSuccess) + + return nil + } + if err != nil { + telemetry.ReportError(ctx, "auth user sync get source user", err, attrs...) + w.observeJob(ctx, job.Args.Operation, jobResultError) + + return fmt.Errorf("get source auth.users %s: %w", userID, err) + } + + if supabaseUser.Email == "" { + err := fmt.Errorf("missing email in source user %s", userID) + telemetry.ReportError(ctx, "auth user sync missing source email", err, attrs...) w.observeJob(ctx, job.Args.Operation, jobResultInvalidArgument) return river.JobCancel(err) } - if err := w.mainDB.UpsertPublicUser(ctx, queries.UpsertPublicUserParams{ + if err := w.authDB.UpsertPublicUser(ctx, queries.UpsertPublicUserParams{ ID: userID, - Email: job.Args.Email, + Email: supabaseUser.Email, }); err != nil { telemetry.ReportError(ctx, "auth user sync upsert public user", err, attrs...) w.observeJob(ctx, job.Args.Operation, jobResultError) diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go index 64f2931429..3f56637fdd 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go @@ -22,7 +22,7 @@ const ( testEventuallyTick = 50 * time.Millisecond testStopTimeout = 5 * time.Second - authMigrationsDir = "packages/db/pkg/auth/migrations" + supabaseMigrationsDir = "packages/db/pkg/supabase/migrations" authCustomSchemaVersion int64 = 20260401000001 ) @@ -128,25 +128,23 @@ func applyAuthUserSyncMigrations(t *testing.T, db *testutils.Database) { // 1. goose migration 20260401000001 creates the shared schema // 2. River library migrations create River tables inside that schema // 3. the remaining auth migrations add triggers that enqueue into River - db.ApplyMigrationsUpTo(t, authCustomSchemaVersion, authMigrationsDir) + db.ApplyMigrationsUpTo(t, authCustomSchemaVersion, supabaseMigrationsDir) - authPool := db.AuthDB.WritePool() - require.NoError(t, RunRiverMigrations(t.Context(), authPool)) + require.NoError(t, RunRiverMigrations(t.Context(), db.SupabaseDB.WritePool())) - db.ApplyMigrations(t, authMigrationsDir) + db.ApplyMigrations(t, supabaseMigrationsDir) } func startRiverWorker(t *testing.T, db *testutils.Database) *riverProcess { t.Helper() ctx := t.Context() - authPool := db.AuthDB.WritePool() l := logger.NewNopLogger() workers := river.NewWorkers() - river.AddWorker(workers, NewAuthUserSyncWorker(ctx, db.SqlcClient, l)) + river.AddWorker(workers, NewAuthUserSyncWorker(ctx, db.SupabaseDB, db.SqlcClient, l)) - client, err := NewRiverClient(authPool, workers) + client, err := NewRiverClient(db.SupabaseDB.WritePool(), workers) require.NoError(t, err) ctx, cancel := context.WithCancel(ctx) @@ -183,7 +181,7 @@ func (p *riverProcess) Stop(t *testing.T) { func insertAuthUser(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, email string) { t.Helper() - err := db.AuthDB.TestsRawSQL(ctx, + err := db.SupabaseDB.TestsRawSQL(ctx, "INSERT INTO auth.users (id, email) VALUES ($1, $2)", userID, email) require.NoError(t, err) } @@ -191,7 +189,7 @@ func insertAuthUser(t *testing.T, ctx context.Context, db *testutils.Database, u func updateAuthUserEmail(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID, email string) { t.Helper() - err := db.AuthDB.TestsRawSQL(ctx, + err := db.SupabaseDB.TestsRawSQL(ctx, "UPDATE auth.users SET email = $1 WHERE id = $2", email, userID) require.NoError(t, err) } @@ -199,7 +197,7 @@ func updateAuthUserEmail(t *testing.T, ctx context.Context, db *testutils.Databa func deleteAuthUser(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID) { t.Helper() - err := db.AuthDB.TestsRawSQL(ctx, + err := db.SupabaseDB.TestsRawSQL(ctx, "DELETE FROM auth.users WHERE id = $1", userID) require.NoError(t, err) } diff --git a/packages/dashboard-api/internal/backgroundworker/river.go b/packages/dashboard-api/internal/backgroundworker/river.go index 39f1bf0df3..a5e07635e9 100644 --- a/packages/dashboard-api/internal/backgroundworker/river.go +++ b/packages/dashboard-api/internal/backgroundworker/river.go @@ -12,6 +12,7 @@ import ( "go.uber.org/zap" sqlcdb "github.com/e2b-dev/infra/packages/db/client" + supabasedb "github.com/e2b-dev/infra/packages/db/pkg/supabase" "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) @@ -40,17 +41,17 @@ func NewRiverClient(pool *pgxpool.Pool, workers *river.Workers) (*river.Client[p }) } -func StartAuthUserSyncWorker(setupCtx, runCtx context.Context, authPool *pgxpool.Pool, mainDB *sqlcdb.Client, l logger.Logger) (*river.Client[pgx.Tx], error) { - if err := RunRiverMigrations(setupCtx, authPool); err != nil { - return nil, fmt.Errorf("run River migrations on auth DB: %w", err) +func StartAuthUserSyncWorker(setupCtx, runCtx context.Context, supabaseDB *supabasedb.Client, authDB *sqlcdb.Client, l logger.Logger) (*river.Client[pgx.Tx], error) { + if err := RunRiverMigrations(setupCtx, supabaseDB.WritePool()); err != nil { + return nil, fmt.Errorf("run River migrations on supabase DB: %w", err) } workerLogger := l.With(zap.String("worker", authUserProjectionKind)) workers := river.NewWorkers() - river.AddWorker(workers, NewAuthUserSyncWorker(setupCtx, mainDB, workerLogger)) + river.AddWorker(workers, NewAuthUserSyncWorker(setupCtx, supabaseDB, authDB, workerLogger)) - riverClient, err := NewRiverClient(authPool, workers) + riverClient, err := NewRiverClient(supabaseDB.WritePool(), workers) if err != nil { return nil, fmt.Errorf("create River client: %w", err) } diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 205f8f1a48..ccad7d6e40 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -36,6 +36,7 @@ import ( sqlcdb "github.com/e2b-dev/infra/packages/db/client" authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" "github.com/e2b-dev/infra/packages/db/pkg/pool" + supabasedb "github.com/e2b-dev/infra/packages/db/pkg/supabase" e2benv "github.com/e2b-dev/infra/packages/shared/pkg/env" "github.com/e2b-dev/infra/packages/shared/pkg/factories" "github.com/e2b-dev/infra/packages/shared/pkg/logger" @@ -132,6 +133,17 @@ func run() int { } defer authDB.Close() + supabaseDB, err := supabasedb.NewClient( + ctx, + config.AuthDBConnectionString, + config.AuthDBReadReplicaConnectionString, + pool.WithMaxConnections(8), + ) + if err != nil { + l.Fatal(ctx, "Initializing supabase database client", zap.Error(err)) + } + defer supabaseDB.Close() + var clickhouseClient clickhouse.Clickhouse if config.ClickhouseConnectionString == "" { clickhouseClient = clickhouse.NewNoopClient() @@ -266,7 +278,7 @@ func run() int { riverClient, err = backgroundworker.StartAuthUserSyncWorker( ctx, signalCtx, - authDB.WritePool(), + supabaseDB, db, l, ) diff --git a/packages/db/pkg/auth/migrations/20260401000001_river_auth_custom_schema.sql b/packages/db/pkg/auth/migrations/20260401000001_river_auth_custom_schema.sql deleted file mode 100644 index 0eba39b6c0..0000000000 --- a/packages/db/pkg/auth/migrations/20260401000001_river_auth_custom_schema.sql +++ /dev/null @@ -1,13 +0,0 @@ --- +goose Up --- +goose StatementBegin - -CREATE SCHEMA IF NOT EXISTS auth_custom; - --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin - -DROP SCHEMA IF EXISTS auth_custom CASCADE; - --- +goose StatementEnd diff --git a/packages/db/pkg/supabase/client.go b/packages/db/pkg/supabase/client.go new file mode 100644 index 0000000000..7162219d09 --- /dev/null +++ b/packages/db/pkg/supabase/client.go @@ -0,0 +1,90 @@ +package supabasedb + +import ( + "context" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + _ "github.com/lib/pq" //nolint:blank-imports + + "github.com/e2b-dev/infra/packages/db/pkg/pool" + supabasequeries "github.com/e2b-dev/infra/packages/db/pkg/supabase/queries" + "github.com/e2b-dev/infra/packages/db/pkg/types" +) + +const ( + poolName = "supabase" + replica = "replica" +) + +type Client struct { + Read *supabasequeries.Queries + Write *supabasequeries.Queries + writeConn *pgxpool.Pool + readConn *pgxpool.Pool +} + +func NewClient(ctx context.Context, databaseURL, replicaURL string, options ...pool.Option) (*Client, error) { + writeClient, writePool, err := pool.New(ctx, databaseURL, poolName, options...) + if err != nil { + return nil, err + } + + writeQueries := supabasequeries.New(writeClient) + readPool := writePool + readQueries := writeQueries + + if strings.TrimSpace(replicaURL) != "" { + var readClient types.DBTX + readClient, readPool, err = pool.New(ctx, replicaURL, strings.Join([]string{poolName, replica}, "-"), options...) + if err != nil { + writePool.Close() + + return nil, err + } + + readQueries = supabasequeries.New(readClient) + } + + return &Client{Read: readQueries, Write: writeQueries, writeConn: writePool, readConn: readPool}, nil +} + +func (db *Client) Close() error { + db.writeConn.Close() + + if db.readConn != nil { + db.readConn.Close() + } + + return nil +} + +func (db *Client) WritePool() *pgxpool.Pool { + return db.writeConn +} + +func (db *Client) WithTx(ctx context.Context) (*supabasequeries.Queries, pgx.Tx, error) { + tx, err := db.writeConn.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return nil, nil, err + } + + return db.Write.WithTx(tx), tx, nil +} + +func (db *Client) TestsRawSQL(ctx context.Context, sql string, args ...any) error { + _, err := db.writeConn.Exec(ctx, sql, args...) + + return err +} + +func (db *Client) TestsRawSQLQuery(ctx context.Context, sql string, processRows func(pgx.Rows) error, args ...any) error { + rows, err := db.writeConn.Query(ctx, sql, args...) + if err != nil { + return err + } + defer rows.Close() + + return processRows(rows) +} diff --git a/packages/db/pkg/supabase/migrations/20260401000001_river_auth_custom_schema.sql b/packages/db/pkg/supabase/migrations/20260401000001_river_auth_custom_schema.sql new file mode 100644 index 0000000000..d5e1a478f5 --- /dev/null +++ b/packages/db/pkg/supabase/migrations/20260401000001_river_auth_custom_schema.sql @@ -0,0 +1,31 @@ +-- +goose Up +-- +goose StatementBegin + +CREATE SCHEMA IF NOT EXISTS auth_custom; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'trigger_user') THEN + CREATE ROLE trigger_user NOLOGIN; + END IF; +END; +$$; + +GRANT USAGE ON SCHEMA auth_custom TO trigger_user; +GRANT CREATE ON SCHEMA auth_custom TO trigger_user; + +DO $$ +BEGIN + EXECUTE format('GRANT TEMP ON DATABASE %I TO trigger_user', current_database()); +END; +$$; + +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin + +DROP ROLE IF EXISTS trigger_user; +DROP SCHEMA IF EXISTS auth_custom; + +-- +goose StatementEnd diff --git a/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql b/packages/db/pkg/supabase/migrations/20260401000003_river_auth_user_sync_triggers.sql similarity index 98% rename from packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql rename to packages/db/pkg/supabase/migrations/20260401000003_river_auth_user_sync_triggers.sql index 93ddb068e9..3f600ec219 100644 --- a/packages/db/pkg/auth/migrations/20260401000003_river_auth_user_sync_triggers.sql +++ b/packages/db/pkg/supabase/migrations/20260401000003_river_auth_user_sync_triggers.sql @@ -9,7 +9,7 @@ AS $$ BEGIN INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) VALUES ( - jsonb_build_object('user_id', NEW.id, 'operation', 'upsert', 'email', NEW.email), + jsonb_build_object('user_id', NEW.id, 'operation', 'upsert'), 'auth_user_projection', 20, 'auth_user_projection', @@ -31,7 +31,7 @@ BEGIN IF OLD.email IS DISTINCT FROM NEW.email THEN INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) VALUES ( - jsonb_build_object('user_id', NEW.id, 'operation', 'upsert', 'email', NEW.email), + jsonb_build_object('user_id', NEW.id, 'operation', 'upsert'), 'auth_user_projection', 20, 'auth_user_projection', diff --git a/packages/db/pkg/supabase/queries/db.go b/packages/db/pkg/supabase/queries/db.go new file mode 100644 index 0000000000..9dbacdf919 --- /dev/null +++ b/packages/db/pkg/supabase/queries/db.go @@ -0,0 +1,32 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 + +package supabasequeries + +import ( + "context" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +type DBTX interface { + Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error) + Query(context.Context, string, ...interface{}) (pgx.Rows, error) + QueryRow(context.Context, string, ...interface{}) pgx.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx pgx.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/packages/db/pkg/auth/queries/get_auth_user.sql.go b/packages/db/pkg/supabase/queries/get_auth_user.sql.go similarity index 59% rename from packages/db/pkg/auth/queries/get_auth_user.sql.go rename to packages/db/pkg/supabase/queries/get_auth_user.sql.go index 4c34da8bdf..4f881e7ff3 100644 --- a/packages/db/pkg/auth/queries/get_auth_user.sql.go +++ b/packages/db/pkg/supabase/queries/get_auth_user.sql.go @@ -3,7 +3,7 @@ // sqlc v1.29.0 // source: get_auth_user.sql -package authqueries +package supabasequeries import ( "context" @@ -12,13 +12,13 @@ import ( ) const getAuthUserByID = `-- name: GetAuthUserByID :one -SELECT id, email +SELECT id, COALESCE(email, '') AS email FROM auth.users WHERE id = $1::uuid ` -func (q *Queries) GetAuthUserByID(ctx context.Context, userID uuid.UUID) (AuthUser, error) { - row := q.db.QueryRow(ctx, getAuthUserByID, userID) +func (q *Queries) GetAuthUserByID(ctx context.Context, dollar_1 uuid.UUID) (AuthUser, error) { + row := q.db.QueryRow(ctx, getAuthUserByID, dollar_1) var i AuthUser err := row.Scan(&i.ID, &i.Email) return i, err diff --git a/packages/db/pkg/supabase/queries/models.go b/packages/db/pkg/supabase/queries/models.go new file mode 100644 index 0000000000..4b44a1aa44 --- /dev/null +++ b/packages/db/pkg/supabase/queries/models.go @@ -0,0 +1,238 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 + +package supabasequeries + +import ( + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +type AccessToken struct { + UserID uuid.UUID + CreatedAt time.Time + ID uuid.UUID + // sensitive + AccessTokenHash string + Name string + AccessTokenPrefix string + AccessTokenLength int32 + AccessTokenMaskPrefix string + AccessTokenMaskSuffix string +} + +type ActiveTemplateBuild struct { + BuildID uuid.UUID + TeamID uuid.UUID + TemplateID string + Tags []string + CreatedAt time.Time +} + +type Addon struct { + ID uuid.UUID + TeamID uuid.UUID + Name string + Description *string + ExtraConcurrentSandboxes int64 + ExtraConcurrentTemplateBuilds int64 + ExtraMaxVcpu int64 + ExtraMaxRamMb int64 + ExtraDiskMb int64 + ValidFrom time.Time + ValidTo *time.Time + AddedBy uuid.UUID + IdempotencyKey *string +} + +type AuthUser struct { + ID uuid.UUID + Email string +} + +type BillingSandboxLog struct { + SandboxID string + EnvID string + Vcpu int64 + RamMb int64 + TotalDiskSizeMb int64 + StartedAt time.Time + StoppedAt *time.Time + CreatedAt time.Time + TeamID uuid.UUID +} + +type Cluster struct { + ID uuid.UUID + Endpoint string + EndpointTls bool + Token string + SandboxProxyDomain *string +} + +type Env struct { + ID string + CreatedAt time.Time + UpdatedAt time.Time + Public bool + BuildCount int32 + // Number of times the env was spawned + SpawnCount int64 + // Timestamp of the last time the env was spawned + LastSpawnedAt *time.Time + TeamID uuid.UUID + CreatedBy *uuid.UUID + ClusterID *uuid.UUID + Source string +} + +type EnvAlias struct { + Alias string + IsRenamable bool + EnvID string + Namespace *string + ID uuid.UUID +} + +type EnvBuild struct { + ID uuid.UUID + CreatedAt time.Time + UpdatedAt time.Time + FinishedAt *time.Time + Status string + Dockerfile *string + StartCmd *string + Vcpu int64 + RamMb int64 + FreeDiskSizeMb int64 + TotalDiskSizeMb *int64 + KernelVersion string + FirecrackerVersion string + EnvID *string + EnvdVersion *string + ReadyCmd *string + ClusterNodeID *string + Reason []byte + Version *string + CpuArchitecture *string + CpuFamily *string + CpuModel *string + CpuModelName *string + CpuFlags []string + StatusGroup string + TeamID *uuid.UUID +} + +type EnvBuildAssignment struct { + ID uuid.UUID + EnvID string + BuildID uuid.UUID + Tag string + Source string + CreatedAt pgtype.Timestamptz +} + +type Snapshot struct { + CreatedAt pgtype.Timestamptz + EnvID string + SandboxID string + ID uuid.UUID + Metadata []byte + BaseEnvID string + SandboxStartedAt pgtype.Timestamptz + EnvSecure bool + OriginNodeID string + AllowInternetAccess *bool + AutoPause bool + TeamID uuid.UUID + Config []byte +} + +type SnapshotTemplate struct { + EnvID string + SandboxID string + CreatedAt pgtype.Timestamptz + OriginNodeID *string + BuildID *uuid.UUID +} + +type Team struct { + ID uuid.UUID + CreatedAt time.Time + IsBlocked bool + Name string + Tier string + Email string + IsBanned bool + BlockedReason *string + ClusterID *uuid.UUID + SandboxSchedulingLabels []string + Slug string +} + +type TeamApiKey struct { + CreatedAt time.Time + TeamID uuid.UUID + UpdatedAt *time.Time + Name string + LastUsed *time.Time + CreatedBy *uuid.UUID + ID uuid.UUID + // sensitive + ApiKeyHash string + ApiKeyPrefix string + ApiKeyLength int32 + ApiKeyMaskPrefix string + ApiKeyMaskSuffix string +} + +type TeamLimit struct { + ID uuid.UUID + MaxLengthHours int64 + ConcurrentSandboxes int32 + ConcurrentTemplateBuilds int32 + MaxVcpu int32 + MaxRamMb int32 + DiskMb int32 +} + +type Tier struct { + ID string + Name string + DiskMb int64 + // The number of instances the team can run concurrently + ConcurrentInstances int64 + MaxLengthHours int64 + MaxVcpu int64 + MaxRamMb int64 + // The number of concurrent template builds the team can run + ConcurrentTemplateBuilds int64 +} + +type User struct { + CreatedAt time.Time + UpdatedAt time.Time + ID uuid.UUID + Email string +} + +type UsersTeam struct { + ID int64 + UserID uuid.UUID + TeamID uuid.UUID + IsDefault bool + AddedBy *uuid.UUID + CreatedAt pgtype.Timestamp + UuidID uuid.UUID +} + +type Volume struct { + ID uuid.UUID + TeamID uuid.UUID + Name string + VolumeType string + CreatedAt pgtype.Timestamptz +} diff --git a/packages/db/pkg/supabase/sql_queries/users/get_auth_user.sql b/packages/db/pkg/supabase/sql_queries/users/get_auth_user.sql new file mode 100644 index 0000000000..f01b84a05c --- /dev/null +++ b/packages/db/pkg/supabase/sql_queries/users/get_auth_user.sql @@ -0,0 +1,4 @@ +-- name: GetAuthUserByID :one +SELECT id, COALESCE(email, '') AS email +FROM auth.users +WHERE id = $1::uuid; diff --git a/packages/db/pkg/testutils/db.go b/packages/db/pkg/testutils/db.go index aa5199f4ac..58ab04b7b7 100644 --- a/packages/db/pkg/testutils/db.go +++ b/packages/db/pkg/testutils/db.go @@ -20,6 +20,7 @@ import ( db "github.com/e2b-dev/infra/packages/db/client" authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" "github.com/e2b-dev/infra/packages/db/pkg/pool" + supabasedb "github.com/e2b-dev/infra/packages/db/pkg/supabase" "github.com/e2b-dev/infra/packages/db/pkg/testutils/queries" ) @@ -38,6 +39,7 @@ func init() { type Database struct { SqlcClient *db.Client AuthDB *authdb.Client + SupabaseDB *supabasedb.Client TestQueries *queries.Queries connStr string } @@ -107,9 +109,17 @@ func SetupDatabase(t *testing.T) *Database { assert.NoError(t, err) }) + supabaseDB, err := supabasedb.NewClient(t.Context(), connStr, connStr) + require.NoError(t, err, "Failed to create supabase db client") + t.Cleanup(func() { + err := supabaseDB.Close() + assert.NoError(t, err) + }) + return &Database{ SqlcClient: sqlcClient, AuthDB: authDB, + SupabaseDB: supabaseDB, TestQueries: testQueries, connStr: connStr, } diff --git a/packages/db/sqlc.yaml b/packages/db/sqlc.yaml index 4959602d19..b07b8e61eb 100644 --- a/packages/db/sqlc.yaml +++ b/packages/db/sqlc.yaml @@ -63,7 +63,6 @@ sql: schema: - "migrations" - "schema" - - "pkg/auth/migrations" gen: go: emit_pointers_for_null_types: true @@ -109,6 +108,38 @@ sql: - db_type: "jsonb" go_type: "github.com/e2b-dev/infra/packages/db/pkg/types.JSONBStringMap" + - engine: "postgresql" + queries: "pkg/supabase/sql_queries/**" + schema: + - "migrations" + - "schema" + gen: + go: + emit_pointers_for_null_types: true + package: "supabasequeries" + out: "pkg/supabase/queries/" + sql_package: "pgx/v5" + overrides: + - db_type: "uuid" + go_type: + import: "github.com/google/uuid" + type: "UUID" + - db_type: "uuid" + nullable: true + go_type: + import: "github.com/google/uuid" + type: "UUID" + pointer: true + + - db_type: "timestamptz" + go_type: "time.Time" + - db_type: "timestamptz" + go_type: + import: "time" + type: "Time" + pointer: true + nullable: true + - engine: "postgresql" queries: "pkg/testutils/*.sql" gen: From c25cabaac0068634a7aa0e0c59c10233749b5400 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 13:40:29 -0700 Subject: [PATCH 57/92] chore(db): address follow-up nits --- .../backgroundworker/auth_user_sync.go | 5 +- .../dashboard-api/internal/handlers/build.go | 5 +- .../internal/handlers/sandbox_record.go | 4 +- packages/db/pkg/auth/client.go | 4 - packages/db/pkg/supabase/queries/models.go | 224 ------------------ .../supabase/schema/auth_users_override.sql | 18 ++ packages/db/sqlc.yaml | 6 +- 7 files changed, 27 insertions(+), 239 deletions(-) create mode 100644 packages/db/pkg/supabase/schema/auth_users_override.sql diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go index b72755eeb5..a6daeda56b 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go @@ -2,11 +2,9 @@ package backgroundworker import ( "context" - "errors" "fmt" "github.com/google/uuid" - "github.com/jackc/pgx/v5" "github.com/riverqueue/river" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -15,6 +13,7 @@ import ( "go.uber.org/zap" sqlcdb "github.com/e2b-dev/infra/packages/db/client" + "github.com/e2b-dev/infra/packages/db/pkg/dberrors" supabasedb "github.com/e2b-dev/infra/packages/db/pkg/supabase" "github.com/e2b-dev/infra/packages/db/queries" "github.com/e2b-dev/infra/packages/shared/pkg/logger" @@ -101,7 +100,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy case "upsert": supabaseUser, err := w.supabaseDB.Read.GetAuthUserByID(ctx, userID) - if errors.Is(err, pgx.ErrNoRows) { + if dberrors.IsNotFoundError(err) { if err := w.authDB.DeletePublicUser(ctx, userID); err != nil { telemetry.ReportError(ctx, "auth user sync delete stale public user", err, attrs...) w.observeJob(ctx, job.Args.Operation, jobResultError) diff --git a/packages/dashboard-api/internal/handlers/build.go b/packages/dashboard-api/internal/handlers/build.go index 90237884e2..30a586bc3b 100644 --- a/packages/dashboard-api/internal/handlers/build.go +++ b/packages/dashboard-api/internal/handlers/build.go @@ -1,16 +1,15 @@ package handlers import ( - "errors" "net/http" "github.com/gin-gonic/gin" - "github.com/jackc/pgx/v5" "go.uber.org/zap" "github.com/e2b-dev/infra/packages/auth/pkg/auth" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" dashboardutils "github.com/e2b-dev/infra/packages/dashboard-api/internal/utils" + "github.com/e2b-dev/infra/packages/db/pkg/dberrors" "github.com/e2b-dev/infra/packages/db/queries" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -27,7 +26,7 @@ func (s *APIStore) GetBuildsBuildId(c *gin.Context, buildId api.BuildId) { BuildID: buildId, }) if err != nil { - if errors.Is(err, pgx.ErrNoRows) { + if dberrors.IsNotFoundError(err) { s.sendAPIStoreError(c, http.StatusNotFound, "Build not found or you don't have access to it") return diff --git a/packages/dashboard-api/internal/handlers/sandbox_record.go b/packages/dashboard-api/internal/handlers/sandbox_record.go index 823c597ff2..72c4976e4a 100644 --- a/packages/dashboard-api/internal/handlers/sandbox_record.go +++ b/packages/dashboard-api/internal/handlers/sandbox_record.go @@ -6,12 +6,12 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "go.uber.org/zap" "github.com/e2b-dev/infra/packages/auth/pkg/auth" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" + "github.com/e2b-dev/infra/packages/db/pkg/dberrors" "github.com/e2b-dev/infra/packages/db/queries" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -36,7 +36,7 @@ func (s *APIStore) GetSandboxesSandboxIDRecord(c *gin.Context, sandboxID api.San CreatedAfter: time.Now().UTC().Add(-sandboxRecordRetention), }) if err != nil { - if errors.Is(err, pgx.ErrNoRows) || isUndefinedTableError(err) { + if dberrors.IsNotFoundError(err) || isUndefinedTableError(err) { s.sendAPIStoreError(c, http.StatusNotFound, "Sandbox not found or you don't have access to it") return diff --git a/packages/db/pkg/auth/client.go b/packages/db/pkg/auth/client.go index fe9f33d666..02a1dcf2a0 100644 --- a/packages/db/pkg/auth/client.go +++ b/packages/db/pkg/auth/client.go @@ -60,10 +60,6 @@ func (db *Client) Close() error { return nil } -func (db *Client) WritePool() *pgxpool.Pool { - return db.writeConn -} - // WithTx runs the given function in a transaction. func (db *Client) WithTx(ctx context.Context) (*authqueries.Queries, pgx.Tx, error) { tx, err := db.writeConn.BeginTx(ctx, pgx.TxOptions{}) diff --git a/packages/db/pkg/supabase/queries/models.go b/packages/db/pkg/supabase/queries/models.go index 4b44a1aa44..4bbb324fa3 100644 --- a/packages/db/pkg/supabase/queries/models.go +++ b/packages/db/pkg/supabase/queries/models.go @@ -5,234 +5,10 @@ package supabasequeries import ( - "time" - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" ) -type AccessToken struct { - UserID uuid.UUID - CreatedAt time.Time - ID uuid.UUID - // sensitive - AccessTokenHash string - Name string - AccessTokenPrefix string - AccessTokenLength int32 - AccessTokenMaskPrefix string - AccessTokenMaskSuffix string -} - -type ActiveTemplateBuild struct { - BuildID uuid.UUID - TeamID uuid.UUID - TemplateID string - Tags []string - CreatedAt time.Time -} - -type Addon struct { - ID uuid.UUID - TeamID uuid.UUID - Name string - Description *string - ExtraConcurrentSandboxes int64 - ExtraConcurrentTemplateBuilds int64 - ExtraMaxVcpu int64 - ExtraMaxRamMb int64 - ExtraDiskMb int64 - ValidFrom time.Time - ValidTo *time.Time - AddedBy uuid.UUID - IdempotencyKey *string -} - type AuthUser struct { ID uuid.UUID Email string } - -type BillingSandboxLog struct { - SandboxID string - EnvID string - Vcpu int64 - RamMb int64 - TotalDiskSizeMb int64 - StartedAt time.Time - StoppedAt *time.Time - CreatedAt time.Time - TeamID uuid.UUID -} - -type Cluster struct { - ID uuid.UUID - Endpoint string - EndpointTls bool - Token string - SandboxProxyDomain *string -} - -type Env struct { - ID string - CreatedAt time.Time - UpdatedAt time.Time - Public bool - BuildCount int32 - // Number of times the env was spawned - SpawnCount int64 - // Timestamp of the last time the env was spawned - LastSpawnedAt *time.Time - TeamID uuid.UUID - CreatedBy *uuid.UUID - ClusterID *uuid.UUID - Source string -} - -type EnvAlias struct { - Alias string - IsRenamable bool - EnvID string - Namespace *string - ID uuid.UUID -} - -type EnvBuild struct { - ID uuid.UUID - CreatedAt time.Time - UpdatedAt time.Time - FinishedAt *time.Time - Status string - Dockerfile *string - StartCmd *string - Vcpu int64 - RamMb int64 - FreeDiskSizeMb int64 - TotalDiskSizeMb *int64 - KernelVersion string - FirecrackerVersion string - EnvID *string - EnvdVersion *string - ReadyCmd *string - ClusterNodeID *string - Reason []byte - Version *string - CpuArchitecture *string - CpuFamily *string - CpuModel *string - CpuModelName *string - CpuFlags []string - StatusGroup string - TeamID *uuid.UUID -} - -type EnvBuildAssignment struct { - ID uuid.UUID - EnvID string - BuildID uuid.UUID - Tag string - Source string - CreatedAt pgtype.Timestamptz -} - -type Snapshot struct { - CreatedAt pgtype.Timestamptz - EnvID string - SandboxID string - ID uuid.UUID - Metadata []byte - BaseEnvID string - SandboxStartedAt pgtype.Timestamptz - EnvSecure bool - OriginNodeID string - AllowInternetAccess *bool - AutoPause bool - TeamID uuid.UUID - Config []byte -} - -type SnapshotTemplate struct { - EnvID string - SandboxID string - CreatedAt pgtype.Timestamptz - OriginNodeID *string - BuildID *uuid.UUID -} - -type Team struct { - ID uuid.UUID - CreatedAt time.Time - IsBlocked bool - Name string - Tier string - Email string - IsBanned bool - BlockedReason *string - ClusterID *uuid.UUID - SandboxSchedulingLabels []string - Slug string -} - -type TeamApiKey struct { - CreatedAt time.Time - TeamID uuid.UUID - UpdatedAt *time.Time - Name string - LastUsed *time.Time - CreatedBy *uuid.UUID - ID uuid.UUID - // sensitive - ApiKeyHash string - ApiKeyPrefix string - ApiKeyLength int32 - ApiKeyMaskPrefix string - ApiKeyMaskSuffix string -} - -type TeamLimit struct { - ID uuid.UUID - MaxLengthHours int64 - ConcurrentSandboxes int32 - ConcurrentTemplateBuilds int32 - MaxVcpu int32 - MaxRamMb int32 - DiskMb int32 -} - -type Tier struct { - ID string - Name string - DiskMb int64 - // The number of instances the team can run concurrently - ConcurrentInstances int64 - MaxLengthHours int64 - MaxVcpu int64 - MaxRamMb int64 - // The number of concurrent template builds the team can run - ConcurrentTemplateBuilds int64 -} - -type User struct { - CreatedAt time.Time - UpdatedAt time.Time - ID uuid.UUID - Email string -} - -type UsersTeam struct { - ID int64 - UserID uuid.UUID - TeamID uuid.UUID - IsDefault bool - AddedBy *uuid.UUID - CreatedAt pgtype.Timestamp - UuidID uuid.UUID -} - -type Volume struct { - ID uuid.UUID - TeamID uuid.UUID - Name string - VolumeType string - CreatedAt pgtype.Timestamptz -} diff --git a/packages/db/pkg/supabase/schema/auth_users_override.sql b/packages/db/pkg/supabase/schema/auth_users_override.sql new file mode 100644 index 0000000000..83cc73104d --- /dev/null +++ b/packages/db/pkg/supabase/schema/auth_users_override.sql @@ -0,0 +1,18 @@ +CREATE SCHEMA IF NOT EXISTS auth; + +CREATE ROLE authenticated; + +CREATE FUNCTION auth.uid() RETURNS uuid AS $func$ +BEGIN + RETURN gen_random_uuid(); +END; +$func$ LANGUAGE plpgsql; + +-- Grant execute on auth.uid() to postgres role +GRANT EXECUTE ON FUNCTION auth.uid() TO postgres; + +CREATE TABLE auth.users ( + id uuid NOT NULL DEFAULT gen_random_uuid(), + email text NOT NULL, + PRIMARY KEY (id) +); diff --git a/packages/db/sqlc.yaml b/packages/db/sqlc.yaml index b07b8e61eb..efa140c889 100644 --- a/packages/db/sqlc.yaml +++ b/packages/db/sqlc.yaml @@ -1,7 +1,7 @@ version: "2" sql: - engine: "postgresql" - queries: + queries: - "queries/**" - "pkg/dashboard/sql_queries/**" schema: @@ -111,8 +111,8 @@ sql: - engine: "postgresql" queries: "pkg/supabase/sql_queries/**" schema: - - "migrations" - - "schema" + - "pkg/supabase/schema" + - "pkg/supabase/migrations" gen: go: emit_pointers_for_null_types: true From b54987448b297c1c6fd7c7f372678b72e8f47de7 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 14:10:03 -0700 Subject: [PATCH 58/92] chore: make extra env sensitive --- iac/modules/job-dashboard-api/variables.tf | 5 +++-- iac/provider-gcp/nomad/variables.tf | 5 +++-- iac/provider-gcp/variables.tf | 5 +++-- packages/db/sqlc.yaml | 1 - 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/iac/modules/job-dashboard-api/variables.tf b/iac/modules/job-dashboard-api/variables.tf index 99ce63284b..50bc355554 100644 --- a/iac/modules/job-dashboard-api/variables.tf +++ b/iac/modules/job-dashboard-api/variables.tf @@ -45,8 +45,9 @@ variable "supabase_jwt_secrets" { } variable "extra_env" { - type = map(string) - default = {} + type = map(string) + default = {} + sensitive = true } variable "otel_collector_grpc_port" { diff --git a/iac/provider-gcp/nomad/variables.tf b/iac/provider-gcp/nomad/variables.tf index cc70a3bf1a..a6d7a3bbdd 100644 --- a/iac/provider-gcp/nomad/variables.tf +++ b/iac/provider-gcp/nomad/variables.tf @@ -455,8 +455,9 @@ variable "dashboard_api_count" { } variable "dashboard_api_env_vars" { - type = map(string) - default = {} + type = map(string) + default = {} + sensitive = true } variable "volume_token_issuer" { diff --git a/iac/provider-gcp/variables.tf b/iac/provider-gcp/variables.tf index 0a1bc8d7c4..7d8cd2db68 100644 --- a/iac/provider-gcp/variables.tf +++ b/iac/provider-gcp/variables.tf @@ -231,8 +231,9 @@ variable "dashboard_api_count" { } variable "dashboard_api_env_vars" { - type = map(string) - default = {} + type = map(string) + default = {} + sensitive = true } variable "docker_reverse_proxy_port" { diff --git a/packages/db/sqlc.yaml b/packages/db/sqlc.yaml index 4959602d19..206c468907 100644 --- a/packages/db/sqlc.yaml +++ b/packages/db/sqlc.yaml @@ -63,7 +63,6 @@ sql: schema: - "migrations" - "schema" - - "pkg/auth/migrations" gen: go: emit_pointers_for_null_types: true From 581afccb3db16b766f8c337059e650eddb2522ae Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 14:15:51 -0700 Subject: [PATCH 59/92] improve: pass billing env explicitly instead of implicitly --- iac/modules/job-dashboard-api/main.tf | 2 ++ iac/modules/job-dashboard-api/variables.tf | 11 +++++++++++ iac/provider-gcp/nomad/main.tf | 12 +++++------- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/iac/modules/job-dashboard-api/main.tf b/iac/modules/job-dashboard-api/main.tf index 68476d109a..85561e1cdc 100644 --- a/iac/modules/job-dashboard-api/main.tf +++ b/iac/modules/job-dashboard-api/main.tf @@ -12,6 +12,8 @@ locals { REDIS_URL = var.redis_url REDIS_CLUSTER_URL = var.redis_cluster_url REDIS_TLS_CA_BASE64 = var.redis_tls_ca_base64 + BILLING_SERVER_URL = var.billing_server_url + BILLING_SERVER_API_TOKEN = var.billing_server_api_token OTEL_COLLECTOR_GRPC_ENDPOINT = "localhost:${var.otel_collector_grpc_port}" LOGS_COLLECTOR_ADDRESS = "http://localhost:${var.logs_proxy_port.port}" } diff --git a/iac/modules/job-dashboard-api/variables.tf b/iac/modules/job-dashboard-api/variables.tf index 50bc355554..c5d4ca72a7 100644 --- a/iac/modules/job-dashboard-api/variables.tf +++ b/iac/modules/job-dashboard-api/variables.tf @@ -71,6 +71,17 @@ variable "redis_tls_ca_base64" { default = "" } +variable "billing_server_url" { + type = string + default = "" +} + +variable "billing_server_api_token" { + type = string + sensitive = true + default = "" +} + variable "logs_proxy_port" { type = object({ name = string diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index 0ac03f5d7b..bea62e8ea3 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -7,13 +7,9 @@ locals { var.dashboard_api_count > 0 && lower(trimspace(lookup(var.dashboard_api_env_vars, "ENABLE_BILLING_HTTP_TEAM_PROVISION_SINK", "false"))) == "true" ) - dashboard_api_extra_env = merge( - local.enable_billing_http_team_provision_sink ? { - BILLING_SERVER_URL = data.google_cloud_run_v2_service.billing_server[0].uri - BILLING_SERVER_API_TOKEN = data.google_secret_manager_secret_version.billing_server_api_token[0].secret_data - } : {}, - var.dashboard_api_env_vars, - ) + dashboard_api_extra_env = var.dashboard_api_env_vars + dashboard_api_billing_server_url = local.enable_billing_http_team_provision_sink ? data.google_cloud_run_v2_service.billing_server[0].uri : "" + dashboard_api_billing_server_api_token = local.enable_billing_http_team_provision_sink ? data.google_secret_manager_secret_version.billing_server_api_token[0].secret_data : "" } # API @@ -164,6 +160,8 @@ module "dashboard_api" { redis_url = local.redis_url redis_cluster_url = local.redis_cluster_url redis_tls_ca_base64 = trimspace(data.google_secret_manager_secret_version.redis_tls_ca_base64.secret_data) + billing_server_url = local.dashboard_api_billing_server_url + billing_server_api_token = local.dashboard_api_billing_server_api_token extra_env = local.dashboard_api_extra_env otel_collector_grpc_port = var.otel_collector_grpc_port From 18f7839cf0110c88f3289818a111cbfa60b7a358 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 14:19:24 -0700 Subject: [PATCH 60/92] fix(dashboard-api): stop river worker gracefully on shutdown --- packages/dashboard-api/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index ccad7d6e40..6df6b54a95 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -277,7 +277,7 @@ func run() int { if config.EnableAuthUserSyncBackgroundWorker { riverClient, err = backgroundworker.StartAuthUserSyncWorker( ctx, - signalCtx, + ctx, supabaseDB, db, l, From 9667b772b336f59fd192c35deec92a19b3f8c217 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 14:26:01 -0700 Subject: [PATCH 61/92] refactor(dashboard-api): simplify service lifecycle orchestration --- packages/dashboard-api/main.go | 162 +++++++++++++++++++++++---------- 1 file changed, 113 insertions(+), 49 deletions(-) diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 6df6b54a95..349794838a 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -10,8 +10,6 @@ import ( "os/signal" "strconv" "strings" - "sync" - "sync/atomic" "syscall" "time" @@ -25,6 +23,7 @@ import ( "github.com/riverqueue/river" "go.uber.org/zap" "go.uber.org/zap/zapcore" + "golang.org/x/sync/errgroup" sharedauth "github.com/e2b-dev/infra/packages/auth/pkg/auth" "github.com/e2b-dev/infra/packages/auth/pkg/types" @@ -55,6 +54,7 @@ const ( writeTimeout = 75 * time.Second requestTimeout = 70 * time.Second idleTimeout = 620 * time.Second + shutdownTimeout = 30 * time.Second ) var ( @@ -66,8 +66,6 @@ func run() int { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - errorCode := atomic.Int32{} - serviceInstanceID := uuid.New().String() nodeID := e2benv.GetNodeID() @@ -190,6 +188,67 @@ func run() int { nil, ) + s, err := newHTTPServer(config.Port, l, tel, swagger, authenticationFunc, apiStore) + if err != nil { + l.Fatal(ctx, "failed to create HTTP server", zap.Error(err)) + } + + signalCtx, sigCancel := signal.NotifyContext(ctx, syscall.SIGTERM, syscall.SIGINT) + defer sigCancel() + + var riverClient *river.Client[pgx.Tx] + + if config.EnableAuthUserSyncBackgroundWorker { + riverClient, err = backgroundworker.StartAuthUserSyncWorker( + ctx, + ctx, + supabaseDB, + db, + l, + ) + if err != nil { + l.Fatal(ctx, "failed to start auth user sync worker", zap.Error(err)) + } + } + + l.Info(ctx, "HTTP service starting", zap.Int("port", config.Port)) + runErr := waitForServiceStop(signalCtx, startHTTPServer(s), riverStoppedChan(riverClient)) + if runErr != nil { + l.Error(ctx, "dashboard-api runtime error", zap.Error(runErr)) + } else { + l.Info(ctx, "Shutting down dashboard-api service...") + } + + shutdownCtx, shutdownCancel := context.WithTimeout(context.WithoutCancel(ctx), shutdownTimeout) + defer shutdownCancel() + + if err := shutdownService(shutdownCtx, s, riverClient); err != nil { + l.Error(ctx, "dashboard-api shutdown error", zap.Error(err)) + + return 1 + } + + if runErr != nil { + return 1 + } + + l.Info(ctx, "dashboard-api service stopped") + + return 0 +} + +func main() { + os.Exit(run()) +} + +func newHTTPServer( + port int, + l logger.Logger, + tel *telemetry.Client, + swagger *openapi3.T, + authenticationFunc openapi3filter.AuthenticationFunc, + apiStore *handlers.APIStore, +) (*http.Server, error) { r := gin.New() r.Use(gin.Recovery()) @@ -258,71 +317,76 @@ func run() int { api.RegisterHandlers(r, apiStore) - s := &http.Server{ + return &http.Server{ Handler: r, - Addr: fmt.Sprintf("0.0.0.0:%d", config.Port), + Addr: fmt.Sprintf("0.0.0.0:%d", port), ReadHeaderTimeout: readHeaderTimeout, ReadTimeout: readTimeout, WriteTimeout: writeTimeout, IdleTimeout: idleTimeout, - } + }, nil +} - signalCtx, sigCancel := signal.NotifyContext(ctx, syscall.SIGTERM, syscall.SIGINT) - defer sigCancel() +func startHTTPServer(s *http.Server) <-chan error { + errCh := make(chan error, 1) - wg := sync.WaitGroup{} + go func() { + err := s.ListenAndServe() + if errors.Is(err, http.ErrServerClosed) { + errCh <- nil - var riverClient *river.Client[pgx.Tx] + return + } - if config.EnableAuthUserSyncBackgroundWorker { - riverClient, err = backgroundworker.StartAuthUserSyncWorker( - ctx, - ctx, - supabaseDB, - db, - l, - ) - if err != nil { - l.Fatal(ctx, "failed to start auth user sync worker", zap.Error(err)) + errCh <- err + }() + + return errCh +} + +func waitForServiceStop(signalCtx context.Context, httpErrCh <-chan error, riverStoppedCh <-chan struct{}) error { + select { + case <-signalCtx.Done(): + return nil + case err := <-httpErrCh: + if err == nil { + return errors.New("http service stopped unexpectedly") } + + return fmt.Errorf("http service error: %w", err) + case <-riverStoppedCh: + return errors.New("auth user sync worker stopped unexpectedly") } +} - wg.Go(func() { - <-signalCtx.Done() - l.Info(ctx, "Shutting down dashboard-api service...") +func riverStoppedChan(riverClient *river.Client[pgx.Tx]) <-chan struct{} { + if riverClient == nil { + return nil + } - shutdownCtx, shutdownCancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) - defer shutdownCancel() + return riverClient.Stopped() +} - if riverClient != nil { - if err := riverClient.Stop(shutdownCtx); err != nil { - l.Error(ctx, "River client shutdown error", zap.Error(err)) +func shutdownService(ctx context.Context, s *http.Server, riverClient *river.Client[pgx.Tx]) error { + var g errgroup.Group - errorCode.Add(1) - } + g.Go(func() error { + if err := s.Shutdown(ctx); err != nil { + return fmt.Errorf("shutdown HTTP server: %w", err) } - if err := s.Shutdown(shutdownCtx); err != nil { - l.Error(ctx, "HTTP server shutdown error", zap.Error(err)) - - errorCode.Add(1) - } + return nil }) - l.Info(ctx, "HTTP service starting", zap.Int("port", config.Port)) - if err := s.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { - l.Error(ctx, "HTTP service error", zap.Error(err)) + if riverClient != nil { + g.Go(func() error { + if err := riverClient.Stop(ctx); err != nil { + return fmt.Errorf("shutdown River client: %w", err) + } - errorCode.Add(1) - } else { - l.Info(ctx, "HTTP service stopped") + return nil + }) } - wg.Wait() - - return int(errorCode.Load()) -} - -func main() { - os.Exit(run()) + return g.Wait() } From 37eb463da51e69a50cba85e85ecbe2ce800a855d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Apr 2026 21:27:57 +0000 Subject: [PATCH 62/92] chore: auto-commit generated changes --- packages/dashboard-api/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dashboard-api/go.mod b/packages/dashboard-api/go.mod index 7d35bc5877..1262430745 100644 --- a/packages/dashboard-api/go.mod +++ b/packages/dashboard-api/go.mod @@ -30,6 +30,7 @@ require ( go.opentelemetry.io/otel/metric v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 go.uber.org/zap v1.27.1 + golang.org/x/sync v0.20.0 ) require ( @@ -165,7 +166,6 @@ require ( golang.org/x/crypto v0.49.0 // indirect golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect - golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/text v0.35.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2 // indirect From c5d82f0c9e50c324e37a238f5517bfe7d7a11918 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 14:28:37 -0700 Subject: [PATCH 63/92] refactor(db): drop supabase replica client plumbing --- packages/dashboard-api/main.go | 1 - packages/db/pkg/supabase/client.go | 31 +++--------------------------- packages/db/pkg/testutils/db.go | 2 +- 3 files changed, 4 insertions(+), 30 deletions(-) diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 349794838a..11c00e58f5 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -134,7 +134,6 @@ func run() int { supabaseDB, err := supabasedb.NewClient( ctx, config.AuthDBConnectionString, - config.AuthDBReadReplicaConnectionString, pool.WithMaxConnections(8), ) if err != nil { diff --git a/packages/db/pkg/supabase/client.go b/packages/db/pkg/supabase/client.go index 7162219d09..d84a3dc88a 100644 --- a/packages/db/pkg/supabase/client.go +++ b/packages/db/pkg/supabase/client.go @@ -2,7 +2,6 @@ package supabasedb import ( "context" - "strings" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" @@ -10,53 +9,29 @@ import ( "github.com/e2b-dev/infra/packages/db/pkg/pool" supabasequeries "github.com/e2b-dev/infra/packages/db/pkg/supabase/queries" - "github.com/e2b-dev/infra/packages/db/pkg/types" ) -const ( - poolName = "supabase" - replica = "replica" -) +const poolName = "supabase" type Client struct { - Read *supabasequeries.Queries Write *supabasequeries.Queries writeConn *pgxpool.Pool - readConn *pgxpool.Pool } -func NewClient(ctx context.Context, databaseURL, replicaURL string, options ...pool.Option) (*Client, error) { +func NewClient(ctx context.Context, databaseURL string, options ...pool.Option) (*Client, error) { writeClient, writePool, err := pool.New(ctx, databaseURL, poolName, options...) if err != nil { return nil, err } writeQueries := supabasequeries.New(writeClient) - readPool := writePool - readQueries := writeQueries - - if strings.TrimSpace(replicaURL) != "" { - var readClient types.DBTX - readClient, readPool, err = pool.New(ctx, replicaURL, strings.Join([]string{poolName, replica}, "-"), options...) - if err != nil { - writePool.Close() - return nil, err - } - - readQueries = supabasequeries.New(readClient) - } - - return &Client{Read: readQueries, Write: writeQueries, writeConn: writePool, readConn: readPool}, nil + return &Client{Write: writeQueries, writeConn: writePool}, nil } func (db *Client) Close() error { db.writeConn.Close() - if db.readConn != nil { - db.readConn.Close() - } - return nil } diff --git a/packages/db/pkg/testutils/db.go b/packages/db/pkg/testutils/db.go index 58ab04b7b7..508de6f545 100644 --- a/packages/db/pkg/testutils/db.go +++ b/packages/db/pkg/testutils/db.go @@ -109,7 +109,7 @@ func SetupDatabase(t *testing.T) *Database { assert.NoError(t, err) }) - supabaseDB, err := supabasedb.NewClient(t.Context(), connStr, connStr) + supabaseDB, err := supabasedb.NewClient(t.Context(), connStr) require.NoError(t, err, "Failed to create supabase db client") t.Cleanup(func() { err := supabaseDB.Close() From e4d5bff433d122f8867de40ccb38f2873d3159ff Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 14:29:30 -0700 Subject: [PATCH 64/92] fix: use write db for supabase db get user read --- .../dashboard-api/internal/backgroundworker/auth_user_sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go index a6daeda56b..3ca466a8ef 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go @@ -99,7 +99,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy } case "upsert": - supabaseUser, err := w.supabaseDB.Read.GetAuthUserByID(ctx, userID) + supabaseUser, err := w.supabaseDB.Write.GetAuthUserByID(ctx, userID) if dberrors.IsNotFoundError(err) { if err := w.authDB.DeletePublicUser(ctx, userID); err != nil { telemetry.ReportError(ctx, "auth user sync delete stale public user", err, attrs...) From 4862139bbf8e0f773995ea8fe05758a457b9195c Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 14:42:07 -0700 Subject: [PATCH 65/92] chore: fix lint --- packages/dashboard-api/main.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 11c00e58f5..956dafc1cf 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -187,10 +187,7 @@ func run() int { nil, ) - s, err := newHTTPServer(config.Port, l, tel, swagger, authenticationFunc, apiStore) - if err != nil { - l.Fatal(ctx, "failed to create HTTP server", zap.Error(err)) - } + s := newHTTPServer(config.Port, l, tel, swagger, authenticationFunc, apiStore) signalCtx, sigCancel := signal.NotifyContext(ctx, syscall.SIGTERM, syscall.SIGINT) defer sigCancel() @@ -247,7 +244,7 @@ func newHTTPServer( swagger *openapi3.T, authenticationFunc openapi3filter.AuthenticationFunc, apiStore *handlers.APIStore, -) (*http.Server, error) { +) *http.Server { r := gin.New() r.Use(gin.Recovery()) @@ -323,7 +320,7 @@ func newHTTPServer( ReadTimeout: readTimeout, WriteTimeout: writeTimeout, IdleTimeout: idleTimeout, - }, nil + } } func startHTTPServer(s *http.Server) <-chan error { From 449df8701d07dda67299d4bdf81aeb1bf3bd24a1 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 14:50:13 -0700 Subject: [PATCH 66/92] chore: address comments --- .../internal/handlers/team_handlers_test.go | 31 +++++++++++++++++++ .../internal/handlers/team_provisioning.go | 6 ++++ .../db/queries/team_creation_guard.sql.go | 27 ---------------- .../db/queries/teams/team_creation_guard.sql | 6 ---- 4 files changed, 37 insertions(+), 33 deletions(-) diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index 8798cb234f..7c9527b51f 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -749,6 +749,37 @@ func TestPostTeams_ProvisioningSuccessReturnsTeam(t *testing.T) { } } +func TestPostTeams_InvalidNameReturnsBadRequest(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUser(t, testDB) + + for _, body := range []string{`{}`, `{"name":""}`, `{"name":" "}`} { + recorder := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(recorder) + ginCtx.Request = httptest.NewRequestWithContext(ctx, http.MethodPost, "/", strings.NewReader(body)) + ginCtx.Request.Header.Set("Content-Type", "application/json") + auth.SetUserID(ginCtx, userID) + + sink := &fakeTeamProvisionSink{} + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDB, + teamProvisionSink: sink, + } + store.PostTeams(ginCtx) + + if recorder.Code != http.StatusBadRequest { + t.Fatalf("expected status 400 for body %s, got %d", body, recorder.Code) + } + if len(sink.requests) != 0 { + t.Fatalf("expected no provisioning call for body %s, got %d", body, len(sink.requests)) + } + } +} + func TestPostTeams_ProvisioningFailureStillReturnsTeam(t *testing.T) { t.Parallel() diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index 9487d71546..6745528b20 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "strings" "time" "github.com/gin-gonic/gin" @@ -68,6 +69,11 @@ func (s *APIStore) PostTeams(c *gin.Context) { return } + if strings.TrimSpace(body.Name) == "" { + s.sendAPIStoreError(c, http.StatusBadRequest, "Team name is required") + + return + } team, err := s.createTeam(ctx, userID, body.Name) if err != nil { diff --git a/packages/db/queries/team_creation_guard.sql.go b/packages/db/queries/team_creation_guard.sql.go index 37226ba423..8747e60310 100644 --- a/packages/db/queries/team_creation_guard.sql.go +++ b/packages/db/queries/team_creation_guard.sql.go @@ -114,30 +114,3 @@ func (q *Queries) LockPublicUserForUpdate(ctx context.Context, id uuid.UUID) (uu err := row.Scan(&id) return id, err } - -const lockUserTeamMembershipsForUpdate = `-- name: LockUserTeamMembershipsForUpdate :many -SELECT team_id -FROM public.users_teams -WHERE user_id = $1::uuid -FOR UPDATE -` - -func (q *Queries) LockUserTeamMembershipsForUpdate(ctx context.Context, userID uuid.UUID) ([]uuid.UUID, error) { - rows, err := q.db.Query(ctx, lockUserTeamMembershipsForUpdate, userID) - if err != nil { - return nil, err - } - defer rows.Close() - var items []uuid.UUID - for rows.Next() { - var team_id uuid.UUID - if err := rows.Scan(&team_id); err != nil { - return nil, err - } - items = append(items, team_id) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} diff --git a/packages/db/queries/teams/team_creation_guard.sql b/packages/db/queries/teams/team_creation_guard.sql index b64aaaa0fe..cb0c2a7c0f 100644 --- a/packages/db/queries/teams/team_creation_guard.sql +++ b/packages/db/queries/teams/team_creation_guard.sql @@ -1,9 +1,3 @@ --- name: LockUserTeamMembershipsForUpdate :many -SELECT team_id -FROM public.users_teams -WHERE user_id = sqlc.arg(user_id)::uuid -FOR UPDATE; - -- name: LockPublicUserForUpdate :one SELECT id FROM public.users From 4ecc62741a6595cf29bd05f614cfae8d883eb8de Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 14:51:49 -0700 Subject: [PATCH 67/92] fix(dashboard-api): lazy-init supabase worker client --- packages/dashboard-api/main.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 956dafc1cf..ec422875f9 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -131,16 +131,6 @@ func run() int { } defer authDB.Close() - supabaseDB, err := supabasedb.NewClient( - ctx, - config.AuthDBConnectionString, - pool.WithMaxConnections(8), - ) - if err != nil { - l.Fatal(ctx, "Initializing supabase database client", zap.Error(err)) - } - defer supabaseDB.Close() - var clickhouseClient clickhouse.Clickhouse if config.ClickhouseConnectionString == "" { clickhouseClient = clickhouse.NewNoopClient() @@ -193,8 +183,19 @@ func run() int { defer sigCancel() var riverClient *river.Client[pgx.Tx] + var supabaseDB *supabasedb.Client if config.EnableAuthUserSyncBackgroundWorker { + supabaseDB, err = supabasedb.NewClient( + ctx, + config.AuthDBConnectionString, + pool.WithMaxConnections(8), + ) + if err != nil { + l.Fatal(ctx, "Initializing supabase database client", zap.Error(err)) + } + defer supabaseDB.Close() + riverClient, err = backgroundworker.StartAuthUserSyncWorker( ctx, ctx, From 29bbf59ffbf34ca86c538f4dda471cbfc24d5468 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 15:03:20 -0700 Subject: [PATCH 68/92] chore(dashboard-api): reduce supabase worker pool size --- packages/dashboard-api/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index ec422875f9..79fdd55e0c 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -189,7 +189,7 @@ func run() int { supabaseDB, err = supabasedb.NewClient( ctx, config.AuthDBConnectionString, - pool.WithMaxConnections(8), + pool.WithMaxConnections(4), ) if err != nil { l.Fatal(ctx, "Initializing supabase database client", zap.Error(err)) From dc36b119e838d179ca08c21edc85f55eb7ada489 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 17:38:17 -0700 Subject: [PATCH 69/92] fix(dashboard-api): validate and normalize team names --- .../internal/handlers/team_handlers_test.go | 46 +++++++++++++++++++ .../internal/handlers/team_provisioning.go | 5 +- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index 7c9527b51f..b4ba902ae5 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -780,6 +780,52 @@ func TestPostTeams_InvalidNameReturnsBadRequest(t *testing.T) { } } +func TestPostTeams_TrimsNameBeforeCreate(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUser(t, testDB) + sink := &fakeTeamProvisionSink{} + + recorder := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(recorder) + ginCtx.Request = httptest.NewRequestWithContext(ctx, http.MethodPost, "/", strings.NewReader(`{"name":" Acme "}`)) + ginCtx.Request.Header.Set("Content-Type", "application/json") + auth.SetUserID(ginCtx, userID) + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDB, + teamProvisionSink: sink, + } + store.PostTeams(ginCtx) + + if recorder.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d", recorder.Code) + } + + rows, err := testDB.AuthDB.Read.GetTeamsWithUsersTeamsWithTier(ctx, userID) + if err != nil { + t.Fatalf("failed to query user teams: %v", err) + } + + foundCreatedTeam := false + for _, row := range rows { + if row.IsDefault { + continue + } + + foundCreatedTeam = true + if row.Team.Name != "Acme" { + t.Fatalf("expected trimmed team name %q, got %q", "Acme", row.Team.Name) + } + } + if !foundCreatedTeam { + t.Fatal("expected created team to exist") + } +} + func TestPostTeams_ProvisioningFailureStillReturnsTeam(t *testing.T) { t.Parallel() diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index 6745528b20..130a430c75 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -69,13 +69,14 @@ func (s *APIStore) PostTeams(c *gin.Context) { return } - if strings.TrimSpace(body.Name) == "" { + name := strings.TrimSpace(body.Name) + if name == "" { s.sendAPIStoreError(c, http.StatusBadRequest, "Team name is required") return } - team, err := s.createTeam(ctx, userID, body.Name) + team, err := s.createTeam(ctx, userID, name) if err != nil { s.handleProvisioningError(ctx, c, "create team", err) From 48a620cd1774b85626fe0f956d530b9c047ba3f3 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 18:05:08 -0700 Subject: [PATCH 70/92] test(dashboard-api): tighten auth user sync coverage Replace the broad backlog case with smaller tests that cover stale upsert cleanup and same-email trigger behavior. This keeps the worker coverage focused on the branch's critical correctness signals without extra soak-style runtime. --- .../backgroundworker/auth_user_sync_test.go | 170 ++++++++++-------- 1 file changed, 94 insertions(+), 76 deletions(-) diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go index 3f56637fdd..6e01d27ad7 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go @@ -10,10 +10,12 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/riverqueue/river" + "github.com/riverqueue/river/rivertype" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/e2b-dev/infra/packages/db/pkg/testutils" + "github.com/e2b-dev/infra/packages/db/queries" "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) @@ -38,87 +40,69 @@ func TestAuthUserSync_EndToEnd(t *testing.T) { db := testutils.SetupDatabase(t) applyAuthUserSyncMigrations(t, db) - t.Run("upsert projects auth users into public users", func(t *testing.T) { - t.Parallel() - - ctx := t.Context() - userID := uuid.New() - email := fmt.Sprintf("river-sync-%s@example.com", userID.String()[:8]) + ctx := t.Context() + userID := uuid.New() + email := fmt.Sprintf("river-sync-%s@example.com", userID.String()[:8]) - proc := startRiverWorker(t, db) - t.Cleanup(func() { proc.Stop(t) }) + proc := startRiverWorker(t, db) + t.Cleanup(func() { proc.Stop(t) }) - insertAuthUser(t, ctx, db, userID, email) - waitForPublicUser(t, ctx, db, userID, email) + insertAuthUser(t, ctx, db, userID, email) + waitForPublicUser(t, ctx, db, userID, email) - updatedEmail := fmt.Sprintf("river-sync-%s-updated@example.com", userID.String()[:8]) - updateAuthUserEmail(t, ctx, db, userID, updatedEmail) - waitForPublicUser(t, ctx, db, userID, updatedEmail) - }) + updatedEmail := fmt.Sprintf("river-sync-%s-updated@example.com", userID.String()[:8]) + updateAuthUserEmail(t, ctx, db, userID, updatedEmail) + waitForPublicUser(t, ctx, db, userID, updatedEmail) - t.Run("delete removes projected public users", func(t *testing.T) { - t.Parallel() + deleteAuthUser(t, ctx, db, userID) + waitForPublicUserGone(t, ctx, db, userID) +} - ctx := t.Context() - userID := uuid.New() - email := fmt.Sprintf("river-del-%s@example.com", userID.String()[:8]) +func TestAuthUserSyncWorker_UpsertDeletesStaleProjectedUser(t *testing.T) { + t.Parallel() - proc := startRiverWorker(t, db) - t.Cleanup(func() { proc.Stop(t) }) + ctx := t.Context() + db := testutils.SetupDatabase(t) + applyAuthUserSyncMigrations(t, db) - insertAuthUser(t, ctx, db, userID, email) - waitForPublicUser(t, ctx, db, userID, email) + userID := uuid.New() + staleEmail := fmt.Sprintf("stale-%s@example.com", userID.String()[:8]) - deleteAuthUser(t, ctx, db, userID) - waitForPublicUserGone(t, ctx, db, userID) + err := db.SqlcClient.UpsertPublicUser(ctx, queries.UpsertPublicUserParams{ + ID: userID, + Email: staleEmail, }) + require.NoError(t, err) + require.Equal(t, 1, publicUserCount(t, ctx, db, userID)) - t.Run("burst backlog drains mixed upsert and delete work", func(t *testing.T) { - t.Parallel() - - ctx := t.Context() - const userCount = 40 - - type testUser struct { - id uuid.UUID - email string - shouldDel bool - } - - users := make([]testUser, 0, userCount) - for i := range userCount { - u := testUser{ - id: uuid.New(), - email: fmt.Sprintf("river-burst-%02d@example.com", i), - shouldDel: i%3 == 0, - } - users = append(users, u) - insertAuthUser(t, ctx, db, u.id, u.email) - } + worker := NewAuthUserSyncWorker(ctx, db.SupabaseDB, db.SqlcClient, logger.NewNopLogger()) - proc := startRiverWorker(t, db) - t.Cleanup(func() { proc.Stop(t) }) + err = worker.Work(ctx, &river.Job[AuthUserSyncArgs]{ + JobRow: &rivertype.JobRow{ID: 1, Attempt: 1}, + Args: AuthUserSyncArgs{ + UserID: userID.String(), + Operation: "upsert", + }, + }) + require.NoError(t, err) + assert.Equal(t, 0, publicUserCount(t, ctx, db, userID)) +} - for _, u := range users { - waitForPublicUser(t, ctx, db, u.id, u.email) - } +func TestAuthUserSyncTrigger_SameEmailUpdateDoesNotEnqueueJob(t *testing.T) { + t.Parallel() - for _, u := range users { - if u.shouldDel { - deleteAuthUser(t, ctx, db, u.id) - } - } + ctx := t.Context() + db := testutils.SetupDatabase(t) + applyAuthUserSyncMigrations(t, db) - for _, u := range users { - if u.shouldDel { - waitForPublicUserGone(t, ctx, db, u.id) + userID := uuid.New() + email := fmt.Sprintf("trigger-%s@example.com", userID.String()[:8]) - continue - } + insertAuthUser(t, ctx, db, userID, email) + require.Equal(t, 1, riverJobCountForUser(t, ctx, db, userID)) - waitForPublicUser(t, ctx, db, u.id, u.email) - } - }) + updateAuthUserEmail(t, ctx, db, userID, email) + assert.Equal(t, 1, riverJobCountForUser(t, ctx, db, userID)) } func applyAuthUserSyncMigrations(t *testing.T, db *testutils.Database) { @@ -230,17 +214,7 @@ func waitForPublicUserGone(t *testing.T, ctx context.Context, db *testutils.Data t.Helper() require.EventuallyWithT(t, func(c *assert.CollectT) { - var count int - - err := db.AuthDB.TestsRawSQLQuery(ctx, - "SELECT count(*) FROM public.users WHERE id = $1", - func(rows pgx.Rows) error { - if !rows.Next() { - return nil - } - - return rows.Scan(&count) - }, userID) + count, err := publicUserCountE(ctx, db, userID) if !assert.NoError(c, err) { return @@ -249,3 +223,47 @@ func waitForPublicUserGone(t *testing.T, ctx context.Context, db *testutils.Data assert.Equal(c, 0, count) }, testEventuallyTimeout, testEventuallyTick) } + +func publicUserCount(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID) int { + t.Helper() + + count, err := publicUserCountE(ctx, db, userID) + require.NoError(t, err) + + return count +} + +func publicUserCountE(ctx context.Context, db *testutils.Database, userID uuid.UUID) (int, error) { + var count int + + err := db.AuthDB.TestsRawSQLQuery(ctx, + "SELECT count(*) FROM public.users WHERE id = $1", + func(rows pgx.Rows) error { + if !rows.Next() { + return nil + } + + return rows.Scan(&count) + }, userID) + + return count, err +} + +func riverJobCountForUser(t *testing.T, ctx context.Context, db *testutils.Database, userID uuid.UUID) int { + t.Helper() + + var count int + + err := db.SupabaseDB.TestsRawSQLQuery(ctx, + "SELECT count(*) FROM auth_custom.river_job WHERE args->>'user_id' = $1", + func(rows pgx.Rows) error { + if !rows.Next() { + return nil + } + + return rows.Scan(&count) + }, userID.String()) + require.NoError(t, err) + + return count +} From 2eda1d4f5967914adca4d0c8f880074bf525b985 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 10 Apr 2026 01:09:46 +0000 Subject: [PATCH 71/92] chore: auto-commit generated changes --- packages/dashboard-api/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dashboard-api/go.mod b/packages/dashboard-api/go.mod index 1262430745..46742fe24e 100644 --- a/packages/dashboard-api/go.mod +++ b/packages/dashboard-api/go.mod @@ -25,6 +25,7 @@ require ( github.com/oapi-codegen/runtime v1.1.1 github.com/riverqueue/river v0.33.0 github.com/riverqueue/river/riverdriver/riverpgxv5 v0.33.0 + github.com/riverqueue/river/rivertype v0.33.0 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/metric v1.43.0 @@ -127,7 +128,6 @@ require ( github.com/redis/go-redis/v9 v9.17.3 // indirect github.com/riverqueue/river/riverdriver v0.33.0 // indirect github.com/riverqueue/river/rivershared v0.33.0 // indirect - github.com/riverqueue/river/rivertype v0.33.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/shirou/gopsutil/v4 v4.25.9 // indirect From 17d09f9a66b7d267f32c8eca3daf54fdcafeb2df Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 18:23:43 -0700 Subject: [PATCH 72/92] test(dashboard-api): trim low-signal provisioning coverage --- .../backgroundworker/auth_user_sync_test.go | 47 --- .../internal/handlers/team_handlers_test.go | 286 ------------------ .../internal/teamprovision/factory_test.go | 79 +++-- .../internal/teamprovision/http_sink_test.go | 82 ----- 4 files changed, 51 insertions(+), 443 deletions(-) diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go index 5ce6eca0db..1d4ddb9bf6 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go @@ -74,53 +74,6 @@ func TestAuthUserSync_EndToEnd(t *testing.T) { deleteAuthUser(t, ctx, db, userID) waitForPublicUserGone(t, ctx, db, userID) }) - - t.Run("burst backlog drains mixed upsert and delete work", func(t *testing.T) { - t.Parallel() - - ctx := t.Context() - const userCount = 40 - - type testUser struct { - id uuid.UUID - email string - shouldDel bool - } - - users := make([]testUser, 0, userCount) - for i := range userCount { - u := testUser{ - id: uuid.New(), - email: fmt.Sprintf("river-burst-%02d@example.com", i), - shouldDel: i%3 == 0, - } - users = append(users, u) - insertAuthUser(t, ctx, db, u.id, u.email) - } - - proc := startRiverWorker(t, db) - t.Cleanup(func() { proc.Stop(t) }) - - for _, u := range users { - waitForPublicUser(t, ctx, db, u.id, u.email) - } - - for _, u := range users { - if u.shouldDel { - deleteAuthUser(t, ctx, db, u.id) - } - } - - for _, u := range users { - if u.shouldDel { - waitForPublicUserGone(t, ctx, db, u.id) - - continue - } - - waitForPublicUser(t, ctx, db, u.id, u.email) - } - }) } func TestAuthUserSyncMigrations_AuthWritesSucceedBeforeRiverTablesExist(t *testing.T) { diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index b4ba902ae5..e468a2b4dc 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -419,50 +419,6 @@ func TestPostUsersBootstrap_ProvisioningFailureKeepsCreatedDefaultTeam(t *testin } } -func TestBootstrapUser_ProvisioningFailureWithExistingDefaultTeamStillSucceeds(t *testing.T) { - t.Parallel() - - testDB := testutils.SetupDatabase(t) - ctx := t.Context() - userID := createHandlerTestUser(t, testDB) - sink := &fakeTeamProvisionSink{ - err: &internalteamprovision.ProvisionError{ - StatusCode: http.StatusInternalServerError, - Message: "boom", - }, - } - - existingTeam, err := testDB.SqlcClient.GetDefaultTeamByUserID(ctx, userID) - if err != nil { - t.Fatalf("expected trigger-created default team: %v", err) - } - - store := &APIStore{ - db: testDB.SqlcClient, - authDB: testDB.AuthDB, - teamProvisionSink: sink, - } - - team, err := store.bootstrapUser(ctx, userID) - if err != nil { - t.Fatalf("expected bootstrap to succeed, got %v", err) - } - if len(sink.requests) != 1 { - t.Fatalf("expected one provisioning call, got %d", len(sink.requests)) - } - if team.ID != existingTeam.ID { - t.Fatalf("expected existing default team %s, got %s", existingTeam.ID, team.ID) - } - - rows, err := testDB.AuthDB.Read.GetTeamsWithUsersTeamsWithTier(ctx, userID) - if err != nil { - t.Fatalf("failed to query user teams: %v", err) - } - if len(rows) != 1 { - t.Fatalf("expected existing team to remain unchanged, got %d rows", len(rows)) - } -} - func TestBootstrapUser_ConcurrentRequestsCreateSingleDefaultTeam(t *testing.T) { t.Parallel() @@ -545,52 +501,6 @@ func TestBootstrapUser_ConcurrentRequestsCreateSingleDefaultTeam(t *testing.T) { } } -func TestCreateTeam_NoProvisionSinkLeavesCreatedTeam(t *testing.T) { - t.Parallel() - - testDB := testutils.SetupDatabase(t) - ctx := t.Context() - userID := createHandlerTestUser(t, testDB) - - store := &APIStore{ - db: testDB.SqlcClient, - authDB: testDB.AuthDB, - teamProvisionSink: &fakeTeamProvisionSink{}, - } - - team, err := store.createTeam(ctx, userID, "Acme") - if err != nil { - t.Fatalf("expected team creation to succeed without external provisioning, got %v", err) - } - - rows, err := testDB.AuthDB.Read.GetTeamsWithUsersTeamsWithTier(ctx, userID) - if err != nil { - t.Fatalf("failed to query user teams: %v", err) - } - if len(rows) != 2 { - t.Fatalf("expected default team and created team, got %d rows", len(rows)) - } - - found := false - for _, row := range rows { - if row.Team.ID == team.ID { - found = true - if row.IsDefault { - t.Fatal("expected manually created team not to be default") - } - } - } - if !found { - t.Fatal("expected created team to remain in local state") - } - if team.IsBlocked { - t.Fatal("expected older user team to remain unblocked") - } - if team.BlockedReason != nil { - t.Fatalf("expected no blocked reason, got %v", *team.BlockedReason) - } -} - func TestCreateTeam_RecentUserCreatesBlockedTeam(t *testing.T) { t.Parallel() @@ -616,54 +526,6 @@ func TestCreateTeam_RecentUserCreatesBlockedTeam(t *testing.T) { } } -func TestCreateTeam_BillingBadRequestStillReturnsCreatedTeam(t *testing.T) { - t.Parallel() - - testDB := testutils.SetupDatabase(t) - ctx := t.Context() - userID := createHandlerTestUser(t, testDB) - - store := &APIStore{ - db: testDB.SqlcClient, - authDB: testDB.AuthDB, - teamProvisionSink: &fakeTeamProvisionSink{ - err: &internalteamprovision.ProvisionError{ - StatusCode: http.StatusBadRequest, - Message: "limit reached", - }, - }, - } - - team, err := store.createTeam(ctx, userID, "Acme") - if err != nil { - t.Fatalf("expected createTeam to succeed, got %v", err) - } - if len(store.teamProvisionSink.(*fakeTeamProvisionSink).requests) != 1 { - t.Fatalf("expected one provisioning call, got %d", len(store.teamProvisionSink.(*fakeTeamProvisionSink).requests)) - } - - rows, err := testDB.AuthDB.Read.GetTeamsWithUsersTeamsWithTier(ctx, userID) - if err != nil { - t.Fatalf("failed to query user teams: %v", err) - } - if len(rows) != 2 { - t.Fatalf("expected default team and created team to remain, got %d rows", len(rows)) - } - - foundCreatedTeam := false - for _, row := range rows { - if row.Team.ID == team.ID { - foundCreatedTeam = true - if row.IsDefault { - t.Fatal("expected created team not to be default") - } - } - } - if !foundCreatedTeam { - t.Fatalf("expected created team %s to remain", team.ID) - } -} - func TestPostTeams_LocalPolicyDeniedReturnsBadRequestWithoutCreatingTeam(t *testing.T) { t.Parallel() @@ -720,35 +582,6 @@ func TestPostTeams_LocalPolicyDeniedReturnsBadRequestWithoutCreatingTeam(t *test } } -func TestPostTeams_ProvisioningSuccessReturnsTeam(t *testing.T) { - t.Parallel() - - testDB := testutils.SetupDatabase(t) - ctx := t.Context() - userID := createHandlerTestUser(t, testDB) - sink := &fakeTeamProvisionSink{} - - recorder := httptest.NewRecorder() - ginCtx, _ := gin.CreateTestContext(recorder) - ginCtx.Request = httptest.NewRequestWithContext(ctx, http.MethodPost, "/", strings.NewReader(`{"name":"Acme"}`)) - ginCtx.Request.Header.Set("Content-Type", "application/json") - auth.SetUserID(ginCtx, userID) - - store := &APIStore{ - db: testDB.SqlcClient, - authDB: testDB.AuthDB, - teamProvisionSink: sink, - } - store.PostTeams(ginCtx) - - if recorder.Code != http.StatusOK { - t.Fatalf("expected status 200, got %d", recorder.Code) - } - if len(sink.requests) != 1 { - t.Fatalf("expected one provisioning call, got %d", len(sink.requests)) - } -} - func TestPostTeams_InvalidNameReturnsBadRequest(t *testing.T) { t.Parallel() @@ -826,125 +659,6 @@ func TestPostTeams_TrimsNameBeforeCreate(t *testing.T) { } } -func TestPostTeams_ProvisioningFailureStillReturnsTeam(t *testing.T) { - t.Parallel() - - testDB := testutils.SetupDatabase(t) - ctx := t.Context() - userID := createHandlerTestUser(t, testDB) - sink := &fakeTeamProvisionSink{ - err: &internalteamprovision.ProvisionError{ - StatusCode: http.StatusInternalServerError, - Message: "boom", - }, - } - - recorder := httptest.NewRecorder() - ginCtx, _ := gin.CreateTestContext(recorder) - ginCtx.Request = httptest.NewRequestWithContext(ctx, http.MethodPost, "/", strings.NewReader(`{"name":"Acme"}`)) - ginCtx.Request.Header.Set("Content-Type", "application/json") - auth.SetUserID(ginCtx, userID) - - store := &APIStore{ - db: testDB.SqlcClient, - authDB: testDB.AuthDB, - teamProvisionSink: sink, - } - store.PostTeams(ginCtx) - - if recorder.Code != http.StatusOK { - t.Fatalf("expected status 200, got %d", recorder.Code) - } - if len(sink.requests) != 1 { - t.Fatalf("expected one provisioning call, got %d", len(sink.requests)) - } - - rows, err := testDB.AuthDB.Read.GetTeamsWithUsersTeamsWithTier(ctx, userID) - if err != nil { - t.Fatalf("failed to query user teams: %v", err) - } - if len(rows) != 2 { - t.Fatalf("expected default team and created team to remain, got %d rows", len(rows)) - } -} - -func TestCreateTeam_ConcurrentRequestsRespectLocalPolicy(t *testing.T) { - t.Parallel() - - testDB := testutils.SetupDatabase(t) - ctx := t.Context() - userID := createHandlerTestUser(t, testDB) - - for range 1 { - team, err := testDB.SqlcClient.CreateTeam(ctx, queries.CreateTeamParams{ - Name: "extra", - Tier: baseTierID, - Email: handlerTestUserEmail(userID), - }) - if err != nil { - t.Fatalf("failed to create extra team: %v", err) - } - if err := testDB.SqlcClient.CreateTeamMembership(ctx, queries.CreateTeamMembershipParams{ - UserID: userID, - TeamID: team.ID, - IsDefault: false, - AddedBy: &userID, - }); err != nil { - t.Fatalf("failed to attach extra team membership: %v", err) - } - } - - store := &APIStore{ - db: testDB.SqlcClient, - authDB: testDB.AuthDB, - teamProvisionSink: &fakeTeamProvisionSink{}, - } - - var wg sync.WaitGroup - results := make(chan error, 2) - - for _, name := range []string{"Acme-1", "Acme-2"} { - wg.Add(1) - go func(teamName string) { - defer wg.Done() - _, err := store.createTeam(ctx, userID, teamName) - results <- err - }(name) - } - - wg.Wait() - close(results) - - var successCount int - var badRequestCount int - for err := range results { - if err == nil { - successCount++ - - continue - } - - var provisionErr *internalteamprovision.ProvisionError - if !errors.As(err, &provisionErr) { - t.Fatalf("expected provisioning error, got %T: %v", err, err) - } - if provisionErr.StatusCode == http.StatusBadRequest { - badRequestCount++ - - continue - } - - t.Fatalf("expected bad request or success, got %d", provisionErr.StatusCode) - } - - if successCount != 1 { - t.Fatalf("expected one success, got %d", successCount) - } - if badRequestCount != 1 { - t.Fatalf("expected one bad request, got %d", badRequestCount) - } -} - func TestCreateTeam_ConcurrentRequestsRespectLocalPolicyWithZeroMemberships(t *testing.T) { t.Parallel() diff --git a/packages/dashboard-api/internal/teamprovision/factory_test.go b/packages/dashboard-api/internal/teamprovision/factory_test.go index 407c608d4a..2a9d5f6796 100644 --- a/packages/dashboard-api/internal/teamprovision/factory_test.go +++ b/packages/dashboard-api/internal/teamprovision/factory_test.go @@ -6,34 +6,57 @@ import ( "github.com/stretchr/testify/require" ) -func TestNewProvisionSink_DisabledReturnsNoop(t *testing.T) { +func TestNewProvisionSink(t *testing.T) { t.Parallel() - sink, err := NewProvisionSink(false, "", "") - require.NoError(t, err) - require.IsType(t, &NoopProvisionSink{}, sink) -} - -func TestNewProvisionSink_EnabledRequiresBaseURL(t *testing.T) { - t.Parallel() - - sink, err := NewProvisionSink(true, "", "token") - require.Nil(t, sink) - require.ErrorIs(t, err, ErrMissingBaseURL) -} - -func TestNewProvisionSink_EnabledRequiresAPIToken(t *testing.T) { - t.Parallel() - - sink, err := NewProvisionSink(true, "https://billing.example.com", "") - require.Nil(t, sink) - require.ErrorIs(t, err, ErrMissingAPIToken) -} - -func TestNewProvisionSink_EnabledReturnsHTTPSink(t *testing.T) { - t.Parallel() - - sink, err := NewProvisionSink(true, "https://billing.example.com", "token") - require.NoError(t, err) - require.IsType(t, &HTTPProvisionSink{}, sink) + tests := []struct { + name string + enabled bool + baseURL string + apiToken string + wantErr error + wantType any + }{ + { + name: "disabled returns noop", + enabled: false, + wantType: &NoopProvisionSink{}, + }, + { + name: "enabled requires base url", + enabled: true, + apiToken: "token", + wantErr: ErrMissingBaseURL, + }, + { + name: "enabled requires api token", + enabled: true, + baseURL: "https://billing.example.com", + wantErr: ErrMissingAPIToken, + }, + { + name: "enabled returns http sink", + enabled: true, + baseURL: "https://billing.example.com", + apiToken: "token", + wantType: &HTTPProvisionSink{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + sink, err := NewProvisionSink(tt.enabled, tt.baseURL, tt.apiToken) + if tt.wantErr != nil { + require.Nil(t, sink) + require.ErrorIs(t, err, tt.wantErr) + + return + } + + require.NoError(t, err) + require.IsType(t, tt.wantType, sink) + }) + } } diff --git a/packages/dashboard-api/internal/teamprovision/http_sink_test.go b/packages/dashboard-api/internal/teamprovision/http_sink_test.go index eda7f843ce..ca8fb22d7c 100644 --- a/packages/dashboard-api/internal/teamprovision/http_sink_test.go +++ b/packages/dashboard-api/internal/teamprovision/http_sink_test.go @@ -1,11 +1,8 @@ package teamprovision import ( - "errors" - "io" "net/http" "net/http/httptest" - "strings" "sync/atomic" "testing" "time" @@ -38,53 +35,6 @@ func TestHTTPProvisionSink_ReturnsJSONErrorMessage(t *testing.T) { require.EqualValues(t, 1, requestCount.Load()) } -func TestHTTPProvisionSink_FallsBackToPlainTextResponse(t *testing.T) { - t.Parallel() - - var requestCount atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - requestCount.Add(1) - w.WriteHeader(http.StatusBadGateway) - _, _ = w.Write([]byte("upstream gateway exploded")) - })) - defer server.Close() - - sink := NewHTTPProvisionSink(server.URL, "token") - sink.client.RetryWaitMin = time.Millisecond - sink.client.RetryWaitMax = time.Millisecond - err := sink.ProvisionTeam(t.Context(), testProvisionRequest()) - require.Error(t, err) - - var provisionErr *ProvisionError - require.ErrorAs(t, err, &provisionErr) - require.Equal(t, http.StatusBadGateway, provisionErr.StatusCode) - require.Equal(t, "upstream gateway exploded", provisionErr.Message) - require.EqualValues(t, sink.client.RetryMax+1, requestCount.Load()) -} - -func TestHTTPProvisionSink_FallsBackToStatusTextForEmptyBody(t *testing.T) { - t.Parallel() - - var requestCount atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - requestCount.Add(1) - w.WriteHeader(http.StatusServiceUnavailable) - })) - defer server.Close() - - sink := NewHTTPProvisionSink(server.URL, "token") - sink.client.RetryWaitMin = time.Millisecond - sink.client.RetryWaitMax = time.Millisecond - err := sink.ProvisionTeam(t.Context(), testProvisionRequest()) - require.Error(t, err) - - var provisionErr *ProvisionError - require.ErrorAs(t, err, &provisionErr) - require.Equal(t, http.StatusServiceUnavailable, provisionErr.StatusCode) - require.Equal(t, http.StatusText(http.StatusServiceUnavailable), provisionErr.Message) - require.EqualValues(t, sink.client.RetryMax+1, requestCount.Load()) -} - func TestHTTPProvisionSink_RetriesRetryableResponsesAndSucceeds(t *testing.T) { t.Parallel() @@ -110,32 +60,6 @@ func TestHTTPProvisionSink_RetriesRetryableResponsesAndSucceeds(t *testing.T) { require.EqualValues(t, 2, requestCount.Load()) } -func TestHTTPProvisionSink_RetriesTransportErrorsAndSucceeds(t *testing.T) { - t.Parallel() - - sink := NewHTTPProvisionSink("http://billing.example", "token") - sink.client.RetryWaitMin = time.Millisecond - sink.client.RetryWaitMax = time.Millisecond - - var attemptCount atomic.Int32 - sink.client.HTTPClient.Transport = roundTripFunc(func(_ *http.Request) (*http.Response, error) { - attempt := attemptCount.Add(1) - if attempt == 1 { - return nil, errors.New("temporary dial failure") - } - - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(strings.NewReader("")), - Header: make(http.Header), - }, nil - }) - - err := sink.ProvisionTeam(t.Context(), testProvisionRequest()) - require.NoError(t, err) - require.EqualValues(t, 2, attemptCount.Load()) -} - func TestHTTPProvisionSink_RetriesRequestTimeoutWithinOverallBudget(t *testing.T) { t.Parallel() @@ -170,9 +94,3 @@ func testProvisionRequest() sharedteamprovision.TeamBillingProvisionRequestedV1 Reason: sharedteamprovision.ReasonAdditionalTeam, } } - -type roundTripFunc func(req *http.Request) (*http.Response, error) - -func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { - return f(req) -} From bb310945688053fdb16c6283e83437b010592c3d Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 18:40:41 -0700 Subject: [PATCH 73/92] feat(dashboard-api): add supabase db config override Allow dashboard-api to use a dedicated SUPABASE_DB_CONNECTION_STRING while keeping the existing fallback to POSTGRES_CONNECTION_STRING. Thread the new setting through Terraform so deployments can configure the worker without reusing the auth DB connection string. --- .env.gcp.template | 2 ++ iac/modules/job-dashboard-api/main.tf | 1 + iac/modules/job-dashboard-api/variables.tf | 6 ++++++ iac/provider-gcp/Makefile | 1 + iac/provider-gcp/main.tf | 5 +++-- iac/provider-gcp/nomad/main.tf | 1 + iac/provider-gcp/nomad/variables.tf | 6 ++++++ iac/provider-gcp/variables.tf | 6 ++++++ packages/dashboard-api/internal/cfg/model.go | 5 +++++ packages/dashboard-api/main.go | 2 +- 10 files changed, 32 insertions(+), 3 deletions(-) diff --git a/.env.gcp.template b/.env.gcp.template index 034e1d45a5..65eba5e604 100644 --- a/.env.gcp.template +++ b/.env.gcp.template @@ -77,6 +77,8 @@ CLICKHOUSE_CLUSTER_SIZE=1 # Dashboard API instance count (default: 0) DASHBOARD_API_COUNT= +# Dashboard API Supabase DB connection string (default: POSTGRES_CONNECTION_STRING) +SUPABASE_DB_CONNECTION_STRING= # Additional dashboard-api env vars passed directly to the Nomad job (default: {}) # Values here are merged into the job env and can override module defaults. # Example: '{"ENABLE_AUTH_USER_SYNC_BACKGROUND_WORKER":"true"}' diff --git a/iac/modules/job-dashboard-api/main.tf b/iac/modules/job-dashboard-api/main.tf index 68476d109a..3131278064 100644 --- a/iac/modules/job-dashboard-api/main.tf +++ b/iac/modules/job-dashboard-api/main.tf @@ -7,6 +7,7 @@ locals { POSTGRES_CONNECTION_STRING = var.postgres_connection_string AUTH_DB_CONNECTION_STRING = var.auth_db_connection_string AUTH_DB_READ_REPLICA_CONNECTION_STRING = var.auth_db_read_replica_connection_string + SUPABASE_DB_CONNECTION_STRING = var.supabase_db_connection_string CLICKHOUSE_CONNECTION_STRING = var.clickhouse_connection_string SUPABASE_JWT_SECRETS = var.supabase_jwt_secrets REDIS_URL = var.redis_url diff --git a/iac/modules/job-dashboard-api/variables.tf b/iac/modules/job-dashboard-api/variables.tf index 99ce63284b..1bbb149914 100644 --- a/iac/modules/job-dashboard-api/variables.tf +++ b/iac/modules/job-dashboard-api/variables.tf @@ -34,6 +34,12 @@ variable "auth_db_read_replica_connection_string" { default = "" } +variable "supabase_db_connection_string" { + type = string + sensitive = true + default = "" +} + variable "clickhouse_connection_string" { type = string sensitive = true diff --git a/iac/provider-gcp/Makefile b/iac/provider-gcp/Makefile index dfd51d5c83..b30c8859db 100644 --- a/iac/provider-gcp/Makefile +++ b/iac/provider-gcp/Makefile @@ -77,6 +77,7 @@ tf_vars := \ $(call tfvar, LOKI_USE_V13_SCHEMA_FROM) \ $(call tfvar, DASHBOARD_API_COUNT) \ $(call tfvar, DASHBOARD_API_ENV_VARS) \ + $(call tfvar, SUPABASE_DB_CONNECTION_STRING) \ $(call tfvar, DEFAULT_PERSISTENT_VOLUME_TYPE) \ $(call tfvar, PERSISTENT_VOLUME_TYPES) \ $(call tfvar, DB_MAX_OPEN_CONNECTIONS) \ diff --git a/iac/provider-gcp/main.tf b/iac/provider-gcp/main.tf index b25d0ae0f5..7e77aea9ad 100644 --- a/iac/provider-gcp/main.tf +++ b/iac/provider-gcp/main.tf @@ -275,8 +275,9 @@ module "nomad" { otel_collector_resources_cpu_count = var.otel_collector_resources_cpu_count # Dashboard API - dashboard_api_count = var.dashboard_api_count - dashboard_api_env_vars = var.dashboard_api_env_vars + dashboard_api_count = var.dashboard_api_count + dashboard_api_env_vars = var.dashboard_api_env_vars + supabase_db_connection_string = var.supabase_db_connection_string # Docker reverse proxy docker_reverse_proxy_port = var.docker_reverse_proxy_port diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index 5f937179fe..0107c701a2 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -133,6 +133,7 @@ module "dashboard_api" { postgres_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data auth_db_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data auth_db_read_replica_connection_string = trimspace(data.google_secret_manager_secret_version.postgres_read_replica_connection_string.secret_data) + supabase_db_connection_string = var.supabase_db_connection_string clickhouse_connection_string = local.clickhouse_connection_string supabase_jwt_secrets = trimspace(data.google_secret_manager_secret_version.supabase_jwt_secrets.secret_data) redis_url = local.redis_url diff --git a/iac/provider-gcp/nomad/variables.tf b/iac/provider-gcp/nomad/variables.tf index cc70a3bf1a..63caaa5207 100644 --- a/iac/provider-gcp/nomad/variables.tf +++ b/iac/provider-gcp/nomad/variables.tf @@ -459,6 +459,12 @@ variable "dashboard_api_env_vars" { default = {} } +variable "supabase_db_connection_string" { + type = string + default = "" + sensitive = true +} + variable "volume_token_issuer" { type = string } diff --git a/iac/provider-gcp/variables.tf b/iac/provider-gcp/variables.tf index 0a1bc8d7c4..0f3c1459b9 100644 --- a/iac/provider-gcp/variables.tf +++ b/iac/provider-gcp/variables.tf @@ -235,6 +235,12 @@ variable "dashboard_api_env_vars" { default = {} } +variable "supabase_db_connection_string" { + type = string + default = "" + sensitive = true +} + variable "docker_reverse_proxy_port" { type = object({ name = string diff --git a/packages/dashboard-api/internal/cfg/model.go b/packages/dashboard-api/internal/cfg/model.go index 94365f06c9..3743f45372 100644 --- a/packages/dashboard-api/internal/cfg/model.go +++ b/packages/dashboard-api/internal/cfg/model.go @@ -14,6 +14,7 @@ type Config struct { AuthDBConnectionString string `env:"AUTH_DB_CONNECTION_STRING"` AuthDBReadReplicaConnectionString string `env:"AUTH_DB_READ_REPLICA_CONNECTION_STRING"` + SupabaseDBConnectionString string `env:"SUPABASE_DB_CONNECTION_STRING"` RedisURL string `env:"REDIS_URL"` RedisClusterURL string `env:"REDIS_CLUSTER_URL"` @@ -30,6 +31,10 @@ func Parse() (Config, error) { config.AuthDBConnectionString = config.PostgresConnectionString } + if config.SupabaseDBConnectionString == "" { + config.SupabaseDBConnectionString = config.PostgresConnectionString + } + if err == nil && config.RedisURL == "" && config.RedisClusterURL == "" { err = fmt.Errorf("at least one of REDIS_URL or REDIS_CLUSTER_URL must be set") } diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 79fdd55e0c..56d8a98f04 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -188,7 +188,7 @@ func run() int { if config.EnableAuthUserSyncBackgroundWorker { supabaseDB, err = supabasedb.NewClient( ctx, - config.AuthDBConnectionString, + config.SupabaseDBConnectionString, pool.WithMaxConnections(4), ) if err != nil { From 62cac529bf395da3bcb0ea6b22aa403d1e43f12a Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 18:44:07 -0700 Subject: [PATCH 74/92] fix(dashboard-api): require admin auth for bootstrap --- iac/modules/job-dashboard-api/main.tf | 1 + iac/modules/job-dashboard-api/variables.tf | 5 + iac/provider-gcp/api.tf | 20 ++- iac/provider-gcp/main.tf | 5 +- iac/provider-gcp/nomad/main.tf | 1 + iac/provider-gcp/nomad/variables.tf | 4 + .../dashboard-api/internal/api/api.gen.go | 140 +++++++++--------- packages/dashboard-api/internal/cfg/model.go | 1 + .../internal/handlers/team_handlers_test.go | 4 +- .../internal/handlers/team_provisioning.go | 2 +- packages/dashboard-api/main.go | 2 + spec/openapi-dashboard.yml | 9 +- 12 files changed, 118 insertions(+), 76 deletions(-) diff --git a/iac/modules/job-dashboard-api/main.tf b/iac/modules/job-dashboard-api/main.tf index 85561e1cdc..5c130b997c 100644 --- a/iac/modules/job-dashboard-api/main.tf +++ b/iac/modules/job-dashboard-api/main.tf @@ -4,6 +4,7 @@ locals { ENVIRONMENT = var.environment NODE_ID = "$${node.unique.id}" PORT = "$${NOMAD_PORT_api}" + ADMIN_TOKEN = var.admin_token POSTGRES_CONNECTION_STRING = var.postgres_connection_string AUTH_DB_CONNECTION_STRING = var.auth_db_connection_string AUTH_DB_READ_REPLICA_CONNECTION_STRING = var.auth_db_read_replica_connection_string diff --git a/iac/modules/job-dashboard-api/variables.tf b/iac/modules/job-dashboard-api/variables.tf index c5d4ca72a7..15110d04fc 100644 --- a/iac/modules/job-dashboard-api/variables.tf +++ b/iac/modules/job-dashboard-api/variables.tf @@ -23,6 +23,11 @@ variable "postgres_connection_string" { sensitive = true } +variable "admin_token" { + type = string + sensitive = true +} + variable "auth_db_connection_string" { type = string sensitive = true diff --git a/iac/provider-gcp/api.tf b/iac/provider-gcp/api.tf index 26f1a1b408..d62d201601 100644 --- a/iac/provider-gcp/api.tf +++ b/iac/provider-gcp/api.tf @@ -64,6 +64,24 @@ resource "google_secret_manager_secret_version" "api_admin_token_value" { secret_data = random_password.api_admin_secret.result } +resource "random_password" "dashboard_api_admin_secret" { + length = 32 + special = true +} + +resource "google_secret_manager_secret" "dashboard_api_admin_token" { + secret_id = "${var.prefix}dashboard-api-admin-token" + + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "dashboard_api_admin_token_value" { + secret = google_secret_manager_secret.dashboard_api_admin_token.id + secret_data = random_password.dashboard_api_admin_secret.result +} + resource "random_password" "sandbox_access_token_hash_seed" { length = 32 special = false @@ -80,4 +98,4 @@ resource "google_secret_manager_secret" "sandbox_access_token_hash_seed" { resource "google_secret_manager_secret_version" "sandbox_access_token_hash_seed" { secret = google_secret_manager_secret.sandbox_access_token_hash_seed.id secret_data = random_password.sandbox_access_token_hash_seed.result -} \ No newline at end of file +} diff --git a/iac/provider-gcp/main.tf b/iac/provider-gcp/main.tf index 251f251609..e35d0a7e6b 100644 --- a/iac/provider-gcp/main.tf +++ b/iac/provider-gcp/main.tf @@ -274,8 +274,9 @@ module "nomad" { otel_collector_resources_cpu_count = var.otel_collector_resources_cpu_count # Dashboard API - dashboard_api_count = var.dashboard_api_count - dashboard_api_env_vars = var.dashboard_api_env_vars + dashboard_api_count = var.dashboard_api_count + dashboard_api_admin_token = random_password.dashboard_api_admin_secret.result + dashboard_api_env_vars = var.dashboard_api_env_vars # Docker reverse proxy docker_reverse_proxy_port = var.docker_reverse_proxy_port diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index bea62e8ea3..7a8d5c8e7c 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -152,6 +152,7 @@ module "dashboard_api" { image = data.google_artifact_registry_docker_image.dashboard_api_image[0].self_link + admin_token = var.dashboard_api_admin_token postgres_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data auth_db_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data auth_db_read_replica_connection_string = trimspace(data.google_secret_manager_secret_version.postgres_read_replica_connection_string.secret_data) diff --git a/iac/provider-gcp/nomad/variables.tf b/iac/provider-gcp/nomad/variables.tf index a6d7a3bbdd..d37e02f531 100644 --- a/iac/provider-gcp/nomad/variables.tf +++ b/iac/provider-gcp/nomad/variables.tf @@ -98,6 +98,10 @@ variable "api_admin_token" { type = string } +variable "dashboard_api_admin_token" { + type = string +} + variable "sandbox_access_token_hash_seed" { type = string } diff --git a/packages/dashboard-api/internal/api/api.gen.go b/packages/dashboard-api/internal/api/api.gen.go index afaf2c5fd1..1d92e1413b 100644 --- a/packages/dashboard-api/internal/api/api.gen.go +++ b/packages/dashboard-api/internal/api/api.gen.go @@ -21,6 +21,7 @@ import ( ) const ( + AdminTokenAuthScopes = "AdminTokenAuth.Scopes" Supabase1TokenAuthScopes = "Supabase1TokenAuth.Scopes" Supabase2TeamAuthScopes = "Supabase2TeamAuth.Scopes" ) @@ -342,6 +343,9 @@ type PostTeamsTeamIDMembersJSONRequestBody = AddTeamMemberRequest // ServerInterface represents all server handlers. type ServerInterface interface { + // Bootstrap user + // (POST /admin/users/bootstrap) + PostAdminUsersBootstrap(c *gin.Context) // List team builds // (GET /builds) GetBuilds(c *gin.Context, params GetBuildsParams) @@ -381,9 +385,6 @@ type ServerInterface interface { // List default templates // (GET /templates/defaults) GetTemplatesDefaults(c *gin.Context) - // Bootstrap user - // (POST /users/bootstrap) - PostUsersBootstrap(c *gin.Context) } // ServerInterfaceWrapper converts contexts to parameters. @@ -395,6 +396,23 @@ type ServerInterfaceWrapper struct { type MiddlewareFunc func(c *gin.Context) +// PostAdminUsersBootstrap operation middleware +func (siw *ServerInterfaceWrapper) PostAdminUsersBootstrap(c *gin.Context) { + + c.Set(AdminTokenAuthScopes, []string{}) + + c.Set(Supabase1TokenAuthScopes, []string{}) + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.PostAdminUsersBootstrap(c) +} + // GetBuilds operation middleware func (siw *ServerInterfaceWrapper) GetBuilds(c *gin.Context) { @@ -756,21 +774,6 @@ func (siw *ServerInterfaceWrapper) GetTemplatesDefaults(c *gin.Context) { siw.Handler.GetTemplatesDefaults(c) } -// PostUsersBootstrap operation middleware -func (siw *ServerInterfaceWrapper) PostUsersBootstrap(c *gin.Context) { - - c.Set(Supabase1TokenAuthScopes, []string{}) - - for _, middleware := range siw.HandlerMiddlewares { - middleware(c) - if c.IsAborted() { - return - } - } - - siw.Handler.PostUsersBootstrap(c) -} - // GinServerOptions provides options for the Gin server. type GinServerOptions struct { BaseURL string @@ -798,6 +801,7 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options ErrorHandler: errorHandler, } + router.POST(options.BaseURL+"/admin/users/bootstrap", wrapper.PostAdminUsersBootstrap) router.GET(options.BaseURL+"/builds", wrapper.GetBuilds) router.GET(options.BaseURL+"/builds/statuses", wrapper.GetBuildsStatuses) router.GET(options.BaseURL+"/builds/:build_id", wrapper.GetBuildsBuildId) @@ -811,61 +815,61 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options router.POST(options.BaseURL+"/teams/:teamID/members", wrapper.PostTeamsTeamIDMembers) router.DELETE(options.BaseURL+"/teams/:teamID/members/:userId", wrapper.DeleteTeamsTeamIDMembersUserId) router.GET(options.BaseURL+"/templates/defaults", wrapper.GetTemplatesDefaults) - router.POST(options.BaseURL+"/users/bootstrap", wrapper.PostUsersBootstrap) } // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xcWXPcuPH/Kij+/1V5oWdGlp1K9KZjd62Klah0bKXKpZIwZM8M1iTABUAdq8x3T+Ek", - "OQSPkTWOstkna0gAff260d0A/RwlLC8YBSpFdPAcFZjjHCRw/Wtekiy9Jan6OwWRcFJIwmh0EJ2mQCVZ", - "EOCILZBcAdJjJ1EcEfW+wHIVxRHFOUQH1TpxxOHXknBIowPJS4gjkawgx4rAgvEcy+ggKks9Uj4Vaq6Q", - "nNBltF7Hfplbxm8l5EWGJbRZ+4f+A2doQTIJHM2fDG+IeJ5j5KY3HjJePccZwcKL82sJ/KktT4ORuizd", - "vIs2w8csz/E7AUr3ElKUESGVVg3XpycCSYaWIJGQWJYCBFowrliDxyJjKUQHC5wJ6GdV9OqeSMjFCCPE", - "UY4fT83gvdnMv8ecY0W0pOTXEuwARWQdR0I+ZWqMWjrymnCybKsOrwPJEKFJVqYwVhWeZFDy/+ewiA6i", - "/5tWDjE1w8T0SJG+1NOVBA2huyQUt0nJBeMBAfVzxEGWnEKqAKocqOBwT1gpjMAcRMGoAEQouks4KFXc", - "YvkvZ887ZEzVBVFLfAQoxW1GciLbfJ7hR5KXOaJlPjd+rpWlNG94RwVwVOAldDFhFq7zkMICl5mMDj7O", - "4gpshMr995EGl6JosZUTan95lRMqYQlcMy8wTefs8fRkTHSygzviU7VUn5O09ScB5+Poq5EdxO0i3xYa", - "1SKXWbls83IFOEciK5fGboJl9532UsO2VEEpgJ+O2iDUyA4V2EW+RQVrNdm4jHbnD7OZ+idhVALV4MZF", - "kZEEK/6mvwjF5HNt/T73/4Fzxg2NppBHOEWKZRBS+f2H2d7uaR6WcqVUa1ZFYMYp4vu7J/4j43OSpkAN", - "xQ+7p/h3JtGClTRVFD9+D6NeAr8H7hS7diDUqDpMU+VPZ6Ai4oW1vEqbOCuAS2KwBzkmWQO05knIcSvE", - "f7GjbvwwNv8FEo0svQGd0gVrE7N7w2EggOtZSA9QUJEkByFxXqg95eLH4/39/b/WdhHPbIolvFODQ/v/", - "glAiVr30WF5kMEQxRmSB3GKd5GmZZXiuNlcTD1rsqAAiQkHPpnH6PeKQ6VRCMiRXRJhUQnOA7zHRFHRk", - "crlAm0yYj1oGoHOD7dIIM+kMhMDLQB77IyZZyQHlZgB6WAG16Q8iAt0tMMkgvYsRkyvgD0QAulN83k2G", - "FbcBvApDDQN7uTZ57YTopddDCBmW+RwXBaQKByjFYjVnmKcoyYjSlc7lqNr0v5jsRLEbR0ZWxUeZJCBE", - "jYPKSDUOVAbadpU3ht0t66rB1Py/HIRaKA+4AAwH0Sc+EyEvbBbQNn+K5fiUXy0FqV42lPJTeJTH/fm9", - "ZKjAQpigA0jNcKm93jd0vWmUpfCk9AdKp5SZsS6x3k6LWsgGf93qurQFUbfK5hVYQmH2c7A0q0fSkUjU", - "/tpS84ZoTWZCYh2fXx+zkgbc+/j8GiWMm9q5XhFEzTLkzx+i/sIjjo51sFRpQGcCYNLaZ1XPfAa6lKvo", - "4P3Hj3ph93tvyJB6jZCQJ6aEuqo1QJrUdevC/DnKDhsLHqrpIcxr/Xv9toq3tqb0BFMcDAavRhozLhUB", - "ep/+DFwQk/aNjLetx0U5z0hSezVnLAOsU1yO87P5prQaI21pRYEfaFA9HRMkkzg7IeLrJfkNOsh0CFVb", - "5T4pylEEQ+HWQaWylZPZLtzmMm5kC1Z5DXA0VDECwQZwYRiHszGV1BU4gRFm35DaLDqCqZ6g6Dp+L/aw", - "wUhXUQhy6oxx1I5z6h0S5DfYjHMqizkjR73hbhbCl6mT2mWH7rZtkteDkXo3ieIxISLvSjzMSvb1ZLB0", - "0uxUy4XU9glwJlfdZu1k5VOZY/qOA04VztBKr4OSFSRfEQdRZnKYvz7G6qnGH+XdHylyg+Puc46rxlGF", - "IVtwEEBlnZY/0Tg9mUQ9BMY18dzoYcQbA1SHIzU6nXVl3FWJhtzmDHLGn0JB0Lx5SQTce/+XUJS6NCtc", - "QMJ42rNTbXTqtF02FBfMfYrS5w19uPTp7TqO0sYm0Lv5VCPVPJZjQgPOjQUg81JBiUNDdZLjxYIkCs9Y", - "F+BEYXYEfPOakfqY9MZ8YWO/w9l5R+i8Irl11LqUD1ggO2l0wBSSFcX2RPSkF4dF70sn2/gsWnCWo4cV", - "SVbKkHWmrNsNOnWNcNw4Nal0XYNzzfwNwIa8uWqrBvwrTSE9egrVEYOqGq4rBpfw7dyO7Wlw1yHixB17", - "tYuMUNh07eJqYl2QfvWJvgxHDxidttZsMpSxuqW7eLswB0/dvI1UpbBnXGNaSWpoiJ/rIm0X8Dmh5zWG", - "9uLXKOn1IguSwTlJZMnhmmfjSpZenr9RhU6S1+K1pfjOzsW1AK5ECPSZMpZ8hfQCsBhZzL+CUx5hSiEN", - "F/5EHBmWul73eHRszr0H3cup47MZ/dqm6XSWOJLEhNmxxozdybCeWMWnNlt1zdV0HG9YuBnarLr6IPPZ", - "a3SzDKVJyTlQaXM0ECObU9VMl0ibpujI6Wo7a/dsuqpcFzQ+sZKLke2hHD9ehNpP3TR+DrSCgqM3g3eT", - "vTio1R6NVcRrXHsV9Zm1t8uC8/FblQ8tw60VtWybJ+UukJScyKdLtaZh4rIs8BwL2LtiX4EelirMP5sb", - "DCvAqXYGe4fhn+/c4Hd6cKV2XJC/ge6guhHvFaujV1NitRZTDBN7ICyJ1PePfnh/hE78idrh+WkUR/eu", - "QRrNJnuTmeKCFUBxQaKDaH8ym8yUH2O50vJO594HlqCDmzKJ7i+o+jD6CaS3ef2q4Jewdaoh0+CVuXU8", - "cp4/Whg7w11qGj/eXpha32xcJHn/incOAqdUoQsI5oxzUWbZU3VLrMBLQvUptmF4Yq5gzLpoeiGmalB1", - "O2Vo7F7tMsnQ2P3apYz+sWpQ3cc0ZELe9eUm6CZfbpRhRJnnmD+5kyfly1YbykHwUvhjIhHdKHLWuNP6", - "bcN+YF9Wh1gvA7j4HhBqndyNhtHGUd3/MoZ+Atk+uexD0bOz8XoYR0f+POVlMPoOKNL3ibYETgoSk8wF", - "n92Awd4rGxr74Q0Ax6qjCzfmqKAPLOZQItqhqTeOPQL2/lQ/0BDe+EZlXuj6KP1qKlxuOH32raD1lPsm", - "aZfMPqe8dLNsY3VbX6kaUDt1lmb3d7TDuN7aHy5jXcYphDtrO5/xQLJu41N/i6Cmti+0ggXCWaYzANPJ", - "xNW1WEj1XWM0h4zRpUCSxeiByBUydSbCVDmuLj7RIsPLSRS3Qaqrk136ZbsEGo0sLZ0WfWtQvbbxA1lZ", - "xV3NxLbsWsdRwUQgLJwzUVO57sodsfTp1bTdvrSzblaG9mOVnZk71AQdMrht+dqPB3aYqu0WFUb3WooA", - "IrzDT+3HCT2Or98LhE3m7z5qcN9X/EnYr7nkU4zucUZSLAld+o8P9OkVMp3qbp+3VLbejPwXGDvdi14C", - "I6vXBo5+f9tQE3NWRwYoDhW96Hs23+GszVeQMlkFgpR6rEFy5b7Z2R4jPlt5/SDXPtj4zkEucEoxBM5S", - "T/kOMe7Nl6NGecNh0gF1WjvO60q1a2C1p4PfhtkdRrXN08vR2ZB2cauLye+40ZV7A26dVL0iAl4/agW/", - "qRoVuPbaOUIDIvreQF15bybAvPl67TBtKG6rgDR9Nt9zro15MjC315rYPNHP2+i8dp+Cvgyjw+1++61p", - "IJ59GIATh5zdv1FA/UdAcqEVMgon9j7z1Nbdw9W9Strdl/+uWPfLmHJeroBwpB+4fhyhC6bre3uxvSPN", - "t8ucOGZ2uLV1Xisfvb+1pH+LRX+LyQYS/G12jQblgWI6Z0wKyXGhE+7O/UvFA3Hkx76x0soLUdi2zBuz", - "jVecZi7knWv/7LnxX4aYo9bm/48AjYdmicYDZ+f1zfrfAQAA//+hahcBWUYAAA==", + "H4sIAAAAAAAC/+xcW2/juPX/KoT+f6AvGtu5TNHmLZfdnaCTNoiTRYFgkNDSsc0didSQVBJv6u9e8CJK", + "sqiLM3GabudpPBLJc/udw3MOqTwHEUszRoFKERw9BxnmOAUJXP9vlpMkviOx+h2DiDjJJGE0OArOY6CS", + "zAlwxOZILgHpsaMgDIh6n2G5DMKA4hSCo3KdMODwLScc4uBI8hzCQERLSLEiMGc8xTI4CvJcj5SrTM0V", + "khO6CNbr0C1zx/idhDRLsIQma//QP3CC5iSRwNFsZXhDxPEcomJ67SHj5XOcECycON9y4KumPDVGqrK0", + "8y6aDJ+yNMUfBCjdS4hRQoRUWjVcn58JJBlagERCYpkLEGjOuGINnrKExRAczXEioJtV0al7IiEVA4wQ", + "Bil+OjeD9yYT9x5zjhXRnJJvOdgBisg6DIRcJWqMWjpwmihk2VYdTgeSIUKjJI9hqCocSa/k/89hHhwF", + "/zcuHWJshonxiSI91dOVBDWh2yQUd1HOBeMeAfVzxEHmnEKsAKocKOPwQFgujMAcRMaoAEQouo84KFXc", + "Yfmvwp73yJiqDaKW+ABQiruEpEQ2+bzATyTNU0TzdGb8XCtLad7wjjLgKMMLaGPCLFzlIYY5zhMZHH2c", + "hCXYCJUH+4EGl6JosZUSav/nVE6ohAVwzbzANJ6xp/OzIdHJDm6JT+VSXU7S1J8EnA6jr0a2ELeLfF9o", + "VItMk3zR5OUacIpEki+M3QRLHlrtpYZtqYJcAD8ftEGokS0qsIt8jwrWarJxGe3Oh5OJ+idiVALV4MZZ", + "lpAIK/7GvwnF5HNl/S73/4lzxg2NupAnOEaKZRBS+f3hZG/3NI9zuVSqNasiMOMU8YPdE/+Z8RmJY6CG", + "4uHuKf6dSTRnOY0VxY9vYdQp8AfghWLXBQg1qo7jWPnTBaiIeGUtr9ImzjLgkhjsQYpJUgOteeJz3BLx", + "t3bUFzeMzX6DSCNLb0DndM6axOzecOwJ4HoW0gMUVCRJQUicZmpPufr59ODg4K+VXcQxG2MJH9Rg3/4/", + "J5SIZSc9lmYJ9FEMEZmjYrFW8jRPEjxTm6uJBw12VAARvqBn0zj9HnFIdCohGZJLIkwqoTnAD5hoCjoy", + "FblAk4yfj0oGoHOD7dIIM+kChMALTx77MyZJzgGlZgB6XAK16Q8iAt3PMUkgvg8Rk0vgj0QAuld83o/6", + "FbcBvBJDNQM7uTZ5bYXo1OnBhwzLfIqzDGKFAxRjsZwxzGMUJUTpSudyVG36tyY7UeyGgZFV8ZFHEQhR", + "4aA0UoUDlYE2XeWdYXfLuqo3Nf8vB6EWygHOA8Ne9InPRMgrmwU0zR9jOTzlV0tBrJf1pfwUnuRpd34v", + "GcqwECboAFIzitRe7xu63jTKUnhS+gOlU8rM2CKx3k6LWsgaf+3qmtqCqF1lsxIsvjD72VuaVSPpQCRq", + "f22oeUO0OjM+sU4vb05ZTj3ufXp5gyLGTe1crQiCehny58Ogu/AIg1MdLFUa0JoAmLT2WdUzn4Eu5DI4", + "2v/4US9c/H+vz5B6DZ+QZ6aEuq40QOrUdevC/Bxkh40Fj9V0H+a1/p1+G8VbU1N6gikOeoNXLY0ZlooA", + "fYh/BS6ISfsGxtvG4yyfJSSqvJoxlgDWKS7H6cVsU1qNkaa0IsOP1KuelgmSSZycEfF1Sn6HFjItQlVW", + "eYiyfBBBX7gtoFLaqpDZLtzkMqxlC1Z5NXDUVDEAwQZwfhj7szGV1GU4ggFm35DaLDqAqY6gWHT8Xuxh", + "vZGupODltDDGSTPOqXdIkN9hM86pLOaCnHSGu4kPX6ZOapYdutu2SV4PRurdKAiHhIi0LfEwK9nXo97S", + "SbNTLudT2yfAiVy2m7WVlU95iukHDjhWOENLvQ6KlhB9RRxEnsh+/roYq6YaP8q7HylyjeP2c47r2lGF", + "IZtxEEBllZY70Tg/GwUdBIY18YrR/Yg3BigPRyp0WuvKsK0S9bnNBaSMr3xB0Lx5SQTc2/+LL0pNzQpX", + "EDEed+xUG506bZcNxXlznyx3eUMXLl16uw6DuLYJdG4+5Ug1j6WYUI9zYwHIvFRQ4lBTneR4PieRwjPW", + "BThRmB0A37RipC4mnTFf2NhvcXbeEjqvSWodtSrlIxbIThocMIVkWbY9ET3pxWHR+dLZNj6L5pyl6HFJ", + "oqUyZJUp63a9Tl0hHNZOTUpdV+BcMX8NsD5vLtuqHv+KY4hPVr46oldV/XVF7xKunduyPfXuOkScFcde", + "zSLDFzaLdnE5sSpIt/pEV4ajBwxOWys26ctYi6XbeLsyB0/tvA1UpbBnXENaSWqoj5+bLG4W8CmhlxWG", + "9sLXKOn1InOSwCWJZM7hhifDSpZOnr9ThYUkr8VrQ/GtnYsbAVyJ4OkzJSz6CvEVYDGwmH8FpzzBlELs", + "L/yJODEstb3u8OjQnHv3ulehjs9m9GubptVZwkASE2aHGjMsTob1xDI+Ndmqaq6i43DDwvXQZtXVBZnP", + "TqObZSiNcs6BSpujgRjYnCpnFom0aYoOnK62s2bPpq3KLYLGJ5ZzMbA9lOKnK1/7qZ3Gr55WkHf0ZvCu", + "sxd6tdqhsZJ4hWunoi6zdnZZcDp8q3Khpb+1opZt8qTcBaKcE7maqjXBnvemhF6zr0CPcxXin83thSXg", + "WDuCvb/wzw964Ac9stQ3zsjfQLdOp3mGZ1jA3pC1isH9y+0rkQevptTTWEwJTuzBsiRS32P6af8EnbmT", + "uePL8yAMHopGazAZ7Y0miguWAcUZCY6Cg9FkNFHxAMul1tsYK32McwFcjGeMSSE5zrSRmdlvlal130LV", + "ncElE1KrUNlRnLgJG5c69l/x/N+XlfhuA5gDx3meJCvkJMkgtvdZymsfPmKO+7EaVN5g6B6rBlUBGRzd", + "NqF4+8UPq9sv6y9hIPI0xXylCruCZ82wAgBeiIonKELjmQt9C/CY5xeQztWrN0Rv/ZKUQ8bem5LrcOA8", + "d6I0dEZxl234eHtPTulsZ1DzHE72Ic1dDszwglB9ecEwbBE3GYK4ybbotHeI+sYefB+S/aj1RrVNMOsD", + "RwVcq40KnO2DKp7H1Uum3cCelmeXLwO4eAsINQ5sB8No44T2fxlDv4BsHlh3oei5sPG6H0cn7hjtZTB6", + "AxTpa2RbAicGiUkiRrsEg71O2Df28B0Ax6qjDTfmhKgLLOYsapfpzcZpl8fen6rnWMIZ36jMCV0dpV+N", + "RVESjJ9dB3A95q433iazKyWmxSzbT9/WV8q+406dpd70H+wwRUv1h8tYlykUwgtrFz7jgGTdxlV8FkF1", + "bV9pBQuEk0RnAKaBjcvb0DYlRzNIGF0IJFmIHolcItNeQJgqx9U9BzRP8GIUhE2Q6qJ0l37ZrHwHI0tL", + "p0V/w7JjSImhs7KSO0+NEXbUfaXKdTP2hMWrV9N2867Wut4QsN8ovasq03b67TcjO0zVdosKo3stRUvV", + "qX+P7TcpHY6v3wuETeZffMtSfFbzJ2E/4pOrED3ghMRYErpw35zoQ0tkDijafd5S2Xozch/e7HQvegmM", + "rF5rOPrjbUN1zFkdGaAUqOhE37P5/GptPn6V0dITpNRjDZLr4lOt7THispXXD3LN86w3DnKew6k+cOZ6", + "yhvEuHdfjhrl9YfJAqjjyiluW6pdAas9FP4+zO4wqm0eWg/OhrSLW12M/sCNrtQZcOuk6hUR8PpRy/sp", + "3aDAtdfMEWoQ0ddFqsp7NwHm3ddrx3FNcVsFpPGz+Yx3bcyTgLm0WMfmmX7eROdN8QXwyzDa3+63nxh7", + "4tlhD5w4pOzhnQLqPwKSK62QQTix19jHtu7ur+5V0l78wYeiWHfLmHJeLoFwpB8U/ThC50zX9/Z7hpY0", + "3y5zVjCzw62t9WuCwftbQ/r3WPQ3mKwhwX3EoIW2z59rf5nEHO3V/wwD1B4aQNUeFOuuv6z/HQAA///j", + "IbitwEYAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/packages/dashboard-api/internal/cfg/model.go b/packages/dashboard-api/internal/cfg/model.go index f77bc5ed57..a205007a37 100644 --- a/packages/dashboard-api/internal/cfg/model.go +++ b/packages/dashboard-api/internal/cfg/model.go @@ -10,6 +10,7 @@ type Config struct { Port int `env:"PORT" envDefault:"3010"` PostgresConnectionString string `env:"POSTGRES_CONNECTION_STRING,required,notEmpty"` ClickhouseConnectionString string `env:"CLICKHOUSE_CONNECTION_STRING"` + AdminToken string `env:"ADMIN_TOKEN"` SupabaseJWTSecrets []string `env:"SUPABASE_JWT_SECRETS"` AuthDBConnectionString string `env:"AUTH_DB_CONNECTION_STRING"` diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index e468a2b4dc..bf4066683b 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -324,7 +324,7 @@ func TestPostUsersBootstrap_CreatesDefaultTeamAndCallsSink(t *testing.T) { authDB: testDB.AuthDB, teamProvisionSink: sink, } - store.PostUsersBootstrap(ginCtx) + store.PostAdminUsersBootstrap(ginCtx) if recorder.Code != http.StatusOK { t.Fatalf("expected status 200, got %d", recorder.Code) @@ -390,7 +390,7 @@ func TestPostUsersBootstrap_ProvisioningFailureKeepsCreatedDefaultTeam(t *testin authDB: testDB.AuthDB, teamProvisionSink: sink, } - store.PostUsersBootstrap(ginCtx) + store.PostAdminUsersBootstrap(ginCtx) if recorder.Code != http.StatusOK { t.Fatalf("expected status 200, got %d", recorder.Code) diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index 130a430c75..439f29cee0 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -40,7 +40,7 @@ type provisionedTeam struct { BlockedReason *string } -func (s *APIStore) PostUsersBootstrap(c *gin.Context) { +func (s *APIStore) PostAdminUsersBootstrap(c *gin.Context) { ctx := c.Request.Context() telemetry.ReportEvent(ctx, "bootstrap user") diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index b2c97f2186..887a90e51c 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -182,6 +182,7 @@ func run() int { authenticationFunc := sharedauth.CreateAuthenticationFunc( []sharedauth.Authenticator{ + sharedauth.NewAdminTokenAuthenticator(config.AdminToken), sharedauth.NewSupabaseTokenAuthenticator(apiStore.GetUserIDFromSupabaseToken), sharedauth.NewSupabaseTeamAuthenticator(apiStore.GetTeamFromSupabaseToken), }, @@ -197,6 +198,7 @@ func run() int { "Origin", "Content-Length", "Content-Type", + sharedauth.HeaderAdminToken, sharedauth.HeaderSupabaseToken, sharedauth.HeaderSupabaseTeam, } diff --git a/spec/openapi-dashboard.yml b/spec/openapi-dashboard.yml index b6962d65b8..24d59ca121 100644 --- a/spec/openapi-dashboard.yml +++ b/spec/openapi-dashboard.yml @@ -6,6 +6,10 @@ info: components: securitySchemes: + AdminTokenAuth: + type: apiKey + in: header + name: X-Admin-Token Supabase1TokenAuth: type: apiKey in: header @@ -749,12 +753,13 @@ paths: "500": $ref: "#/components/responses/500" - /users/bootstrap: + /admin/users/bootstrap: post: summary: Bootstrap user tags: [teams] security: - - Supabase1TokenAuth: [] + - AdminTokenAuth: [] + Supabase1TokenAuth: [] responses: "200": description: Successfully bootstrapped user. From 11d0f0f3e1e7721f127e49e8044c717a9b759ac8 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 20:39:38 -0700 Subject: [PATCH 75/92] refactor(dashboard-api): narrow infra sync runner changes Keep dashboard-api specific env wiring explicit, restore NODE_ID and PORT in the Nomad job spec, and remove generic dashboard-api env passthrough. Also trim unrelated lint churn from the branch and scope the remaining test/migration helpers to the auth user sync worker flow. --- .env.gcp.template | 6 +-- .../job-dashboard-api/jobs/dashboard-api.hcl | 3 ++ iac/modules/job-dashboard-api/main.tf | 33 ++++++++--------- iac/modules/job-dashboard-api/variables.tf | 6 +-- iac/provider-gcp/Makefile | 2 +- iac/provider-gcp/main.tf | 6 +-- iac/provider-gcp/nomad/main.tf | 20 +++++----- iac/provider-gcp/nomad/variables.tf | 10 ++--- iac/provider-gcp/variables.tf | 10 ++--- packages/api/go.mod | 4 +- packages/api/go.sum | 8 ++-- packages/api/internal/db/snapshots_test.go | 2 +- .../internal/handlers/template_alias_test.go | 12 +++--- .../placement/placement_benchmark_test.go | 12 +++--- .../sandbox/storage/memory/operations_test.go | 18 ++++----- packages/auth/go.mod | 2 +- packages/auth/go.sum | 4 +- packages/clickhouse/go.mod | 4 +- packages/clickhouse/go.sum | 8 ++-- packages/client-proxy/go.mod | 2 +- packages/client-proxy/go.sum | 4 +- .../backgroundworker/auth_user_sync.go | 19 +++++----- .../backgroundworker/auth_user_sync_test.go | 26 +++++++++++-- .../dashboard-api/internal/handlers/build.go | 5 ++- .../internal/handlers/sandbox_record.go | 4 +- .../internal/handlers/team_handlers_test.go | 4 +- packages/db/pkg/testutils/db.go | 37 ++++++------------- packages/db/pkg/testutils/queries.go | 4 +- packages/docker-reverse-proxy/go.mod | 4 +- packages/docker-reverse-proxy/go.sum | 8 ++-- .../internal/auth/validate_test.go | 8 ++-- packages/envd/go.mod | 2 +- packages/envd/go.sum | 4 +- packages/local-dev/go.mod | 2 +- packages/local-dev/go.sum | 4 +- packages/orchestrator/go.mod | 2 +- packages/orchestrator/go.sum | 4 +- packages/shared/go.mod | 2 +- packages/shared/go.sum | 4 +- tests/integration/go.mod | 4 +- tests/integration/go.sum | 8 ++-- tests/integration/internal/setup/db_client.go | 4 +- tests/integration/internal/tests/team_test.go | 4 +- tests/integration/internal/utils/process.go | 15 ++++---- tests/integration/internal/utils/team.go | 8 ++-- tests/integration/internal/utils/user.go | 6 +-- 46 files changed, 184 insertions(+), 184 deletions(-) diff --git a/.env.gcp.template b/.env.gcp.template index 65eba5e604..7674763b5c 100644 --- a/.env.gcp.template +++ b/.env.gcp.template @@ -79,10 +79,8 @@ CLICKHOUSE_CLUSTER_SIZE=1 DASHBOARD_API_COUNT= # Dashboard API Supabase DB connection string (default: POSTGRES_CONNECTION_STRING) SUPABASE_DB_CONNECTION_STRING= -# Additional dashboard-api env vars passed directly to the Nomad job (default: {}) -# Values here are merged into the job env and can override module defaults. -# Example: '{"ENABLE_AUTH_USER_SYNC_BACKGROUND_WORKER":"true"}' -DASHBOARD_API_ENV_VARS= +# Enable dashboard-api auth user sync background worker (default: false) +ENABLE_AUTH_USER_SYNC_BACKGROUND_WORKER= # Filestore cache for builds shared across cluster (default:false) FILESTORE_CACHE_ENABLED= diff --git a/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl b/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl index 885b59a7c5..45c80fac45 100644 --- a/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl +++ b/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl @@ -71,6 +71,9 @@ job "dashboard-api" { } env { + NODE_ID = "$${node.unique.id}" + PORT = "$${NOMAD_PORT_api}" + %{ for key, value in env ~} ${key} = "${value}" %{ endfor ~} diff --git a/iac/modules/job-dashboard-api/main.tf b/iac/modules/job-dashboard-api/main.tf index 3131278064..7797675d6b 100644 --- a/iac/modules/job-dashboard-api/main.tf +++ b/iac/modules/job-dashboard-api/main.tf @@ -1,23 +1,20 @@ locals { base_env = { - GIN_MODE = "release" - ENVIRONMENT = var.environment - NODE_ID = "$${node.unique.id}" - PORT = "$${NOMAD_PORT_api}" - POSTGRES_CONNECTION_STRING = var.postgres_connection_string - AUTH_DB_CONNECTION_STRING = var.auth_db_connection_string - AUTH_DB_READ_REPLICA_CONNECTION_STRING = var.auth_db_read_replica_connection_string - SUPABASE_DB_CONNECTION_STRING = var.supabase_db_connection_string - CLICKHOUSE_CONNECTION_STRING = var.clickhouse_connection_string - SUPABASE_JWT_SECRETS = var.supabase_jwt_secrets - REDIS_URL = var.redis_url - REDIS_CLUSTER_URL = var.redis_cluster_url - REDIS_TLS_CA_BASE64 = var.redis_tls_ca_base64 - OTEL_COLLECTOR_GRPC_ENDPOINT = "localhost:${var.otel_collector_grpc_port}" - LOGS_COLLECTOR_ADDRESS = "http://localhost:${var.logs_proxy_port.port}" + GIN_MODE = "release" + ENVIRONMENT = var.environment + POSTGRES_CONNECTION_STRING = var.postgres_connection_string + AUTH_DB_CONNECTION_STRING = var.auth_db_connection_string + AUTH_DB_READ_REPLICA_CONNECTION_STRING = var.auth_db_read_replica_connection_string + SUPABASE_DB_CONNECTION_STRING = var.supabase_db_connection_string + CLICKHOUSE_CONNECTION_STRING = var.clickhouse_connection_string + SUPABASE_JWT_SECRETS = var.supabase_jwt_secrets + REDIS_URL = var.redis_url + REDIS_CLUSTER_URL = var.redis_cluster_url + REDIS_TLS_CA_BASE64 = var.redis_tls_ca_base64 + ENABLE_AUTH_USER_SYNC_BACKGROUND_WORKER = tostring(var.enable_auth_user_sync_background_worker) + OTEL_COLLECTOR_GRPC_ENDPOINT = "localhost:${var.otel_collector_grpc_port}" + LOGS_COLLECTOR_ADDRESS = "http://localhost:${var.logs_proxy_port.port}" } - - env = merge(local.base_env, var.extra_env) } resource "nomad_job" "dashboard_api" { @@ -31,7 +28,7 @@ resource "nomad_job" "dashboard_api" { memory_mb = 512 cpu_count = 1 - env = local.env + env = local.base_env subdomain = "dashboard-api" }) diff --git a/iac/modules/job-dashboard-api/variables.tf b/iac/modules/job-dashboard-api/variables.tf index 1bbb149914..063cff473f 100644 --- a/iac/modules/job-dashboard-api/variables.tf +++ b/iac/modules/job-dashboard-api/variables.tf @@ -50,9 +50,9 @@ variable "supabase_jwt_secrets" { sensitive = true } -variable "extra_env" { - type = map(string) - default = {} +variable "enable_auth_user_sync_background_worker" { + type = bool + default = false } variable "otel_collector_grpc_port" { diff --git a/iac/provider-gcp/Makefile b/iac/provider-gcp/Makefile index b30c8859db..625f1febf9 100644 --- a/iac/provider-gcp/Makefile +++ b/iac/provider-gcp/Makefile @@ -76,8 +76,8 @@ tf_vars := \ $(call tfvar, LOKI_BOOT_DISK_TYPE) \ $(call tfvar, LOKI_USE_V13_SCHEMA_FROM) \ $(call tfvar, DASHBOARD_API_COUNT) \ - $(call tfvar, DASHBOARD_API_ENV_VARS) \ $(call tfvar, SUPABASE_DB_CONNECTION_STRING) \ + $(call tfvar, ENABLE_AUTH_USER_SYNC_BACKGROUND_WORKER) \ $(call tfvar, DEFAULT_PERSISTENT_VOLUME_TYPE) \ $(call tfvar, PERSISTENT_VOLUME_TYPES) \ $(call tfvar, DB_MAX_OPEN_CONNECTIONS) \ diff --git a/iac/provider-gcp/main.tf b/iac/provider-gcp/main.tf index 7e77aea9ad..18855409c9 100644 --- a/iac/provider-gcp/main.tf +++ b/iac/provider-gcp/main.tf @@ -275,9 +275,9 @@ module "nomad" { otel_collector_resources_cpu_count = var.otel_collector_resources_cpu_count # Dashboard API - dashboard_api_count = var.dashboard_api_count - dashboard_api_env_vars = var.dashboard_api_env_vars - supabase_db_connection_string = var.supabase_db_connection_string + dashboard_api_count = var.dashboard_api_count + supabase_db_connection_string = var.supabase_db_connection_string + enable_auth_user_sync_background_worker = var.enable_auth_user_sync_background_worker # Docker reverse proxy docker_reverse_proxy_port = var.docker_reverse_proxy_port diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index 0107c701a2..edd9e6a318 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -130,16 +130,16 @@ module "dashboard_api" { image = data.google_artifact_registry_docker_image.dashboard_api_image[0].self_link - postgres_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data - auth_db_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data - auth_db_read_replica_connection_string = trimspace(data.google_secret_manager_secret_version.postgres_read_replica_connection_string.secret_data) - supabase_db_connection_string = var.supabase_db_connection_string - clickhouse_connection_string = local.clickhouse_connection_string - supabase_jwt_secrets = trimspace(data.google_secret_manager_secret_version.supabase_jwt_secrets.secret_data) - redis_url = local.redis_url - redis_cluster_url = local.redis_cluster_url - redis_tls_ca_base64 = trimspace(data.google_secret_manager_secret_version.redis_tls_ca_base64.secret_data) - extra_env = var.dashboard_api_env_vars + postgres_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data + auth_db_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data + auth_db_read_replica_connection_string = trimspace(data.google_secret_manager_secret_version.postgres_read_replica_connection_string.secret_data) + supabase_db_connection_string = var.supabase_db_connection_string + clickhouse_connection_string = local.clickhouse_connection_string + supabase_jwt_secrets = trimspace(data.google_secret_manager_secret_version.supabase_jwt_secrets.secret_data) + redis_url = local.redis_url + redis_cluster_url = local.redis_cluster_url + redis_tls_ca_base64 = trimspace(data.google_secret_manager_secret_version.redis_tls_ca_base64.secret_data) + enable_auth_user_sync_background_worker = var.enable_auth_user_sync_background_worker otel_collector_grpc_port = var.otel_collector_grpc_port logs_proxy_port = var.logs_proxy_port diff --git a/iac/provider-gcp/nomad/variables.tf b/iac/provider-gcp/nomad/variables.tf index 63caaa5207..074e4b0d1f 100644 --- a/iac/provider-gcp/nomad/variables.tf +++ b/iac/provider-gcp/nomad/variables.tf @@ -454,17 +454,17 @@ variable "dashboard_api_count" { default = 0 } -variable "dashboard_api_env_vars" { - type = map(string) - default = {} -} - variable "supabase_db_connection_string" { type = string default = "" sensitive = true } +variable "enable_auth_user_sync_background_worker" { + type = bool + default = false +} + variable "volume_token_issuer" { type = string } diff --git a/iac/provider-gcp/variables.tf b/iac/provider-gcp/variables.tf index 0f3c1459b9..f9438b7080 100644 --- a/iac/provider-gcp/variables.tf +++ b/iac/provider-gcp/variables.tf @@ -230,17 +230,17 @@ variable "dashboard_api_count" { default = 0 } -variable "dashboard_api_env_vars" { - type = map(string) - default = {} -} - variable "supabase_db_connection_string" { type = string default = "" sensitive = true } +variable "enable_auth_user_sync_background_worker" { + type = bool + default = false +} + variable "docker_reverse_proxy_port" { type = object({ name = string diff --git a/packages/api/go.mod b/packages/api/go.mod index bf2248bc1a..f49287dece 100644 --- a/packages/api/go.mod +++ b/packages/api/go.mod @@ -35,7 +35,7 @@ require ( github.com/google/uuid v1.6.0 github.com/grafana/loki/v3 v3.6.4 github.com/hashicorp/nomad/api v0.0.0-20251216171439-1dee0671280e - github.com/jackc/pgx/v5 v5.9.1 + github.com/jackc/pgx/v5 v5.7.5 github.com/launchdarkly/go-sdk-common/v3 v3.3.0 github.com/launchdarkly/go-server-sdk/v7 v7.13.0 github.com/oapi-codegen/gin-middleware v1.0.2 @@ -386,7 +386,7 @@ require ( golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect golang.org/x/image v0.38.0 // indirect - golang.org/x/mod v0.34.0 // indirect + golang.org/x/mod v0.33.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/text v0.35.0 // indirect diff --git a/packages/api/go.sum b/packages/api/go.sum index 129d2ffb53..4508575a05 100644 --- a/packages/api/go.sum +++ b/packages/api/go.sum @@ -559,8 +559,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= -github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= +github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jaegertracing/jaeger-idl v0.5.0 h1:zFXR5NL3Utu7MhPg8ZorxtCBjHrL3ReM1VoB65FOFGE= @@ -1168,8 +1168,8 @@ golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= -golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/api/internal/db/snapshots_test.go b/packages/api/internal/db/snapshots_test.go index 6550885042..83f7d2a872 100644 --- a/packages/api/internal/db/snapshots_test.go +++ b/packages/api/internal/db/snapshots_test.go @@ -23,7 +23,7 @@ func createTestTeam(t *testing.T, db *testutils.Database) uuid.UUID { // Insert a team directly into the database using raw SQL // Using the default tier 'base_v1' that is created in migrations - err := db.AuthDB.TestsRawSQL(t.Context(), + err := db.AuthDb.TestsRawSQL(t.Context(), "INSERT INTO public.teams (id, name, tier, email, slug) VALUES ($1, $2, $3, $4, $5)", teamID, "Test Team "+teamID.String(), "base_v1", "test-"+teamID.String()+"@example.com", slug, ) diff --git a/packages/api/internal/handlers/template_alias_test.go b/packages/api/internal/handlers/template_alias_test.go index 52d39d296b..e89a660078 100644 --- a/packages/api/internal/handlers/template_alias_test.go +++ b/packages/api/internal/handlers/template_alias_test.go @@ -28,7 +28,7 @@ func TestQueryNotExistingTemplateAlias(t *testing.T) { store := &APIStore{ sqlcDB: testDB.SqlcClient, - authDB: testDB.AuthDB, + authDB: testDB.AuthDb, templateCache: templatecache.NewTemplateCache(testDB.SqlcClient, redis), } @@ -38,7 +38,7 @@ func TestQueryNotExistingTemplateAlias(t *testing.T) { w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) - c.Request = httptest.NewRequestWithContext(t.Context(), http.MethodGet, fmt.Sprintf("/templates/aliases/%s", alias), nil) + c.Request = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/templates/aliases/%s", alias), nil) auth.SetTeamInfo(c, &types.Team{ Team: &authqueries.Team{ ID: teamID, @@ -69,13 +69,13 @@ func TestQueryExistingTemplateAlias(t *testing.T) { store := &APIStore{ sqlcDB: testDB.SqlcClient, - authDB: testDB.AuthDB, + authDB: testDB.AuthDb, templateCache: templatecache.NewTemplateCache(testDB.SqlcClient, redis), } w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) - c.Request = httptest.NewRequestWithContext(t.Context(), http.MethodGet, fmt.Sprintf("/templates/aliases/%s", alias), nil) + c.Request = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/templates/aliases/%s", alias), nil) auth.SetTeamInfo(c, &types.Team{ Team: &authqueries.Team{ ID: teamID, @@ -114,13 +114,13 @@ func TestQueryExistingTemplateAliasAsNotOwnerTeam(t *testing.T) { store := &APIStore{ sqlcDB: testDB.SqlcClient, - authDB: testDB.AuthDB, + authDB: testDB.AuthDb, templateCache: templatecache.NewTemplateCache(testDB.SqlcClient, redis), } w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) - c.Request = httptest.NewRequestWithContext(t.Context(), http.MethodGet, fmt.Sprintf("/templates/aliases/%s", alias), nil) + c.Request = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/templates/aliases/%s", alias), nil) auth.SetTeamInfo(c, &types.Team{ Team: &authqueries.Team{ diff --git a/packages/api/internal/orchestrator/placement/placement_benchmark_test.go b/packages/api/internal/orchestrator/placement/placement_benchmark_test.go index 9d0bbf9575..2b14da9ba2 100644 --- a/packages/api/internal/orchestrator/placement/placement_benchmark_test.go +++ b/packages/api/internal/orchestrator/placement/placement_benchmark_test.go @@ -334,12 +334,12 @@ func runBenchmark(b *testing.B, algorithm Algorithm, config BenchmarkConfig, nod mu sync.Mutex placementTimes []time.Duration activeSandboxes sync.Map // sandboxID -> *LiveSandbox - sandboxIDCounter atomic.Int64 + sandboxIDCounter int64 // Metrics for time series recentPlacements []time.Duration - recentSuccesses atomic.Int64 - recentFailures atomic.Int64 + recentSuccesses int64 + recentFailures int64 ) // Start sandbox cleanup goroutine @@ -384,7 +384,7 @@ func runBenchmark(b *testing.B, algorithm Algorithm, config BenchmarkConfig, nod select { case <-ticker.C: // Generate sandbox with variance - sandboxID := sandboxIDCounter.Add(1) + sandboxID := atomic.AddInt64(&sandboxIDCounter, 1) cpuVariance := (rand.Float64()*2 - 1) * config.CPUVariance requestedCPU := max(int64(float64(config.AvgSandboxCPU)*(1+cpuVariance)), 1) @@ -448,7 +448,7 @@ func runBenchmark(b *testing.B, algorithm Algorithm, config BenchmarkConfig, nod if simNode.PlaceSandbox(sbx) { activeSandboxes.Store(sbx.ID, sbx) metrics.SuccessfulPlacements++ - recentSuccesses.Add(1) + atomic.AddInt64(&recentSuccesses, 1) success = true } } @@ -456,7 +456,7 @@ func runBenchmark(b *testing.B, algorithm Algorithm, config BenchmarkConfig, nod if !success { metrics.FailedPlacements++ - recentFailures.Add(1) + atomic.AddInt64(&recentFailures, 1) } mu.Unlock() } diff --git a/packages/api/internal/sandbox/storage/memory/operations_test.go b/packages/api/internal/sandbox/storage/memory/operations_test.go index e4e364ae61..7d995e8ce4 100644 --- a/packages/api/internal/sandbox/storage/memory/operations_test.go +++ b/packages/api/internal/sandbox/storage/memory/operations_test.go @@ -832,8 +832,8 @@ func TestConcurrency_StressTest(t *testing.T) { stopCh := make(chan struct{}) // Metrics - var opsCompleted atomic.Uint64 - var errorCount atomic.Uint64 + var opsCompleted uint64 + var errorCount uint64 // Launch workers that continuously perform random operations for i := range 200 { @@ -857,21 +857,21 @@ func TestConcurrency_StressTest(t *testing.T) { if finish != nil { finish(t.Context(), nil) } - opsCompleted.Add(1) + atomic.AddUint64(&opsCompleted, 1) } else if err != nil { - errorCount.Add(1) + atomic.AddUint64(&errorCount, 1) } case 1: // Read state _ = sbx.State() - opsCompleted.Add(1) + atomic.AddUint64(&opsCompleted, 1) case 2: // Wait with timeout waitCtx, cancel := context.WithTimeout(t.Context(), time.Microsecond*10) _ = waitForStateChange(waitCtx, sbx) cancel() - opsCompleted.Add(1) + atomic.AddUint64(&opsCompleted, 1) case 3: // Read _data _ = sbx.Data() - opsCompleted.Add(1) + atomic.AddUint64(&opsCompleted, 1) } } @@ -887,8 +887,8 @@ func TestConcurrency_StressTest(t *testing.T) { close(stopCh) wg.Wait() - finalOps := opsCompleted.Load() - finalErrors := errorCount.Load() + finalOps := atomic.LoadUint64(&opsCompleted) + finalErrors := atomic.LoadUint64(&errorCount) t.Logf("Stress test completed: %d operations, %d errors", finalOps, finalErrors) // Should have completed many operations without panic diff --git a/packages/auth/go.mod b/packages/auth/go.mod index adfdb792b4..8895fdba14 100644 --- a/packages/auth/go.mod +++ b/packages/auth/go.mod @@ -64,7 +64,7 @@ require ( github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.9.1 // indirect + github.com/jackc/pgx/v5 v5.7.5 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jellydator/ttlcache/v3 v3.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/packages/auth/go.sum b/packages/auth/go.sum index bda859ab9b..317c9eca6b 100644 --- a/packages/auth/go.sum +++ b/packages/auth/go.sum @@ -116,8 +116,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= -github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= +github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= diff --git a/packages/clickhouse/go.mod b/packages/clickhouse/go.mod index e7d399d034..932709551a 100644 --- a/packages/clickhouse/go.mod +++ b/packages/clickhouse/go.mod @@ -42,7 +42,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.9.1 // indirect + github.com/jackc/pgx/v5 v5.7.5 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/joho/godotenv v1.5.1 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect @@ -93,7 +93,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.34.0 // indirect + golang.org/x/mod v0.33.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect diff --git a/packages/clickhouse/go.sum b/packages/clickhouse/go.sum index c5f05d320c..a6dacb3c42 100644 --- a/packages/clickhouse/go.sum +++ b/packages/clickhouse/go.sum @@ -128,8 +128,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= -github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= +github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -310,8 +310,8 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= -golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/client-proxy/go.mod b/packages/client-proxy/go.mod index 517bb8ba97..5ac838ffdf 100644 --- a/packages/client-proxy/go.mod +++ b/packages/client-proxy/go.mod @@ -65,7 +65,7 @@ require ( go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.34.0 // indirect + golang.org/x/mod v0.33.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect diff --git a/packages/client-proxy/go.sum b/packages/client-proxy/go.sum index 2987c46eb7..2402ce9653 100644 --- a/packages/client-proxy/go.sum +++ b/packages/client-proxy/go.sum @@ -131,8 +131,8 @@ go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= -golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= -golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go index 3ca466a8ef..4cc42ffbfe 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go @@ -62,20 +62,19 @@ func NewAuthUserSyncWorker(ctx context.Context, supabaseDB *supabasedb.Client, a } func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSyncArgs]) error { - attrs := []attribute.KeyValue{ + ctx, span := workerTracer.Start(ctx, "auth_user_sync.work", trace.WithAttributes( attribute.String("job.kind", authUserProjectionKind), attribute.String("job.operation", job.Args.Operation), attribute.Int64("job.id", job.ID), telemetry.WithUserID(job.Args.UserID), - } - ctx, span := workerTracer.Start(ctx, "auth_user_sync.work", trace.WithAttributes(attrs...)) + )) defer span.End() telemetry.ReportEvent(ctx, "auth_user_sync.job.started") userID, err := uuid.Parse(job.Args.UserID) if err != nil { - telemetry.ReportError(ctx, "auth user sync parse user_id", err, attrs...) + telemetry.ReportError(ctx, "auth user sync parse user_id", err) w.observeJob(ctx, job.Args.Operation, jobResultInvalidArgument) return river.JobCancel(fmt.Errorf("parse user_id %q: %w", job.Args.UserID, err)) @@ -92,7 +91,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy switch job.Args.Operation { case "delete": if err := w.authDB.DeletePublicUser(ctx, userID); err != nil { - telemetry.ReportError(ctx, "auth user sync delete public user", err, attrs...) + telemetry.ReportError(ctx, "auth user sync delete public user", err) w.observeJob(ctx, job.Args.Operation, jobResultError) return fmt.Errorf("delete public.users %s: %w", userID, err) @@ -102,7 +101,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy supabaseUser, err := w.supabaseDB.Write.GetAuthUserByID(ctx, userID) if dberrors.IsNotFoundError(err) { if err := w.authDB.DeletePublicUser(ctx, userID); err != nil { - telemetry.ReportError(ctx, "auth user sync delete stale public user", err, attrs...) + telemetry.ReportError(ctx, "auth user sync delete stale public user", err) w.observeJob(ctx, job.Args.Operation, jobResultError) return fmt.Errorf("delete stale public.users %s: %w", userID, err) @@ -114,7 +113,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy return nil } if err != nil { - telemetry.ReportError(ctx, "auth user sync get source user", err, attrs...) + telemetry.ReportError(ctx, "auth user sync get source user", err) w.observeJob(ctx, job.Args.Operation, jobResultError) return fmt.Errorf("get source auth.users %s: %w", userID, err) @@ -122,7 +121,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy if supabaseUser.Email == "" { err := fmt.Errorf("missing email in source user %s", userID) - telemetry.ReportError(ctx, "auth user sync missing source email", err, attrs...) + telemetry.ReportError(ctx, "auth user sync missing source email", err) w.observeJob(ctx, job.Args.Operation, jobResultInvalidArgument) return river.JobCancel(err) @@ -132,7 +131,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy ID: userID, Email: supabaseUser.Email, }); err != nil { - telemetry.ReportError(ctx, "auth user sync upsert public user", err, attrs...) + telemetry.ReportError(ctx, "auth user sync upsert public user", err) w.observeJob(ctx, job.Args.Operation, jobResultError) return fmt.Errorf("upsert public.users %s: %w", userID, err) @@ -140,7 +139,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy default: err := fmt.Errorf("unknown operation %q for user %s", job.Args.Operation, userID) - telemetry.ReportError(ctx, "auth user sync unknown operation", err, attrs...) + telemetry.ReportError(ctx, "auth user sync unknown operation", err) w.observeJob(ctx, job.Args.Operation, jobResultInvalidArgument) return river.JobCancel(err) diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go index 6e01d27ad7..a981142987 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go @@ -24,8 +24,7 @@ const ( testEventuallyTick = 50 * time.Millisecond testStopTimeout = 5 * time.Second - supabaseMigrationsDir = "packages/db/pkg/supabase/migrations" - authCustomSchemaVersion int64 = 20260401000001 + supabaseMigrationsDir = "packages/db/pkg/supabase/migrations" ) type riverProcess struct { @@ -109,10 +108,29 @@ func applyAuthUserSyncMigrations(t *testing.T, db *testutils.Database) { t.Helper() // The auth user sync bootstraps `auth_custom` in three steps: - // 1. goose migration 20260401000001 creates the shared schema + // 1. create the shared schema and role needed by River migrations // 2. River library migrations create River tables inside that schema // 3. the remaining auth migrations add triggers that enqueue into River - db.ApplyMigrationsUpTo(t, authCustomSchemaVersion, supabaseMigrationsDir) + require.NoError(t, db.SupabaseDB.TestsRawSQL(t.Context(), ` +CREATE SCHEMA IF NOT EXISTS auth_custom; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'trigger_user') THEN + CREATE ROLE trigger_user NOLOGIN; + END IF; +END; +$$; + +GRANT USAGE ON SCHEMA auth_custom TO trigger_user; +GRANT CREATE ON SCHEMA auth_custom TO trigger_user; + +DO $$ +BEGIN + EXECUTE format('GRANT TEMP ON DATABASE %I TO trigger_user', current_database()); +END; +$$; +`)) require.NoError(t, RunRiverMigrations(t.Context(), db.SupabaseDB.WritePool())) diff --git a/packages/dashboard-api/internal/handlers/build.go b/packages/dashboard-api/internal/handlers/build.go index 30a586bc3b..90237884e2 100644 --- a/packages/dashboard-api/internal/handlers/build.go +++ b/packages/dashboard-api/internal/handlers/build.go @@ -1,15 +1,16 @@ package handlers import ( + "errors" "net/http" "github.com/gin-gonic/gin" + "github.com/jackc/pgx/v5" "go.uber.org/zap" "github.com/e2b-dev/infra/packages/auth/pkg/auth" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" dashboardutils "github.com/e2b-dev/infra/packages/dashboard-api/internal/utils" - "github.com/e2b-dev/infra/packages/db/pkg/dberrors" "github.com/e2b-dev/infra/packages/db/queries" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -26,7 +27,7 @@ func (s *APIStore) GetBuildsBuildId(c *gin.Context, buildId api.BuildId) { BuildID: buildId, }) if err != nil { - if dberrors.IsNotFoundError(err) { + if errors.Is(err, pgx.ErrNoRows) { s.sendAPIStoreError(c, http.StatusNotFound, "Build not found or you don't have access to it") return diff --git a/packages/dashboard-api/internal/handlers/sandbox_record.go b/packages/dashboard-api/internal/handlers/sandbox_record.go index 72c4976e4a..823c597ff2 100644 --- a/packages/dashboard-api/internal/handlers/sandbox_record.go +++ b/packages/dashboard-api/internal/handlers/sandbox_record.go @@ -6,12 +6,12 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "go.uber.org/zap" "github.com/e2b-dev/infra/packages/auth/pkg/auth" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" - "github.com/e2b-dev/infra/packages/db/pkg/dberrors" "github.com/e2b-dev/infra/packages/db/queries" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -36,7 +36,7 @@ func (s *APIStore) GetSandboxesSandboxIDRecord(c *gin.Context, sandboxID api.San CreatedAfter: time.Now().UTC().Add(-sandboxRecordRetention), }) if err != nil { - if dberrors.IsNotFoundError(err) || isUndefinedTableError(err) { + if errors.Is(err, pgx.ErrNoRows) || isUndefinedTableError(err) { s.sendAPIStoreError(c, http.StatusNotFound, "Sandbox not found or you don't have access to it") return diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index 13565544e6..6eb42fe775 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -255,7 +255,7 @@ func createHandlerTestUser(t *testing.T, db *testutils.Database) uuid.UUID { userID := uuid.New() email := handlerTestUserEmail(userID) - err := db.AuthDB.TestsRawSQL(t.Context(), ` + err := db.AuthDb.TestsRawSQL(t.Context(), ` INSERT INTO auth.users (id, email) VALUES ($1, $2) `, userID, email) @@ -273,7 +273,7 @@ func handlerTestUserEmail(userID uuid.UUID) string { func insertHandlerTestTeamMember(t *testing.T, db *testutils.Database, userID, teamID uuid.UUID, isDefault bool) { t.Helper() - err := db.AuthDB.TestsRawSQL(t.Context(), ` + err := db.AuthDb.TestsRawSQL(t.Context(), ` INSERT INTO public.users_teams (user_id, team_id, is_default) VALUES ($1, $2, $3) `, userID, teamID, isDefault) diff --git a/packages/db/pkg/testutils/db.go b/packages/db/pkg/testutils/db.go index 508de6f545..6e0f1307f1 100644 --- a/packages/db/pkg/testutils/db.go +++ b/packages/db/pkg/testutils/db.go @@ -38,6 +38,7 @@ func init() { // Database encapsulates the test database container and clients type Database struct { SqlcClient *db.Client + AuthDb *authdb.Client AuthDB *authdb.Client SupabaseDB *supabasedb.Client TestQueries *queries.Queries @@ -118,6 +119,7 @@ func SetupDatabase(t *testing.T) *Database { return &Database{ SqlcClient: sqlcClient, + AuthDb: authDB, AuthDB: authDB, SupabaseDB: supabaseDB, TestQueries: testQueries, @@ -128,22 +130,14 @@ func SetupDatabase(t *testing.T) *Database { func (db *Database) ApplyMigrations(t *testing.T, migrationDirs ...string) { t.Helper() - db.applyGooseMigrations(t, 0, migrationDirs...) -} - -func (db *Database) ApplyMigrationsUpTo(t *testing.T, version int64, migrationDirs ...string) { - t.Helper() - - // This is only used for staged bootstrap flows that must interleave - // third-party migrations with goose-managed SQL migrations. - db.applyGooseMigrations(t, version, migrationDirs...) + db.applyGooseMigrations(t, migrationDirs...) } func (db *Database) ConnStr() string { return db.connStr } -func (db *Database) applyGooseMigrations(t *testing.T, upToVersion int64, migrationDirs ...string) { +func (db *Database) applyGooseMigrations(t *testing.T, migrationDirs ...string) { t.Helper() cmd := exec.CommandContext(t.Context(), "git", "rev-parse", "--show-toplevel") @@ -162,22 +156,13 @@ func (db *Database) applyGooseMigrations(t *testing.T, upToVersion int64, migrat }) for _, migrationsDir := range migrationDirs { - if upToVersion > 0 { - err = goose.UpToContext( - t.Context(), - sqlDB, - filepath.Join(repoRoot, migrationsDir), - upToVersion, - ) - } else { - err = goose.RunWithOptionsContext( - t.Context(), - "up", - sqlDB, - filepath.Join(repoRoot, migrationsDir), - nil, - ) - } + err = goose.RunWithOptionsContext( + t.Context(), + "up", + sqlDB, + filepath.Join(repoRoot, migrationsDir), + nil, + ) require.NoError(t, err) } diff --git a/packages/db/pkg/testutils/queries.go b/packages/db/pkg/testutils/queries.go index ab3c5529f7..c264ef69d8 100644 --- a/packages/db/pkg/testutils/queries.go +++ b/packages/db/pkg/testutils/queries.go @@ -23,7 +23,7 @@ func CreateTestTeam(t *testing.T, db *Database) uuid.UUID { // Insert a team directly into the database using raw SQL // Using the default tier 'base_v1' that is created in migrations - err := db.AuthDB.TestsRawSQL(t.Context(), + err := db.AuthDb.TestsRawSQL(t.Context(), "INSERT INTO public.teams (id, name, tier, email, slug) VALUES ($1, $2, $3, $4, $5)", teamID, "Test Team "+teamID.String(), "base_v1", "test-"+teamID.String()+"@example.com", slug, ) @@ -92,7 +92,7 @@ func GetTeamSlug(t *testing.T, ctx context.Context, db *Database, teamID uuid.UU t.Helper() var slug string - err := db.AuthDB.TestsRawSQLQuery(ctx, + err := db.AuthDb.TestsRawSQLQuery(ctx, "SELECT slug FROM public.teams WHERE id = $1", func(rows pgx.Rows) error { if rows.Next() { diff --git a/packages/docker-reverse-proxy/go.mod b/packages/docker-reverse-proxy/go.mod index 4b07e1c717..b57f92bc28 100644 --- a/packages/docker-reverse-proxy/go.mod +++ b/packages/docker-reverse-proxy/go.mod @@ -40,7 +40,7 @@ require ( github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.9.1 // indirect + github.com/jackc/pgx/v5 v5.7.5 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/klauspost/compress v1.18.2 // indirect github.com/lib/pq v1.11.2 // indirect @@ -79,7 +79,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect golang.org/x/crypto v0.49.0 // indirect - golang.org/x/mod v0.34.0 // indirect + golang.org/x/mod v0.33.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect diff --git a/packages/docker-reverse-proxy/go.sum b/packages/docker-reverse-proxy/go.sum index ca2293ebd5..b17ae0b6e7 100644 --- a/packages/docker-reverse-proxy/go.sum +++ b/packages/docker-reverse-proxy/go.sum @@ -69,8 +69,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= -github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= +github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= @@ -189,8 +189,8 @@ golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= -golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= -golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= diff --git a/packages/docker-reverse-proxy/internal/auth/validate_test.go b/packages/docker-reverse-proxy/internal/auth/validate_test.go index fa7e62276f..29ff34d215 100644 --- a/packages/docker-reverse-proxy/internal/auth/validate_test.go +++ b/packages/docker-reverse-proxy/internal/auth/validate_test.go @@ -111,27 +111,27 @@ func setupValidateTest(tb testing.TB, db *testutils.Database, userID, teamID uui tb.Helper() // Create team - err := db.AuthDB.TestsRawSQL(tb.Context(), ` + err := db.AuthDb.TestsRawSQL(tb.Context(), ` INSERT INTO "auth"."users" (id, email) VALUES ($1, 'test@e2b.dev') `, userID) require.NoError(tb, err) - err = db.AuthDB.TestsRawSQL(tb.Context(), ` + err = db.AuthDb.TestsRawSQL(tb.Context(), ` INSERT INTO teams (id, name, email, tier, slug) VALUES ($1, 'test-team', 'test@e2b.dev', 'base_v1', 'test-team-slug') `, teamID) require.NoError(tb, err) // Link user to team - err = db.AuthDB.TestsRawSQL(tb.Context(), ` + err = db.AuthDb.TestsRawSQL(tb.Context(), ` INSERT INTO users_teams (user_id, team_id, is_default) VALUES ($1, $2, true) `, userID, teamID) require.NoError(tb, err) // Create access token - _, err = db.AuthDB.Write.CreateAccessToken(tb.Context(), authqueries.CreateAccessTokenParams{ + _, err = db.AuthDb.Write.CreateAccessToken(tb.Context(), authqueries.CreateAccessTokenParams{ ID: uuid.New(), UserID: userID, AccessTokenHash: accessToken.HashedValue, diff --git a/packages/envd/go.mod b/packages/envd/go.mod index bcae5ece5e..42241adf20 100644 --- a/packages/envd/go.mod +++ b/packages/envd/go.mod @@ -73,7 +73,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect golang.org/x/crypto v0.49.0 // indirect - golang.org/x/mod v0.34.0 // indirect + golang.org/x/mod v0.33.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/text v0.35.0 // indirect diff --git a/packages/envd/go.sum b/packages/envd/go.sum index 384d294119..8bd0104101 100644 --- a/packages/envd/go.sum +++ b/packages/envd/go.sum @@ -213,8 +213,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= -golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/packages/local-dev/go.mod b/packages/local-dev/go.mod index 0a0a79a15f..32a7ce3132 100644 --- a/packages/local-dev/go.mod +++ b/packages/local-dev/go.mod @@ -10,7 +10,7 @@ require ( github.com/e2b-dev/infra/packages/db v0.0.0 github.com/e2b-dev/infra/packages/shared v0.0.0 github.com/google/uuid v1.6.0 - github.com/jackc/pgx/v5 v5.9.1 + github.com/jackc/pgx/v5 v5.7.5 github.com/pressly/goose/v3 v3.26.0 github.com/stretchr/testify v1.11.1 github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 diff --git a/packages/local-dev/go.sum b/packages/local-dev/go.sum index 2666677cdf..717dbdfdb8 100644 --- a/packages/local-dev/go.sum +++ b/packages/local-dev/go.sum @@ -69,8 +69,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= -github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= +github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= diff --git a/packages/orchestrator/go.mod b/packages/orchestrator/go.mod index 34e58ce46b..7c719201b3 100644 --- a/packages/orchestrator/go.mod +++ b/packages/orchestrator/go.mod @@ -314,7 +314,7 @@ require ( golang.org/x/arch v0.25.0 // indirect golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.34.0 // indirect + golang.org/x/mod v0.33.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/term v0.41.0 // indirect diff --git a/packages/orchestrator/go.sum b/packages/orchestrator/go.sum index 8db15eee62..6879e76f78 100644 --- a/packages/orchestrator/go.sum +++ b/packages/orchestrator/go.sum @@ -1439,8 +1439,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= -golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/shared/go.mod b/packages/shared/go.mod index 08442cb799..0a92ccac46 100644 --- a/packages/shared/go.mod +++ b/packages/shared/go.mod @@ -55,7 +55,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 go.uber.org/zap v1.27.1 - golang.org/x/mod v0.34.0 + golang.org/x/mod v0.33.0 golang.org/x/oauth2 v0.34.0 golang.org/x/sync v0.20.0 google.golang.org/api v0.257.0 diff --git a/packages/shared/go.sum b/packages/shared/go.sum index 7b75861019..c5d23cd03d 100644 --- a/packages/shared/go.sum +++ b/packages/shared/go.sum @@ -1036,8 +1036,8 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= -golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/tests/integration/go.mod b/tests/integration/go.mod index 4bc14e49c4..e81b2fe7ef 100644 --- a/tests/integration/go.mod +++ b/tests/integration/go.mod @@ -25,7 +25,7 @@ require ( github.com/e2b-dev/infra/packages/envd v0.0.0-00010101000000-000000000000 github.com/e2b-dev/infra/packages/shared v0.0.0 github.com/google/uuid v1.6.0 - github.com/jackc/pgx/v5 v5.9.1 + github.com/jackc/pgx/v5 v5.7.5 github.com/oapi-codegen/runtime v1.1.1 github.com/stretchr/testify v1.11.1 golang.org/x/sync v0.20.0 @@ -181,7 +181,7 @@ require ( go.uber.org/zap v1.27.1 // indirect golang.org/x/arch v0.25.0 // indirect golang.org/x/crypto v0.49.0 // indirect - golang.org/x/mod v0.34.0 // indirect + golang.org/x/mod v0.33.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/term v0.41.0 // indirect diff --git a/tests/integration/go.sum b/tests/integration/go.sum index a94cd4a961..db41922318 100644 --- a/tests/integration/go.sum +++ b/tests/integration/go.sum @@ -184,8 +184,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= -github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= +github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= @@ -458,8 +458,8 @@ golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkN golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= -golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/tests/integration/internal/setup/db_client.go b/tests/integration/internal/setup/db_client.go index 2092e28bd8..97cbf5fe26 100644 --- a/tests/integration/internal/setup/db_client.go +++ b/tests/integration/internal/setup/db_client.go @@ -12,7 +12,7 @@ import ( type Database struct { Db *client.Client - AuthDB *authdb.Client + AuthDb *authdb.Client } func GetTestDBClient(tb testing.TB) *Database { @@ -33,6 +33,6 @@ func GetTestDBClient(tb testing.TB) *Database { return &Database{ Db: db, - AuthDB: authDb, + AuthDb: authDb, } } diff --git a/tests/integration/internal/tests/team_test.go b/tests/integration/internal/tests/team_test.go index 93d8871d3c..63104897f4 100644 --- a/tests/integration/internal/tests/team_test.go +++ b/tests/integration/internal/tests/team_test.go @@ -27,7 +27,7 @@ func TestBannedTeam(t *testing.T) { teamID := utils.CreateTeamWithUser(t, db, teamName, setup.UserID) apiKey := utils.CreateAPIKey(t, ctx, c, setup.UserID, teamID) - err := db.AuthDB.TestsRawSQL(ctx, ` + err := db.AuthDb.TestsRawSQL(ctx, ` UPDATE teams SET is_banned = $1 WHERE id = $2 `, true, teamID) require.NoError(t, err) @@ -56,7 +56,7 @@ func TestBlockedTeam(t *testing.T) { teamID := utils.CreateTeamWithUser(t, db, teamName, setup.UserID) apiKey := utils.CreateAPIKey(t, ctx, c, setup.UserID, teamID) - err := db.AuthDB.TestsRawSQL(ctx, ` + err := db.AuthDb.TestsRawSQL(ctx, ` UPDATE teams SET is_blocked = $1, blocked_reason = $2 WHERE id = $3 `, true, blockReason, teamID) require.NoError(t, err) diff --git a/tests/integration/internal/utils/process.go b/tests/integration/internal/utils/process.go index 69a04b54f7..765a095539 100644 --- a/tests/integration/internal/utils/process.go +++ b/tests/integration/internal/utils/process.go @@ -3,7 +3,6 @@ package utils import ( "context" "fmt" - "strings" "testing" "connectrpc.com/connect" @@ -83,7 +82,7 @@ func ExecCommandWithOutput(tb testing.TB, ctx context.Context, sbx *api.Sandbox, } }() - var output strings.Builder + var output string for stream.Receive() { select { case <-ctx.Done(): @@ -96,28 +95,28 @@ func ExecCommandWithOutput(tb testing.TB, ctx context.Context, sbx *api.Sandbox, // Capture stdout if msg.GetEvent().GetData() != nil { if stdout := msg.GetEvent().GetData().GetStdout(); stdout != nil { - output.Write(stdout) + output += string(stdout) } if stderr := msg.GetEvent().GetData().GetStderr(); stderr != nil { - output.Write(stderr) + output += string(stderr) } } if msg.GetEvent().GetEnd() != nil { if msg.GetEvent().GetEnd().GetExitCode() != 0 { - return output.String(), fmt.Errorf("command %s in sandbox %s failed with exit code %d", command, sbx.SandboxID, msg.GetEvent().GetEnd().GetExitCode()) + return output, fmt.Errorf("command %s in sandbox %s failed with exit code %d", command, sbx.SandboxID, msg.GetEvent().GetEnd().GetExitCode()) } tb.Logf("Command [%s] completed successfully in sandbox %s", command, sbx.SandboxID) - return output.String(), nil + return output, nil } } } if err := stream.Err(); err != nil { - return output.String(), fmt.Errorf("failed to execute command %s in sandbox %s: %w", command, sbx.SandboxID, err) + return output, fmt.Errorf("failed to execute command %s in sandbox %s: %w", command, sbx.SandboxID, err) } - return output.String(), nil + return output, nil } diff --git a/tests/integration/internal/utils/team.go b/tests/integration/internal/utils/team.go index 1f21fa94ad..7628247fd1 100644 --- a/tests/integration/internal/utils/team.go +++ b/tests/integration/internal/utils/team.go @@ -28,7 +28,7 @@ func CreateTeamWithUser( teamID := uuid.New() slug := fmt.Sprintf("test-%s", teamID.String()[:8]) - err := db.AuthDB.TestsRawSQL(t.Context(), ` + err := db.AuthDb.TestsRawSQL(t.Context(), ` INSERT INTO teams (id, email, name, tier, is_blocked, slug) VALUES ($1, $2, $3, $4, $5, $6) `, teamID, fmt.Sprintf("test-integration-%s@e2b.dev", teamID), teamName, "base_v1", false, slug) @@ -39,7 +39,7 @@ VALUES ($1, $2, $3, $4, $5, $6) } t.Cleanup(func() { - db.AuthDB.TestsRawSQL(t.Context(), ` + db.AuthDb.TestsRawSQL(t.Context(), ` DELETE FROM teams WHERE id = $1 `, teamID) }) @@ -53,14 +53,14 @@ func AddUserToTeam(t *testing.T, db *setup.Database, teamID uuid.UUID, userID st userUUID, err := uuid.Parse(userID) require.NoError(t, err) - err = db.AuthDB.TestsRawSQL(t.Context(), ` + err = db.AuthDb.TestsRawSQL(t.Context(), ` INSERT INTO users_teams (user_id, team_id, is_default) VALUES ($1, $2, $3) `, userUUID, teamID, false) require.NoError(t, err) t.Cleanup(func() { - db.AuthDB.TestsRawSQL(t.Context(), ` + db.AuthDb.TestsRawSQL(t.Context(), ` DELETE FROM users_teams WHERE user_id = $1 and team_id = $2 `, userUUID, teamID) }) diff --git a/tests/integration/internal/utils/user.go b/tests/integration/internal/utils/user.go index 4073959ba1..d766adceb9 100644 --- a/tests/integration/internal/utils/user.go +++ b/tests/integration/internal/utils/user.go @@ -19,14 +19,14 @@ func CreateUser(t *testing.T, db *setup.Database) uuid.UUID { userID := uuid.New() - err := db.AuthDB.TestsRawSQL(t.Context(), ` + err := db.AuthDb.TestsRawSQL(t.Context(), ` INSERT INTO auth.users (id, email) VALUES ($1, $2) `, userID, fmt.Sprintf("user-test-integration-%s@e2b.dev", userID)) require.NoError(t, err) t.Cleanup(func() { - db.AuthDB.TestsRawSQL(t.Context(), ` + db.AuthDb.TestsRawSQL(t.Context(), ` DELETE FROM auth.users WHERE id = $1 `, userID) }) @@ -49,7 +49,7 @@ func CreateAccessToken(t *testing.T, db *setup.Database, userID uuid.UUID) strin accessTokenMask, err := keys.MaskKey(keys.AccessTokenPrefix, tokenWithoutPrefix) require.NoError(t, err) - _, err = db.AuthDB.Write.CreateAccessToken(t.Context(), authqueries.CreateAccessTokenParams{ + _, err = db.AuthDb.Write.CreateAccessToken(t.Context(), authqueries.CreateAccessTokenParams{ ID: uuid.New(), UserID: userID, AccessTokenHash: accessTokenHash, From 268400f413d40ee6f2bb1fc5ee22f179f7b95554 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 20:41:57 -0700 Subject: [PATCH 76/92] fix(dashboard-api): harden bootstrap auth and provisioning --- packages/auth/pkg/auth/middleware.go | 3 +- packages/auth/pkg/auth/middleware_test.go | 28 ++++++++++++ packages/dashboard-api/internal/cfg/model.go | 2 +- .../dashboard-api/internal/cfg/model_test.go | 34 ++++++++++++++ .../internal/handlers/team_handlers_test.go | 45 +++++++++++++++++++ .../internal/handlers/team_provisioning.go | 12 ++++- 6 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 packages/auth/pkg/auth/middleware_test.go create mode 100644 packages/dashboard-api/internal/cfg/model_test.go diff --git a/packages/auth/pkg/auth/middleware.go b/packages/auth/pkg/auth/middleware.go index 49fee38a81..6d4e9ecc2a 100644 --- a/packages/auth/pkg/auth/middleware.go +++ b/packages/auth/pkg/auth/middleware.go @@ -2,6 +2,7 @@ package auth import ( "context" + "crypto/subtle" "errors" "fmt" "net/http" @@ -127,7 +128,7 @@ func (a *CommonAuthenticator[T]) SecuritySchemeName() string { func adminValidationFunction(adminToken string) func(ctx context.Context, ginCtx *gin.Context, token string) (struct{}, *APIError) { return func(_ context.Context, _ *gin.Context, token string) (struct{}, *APIError) { - if token != adminToken { + if subtle.ConstantTimeCompare([]byte(token), []byte(adminToken)) != 1 { return struct{}{}, &APIError{ Code: http.StatusUnauthorized, Err: errors.New("invalid access token"), diff --git a/packages/auth/pkg/auth/middleware_test.go b/packages/auth/pkg/auth/middleware_test.go new file mode 100644 index 0000000000..c3ba82263b --- /dev/null +++ b/packages/auth/pkg/auth/middleware_test.go @@ -0,0 +1,28 @@ +package auth + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAdminValidationFunction(t *testing.T) { + t.Parallel() + + validate := adminValidationFunction("super-secret-token") + + t.Run("accepts matching token", func(t *testing.T) { + t.Parallel() + + _, err := validate(t.Context(), nil, "super-secret-token") + require.Nil(t, err) + }) + + t.Run("rejects non-matching token", func(t *testing.T) { + t.Parallel() + + _, err := validate(t.Context(), nil, "super-secret-tokem") + require.NotNil(t, err) + require.Equal(t, 401, err.Code) + }) +} diff --git a/packages/dashboard-api/internal/cfg/model.go b/packages/dashboard-api/internal/cfg/model.go index a205007a37..cee25796a5 100644 --- a/packages/dashboard-api/internal/cfg/model.go +++ b/packages/dashboard-api/internal/cfg/model.go @@ -10,7 +10,7 @@ type Config struct { Port int `env:"PORT" envDefault:"3010"` PostgresConnectionString string `env:"POSTGRES_CONNECTION_STRING,required,notEmpty"` ClickhouseConnectionString string `env:"CLICKHOUSE_CONNECTION_STRING"` - AdminToken string `env:"ADMIN_TOKEN"` + AdminToken string `env:"ADMIN_TOKEN,required,notEmpty"` SupabaseJWTSecrets []string `env:"SUPABASE_JWT_SECRETS"` AuthDBConnectionString string `env:"AUTH_DB_CONNECTION_STRING"` diff --git a/packages/dashboard-api/internal/cfg/model_test.go b/packages/dashboard-api/internal/cfg/model_test.go new file mode 100644 index 0000000000..1b8f3d1393 --- /dev/null +++ b/packages/dashboard-api/internal/cfg/model_test.go @@ -0,0 +1,34 @@ +package cfg + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseRequiresAdminToken(t *testing.T) { + t.Setenv("POSTGRES_CONNECTION_STRING", "postgres://test") + t.Setenv("REDIS_URL", "redis://localhost:6379") + t.Setenv("REDIS_CLUSTER_URL", "") + t.Setenv("AUTH_DB_CONNECTION_STRING", "") + t.Setenv("AUTH_DB_READ_REPLICA_CONNECTION_STRING", "") + t.Setenv("ADMIN_TOKEN", "") + + _, err := Parse() + require.Error(t, err) + require.ErrorContains(t, err, "ADMIN_TOKEN") +} + +func TestParseAcceptsAdminToken(t *testing.T) { + t.Setenv("POSTGRES_CONNECTION_STRING", "postgres://test") + t.Setenv("REDIS_URL", "redis://localhost:6379") + t.Setenv("REDIS_CLUSTER_URL", "") + t.Setenv("AUTH_DB_CONNECTION_STRING", "") + t.Setenv("AUTH_DB_READ_REPLICA_CONNECTION_STRING", "") + t.Setenv("ADMIN_TOKEN", "secret") + + config, err := Parse() + require.NoError(t, err) + require.Equal(t, "secret", config.AdminToken) + require.Equal(t, "postgres://test", config.AuthDBConnectionString) +} diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index bf4066683b..0726db7f83 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -659,6 +659,51 @@ func TestPostTeams_TrimsNameBeforeCreate(t *testing.T) { } } +func TestPostTeams_ProvisioningFailureRollsBackCreatedTeam(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUser(t, testDB) + sink := &fakeTeamProvisionSink{ + err: &internalteamprovision.ProvisionError{ + StatusCode: http.StatusBadRequest, + Message: "limit reached", + }, + } + + recorder := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(recorder) + ginCtx.Request = httptest.NewRequestWithContext(ctx, http.MethodPost, "/", strings.NewReader(`{"name":"Acme"}`)) + ginCtx.Request.Header.Set("Content-Type", "application/json") + auth.SetUserID(ginCtx, userID) + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDB, + teamProvisionSink: sink, + } + store.PostTeams(ginCtx) + + if recorder.Code != http.StatusBadRequest { + t.Fatalf("expected status 400, got %d", recorder.Code) + } + if len(sink.requests) != 1 { + t.Fatalf("expected one provisioning call, got %d", len(sink.requests)) + } + + rows, err := testDB.AuthDB.Read.GetTeamsWithUsersTeamsWithTier(ctx, userID) + if err != nil { + t.Fatalf("failed to query user teams: %v", err) + } + if len(rows) != 1 { + t.Fatalf("expected only the default team to remain, got %d rows", len(rows)) + } + if !rows[0].IsDefault { + t.Fatal("expected remaining team to be the default team") + } +} + func TestCreateTeam_ConcurrentRequestsRespectLocalPolicyWithZeroMemberships(t *testing.T) { t.Parallel() diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index 439f29cee0..e80e8bc25a 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -29,6 +29,7 @@ const ( maxTeamsPerUserWithProTier = 10 newUserNewTeamRequireBillingMethodThreshold = 3 * 24 * time.Hour blockedReasonMissingPayment = "missing_payment" + teamProvisionRollbackTimeout = 5 * time.Second ) type provisionedTeam struct { @@ -249,7 +250,16 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string OwnerUserID: userID, Reason: teamprovision.ReasonAdditionalTeam, } - _ = s.teamProvisionSink.ProvisionTeam(ctx, req) + if err := s.teamProvisionSink.ProvisionTeam(ctx, req); err != nil { + rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), teamProvisionRollbackTimeout) + defer cancel() + + if deleteErr := s.db.DeleteTeamByID(rollbackCtx, team.ID); deleteErr != nil { + return provisionedTeam{}, fmt.Errorf("delete team after provisioning failure: provision=%s delete=%w", err.Error(), deleteErr) + } + + return provisionedTeam{}, err + } return provisionedTeam{ ID: team.ID, From 63b4a2b490eaa51ad06cdcda78368d134cefa3c1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 10 Apr 2026 03:51:49 +0000 Subject: [PATCH 77/92] chore: auto-commit generated changes --- packages/api/go.mod | 4 ++-- packages/api/go.sum | 8 ++++---- packages/auth/go.mod | 2 +- packages/auth/go.sum | 4 ++-- packages/clickhouse/go.mod | 2 +- packages/clickhouse/go.sum | 7 +++---- packages/client-proxy/go.sum | 2 -- packages/dashboard-api/go.mod | 2 -- packages/docker-reverse-proxy/go.mod | 2 +- packages/docker-reverse-proxy/go.sum | 6 ++---- packages/envd/go.sum | 4 ++-- packages/local-dev/go.mod | 2 +- packages/local-dev/go.sum | 4 ++-- packages/orchestrator/go.sum | 4 ++-- packages/shared/go.mod | 2 +- packages/shared/go.sum | 3 +-- tests/integration/go.mod | 2 +- tests/integration/go.sum | 8 ++++---- 18 files changed, 30 insertions(+), 38 deletions(-) diff --git a/packages/api/go.mod b/packages/api/go.mod index f49287dece..bf2248bc1a 100644 --- a/packages/api/go.mod +++ b/packages/api/go.mod @@ -35,7 +35,7 @@ require ( github.com/google/uuid v1.6.0 github.com/grafana/loki/v3 v3.6.4 github.com/hashicorp/nomad/api v0.0.0-20251216171439-1dee0671280e - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.9.1 github.com/launchdarkly/go-sdk-common/v3 v3.3.0 github.com/launchdarkly/go-server-sdk/v7 v7.13.0 github.com/oapi-codegen/gin-middleware v1.0.2 @@ -386,7 +386,7 @@ require ( golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect golang.org/x/image v0.38.0 // indirect - golang.org/x/mod v0.33.0 // indirect + golang.org/x/mod v0.34.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/text v0.35.0 // indirect diff --git a/packages/api/go.sum b/packages/api/go.sum index 4508575a05..129d2ffb53 100644 --- a/packages/api/go.sum +++ b/packages/api/go.sum @@ -559,8 +559,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jaegertracing/jaeger-idl v0.5.0 h1:zFXR5NL3Utu7MhPg8ZorxtCBjHrL3ReM1VoB65FOFGE= @@ -1168,8 +1168,8 @@ golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/auth/go.mod b/packages/auth/go.mod index 8895fdba14..adfdb792b4 100644 --- a/packages/auth/go.mod +++ b/packages/auth/go.mod @@ -64,7 +64,7 @@ require ( github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.7.5 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jellydator/ttlcache/v3 v3.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/packages/auth/go.sum b/packages/auth/go.sum index 317c9eca6b..bda859ab9b 100644 --- a/packages/auth/go.sum +++ b/packages/auth/go.sum @@ -116,8 +116,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= diff --git a/packages/clickhouse/go.mod b/packages/clickhouse/go.mod index af431d7702..e7d399d034 100644 --- a/packages/clickhouse/go.mod +++ b/packages/clickhouse/go.mod @@ -42,7 +42,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.7.5 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/joho/godotenv v1.5.1 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect diff --git a/packages/clickhouse/go.sum b/packages/clickhouse/go.sum index a6dacb3c42..1c485d02b4 100644 --- a/packages/clickhouse/go.sum +++ b/packages/clickhouse/go.sum @@ -128,8 +128,7 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -310,8 +309,8 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/client-proxy/go.sum b/packages/client-proxy/go.sum index 588f3d5d51..2987c46eb7 100644 --- a/packages/client-proxy/go.sum +++ b/packages/client-proxy/go.sum @@ -133,8 +133,6 @@ golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05 golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= diff --git a/packages/dashboard-api/go.mod b/packages/dashboard-api/go.mod index 1aaacb18d4..cb4c50b26a 100644 --- a/packages/dashboard-api/go.mod +++ b/packages/dashboard-api/go.mod @@ -131,7 +131,6 @@ require ( github.com/redis/go-redis/v9 v9.17.3 // indirect github.com/riverqueue/river/riverdriver v0.33.0 // indirect github.com/riverqueue/river/rivershared v0.33.0 // indirect - github.com/riverqueue/river/rivertype v0.33.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/shirou/gopsutil/v4 v4.25.9 // indirect @@ -169,7 +168,6 @@ require ( golang.org/x/crypto v0.49.0 // indirect golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect - golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/text v0.35.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2 // indirect diff --git a/packages/docker-reverse-proxy/go.mod b/packages/docker-reverse-proxy/go.mod index 0bd891f836..4b07e1c717 100644 --- a/packages/docker-reverse-proxy/go.mod +++ b/packages/docker-reverse-proxy/go.mod @@ -40,7 +40,7 @@ require ( github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.7.5 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/klauspost/compress v1.18.2 // indirect github.com/lib/pq v1.11.2 // indirect diff --git a/packages/docker-reverse-proxy/go.sum b/packages/docker-reverse-proxy/go.sum index f015853450..ca2293ebd5 100644 --- a/packages/docker-reverse-proxy/go.sum +++ b/packages/docker-reverse-proxy/go.sum @@ -69,8 +69,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= @@ -191,8 +191,6 @@ golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05 golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= diff --git a/packages/envd/go.sum b/packages/envd/go.sum index 8bd0104101..384d294119 100644 --- a/packages/envd/go.sum +++ b/packages/envd/go.sum @@ -213,8 +213,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/packages/local-dev/go.mod b/packages/local-dev/go.mod index 32a7ce3132..0a0a79a15f 100644 --- a/packages/local-dev/go.mod +++ b/packages/local-dev/go.mod @@ -10,7 +10,7 @@ require ( github.com/e2b-dev/infra/packages/db v0.0.0 github.com/e2b-dev/infra/packages/shared v0.0.0 github.com/google/uuid v1.6.0 - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.9.1 github.com/pressly/goose/v3 v3.26.0 github.com/stretchr/testify v1.11.1 github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 diff --git a/packages/local-dev/go.sum b/packages/local-dev/go.sum index 717dbdfdb8..2666677cdf 100644 --- a/packages/local-dev/go.sum +++ b/packages/local-dev/go.sum @@ -69,8 +69,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= diff --git a/packages/orchestrator/go.sum b/packages/orchestrator/go.sum index 6879e76f78..8db15eee62 100644 --- a/packages/orchestrator/go.sum +++ b/packages/orchestrator/go.sum @@ -1439,8 +1439,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/shared/go.mod b/packages/shared/go.mod index 0a92ccac46..08442cb799 100644 --- a/packages/shared/go.mod +++ b/packages/shared/go.mod @@ -55,7 +55,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 go.uber.org/zap v1.27.1 - golang.org/x/mod v0.33.0 + golang.org/x/mod v0.34.0 golang.org/x/oauth2 v0.34.0 golang.org/x/sync v0.20.0 google.golang.org/api v0.257.0 diff --git a/packages/shared/go.sum b/packages/shared/go.sum index c5d23cd03d..fd3b849ea6 100644 --- a/packages/shared/go.sum +++ b/packages/shared/go.sum @@ -1036,8 +1036,7 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/tests/integration/go.mod b/tests/integration/go.mod index 69929f1264..4bc14e49c4 100644 --- a/tests/integration/go.mod +++ b/tests/integration/go.mod @@ -25,7 +25,7 @@ require ( github.com/e2b-dev/infra/packages/envd v0.0.0-00010101000000-000000000000 github.com/e2b-dev/infra/packages/shared v0.0.0 github.com/google/uuid v1.6.0 - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.9.1 github.com/oapi-codegen/runtime v1.1.1 github.com/stretchr/testify v1.11.1 golang.org/x/sync v0.20.0 diff --git a/tests/integration/go.sum b/tests/integration/go.sum index db41922318..a94cd4a961 100644 --- a/tests/integration/go.sum +++ b/tests/integration/go.sum @@ -184,8 +184,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= @@ -458,8 +458,8 @@ golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkN golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= From 06b32749db7089b965f47152529309bb78962f40 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 10 Apr 2026 03:56:47 +0000 Subject: [PATCH 78/92] chore: auto-commit generated changes --- packages/clickhouse/go.sum | 1 + packages/shared/go.sum | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/clickhouse/go.sum b/packages/clickhouse/go.sum index 1c485d02b4..c5f05d320c 100644 --- a/packages/clickhouse/go.sum +++ b/packages/clickhouse/go.sum @@ -129,6 +129,7 @@ github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5ey github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= diff --git a/packages/shared/go.sum b/packages/shared/go.sum index fd3b849ea6..7b75861019 100644 --- a/packages/shared/go.sum +++ b/packages/shared/go.sum @@ -1037,6 +1037,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= From 097db234a2b0b4bc4437093ae300d4c8852dd3cb Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Thu, 9 Apr 2026 21:28:09 -0700 Subject: [PATCH 79/92] refactor(dashboard-api): remove env blob wiring --- .env.gcp.template | 3 +++ iac/modules/job-dashboard-api/main.tf | 7 ++----- iac/modules/job-dashboard-api/variables.tf | 9 ++++----- iac/provider-gcp/main.tf | 2 +- iac/provider-gcp/nomad/main.tf | 20 ++++++++------------ iac/provider-gcp/nomad/variables.tf | 11 +++++------ iac/provider-gcp/variables.tf | 11 +++++------ 7 files changed, 28 insertions(+), 35 deletions(-) diff --git a/.env.gcp.template b/.env.gcp.template index 7674763b5c..e946e9d7e6 100644 --- a/.env.gcp.template +++ b/.env.gcp.template @@ -81,6 +81,9 @@ DASHBOARD_API_COUNT= SUPABASE_DB_CONNECTION_STRING= # Enable dashboard-api auth user sync background worker (default: false) ENABLE_AUTH_USER_SYNC_BACKGROUND_WORKER= +# Enable dashboard-api billing/team provisioning sink (default: false) +# Set via Terraform env var so it doesn't need Makefile wiring. +TF_VAR_enable_billing_http_team_provision_sink= # Filestore cache for builds shared across cluster (default:false) FILESTORE_CACHE_ENABLED= diff --git a/iac/modules/job-dashboard-api/main.tf b/iac/modules/job-dashboard-api/main.tf index dd6b77b849..1156c559fc 100644 --- a/iac/modules/job-dashboard-api/main.tf +++ b/iac/modules/job-dashboard-api/main.tf @@ -2,8 +2,6 @@ locals { base_env = { GIN_MODE = "release" ENVIRONMENT = var.environment - NODE_ID = "$${node.unique.id}" - PORT = "$${NOMAD_PORT_api}" ADMIN_TOKEN = var.admin_token POSTGRES_CONNECTION_STRING = var.postgres_connection_string AUTH_DB_CONNECTION_STRING = var.auth_db_connection_string @@ -15,13 +13,12 @@ locals { REDIS_CLUSTER_URL = var.redis_cluster_url REDIS_TLS_CA_BASE64 = var.redis_tls_ca_base64 ENABLE_AUTH_USER_SYNC_BACKGROUND_WORKER = tostring(var.enable_auth_user_sync_background_worker) + ENABLE_BILLING_HTTP_TEAM_PROVISION_SINK = tostring(var.enable_billing_http_team_provision_sink) BILLING_SERVER_URL = var.billing_server_url BILLING_SERVER_API_TOKEN = var.billing_server_api_token OTEL_COLLECTOR_GRPC_ENDPOINT = "localhost:${var.otel_collector_grpc_port}" LOGS_COLLECTOR_ADDRESS = "http://localhost:${var.logs_proxy_port.port}" } - - env = merge(local.base_env, var.extra_env) } resource "nomad_job" "dashboard_api" { @@ -35,7 +32,7 @@ resource "nomad_job" "dashboard_api" { memory_mb = 512 cpu_count = 1 - env = local.env + env = local.base_env subdomain = "dashboard-api" }) diff --git a/iac/modules/job-dashboard-api/variables.tf b/iac/modules/job-dashboard-api/variables.tf index 8ca67585fb..210bbc2e8c 100644 --- a/iac/modules/job-dashboard-api/variables.tf +++ b/iac/modules/job-dashboard-api/variables.tf @@ -55,13 +55,12 @@ variable "supabase_jwt_secrets" { sensitive = true } -variable "extra_env" { - type = map(string) - default = {} - sensitive = true +variable "enable_auth_user_sync_background_worker" { + type = bool + default = false } -variable "enable_auth_user_sync_background_worker" { +variable "enable_billing_http_team_provision_sink" { type = bool default = false } diff --git a/iac/provider-gcp/main.tf b/iac/provider-gcp/main.tf index 6032e86f5f..d5a1e32000 100644 --- a/iac/provider-gcp/main.tf +++ b/iac/provider-gcp/main.tf @@ -277,9 +277,9 @@ module "nomad" { # Dashboard API dashboard_api_count = var.dashboard_api_count dashboard_api_admin_token = random_password.dashboard_api_admin_secret.result - dashboard_api_env_vars = var.dashboard_api_env_vars supabase_db_connection_string = var.supabase_db_connection_string enable_auth_user_sync_background_worker = var.enable_auth_user_sync_background_worker + enable_billing_http_team_provision_sink = var.enable_billing_http_team_provision_sink # Docker reverse proxy docker_reverse_proxy_port = var.docker_reverse_proxy_port diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index 8f4354758e..f7606ca4d7 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -1,15 +1,11 @@ locals { - clickhouse_connection_string = var.clickhouse_server_count > 0 ? "clickhouse://${var.clickhouse_username}:${random_password.clickhouse_password.result}@clickhouse.service.consul:${var.clickhouse_server_port.port}/${var.clickhouse_database}" : "" - redis_url = trimspace(data.google_secret_manager_secret_version.redis_cluster_url.secret_data) == "" ? "redis.service.consul:${var.redis_port.port}" : "" - redis_cluster_url = trimspace(data.google_secret_manager_secret_version.redis_cluster_url.secret_data) - loki_url = "http://loki.service.consul:${var.loki_service_port.port}" - enable_billing_http_team_provision_sink = ( - var.dashboard_api_count > 0 && - lower(trimspace(lookup(var.dashboard_api_env_vars, "ENABLE_BILLING_HTTP_TEAM_PROVISION_SINK", "false"))) == "true" - ) - dashboard_api_extra_env = var.dashboard_api_env_vars - dashboard_api_billing_server_url = local.enable_billing_http_team_provision_sink ? data.google_cloud_run_v2_service.billing_server[0].uri : "" - dashboard_api_billing_server_api_token = local.enable_billing_http_team_provision_sink ? data.google_secret_manager_secret_version.billing_server_api_token[0].secret_data : "" + clickhouse_connection_string = var.clickhouse_server_count > 0 ? "clickhouse://${var.clickhouse_username}:${random_password.clickhouse_password.result}@clickhouse.service.consul:${var.clickhouse_server_port.port}/${var.clickhouse_database}" : "" + redis_url = trimspace(data.google_secret_manager_secret_version.redis_cluster_url.secret_data) == "" ? "redis.service.consul:${var.redis_port.port}" : "" + redis_cluster_url = trimspace(data.google_secret_manager_secret_version.redis_cluster_url.secret_data) + loki_url = "http://loki.service.consul:${var.loki_service_port.port}" + enable_billing_http_team_provision_sink = var.enable_billing_http_team_provision_sink + dashboard_api_billing_server_url = local.enable_billing_http_team_provision_sink ? data.google_cloud_run_v2_service.billing_server[0].uri : "" + dashboard_api_billing_server_api_token = local.enable_billing_http_team_provision_sink ? data.google_secret_manager_secret_version.billing_server_api_token[0].secret_data : "" } # API @@ -163,9 +159,9 @@ module "dashboard_api" { redis_cluster_url = local.redis_cluster_url redis_tls_ca_base64 = trimspace(data.google_secret_manager_secret_version.redis_tls_ca_base64.secret_data) enable_auth_user_sync_background_worker = var.enable_auth_user_sync_background_worker + enable_billing_http_team_provision_sink = var.enable_billing_http_team_provision_sink billing_server_url = local.dashboard_api_billing_server_url billing_server_api_token = local.dashboard_api_billing_server_api_token - extra_env = local.dashboard_api_extra_env otel_collector_grpc_port = var.otel_collector_grpc_port logs_proxy_port = var.logs_proxy_port diff --git a/iac/provider-gcp/nomad/variables.tf b/iac/provider-gcp/nomad/variables.tf index c47877f617..4cd4d686a2 100644 --- a/iac/provider-gcp/nomad/variables.tf +++ b/iac/provider-gcp/nomad/variables.tf @@ -458,12 +458,6 @@ variable "dashboard_api_count" { default = 0 } -variable "dashboard_api_env_vars" { - type = map(string) - default = {} - sensitive = true -} - variable "supabase_db_connection_string" { type = string default = "" @@ -475,6 +469,11 @@ variable "enable_auth_user_sync_background_worker" { default = false } +variable "enable_billing_http_team_provision_sink" { + type = bool + default = false +} + variable "volume_token_issuer" { type = string } diff --git a/iac/provider-gcp/variables.tf b/iac/provider-gcp/variables.tf index 73d9ecb416..1fa5a2cc1a 100644 --- a/iac/provider-gcp/variables.tf +++ b/iac/provider-gcp/variables.tf @@ -230,12 +230,6 @@ variable "dashboard_api_count" { default = 0 } -variable "dashboard_api_env_vars" { - type = map(string) - default = {} - sensitive = true -} - variable "supabase_db_connection_string" { type = string default = "" @@ -247,6 +241,11 @@ variable "enable_auth_user_sync_background_worker" { default = false } +variable "enable_billing_http_team_provision_sink" { + type = bool + default = false +} + variable "docker_reverse_proxy_port" { type = object({ name = string From 33e45b9d8b16289ae6e96ea9b2bacdedecd10f1a Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Fri, 10 Apr 2026 16:32:42 -0700 Subject: [PATCH 80/92] chore --- iac/provider-gcp/Makefile | 1 + iac/provider-gcp/api.tf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/iac/provider-gcp/Makefile b/iac/provider-gcp/Makefile index 625f1febf9..5edb0e0767 100644 --- a/iac/provider-gcp/Makefile +++ b/iac/provider-gcp/Makefile @@ -78,6 +78,7 @@ tf_vars := \ $(call tfvar, DASHBOARD_API_COUNT) \ $(call tfvar, SUPABASE_DB_CONNECTION_STRING) \ $(call tfvar, ENABLE_AUTH_USER_SYNC_BACKGROUND_WORKER) \ + $(call tfvar, ENABLE_BILLING_HTTP_TEAM_PROVISION_SINK) \ $(call tfvar, DEFAULT_PERSISTENT_VOLUME_TYPE) \ $(call tfvar, PERSISTENT_VOLUME_TYPES) \ $(call tfvar, DB_MAX_OPEN_CONNECTIONS) \ diff --git a/iac/provider-gcp/api.tf b/iac/provider-gcp/api.tf index d62d201601..576b953ba2 100644 --- a/iac/provider-gcp/api.tf +++ b/iac/provider-gcp/api.tf @@ -66,7 +66,7 @@ resource "google_secret_manager_secret_version" "api_admin_token_value" { resource "random_password" "dashboard_api_admin_secret" { length = 32 - special = true + special = false } resource "google_secret_manager_secret" "dashboard_api_admin_token" { From c9828849fd4568807165831b9a4844c358b1cca6 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Fri, 10 Apr 2026 16:45:55 -0700 Subject: [PATCH 81/92] test(dashboard-api): trim low-signal sync coverage --- .../backgroundworker/auth_user_sync_test.go | 19 +----- .../dashboard-api/internal/cfg/model_test.go | 34 ---------- .../internal/teamprovision/factory_test.go | 62 ------------------- ...0260401000001_river_auth_custom_schema.sql | 18 ------ ...01000003_river_auth_user_sync_triggers.sql | 30 --------- 5 files changed, 1 insertion(+), 162 deletions(-) delete mode 100644 packages/dashboard-api/internal/cfg/model_test.go delete mode 100644 packages/dashboard-api/internal/teamprovision/factory_test.go diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go index 2e4824a6e1..17a08740cd 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go @@ -107,28 +107,11 @@ func applyAuthUserSyncMigrations(t *testing.T, db *testutils.Database) { t.Helper() // The auth user sync bootstraps `auth_custom` in three steps: - // 1. create the shared schema and role needed by River migrations + // 1. create the shared schema needed by River migrations // 2. River library migrations create River tables inside that schema // 3. the remaining auth migrations add triggers that enqueue into River require.NoError(t, db.SupabaseDB.TestsRawSQL(t.Context(), ` CREATE SCHEMA IF NOT EXISTS auth_custom; - -DO $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'trigger_user') THEN - CREATE ROLE trigger_user NOLOGIN; - END IF; -END; -$$; - -GRANT USAGE ON SCHEMA auth_custom TO trigger_user; -GRANT CREATE ON SCHEMA auth_custom TO trigger_user; - -DO $$ -BEGIN - EXECUTE format('GRANT TEMP ON DATABASE %I TO trigger_user', current_database()); -END; -$$; `)) require.NoError(t, RunRiverMigrations(t.Context(), db.SupabaseDB.WritePool())) diff --git a/packages/dashboard-api/internal/cfg/model_test.go b/packages/dashboard-api/internal/cfg/model_test.go deleted file mode 100644 index 1b8f3d1393..0000000000 --- a/packages/dashboard-api/internal/cfg/model_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package cfg - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestParseRequiresAdminToken(t *testing.T) { - t.Setenv("POSTGRES_CONNECTION_STRING", "postgres://test") - t.Setenv("REDIS_URL", "redis://localhost:6379") - t.Setenv("REDIS_CLUSTER_URL", "") - t.Setenv("AUTH_DB_CONNECTION_STRING", "") - t.Setenv("AUTH_DB_READ_REPLICA_CONNECTION_STRING", "") - t.Setenv("ADMIN_TOKEN", "") - - _, err := Parse() - require.Error(t, err) - require.ErrorContains(t, err, "ADMIN_TOKEN") -} - -func TestParseAcceptsAdminToken(t *testing.T) { - t.Setenv("POSTGRES_CONNECTION_STRING", "postgres://test") - t.Setenv("REDIS_URL", "redis://localhost:6379") - t.Setenv("REDIS_CLUSTER_URL", "") - t.Setenv("AUTH_DB_CONNECTION_STRING", "") - t.Setenv("AUTH_DB_READ_REPLICA_CONNECTION_STRING", "") - t.Setenv("ADMIN_TOKEN", "secret") - - config, err := Parse() - require.NoError(t, err) - require.Equal(t, "secret", config.AdminToken) - require.Equal(t, "postgres://test", config.AuthDBConnectionString) -} diff --git a/packages/dashboard-api/internal/teamprovision/factory_test.go b/packages/dashboard-api/internal/teamprovision/factory_test.go deleted file mode 100644 index 2a9d5f6796..0000000000 --- a/packages/dashboard-api/internal/teamprovision/factory_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package teamprovision - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestNewProvisionSink(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - enabled bool - baseURL string - apiToken string - wantErr error - wantType any - }{ - { - name: "disabled returns noop", - enabled: false, - wantType: &NoopProvisionSink{}, - }, - { - name: "enabled requires base url", - enabled: true, - apiToken: "token", - wantErr: ErrMissingBaseURL, - }, - { - name: "enabled requires api token", - enabled: true, - baseURL: "https://billing.example.com", - wantErr: ErrMissingAPIToken, - }, - { - name: "enabled returns http sink", - enabled: true, - baseURL: "https://billing.example.com", - apiToken: "token", - wantType: &HTTPProvisionSink{}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - sink, err := NewProvisionSink(tt.enabled, tt.baseURL, tt.apiToken) - if tt.wantErr != nil { - require.Nil(t, sink) - require.ErrorIs(t, err, tt.wantErr) - - return - } - - require.NoError(t, err) - require.IsType(t, tt.wantType, sink) - }) - } -} diff --git a/packages/db/pkg/supabase/migrations/20260401000001_river_auth_custom_schema.sql b/packages/db/pkg/supabase/migrations/20260401000001_river_auth_custom_schema.sql index d5e1a478f5..e4278c9e3d 100644 --- a/packages/db/pkg/supabase/migrations/20260401000001_river_auth_custom_schema.sql +++ b/packages/db/pkg/supabase/migrations/20260401000001_river_auth_custom_schema.sql @@ -3,29 +3,11 @@ CREATE SCHEMA IF NOT EXISTS auth_custom; -DO $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'trigger_user') THEN - CREATE ROLE trigger_user NOLOGIN; - END IF; -END; -$$; - -GRANT USAGE ON SCHEMA auth_custom TO trigger_user; -GRANT CREATE ON SCHEMA auth_custom TO trigger_user; - -DO $$ -BEGIN - EXECUTE format('GRANT TEMP ON DATABASE %I TO trigger_user', current_database()); -END; -$$; - -- +goose StatementEnd -- +goose Down -- +goose StatementBegin -DROP ROLE IF EXISTS trigger_user; DROP SCHEMA IF EXISTS auth_custom; -- +goose StatementEnd diff --git a/packages/db/pkg/supabase/migrations/20260401000003_river_auth_user_sync_triggers.sql b/packages/db/pkg/supabase/migrations/20260401000003_river_auth_user_sync_triggers.sql index 58ce23a7ab..24c0aee26e 100644 --- a/packages/db/pkg/supabase/migrations/20260401000003_river_auth_user_sync_triggers.sql +++ b/packages/db/pkg/supabase/migrations/20260401000003_river_auth_user_sync_triggers.sql @@ -7,10 +7,6 @@ LANGUAGE plpgsql SECURITY DEFINER SET search_path = '' AS $$ BEGIN - IF to_regclass('auth_custom.river_job') IS NULL THEN - RETURN NEW; - END IF; - INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) VALUES ( jsonb_build_object('user_id', NEW.id, 'operation', 'upsert'), @@ -32,10 +28,6 @@ LANGUAGE plpgsql SECURITY DEFINER SET search_path = '' AS $$ BEGIN - IF to_regclass('auth_custom.river_job') IS NULL THEN - RETURN NEW; - END IF; - IF OLD.email IS DISTINCT FROM NEW.email THEN INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) VALUES ( @@ -59,10 +51,6 @@ LANGUAGE plpgsql SECURITY DEFINER SET search_path = '' AS $$ BEGIN - IF to_regclass('auth_custom.river_job') IS NULL THEN - RETURN OLD; - END IF; - INSERT INTO auth_custom.river_job (args, kind, max_attempts, queue, state) VALUES ( jsonb_build_object('user_id', OLD.id, 'operation', 'delete'), @@ -90,15 +78,6 @@ CREATE TRIGGER enqueue_user_sync_on_delete AFTER DELETE ON auth.users FOR EACH ROW EXECUTE FUNCTION auth_custom.enqueue_user_sync_on_delete(); -DO $grant$ -BEGIN - IF to_regclass('auth_custom.river_job') IS NOT NULL THEN - EXECUTE 'GRANT INSERT ON auth_custom.river_job TO trigger_user'; - EXECUTE 'GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA auth_custom TO trigger_user'; - END IF; -END; -$grant$; - -- +goose StatementEnd -- +goose Down @@ -112,13 +91,4 @@ DROP FUNCTION IF EXISTS auth_custom.enqueue_user_sync_on_insert(); DROP FUNCTION IF EXISTS auth_custom.enqueue_user_sync_on_update(); DROP FUNCTION IF EXISTS auth_custom.enqueue_user_sync_on_delete(); -DO $revoke$ -BEGIN - IF to_regclass('auth_custom.river_job') IS NOT NULL THEN - EXECUTE 'REVOKE INSERT ON auth_custom.river_job FROM trigger_user'; - EXECUTE 'REVOKE USAGE, SELECT ON ALL SEQUENCES IN SCHEMA auth_custom FROM trigger_user'; - END IF; -END; -$revoke$; - -- +goose StatementEnd From 5285ea8e9837709d8dad3364760612698204210d Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Fri, 10 Apr 2026 16:50:55 -0700 Subject: [PATCH 82/92] chore: cleanup main exiting --- packages/dashboard-api/main.go | 48 +++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index f6e6d59363..3e19124d80 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -72,7 +72,9 @@ func run() int { tel, err := telemetry.New(ctx, nodeID, serviceName, commitSHA, serviceVersion, serviceInstanceID) if err != nil { - logger.L().Fatal(ctx, "failed to create telemetry", zap.Error(err)) + log.Printf("failed to create telemetry: %v\n", err) + + return 1 } defer func() { if err := tel.Shutdown(ctx); err != nil { @@ -88,14 +90,18 @@ func run() int { EnableConsole: true, }) if err != nil { - logger.L().Fatal(ctx, "failed to create logger", zap.Error(err)) + log.Printf("failed to create logger: %v\n", err) + + return 1 } defer l.Sync() logger.ReplaceGlobals(ctx, l) config, err := cfg.Parse() if err != nil { - l.Fatal(ctx, "failed to parse config", zap.Error(err)) + l.Error(ctx, "failed to parse config", zap.Error(err)) + + return 1 } l.Info(ctx, "Starting dashboard-api service...", zap.String("commit_sha", commitSHA), zap.String("instance_id", serviceInstanceID)) @@ -108,7 +114,9 @@ func run() int { err = sqlcdb.CheckMigrationVersion(ctx, config.PostgresConnectionString, expectedMigration) if err != nil { - l.Fatal(ctx, "failed to check migration version", zap.Error(err)) + l.Error(ctx, "failed to check migration version", zap.Error(err)) + + return 1 } db, err := sqlcdb.NewClient( @@ -117,7 +125,9 @@ func run() int { pool.WithMaxConnections(8), ) if err != nil { - l.Fatal(ctx, "Initializing database client", zap.Error(err)) + l.Error(ctx, "Initializing database client", zap.Error(err)) + + return 1 } defer db.Close() @@ -128,7 +138,9 @@ func run() int { pool.WithMaxConnections(8), ) if err != nil { - l.Fatal(ctx, "Initializing auth database client", zap.Error(err)) + l.Error(ctx, "Initializing auth database client", zap.Error(err)) + + return 1 } defer authDB.Close() @@ -138,7 +150,9 @@ func run() int { } else { clickhouseClient, err = clickhouse.New(config.ClickhouseConnectionString) if err != nil { - l.Fatal(ctx, "Initializing ClickHouse client", zap.Error(err)) + l.Error(ctx, "Initializing ClickHouse client", zap.Error(err)) + + return 1 } defer clickhouseClient.Close(ctx) } @@ -149,7 +163,9 @@ func run() int { RedisTLSCABase64: config.RedisTLSCABase64, }) if err != nil { - l.Fatal(ctx, "Initializing Redis client", zap.Error(err)) + l.Error(ctx, "Initializing Redis client", zap.Error(err)) + + return 1 } defer func() { if err := factories.CloseCleanly(redisClient); err != nil { @@ -168,14 +184,18 @@ func run() int { config.BillingServerAPIToken, ) if err != nil { - l.Fatal(ctx, "initializing team provision sink", zap.Error(err)) + l.Error(ctx, "initializing team provision sink", zap.Error(err)) + + return 1 } apiStore := handlers.NewAPIStore(config, db, authDB, clickhouseClient, authService, teamProvisionSink) swagger, err := api.GetSwagger() if err != nil { - l.Fatal(ctx, "Error loading swagger spec", zap.Error(err)) + l.Error(ctx, "Error loading swagger spec", zap.Error(err)) + + return 1 } swagger.Servers = nil @@ -203,7 +223,9 @@ func run() int { pool.WithMaxConnections(4), ) if err != nil { - l.Fatal(ctx, "Initializing supabase database client", zap.Error(err)) + l.Error(ctx, "Initializing supabase database client", zap.Error(err)) + + return 1 } defer supabaseDB.Close() @@ -215,7 +237,9 @@ func run() int { l, ) if err != nil { - l.Fatal(ctx, "failed to start auth user sync worker", zap.Error(err)) + l.Error(ctx, "failed to start auth user sync worker", zap.Error(err)) + + return 1 } } From 973d3043c046e2bc32579c7cae0b06a61e61bdb5 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Fri, 10 Apr 2026 17:37:14 -0700 Subject: [PATCH 83/92] fix: use supabase db client for auth.users queries --- .../dashboard-api/internal/handlers/store.go | 5 +++- .../internal/handlers/team_handlers_test.go | 11 +++++++- .../internal/handlers/team_provisioning.go | 4 +-- packages/dashboard-api/main.go | 28 +++++++++---------- .../pkg/supabase/queries/get_auth_user.sql.go | 4 +-- packages/db/pkg/supabase/queries/models.go | 7 +++-- .../supabase/schema/auth_users_override.sql | 1 + .../sql_queries/users/get_auth_user.sql | 2 +- 8 files changed, 38 insertions(+), 24 deletions(-) diff --git a/packages/dashboard-api/internal/handlers/store.go b/packages/dashboard-api/internal/handlers/store.go index c1c92f0018..f64013f394 100644 --- a/packages/dashboard-api/internal/handlers/store.go +++ b/packages/dashboard-api/internal/handlers/store.go @@ -15,6 +15,7 @@ import ( internalteamprovision "github.com/e2b-dev/infra/packages/dashboard-api/internal/teamprovision" sqlcdb "github.com/e2b-dev/infra/packages/db/client" authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" + supabasedb "github.com/e2b-dev/infra/packages/db/pkg/supabase" "github.com/e2b-dev/infra/packages/shared/pkg/apierrors" ) @@ -24,16 +25,18 @@ type APIStore struct { config cfg.Config db *sqlcdb.Client authDB *authdb.Client + supabaseDB *supabasedb.Client clickhouse clickhouse.Clickhouse authService *sharedauth.AuthService[*types.Team] teamProvisionSink internalteamprovision.TeamProvisionSink } -func NewAPIStore(config cfg.Config, db *sqlcdb.Client, authDB *authdb.Client, ch clickhouse.Clickhouse, authService *sharedauth.AuthService[*types.Team], teamProvisionSink internalteamprovision.TeamProvisionSink) *APIStore { +func NewAPIStore(config cfg.Config, db *sqlcdb.Client, authDB *authdb.Client, supabaseDB *supabasedb.Client, ch clickhouse.Clickhouse, authService *sharedauth.AuthService[*types.Team], teamProvisionSink internalteamprovision.TeamProvisionSink) *APIStore { return &APIStore{ config: config, db: db, authDB: authDB, + supabaseDB: supabaseDB, clickhouse: ch, authService: authService, teamProvisionSink: teamProvisionSink, diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index 02506c9068..80ac6d8309 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -268,7 +268,7 @@ func createHandlerTestUserAt(t *testing.T, db *testutils.Database, createdAt tim userID := uuid.New() email := handlerTestUserEmail(userID) - err := db.AuthDB.TestsRawSQL(t.Context(), ` + err := db.SupabaseDB.TestsRawSQL(t.Context(), ` INSERT INTO auth.users (id, email, created_at) VALUES ($1, $2, $3) `, userID, email, createdAt) @@ -322,6 +322,7 @@ func TestPostUsersBootstrap_CreatesDefaultTeamAndCallsSink(t *testing.T) { store := &APIStore{ db: testDB.SqlcClient, authDB: testDB.AuthDB, + supabaseDB: testDB.SupabaseDB, teamProvisionSink: sink, } store.PostAdminUsersBootstrap(ginCtx) @@ -388,6 +389,7 @@ func TestPostUsersBootstrap_ProvisioningFailureKeepsCreatedDefaultTeam(t *testin store := &APIStore{ db: testDB.SqlcClient, authDB: testDB.AuthDB, + supabaseDB: testDB.SupabaseDB, teamProvisionSink: sink, } store.PostAdminUsersBootstrap(ginCtx) @@ -438,6 +440,7 @@ func TestBootstrapUser_ConcurrentRequestsCreateSingleDefaultTeam(t *testing.T) { store := &APIStore{ db: testDB.SqlcClient, authDB: testDB.AuthDB, + supabaseDB: testDB.SupabaseDB, teamProvisionSink: sink, } @@ -511,6 +514,7 @@ func TestCreateTeam_RecentUserCreatesBlockedTeam(t *testing.T) { store := &APIStore{ db: testDB.SqlcClient, authDB: testDB.AuthDB, + supabaseDB: testDB.SupabaseDB, teamProvisionSink: &fakeTeamProvisionSink{}, } @@ -562,6 +566,7 @@ func TestPostTeams_LocalPolicyDeniedReturnsBadRequestWithoutCreatingTeam(t *test store := &APIStore{ db: testDB.SqlcClient, authDB: testDB.AuthDB, + supabaseDB: testDB.SupabaseDB, teamProvisionSink: sink, } store.PostTeams(ginCtx) @@ -600,6 +605,7 @@ func TestPostTeams_InvalidNameReturnsBadRequest(t *testing.T) { store := &APIStore{ db: testDB.SqlcClient, authDB: testDB.AuthDB, + supabaseDB: testDB.SupabaseDB, teamProvisionSink: sink, } store.PostTeams(ginCtx) @@ -630,6 +636,7 @@ func TestPostTeams_TrimsNameBeforeCreate(t *testing.T) { store := &APIStore{ db: testDB.SqlcClient, authDB: testDB.AuthDB, + supabaseDB: testDB.SupabaseDB, teamProvisionSink: sink, } store.PostTeams(ginCtx) @@ -681,6 +688,7 @@ func TestPostTeams_ProvisioningFailureRollsBackCreatedTeam(t *testing.T) { store := &APIStore{ db: testDB.SqlcClient, authDB: testDB.AuthDB, + supabaseDB: testDB.SupabaseDB, teamProvisionSink: sink, } store.PostTeams(ginCtx) @@ -722,6 +730,7 @@ func TestCreateTeam_ConcurrentRequestsRespectLocalPolicyWithZeroMemberships(t *t store := &APIStore{ db: testDB.SqlcClient, authDB: testDB.AuthDB, + supabaseDB: testDB.SupabaseDB, teamProvisionSink: &fakeTeamProvisionSink{}, } diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index e80e8bc25a..2599ed65a5 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -91,7 +91,7 @@ func (s *APIStore) PostTeams(c *gin.Context) { } func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisionedTeam, error) { - authUser, err := s.authDB.Write.GetUser(ctx, userID) + authUser, err := s.supabaseDB.Write.GetAuthUserByID(ctx, userID) if err != nil { return provisionedTeam{}, fmt.Errorf("get auth user: %w", err) } @@ -188,7 +188,7 @@ func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisi } func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string) (provisionedTeam, error) { - authUser, err := s.authDB.Write.GetUser(ctx, userID) + authUser, err := s.supabaseDB.Write.GetAuthUserByID(ctx, userID) if err != nil { return provisionedTeam{}, fmt.Errorf("get auth user: %w", err) } diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 3e19124d80..d2b9e2133d 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -144,6 +144,18 @@ func run() int { } defer authDB.Close() + supabaseDB, err := supabasedb.NewClient( + ctx, + config.SupabaseDBConnectionString, + pool.WithMaxConnections(4), + ) + if err != nil { + l.Error(ctx, "Initializing supabase database client", zap.Error(err)) + + return 1 + } + defer supabaseDB.Close() + var clickhouseClient clickhouse.Clickhouse if config.ClickhouseConnectionString == "" { clickhouseClient = clickhouse.NewNoopClient() @@ -189,7 +201,7 @@ func run() int { return 1 } - apiStore := handlers.NewAPIStore(config, db, authDB, clickhouseClient, authService, teamProvisionSink) + apiStore := handlers.NewAPIStore(config, db, authDB, supabaseDB, clickhouseClient, authService, teamProvisionSink) swagger, err := api.GetSwagger() if err != nil { @@ -214,21 +226,7 @@ func run() int { defer sigCancel() var riverClient *river.Client[pgx.Tx] - var supabaseDB *supabasedb.Client - if config.EnableAuthUserSyncBackgroundWorker { - supabaseDB, err = supabasedb.NewClient( - ctx, - config.SupabaseDBConnectionString, - pool.WithMaxConnections(4), - ) - if err != nil { - l.Error(ctx, "Initializing supabase database client", zap.Error(err)) - - return 1 - } - defer supabaseDB.Close() - riverClient, err = backgroundworker.StartAuthUserSyncWorker( ctx, signalCtx, diff --git a/packages/db/pkg/supabase/queries/get_auth_user.sql.go b/packages/db/pkg/supabase/queries/get_auth_user.sql.go index 4f881e7ff3..11c863f8dd 100644 --- a/packages/db/pkg/supabase/queries/get_auth_user.sql.go +++ b/packages/db/pkg/supabase/queries/get_auth_user.sql.go @@ -12,7 +12,7 @@ import ( ) const getAuthUserByID = `-- name: GetAuthUserByID :one -SELECT id, COALESCE(email, '') AS email +SELECT id, COALESCE(email, '') AS email, created_at FROM auth.users WHERE id = $1::uuid ` @@ -20,6 +20,6 @@ WHERE id = $1::uuid func (q *Queries) GetAuthUserByID(ctx context.Context, dollar_1 uuid.UUID) (AuthUser, error) { row := q.db.QueryRow(ctx, getAuthUserByID, dollar_1) var i AuthUser - err := row.Scan(&i.ID, &i.Email) + err := row.Scan(&i.ID, &i.Email, &i.CreatedAt) return i, err } diff --git a/packages/db/pkg/supabase/queries/models.go b/packages/db/pkg/supabase/queries/models.go index 4bbb324fa3..11384b0cad 100644 --- a/packages/db/pkg/supabase/queries/models.go +++ b/packages/db/pkg/supabase/queries/models.go @@ -5,10 +5,13 @@ package supabasequeries import ( + "time" + "github.com/google/uuid" ) type AuthUser struct { - ID uuid.UUID - Email string + ID uuid.UUID + Email string + CreatedAt time.Time } diff --git a/packages/db/pkg/supabase/schema/auth_users_override.sql b/packages/db/pkg/supabase/schema/auth_users_override.sql index 83cc73104d..7522d1902d 100644 --- a/packages/db/pkg/supabase/schema/auth_users_override.sql +++ b/packages/db/pkg/supabase/schema/auth_users_override.sql @@ -14,5 +14,6 @@ GRANT EXECUTE ON FUNCTION auth.uid() TO postgres; CREATE TABLE auth.users ( id uuid NOT NULL DEFAULT gen_random_uuid(), email text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (id) ); diff --git a/packages/db/pkg/supabase/sql_queries/users/get_auth_user.sql b/packages/db/pkg/supabase/sql_queries/users/get_auth_user.sql index f01b84a05c..ca83c61b33 100644 --- a/packages/db/pkg/supabase/sql_queries/users/get_auth_user.sql +++ b/packages/db/pkg/supabase/sql_queries/users/get_auth_user.sql @@ -1,4 +1,4 @@ -- name: GetAuthUserByID :one -SELECT id, COALESCE(email, '') AS email +SELECT id, COALESCE(email, '') AS email, created_at FROM auth.users WHERE id = $1::uuid; From 32d9d9cdc0e63bffdb8ff63403af163984b0c283 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Fri, 10 Apr 2026 18:59:50 -0700 Subject: [PATCH 84/92] chore: template env --- .env.gcp.template | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.env.gcp.template b/.env.gcp.template index e946e9d7e6..94df21dac7 100644 --- a/.env.gcp.template +++ b/.env.gcp.template @@ -82,8 +82,7 @@ SUPABASE_DB_CONNECTION_STRING= # Enable dashboard-api auth user sync background worker (default: false) ENABLE_AUTH_USER_SYNC_BACKGROUND_WORKER= # Enable dashboard-api billing/team provisioning sink (default: false) -# Set via Terraform env var so it doesn't need Makefile wiring. -TF_VAR_enable_billing_http_team_provision_sink= +ENABLE_BILLING_HTTP_TEAM_PROVISION_SINK= # Filestore cache for builds shared across cluster (default:false) FILESTORE_CACHE_ENABLED= From fdea3be657dcf323d6dbd3e5fc2712509e0434c5 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Fri, 10 Apr 2026 20:49:11 -0700 Subject: [PATCH 85/92] refactor(dashboard-api): align auth locking and sink setup --- packages/dashboard-api/Makefile | 8 ++---- .../internal/handlers/team_provisioning.go | 7 ++++-- .../internal/teamprovision/factory.go | 6 ++--- .../internal/teamprovision/http_sink.go | 4 ++- packages/dashboard-api/main.go | 1 + .../lock_public_user_for_update.sql.go | 25 +++++++++++++++++++ .../users/lock_public_user_for_update.sql | 5 ++++ .../db/queries/team_creation_guard.sql.go | 13 ---------- 8 files changed, 44 insertions(+), 25 deletions(-) create mode 100644 packages/db/pkg/auth/queries/lock_public_user_for_update.sql.go create mode 100644 packages/db/pkg/auth/sql_queries/users/lock_public_user_for_update.sql diff --git a/packages/dashboard-api/Makefile b/packages/dashboard-api/Makefile index fb8e5fe69e..514252612b 100644 --- a/packages/dashboard-api/Makefile +++ b/packages/dashboard-api/Makefile @@ -13,10 +13,6 @@ endif HOSTNAME := $(shell hostname 2> /dev/null || hostnamectl hostname 2> /dev/null) $(if $(HOSTNAME),,$(error Failed to determine hostname: both 'hostname' and 'hostnamectl' failed)) -define export_extra_env -export $$(printf '%s' "$$DASHBOARD_API_ENV_VARS" | jq -r '(if .=="" then empty elif type=="string" then (fromjson? // empty) else . end) | to_entries? // [] | map("\(.key)=\(.value|tostring)") | .[]' 2>/dev/null) 2>/dev/null; -endef - .PHONY: generate generate: go generate ./... @@ -36,12 +32,12 @@ build-and-upload: .PHONY: run run: make build - @$(export_extra_env) ./bin/dashboard-api + @./bin/dashboard-api .PHONY: run-local run-local: make build - @$(export_extra_env) NODE_ID=$(HOSTNAME) ./bin/dashboard-api + @NODE_ID=$(HOSTNAME) ./bin/dashboard-api .PHONY: test test: diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index 2599ed65a5..303cff2410 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -16,6 +16,7 @@ import ( "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" internalteamprovision "github.com/e2b-dev/infra/packages/dashboard-api/internal/teamprovision" sqlcdb "github.com/e2b-dev/infra/packages/db/client" + authqueries "github.com/e2b-dev/infra/packages/db/pkg/auth/queries" "github.com/e2b-dev/infra/packages/db/pkg/dberrors" "github.com/e2b-dev/infra/packages/db/queries" "github.com/e2b-dev/infra/packages/shared/pkg/ginutils" @@ -100,6 +101,7 @@ func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisi if err != nil { return provisionedTeam{}, fmt.Errorf("start transaction: %w", err) } + authTxDB := authqueries.New(tx) defer func() { _ = tx.Rollback(ctx) }() @@ -112,7 +114,7 @@ func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisi } // Serialize bootstrap for a user even when they have no team memberships yet. - if _, err := txDB.LockPublicUserForUpdate(ctx, authUser.ID); err != nil { + if _, err := authTxDB.LockPublicUserForUpdate(ctx, authUser.ID); err != nil { return provisionedTeam{}, fmt.Errorf("lock public user: %w", err) } @@ -197,6 +199,7 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string if err != nil { return provisionedTeam{}, fmt.Errorf("start transaction: %w", err) } + authTxDB := authqueries.New(tx) defer func() { _ = tx.Rollback(ctx) }() @@ -209,7 +212,7 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string } // Serialize team creation even when the user currently has no team memberships. - if _, err := txDB.LockPublicUserForUpdate(ctx, authUser.ID); err != nil { + if _, err := authTxDB.LockPublicUserForUpdate(ctx, authUser.ID); err != nil { return provisionedTeam{}, fmt.Errorf("lock public user: %w", err) } diff --git a/packages/dashboard-api/internal/teamprovision/factory.go b/packages/dashboard-api/internal/teamprovision/factory.go index 7c39b88047..05bc4625ce 100644 --- a/packages/dashboard-api/internal/teamprovision/factory.go +++ b/packages/dashboard-api/internal/teamprovision/factory.go @@ -14,9 +14,9 @@ var ( ErrMissingAPIToken = errors.New("billing server api token is required when billing http team provision sink is enabled") ) -func NewProvisionSink(enabled bool, baseURL, apiToken string) (TeamProvisionSink, error) { +func NewProvisionSink(ctx context.Context, enabled bool, baseURL, apiToken string) (TeamProvisionSink, error) { if !enabled { - logger.L().Info(context.Background(), "team provision sink configured", + logger.L().Info(ctx, "team provision sink configured", zap.String("sink", "noop"), zap.String("result", "disabled"), ) @@ -32,7 +32,7 @@ func NewProvisionSink(enabled bool, baseURL, apiToken string) (TeamProvisionSink return nil, ErrMissingAPIToken } - logger.L().Info(context.Background(), "team provision sink configured", + logger.L().Info(ctx, "team provision sink configured", zap.String("sink", "http"), zap.String("result", "enabled"), zap.String("base_url", baseURL), diff --git a/packages/dashboard-api/internal/teamprovision/http_sink.go b/packages/dashboard-api/internal/teamprovision/http_sink.go index 96e0332ec3..b0c7a8df05 100644 --- a/packages/dashboard-api/internal/teamprovision/http_sink.go +++ b/packages/dashboard-api/internal/teamprovision/http_sink.go @@ -30,6 +30,8 @@ const ( defaultProvisionRetryWaitCeiling = 2 * time.Second defaultProvisionAttemptTimeout = defaultProvisionTimeout / defaultProvisionRetryMaxAttempts provisionBackoffMultiplier = 2.0 + // Error responses only need enough body to extract a short API message without buffering large upstream payloads. + provisionErrorMessageReadLimit = 2 * 1024 ) type HTTPProvisionSink struct { @@ -210,7 +212,7 @@ func newRetryableProvisionClient(timeout time.Duration) *retryablehttp.Client { } func readProvisionErrorMessage(resp *http.Response) (string, error) { - body, err := io.ReadAll(io.LimitReader(resp.Body, 2048)) + body, err := io.ReadAll(io.LimitReader(resp.Body, provisionErrorMessageReadLimit)) if err != nil { return "", err } diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index d2b9e2133d..3e3b4df657 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -191,6 +191,7 @@ func run() int { defer authService.Close(ctx) teamProvisionSink, err := internalteamprovision.NewProvisionSink( + ctx, config.EnableBillingHTTPTeamProvisionSink, config.BillingServerURL, config.BillingServerAPIToken, diff --git a/packages/db/pkg/auth/queries/lock_public_user_for_update.sql.go b/packages/db/pkg/auth/queries/lock_public_user_for_update.sql.go new file mode 100644 index 0000000000..d255400753 --- /dev/null +++ b/packages/db/pkg/auth/queries/lock_public_user_for_update.sql.go @@ -0,0 +1,25 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: lock_public_user_for_update.sql + +package authqueries + +import ( + "context" + + "github.com/google/uuid" +) + +const lockPublicUserForUpdate = `-- name: LockPublicUserForUpdate :one +SELECT id +FROM public.users +WHERE id = $1 +FOR UPDATE +` + +func (q *Queries) LockPublicUserForUpdate(ctx context.Context, id uuid.UUID) (uuid.UUID, error) { + row := q.db.QueryRow(ctx, lockPublicUserForUpdate, id) + err := row.Scan(&id) + return id, err +} diff --git a/packages/db/pkg/auth/sql_queries/users/lock_public_user_for_update.sql b/packages/db/pkg/auth/sql_queries/users/lock_public_user_for_update.sql new file mode 100644 index 0000000000..d8201156b8 --- /dev/null +++ b/packages/db/pkg/auth/sql_queries/users/lock_public_user_for_update.sql @@ -0,0 +1,5 @@ +-- name: LockPublicUserForUpdate :one +SELECT id +FROM public.users +WHERE id = @id +FOR UPDATE; diff --git a/packages/db/queries/team_creation_guard.sql.go b/packages/db/queries/team_creation_guard.sql.go index 8747e60310..810ceccda8 100644 --- a/packages/db/queries/team_creation_guard.sql.go +++ b/packages/db/queries/team_creation_guard.sql.go @@ -101,16 +101,3 @@ func (q *Queries) GetTeamsWithUsersTeamsWithTierForUpdate(ctx context.Context, u } return items, nil } - -const lockPublicUserForUpdate = `-- name: LockPublicUserForUpdate :one -SELECT id -FROM public.users -WHERE id = $1::uuid -FOR UPDATE -` - -func (q *Queries) LockPublicUserForUpdate(ctx context.Context, id uuid.UUID) (uuid.UUID, error) { - row := q.db.QueryRow(ctx, lockPublicUserForUpdate, id) - err := row.Scan(&id) - return id, err -} From a486438c9cdc491d77c0b8f6b0b71f17b68ea344 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 11 Apr 2026 03:50:42 +0000 Subject: [PATCH 86/92] chore: auto-commit generated changes --- packages/db/queries/team_creation_guard.sql.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/db/queries/team_creation_guard.sql.go b/packages/db/queries/team_creation_guard.sql.go index 810ceccda8..8747e60310 100644 --- a/packages/db/queries/team_creation_guard.sql.go +++ b/packages/db/queries/team_creation_guard.sql.go @@ -101,3 +101,16 @@ func (q *Queries) GetTeamsWithUsersTeamsWithTierForUpdate(ctx context.Context, u } return items, nil } + +const lockPublicUserForUpdate = `-- name: LockPublicUserForUpdate :one +SELECT id +FROM public.users +WHERE id = $1::uuid +FOR UPDATE +` + +func (q *Queries) LockPublicUserForUpdate(ctx context.Context, id uuid.UUID) (uuid.UUID, error) { + row := q.db.QueryRow(ctx, lockPublicUserForUpdate, id) + err := row.Scan(&id) + return id, err +} From 8d70872af46e92a5bc45c02f5f6601732fd1c389 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Mon, 13 Apr 2026 18:22:59 -0700 Subject: [PATCH 87/92] fix(dashboard-api): preserve create-team provisioning errors Keep upstream billing status codes in team creation responses and report invalid create-team requests so traces retain the real failure cause. --- .../internal/handlers/team_handlers_test.go | 113 ++++++++++++++++++ .../internal/handlers/team_provisioning.go | 18 ++- 2 files changed, 128 insertions(+), 3 deletions(-) diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index 80ac6d8309..146a556fa8 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -619,6 +619,39 @@ func TestPostTeams_InvalidNameReturnsBadRequest(t *testing.T) { } } +func TestPostTeams_InvalidRequestBodyReturnsBadRequest(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUser(t, testDB) + sink := &fakeTeamProvisionSink{} + + recorder := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(recorder) + ginCtx.Request = httptest.NewRequestWithContext(ctx, http.MethodPost, "/", strings.NewReader(`{"name":`)) + ginCtx.Request.Header.Set("Content-Type", "application/json") + auth.SetUserID(ginCtx, userID) + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDB, + supabaseDB: testDB.SupabaseDB, + teamProvisionSink: sink, + } + store.PostTeams(ginCtx) + + if recorder.Code != http.StatusBadRequest { + t.Fatalf("PostTeams(invalid JSON) status = %d, want %d", recorder.Code, http.StatusBadRequest) + } + if !strings.Contains(recorder.Body.String(), "Invalid request body") { + t.Fatalf("PostTeams(invalid JSON) body = %q, want message containing %q", recorder.Body.String(), "Invalid request body") + } + if len(sink.requests) != 0 { + t.Fatalf("PostTeams(invalid JSON) provisioning calls = %d, want %d", len(sink.requests), 0) + } +} + func TestPostTeams_TrimsNameBeforeCreate(t *testing.T) { t.Parallel() @@ -712,6 +745,86 @@ func TestPostTeams_ProvisioningFailureRollsBackCreatedTeam(t *testing.T) { } } +func TestPostTeams_ProvisioningFailurePreservesProvisionErrorStatus(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + status int + message string + }{ + {name: "too_many_requests", status: http.StatusTooManyRequests, message: "rate limited"}, + {name: "service_unavailable", status: http.StatusServiceUnavailable, message: "billing unavailable"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUser(t, testDB) + sink := &fakeTeamProvisionSink{ + err: &internalteamprovision.ProvisionError{ + StatusCode: tt.status, + Message: tt.message, + }, + } + + recorder := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(recorder) + ginCtx.Request = httptest.NewRequestWithContext(ctx, http.MethodPost, "/", strings.NewReader(`{"name":"Acme"}`)) + ginCtx.Request.Header.Set("Content-Type", "application/json") + auth.SetUserID(ginCtx, userID) + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDB, + supabaseDB: testDB.SupabaseDB, + teamProvisionSink: sink, + } + store.PostTeams(ginCtx) + + if recorder.Code != tt.status { + t.Fatalf("PostTeams(provision status %d) status = %d, want %d", tt.status, recorder.Code, tt.status) + } + if len(sink.requests) != 1 { + t.Fatalf("PostTeams(provision status %d) provisioning calls = %d, want %d", tt.status, len(sink.requests), 1) + } + + var responseBody map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &responseBody); err != nil { + t.Fatalf("json.Unmarshal(PostTeams response) error = %v, want nil", err) + } + + codeValue, ok := responseBody["code"].(float64) + if !ok { + t.Fatalf("PostTeams(provision status %d) response code type = %T, want float64", tt.status, responseBody["code"]) + } + if got := int(codeValue); got != tt.status { + t.Fatalf("PostTeams(provision status %d) response code = %d, want %d", tt.status, got, tt.status) + } + + messageValue, ok := responseBody["message"].(string) + if !ok { + t.Fatalf("PostTeams(provision status %d) response message type = %T, want string", tt.status, responseBody["message"]) + } + if messageValue != tt.message { + t.Fatalf("PostTeams(provision status %d) response message = %q, want %q", tt.status, messageValue, tt.message) + } + + rows, err := testDB.AuthDB.Read.GetTeamsWithUsersTeamsWithTier(ctx, userID) + if err != nil { + t.Fatalf("GetTeamsWithUsersTeamsWithTier(userID=%s) error = %v, want nil", userID, err) + } + if len(rows) != 1 { + t.Fatalf("GetTeamsWithUsersTeamsWithTier(userID=%s) rows = %d, want %d", userID, len(rows), 1) + } + if !rows[0].IsDefault { + t.Fatal("expected remaining team to be the default team") + } + }) + } +} + func TestCreateTeam_ConcurrentRequestsRespectLocalPolicyWithZeroMemberships(t *testing.T) { t.Parallel() diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index 303cff2410..0102d8bb76 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -65,14 +65,19 @@ func (s *APIStore) PostTeams(c *gin.Context) { telemetry.ReportEvent(ctx, "create team") userID := auth.MustGetUserID(c) + attrs := []attribute.KeyValue{ + attribute.String("team.provision.operation", "create team"), + } body, err := ginutils.ParseBody[api.CreateTeamRequest](ctx, c) if err != nil { + telemetry.ReportErrorByCode(ctx, http.StatusBadRequest, "create team failed", fmt.Errorf("parse create team request: %w", err), attrs...) s.sendAPIStoreError(c, http.StatusBadRequest, "Invalid request body") return } name := strings.TrimSpace(body.Name) if name == "" { + telemetry.ReportErrorByCode(ctx, http.StatusBadRequest, "create team failed", errors.New("team name is required"), attrs...) s.sendAPIStoreError(c, http.StatusBadRequest, "Team name is required") return @@ -332,9 +337,16 @@ func (s *APIStore) handleProvisioningError(ctx context.Context, c *gin.Context, } var provisionErr *internalteamprovision.ProvisionError - if errors.As(err, &provisionErr) && provisionErr.IsBadRequest() { - telemetry.ReportErrorByCode(ctx, http.StatusBadRequest, operation+" failed", err, attrs...) - s.sendAPIStoreError(c, http.StatusBadRequest, provisionErr.Error()) + if errors.As(err, &provisionErr) { + if provisionErr.StatusCode < http.StatusBadRequest || provisionErr.StatusCode >= 600 { + telemetry.ReportErrorByCode(ctx, http.StatusInternalServerError, operation+" failed", err, attrs...) + s.sendAPIStoreError(c, http.StatusInternalServerError, "Failed to "+operation) + + return + } + + telemetry.ReportErrorByCode(ctx, provisionErr.StatusCode, operation+" failed", err, attrs...) + s.sendAPIStoreError(c, provisionErr.StatusCode, provisionErr.Error()) return } From 1e49371f3e0a6608297080187c1737c9dcae2302 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Mon, 13 Apr 2026 18:31:46 -0700 Subject: [PATCH 88/92] test(dashboard-api): parallelize provisioning subtests Mark the create-team provisioning status subtests parallel so the handler tests match the repo's lint rules and CI passes. --- packages/dashboard-api/internal/handlers/team_handlers_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index 146a556fa8..1a63032cd7 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -759,6 +759,8 @@ func TestPostTeams_ProvisioningFailurePreservesProvisionErrorStatus(t *testing.T for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() + testDB := testutils.SetupDatabase(t) ctx := t.Context() userID := createHandlerTestUser(t, testDB) From e3f6b093b327552712cc0c746ffa305fee3546e0 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Tue, 14 Apr 2026 14:19:15 -0700 Subject: [PATCH 89/92] Use auth DB client for auth queries --- .../backgroundworker/auth_user_sync.go | 14 +- .../backgroundworker/auth_user_sync_test.go | 8 +- .../internal/backgroundworker/river.go | 4 +- .../internal/handlers/team_handlers_test.go | 28 +-- .../internal/handlers/team_provisioning.go | 30 ++- packages/dashboard-api/main.go | 2 +- packages/db/queries/delete_public_user.sql.go | 22 --- .../db/queries/team_creation_guard.sql.go | 116 ------------ packages/db/queries/team_lifecycle.sql.go | 172 ------------------ .../db/queries/teams/team_creation_guard.sql | 32 ---- packages/db/queries/teams/team_lifecycle.sql | 52 ------ packages/db/queries/upsert_public_user.sql.go | 31 ---- .../db/queries/users/delete_public_user.sql | 3 - .../db/queries/users/upsert_public_user.sql | 7 - 14 files changed, 41 insertions(+), 480 deletions(-) delete mode 100644 packages/db/queries/delete_public_user.sql.go delete mode 100644 packages/db/queries/team_creation_guard.sql.go delete mode 100644 packages/db/queries/team_lifecycle.sql.go delete mode 100644 packages/db/queries/teams/team_creation_guard.sql delete mode 100644 packages/db/queries/teams/team_lifecycle.sql delete mode 100644 packages/db/queries/upsert_public_user.sql.go delete mode 100644 packages/db/queries/users/delete_public_user.sql delete mode 100644 packages/db/queries/users/upsert_public_user.sql diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go index 4cc42ffbfe..b937cc6f99 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync.go @@ -12,10 +12,10 @@ import ( "go.opentelemetry.io/otel/trace" "go.uber.org/zap" - sqlcdb "github.com/e2b-dev/infra/packages/db/client" + authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" + authqueries "github.com/e2b-dev/infra/packages/db/pkg/auth/queries" "github.com/e2b-dev/infra/packages/db/pkg/dberrors" supabasedb "github.com/e2b-dev/infra/packages/db/pkg/supabase" - "github.com/e2b-dev/infra/packages/db/queries" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) @@ -38,12 +38,12 @@ type AuthUserSyncWorker struct { river.WorkerDefaults[AuthUserSyncArgs] supabaseDB *supabasedb.Client - authDB *sqlcdb.Client + authDB *authdb.Client l logger.Logger jobsCounter metric.Int64Counter } -func NewAuthUserSyncWorker(ctx context.Context, supabaseDB *supabasedb.Client, authDB *sqlcdb.Client, l logger.Logger) *AuthUserSyncWorker { +func NewAuthUserSyncWorker(ctx context.Context, supabaseDB *supabasedb.Client, authDB *authdb.Client, l logger.Logger) *AuthUserSyncWorker { jobsCounter, err := workerMeter.Int64Counter( "jobs_total", metric.WithDescription("Total auth user sync jobs by operation and result."), @@ -90,7 +90,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy switch job.Args.Operation { case "delete": - if err := w.authDB.DeletePublicUser(ctx, userID); err != nil { + if err := w.authDB.Write.DeletePublicUser(ctx, userID); err != nil { telemetry.ReportError(ctx, "auth user sync delete public user", err) w.observeJob(ctx, job.Args.Operation, jobResultError) @@ -100,7 +100,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy case "upsert": supabaseUser, err := w.supabaseDB.Write.GetAuthUserByID(ctx, userID) if dberrors.IsNotFoundError(err) { - if err := w.authDB.DeletePublicUser(ctx, userID); err != nil { + if err := w.authDB.Write.DeletePublicUser(ctx, userID); err != nil { telemetry.ReportError(ctx, "auth user sync delete stale public user", err) w.observeJob(ctx, job.Args.Operation, jobResultError) @@ -127,7 +127,7 @@ func (w *AuthUserSyncWorker) Work(ctx context.Context, job *river.Job[AuthUserSy return river.JobCancel(err) } - if err := w.authDB.UpsertPublicUser(ctx, queries.UpsertPublicUserParams{ + if err := w.authDB.Write.UpsertPublicUser(ctx, authqueries.UpsertPublicUserParams{ ID: userID, Email: supabaseUser.Email, }); err != nil { diff --git a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go index 17a08740cd..0c82cb2c2b 100644 --- a/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go +++ b/packages/dashboard-api/internal/backgroundworker/auth_user_sync_test.go @@ -14,8 +14,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + authqueries "github.com/e2b-dev/infra/packages/db/pkg/auth/queries" "github.com/e2b-dev/infra/packages/db/pkg/testutils" - "github.com/e2b-dev/infra/packages/db/queries" "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) @@ -66,14 +66,14 @@ func TestAuthUserSyncWorker_UpsertDeletesStaleProjectedUser(t *testing.T) { userID := uuid.New() staleEmail := fmt.Sprintf("stale-%s@example.com", userID.String()[:8]) - err := db.SqlcClient.UpsertPublicUser(ctx, queries.UpsertPublicUserParams{ + err := db.AuthDB.Write.UpsertPublicUser(ctx, authqueries.UpsertPublicUserParams{ ID: userID, Email: staleEmail, }) require.NoError(t, err) require.Equal(t, 1, publicUserCount(t, ctx, db, userID)) - worker := NewAuthUserSyncWorker(ctx, db.SupabaseDB, db.SqlcClient, logger.NewNopLogger()) + worker := NewAuthUserSyncWorker(ctx, db.SupabaseDB, db.AuthDB, logger.NewNopLogger()) err = worker.Work(ctx, &river.Job[AuthUserSyncArgs]{ JobRow: &rivertype.JobRow{ID: 1, Attempt: 1}, @@ -126,7 +126,7 @@ func startRiverWorker(t *testing.T, db *testutils.Database) *riverProcess { l := logger.NewNopLogger() workers := river.NewWorkers() - river.AddWorker(workers, NewAuthUserSyncWorker(ctx, db.SupabaseDB, db.SqlcClient, l)) + river.AddWorker(workers, NewAuthUserSyncWorker(ctx, db.SupabaseDB, db.AuthDB, l)) client, err := NewRiverClient(db.SupabaseDB.WritePool(), workers) require.NoError(t, err) diff --git a/packages/dashboard-api/internal/backgroundworker/river.go b/packages/dashboard-api/internal/backgroundworker/river.go index a5e07635e9..9e2689fdbc 100644 --- a/packages/dashboard-api/internal/backgroundworker/river.go +++ b/packages/dashboard-api/internal/backgroundworker/river.go @@ -11,7 +11,7 @@ import ( "github.com/riverqueue/river/rivermigrate" "go.uber.org/zap" - sqlcdb "github.com/e2b-dev/infra/packages/db/client" + authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" supabasedb "github.com/e2b-dev/infra/packages/db/pkg/supabase" "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) @@ -41,7 +41,7 @@ func NewRiverClient(pool *pgxpool.Pool, workers *river.Workers) (*river.Client[p }) } -func StartAuthUserSyncWorker(setupCtx, runCtx context.Context, supabaseDB *supabasedb.Client, authDB *sqlcdb.Client, l logger.Logger) (*river.Client[pgx.Tx], error) { +func StartAuthUserSyncWorker(setupCtx, runCtx context.Context, supabaseDB *supabasedb.Client, authDB *authdb.Client, l logger.Logger) (*river.Client[pgx.Tx], error) { if err := RunRiverMigrations(setupCtx, supabaseDB.WritePool()); err != nil { return nil, fmt.Errorf("run River migrations on supabase DB: %w", err) } diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index 1a63032cd7..4958f9c847 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -303,14 +303,14 @@ func TestPostUsersBootstrap_CreatesDefaultTeamAndCallsSink(t *testing.T) { userID := createHandlerTestUser(t, testDB) sink := &fakeTeamProvisionSink{} - existingTeam, err := testDB.SqlcClient.GetDefaultTeamByUserID(ctx, userID) + existingTeam, err := testDB.AuthDB.Write.GetDefaultTeamByUserID(ctx, userID) if err != nil { t.Fatalf("expected trigger-created default team: %v", err) } - if err := testDB.SqlcClient.DeleteTeamByID(ctx, existingTeam.ID); err != nil { + if err := testDB.AuthDB.Write.DeleteTeamByID(ctx, existingTeam.ID); err != nil { t.Fatalf("failed to remove trigger-created default team: %v", err) } - if err := testDB.SqlcClient.DeletePublicUser(ctx, userID); err != nil { + if err := testDB.AuthDB.Write.DeletePublicUser(ctx, userID); err != nil { t.Fatalf("failed to remove trigger-created public user: %v", err) } @@ -331,7 +331,7 @@ func TestPostUsersBootstrap_CreatesDefaultTeamAndCallsSink(t *testing.T) { t.Fatalf("expected status 200, got %d", recorder.Code) } - team, err := testDB.SqlcClient.GetDefaultTeamByUserID(ctx, userID) + team, err := testDB.AuthDB.Write.GetDefaultTeamByUserID(ctx, userID) if err != nil { t.Fatalf("expected default team to be created: %v", err) } @@ -370,14 +370,14 @@ func TestPostUsersBootstrap_ProvisioningFailureKeepsCreatedDefaultTeam(t *testin }, } - existingTeam, err := testDB.SqlcClient.GetDefaultTeamByUserID(ctx, userID) + existingTeam, err := testDB.AuthDB.Write.GetDefaultTeamByUserID(ctx, userID) if err != nil { t.Fatalf("expected trigger-created default team: %v", err) } - if err := testDB.SqlcClient.DeleteTeamByID(ctx, existingTeam.ID); err != nil { + if err := testDB.AuthDB.Write.DeleteTeamByID(ctx, existingTeam.ID); err != nil { t.Fatalf("failed to remove trigger-created default team: %v", err) } - if err := testDB.SqlcClient.DeletePublicUser(ctx, userID); err != nil { + if err := testDB.AuthDB.Write.DeletePublicUser(ctx, userID); err != nil { t.Fatalf("failed to remove trigger-created public user: %v", err) } @@ -401,7 +401,7 @@ func TestPostUsersBootstrap_ProvisioningFailureKeepsCreatedDefaultTeam(t *testin t.Fatalf("expected one provisioning call, got %d", len(sink.requests)) } - team, err := testDB.SqlcClient.GetDefaultTeamByUserID(ctx, userID) + team, err := testDB.AuthDB.Write.GetDefaultTeamByUserID(ctx, userID) if err != nil { t.Fatalf("expected default team to remain after provisioning failure: %v", err) } @@ -429,11 +429,11 @@ func TestBootstrapUser_ConcurrentRequestsCreateSingleDefaultTeam(t *testing.T) { userID := createHandlerTestUser(t, testDB) sink := &fakeTeamProvisionSink{} - existingTeam, err := testDB.SqlcClient.GetDefaultTeamByUserID(ctx, userID) + existingTeam, err := testDB.AuthDB.Write.GetDefaultTeamByUserID(ctx, userID) if err != nil { t.Fatalf("expected trigger-created default team: %v", err) } - if err := testDB.SqlcClient.DeleteTeamByID(ctx, existingTeam.ID); err != nil { + if err := testDB.AuthDB.Write.DeleteTeamByID(ctx, existingTeam.ID); err != nil { t.Fatalf("failed to remove trigger-created default team: %v", err) } @@ -539,7 +539,7 @@ func TestPostTeams_LocalPolicyDeniedReturnsBadRequestWithoutCreatingTeam(t *test sink := &fakeTeamProvisionSink{} for range 2 { - team, err := testDB.SqlcClient.CreateTeam(ctx, queries.CreateTeamParams{ + team, err := testDB.AuthDB.Write.CreateTeam(ctx, authqueries.CreateTeamParams{ Name: "extra", Tier: baseTierID, Email: handlerTestUserEmail(userID), @@ -547,7 +547,7 @@ func TestPostTeams_LocalPolicyDeniedReturnsBadRequestWithoutCreatingTeam(t *test if err != nil { t.Fatalf("failed to create extra team: %v", err) } - if err := testDB.SqlcClient.CreateTeamMembership(ctx, queries.CreateTeamMembershipParams{ + if err := testDB.AuthDB.Write.CreateTeamMembership(ctx, authqueries.CreateTeamMembershipParams{ UserID: userID, TeamID: team.ID, IsDefault: false, @@ -834,11 +834,11 @@ func TestCreateTeam_ConcurrentRequestsRespectLocalPolicyWithZeroMemberships(t *t ctx := t.Context() userID := createHandlerTestUser(t, testDB) - existingTeam, err := testDB.SqlcClient.GetDefaultTeamByUserID(ctx, userID) + existingTeam, err := testDB.AuthDB.Write.GetDefaultTeamByUserID(ctx, userID) if err != nil { t.Fatalf("expected trigger-created default team: %v", err) } - if err := testDB.SqlcClient.DeleteTeamByID(ctx, existingTeam.ID); err != nil { + if err := testDB.AuthDB.Write.DeleteTeamByID(ctx, existingTeam.ID); err != nil { t.Fatalf("failed to remove default team: %v", err) } diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index 0102d8bb76..5c5cca1a5e 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -15,10 +15,8 @@ import ( "github.com/e2b-dev/infra/packages/auth/pkg/auth" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" internalteamprovision "github.com/e2b-dev/infra/packages/dashboard-api/internal/teamprovision" - sqlcdb "github.com/e2b-dev/infra/packages/db/client" authqueries "github.com/e2b-dev/infra/packages/db/pkg/auth/queries" "github.com/e2b-dev/infra/packages/db/pkg/dberrors" - "github.com/e2b-dev/infra/packages/db/queries" "github.com/e2b-dev/infra/packages/shared/pkg/ginutils" "github.com/e2b-dev/infra/packages/shared/pkg/teamprovision" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -102,16 +100,15 @@ func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisi return provisionedTeam{}, fmt.Errorf("get auth user: %w", err) } - txDB, tx, err := s.db.WithTx(ctx) + authTxDB, tx, err := s.authDB.WithTx(ctx) if err != nil { return provisionedTeam{}, fmt.Errorf("start transaction: %w", err) } - authTxDB := authqueries.New(tx) defer func() { _ = tx.Rollback(ctx) }() - if err := txDB.UpsertPublicUser(ctx, queries.UpsertPublicUserParams{ + if err := authTxDB.UpsertPublicUser(ctx, authqueries.UpsertPublicUserParams{ ID: authUser.ID, Email: authUser.Email, }); err != nil { @@ -123,7 +120,7 @@ func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisi return provisionedTeam{}, fmt.Errorf("lock public user: %w", err) } - existingTeam, err := txDB.GetDefaultTeamByUserID(ctx, userID) + existingTeam, err := authTxDB.GetDefaultTeamByUserID(ctx, userID) if err == nil { if err := tx.Commit(ctx); err != nil { return provisionedTeam{}, fmt.Errorf("commit existing user bootstrap transaction: %w", err) @@ -151,7 +148,7 @@ func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisi return provisionedTeam{}, fmt.Errorf("get default team: %w", err) } - team, err := txDB.CreateTeam(ctx, queries.CreateTeamParams{ + team, err := authTxDB.CreateTeam(ctx, authqueries.CreateTeamParams{ Name: authUser.Email, Tier: baseTierID, Email: authUser.Email, @@ -162,7 +159,7 @@ func (s *APIStore) bootstrapUser(ctx context.Context, userID uuid.UUID) (provisi return provisionedTeam{}, fmt.Errorf("create default team: %w", err) } - if err := txDB.CreateTeamMembership(ctx, queries.CreateTeamMembershipParams{ + if err := authTxDB.CreateTeamMembership(ctx, authqueries.CreateTeamMembershipParams{ UserID: userID, TeamID: team.ID, IsDefault: true, @@ -200,16 +197,15 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string return provisionedTeam{}, fmt.Errorf("get auth user: %w", err) } - txDB, tx, err := s.db.WithTx(ctx) + authTxDB, tx, err := s.authDB.WithTx(ctx) if err != nil { return provisionedTeam{}, fmt.Errorf("start transaction: %w", err) } - authTxDB := authqueries.New(tx) defer func() { _ = tx.Rollback(ctx) }() - if err := txDB.UpsertPublicUser(ctx, queries.UpsertPublicUserParams{ + if err := authTxDB.UpsertPublicUser(ctx, authqueries.UpsertPublicUserParams{ ID: authUser.ID, Email: authUser.Email, }); err != nil { @@ -221,13 +217,13 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string return provisionedTeam{}, fmt.Errorf("lock public user: %w", err) } - if err := validateTeamCreationAllowed(ctx, txDB, userID); err != nil { + if err := validateTeamCreationAllowed(ctx, authTxDB, userID); err != nil { return provisionedTeam{}, err } isBlocked, blockedReason := teamBlockPolicy(authUser.CreatedAt, time.Now()) - team, err := txDB.CreateTeam(ctx, queries.CreateTeamParams{ + team, err := authTxDB.CreateTeam(ctx, authqueries.CreateTeamParams{ Name: name, Tier: baseTierID, Email: authUser.Email, @@ -238,7 +234,7 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string return provisionedTeam{}, fmt.Errorf("create team: %w", err) } - if err := txDB.CreateTeamMembership(ctx, queries.CreateTeamMembershipParams{ + if err := authTxDB.CreateTeamMembership(ctx, authqueries.CreateTeamMembershipParams{ UserID: userID, TeamID: team.ID, IsDefault: false, @@ -262,7 +258,7 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), teamProvisionRollbackTimeout) defer cancel() - if deleteErr := s.db.DeleteTeamByID(rollbackCtx, team.ID); deleteErr != nil { + if deleteErr := s.authDB.Write.DeleteTeamByID(rollbackCtx, team.ID); deleteErr != nil { return provisionedTeam{}, fmt.Errorf("delete team after provisioning failure: provision=%s delete=%w", err.Error(), deleteErr) } @@ -279,8 +275,8 @@ func (s *APIStore) createTeam(ctx context.Context, userID uuid.UUID, name string }, nil } -func validateTeamCreationAllowed(ctx context.Context, txDB *sqlcdb.Client, ownerUserID uuid.UUID) error { - teams, err := txDB.GetTeamsWithUsersTeamsWithTierForUpdate(ctx, ownerUserID) +func validateTeamCreationAllowed(ctx context.Context, authTxDB *authqueries.Queries, ownerUserID uuid.UUID) error { + teams, err := authTxDB.GetTeamsWithUsersTeamsWithTierForUpdate(ctx, ownerUserID) if err != nil { return fmt.Errorf("query user teams for limit check: %w", err) } diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 3e3b4df657..c8b3c1b7f6 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -232,7 +232,7 @@ func run() int { ctx, signalCtx, supabaseDB, - db, + authDB, l, ) if err != nil { diff --git a/packages/db/queries/delete_public_user.sql.go b/packages/db/queries/delete_public_user.sql.go deleted file mode 100644 index 585d8c977c..0000000000 --- a/packages/db/queries/delete_public_user.sql.go +++ /dev/null @@ -1,22 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.29.0 -// source: delete_public_user.sql - -package queries - -import ( - "context" - - "github.com/google/uuid" -) - -const deletePublicUser = `-- name: DeletePublicUser :exec -DELETE FROM public.users -WHERE id = $1::uuid -` - -func (q *Queries) DeletePublicUser(ctx context.Context, id uuid.UUID) error { - _, err := q.db.Exec(ctx, deletePublicUser, id) - return err -} diff --git a/packages/db/queries/team_creation_guard.sql.go b/packages/db/queries/team_creation_guard.sql.go deleted file mode 100644 index 8747e60310..0000000000 --- a/packages/db/queries/team_creation_guard.sql.go +++ /dev/null @@ -1,116 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.29.0 -// source: team_creation_guard.sql - -package queries - -import ( - "context" - "time" - - "github.com/google/uuid" -) - -const getTeamsWithUsersTeamsWithTierForUpdate = `-- name: GetTeamsWithUsersTeamsWithTierForUpdate :many -SELECT - t.id, - t.created_at, - t.is_blocked, - t.name, - t.tier, - t.email, - t.is_banned, - t.blocked_reason, - t.cluster_id, - t.sandbox_scheduling_labels, - t.slug, - ut.is_default, - tl.id, - tl.max_length_hours, - tl.concurrent_sandboxes, - tl.concurrent_template_builds, - tl.max_vcpu, - tl.max_ram_mb, - tl.disk_mb -FROM public.teams t -JOIN public.users_teams ut ON ut.team_id = t.id -JOIN public.team_limits tl ON tl.id = t.id -WHERE ut.user_id = $1::uuid -FOR UPDATE OF ut -` - -type GetTeamsWithUsersTeamsWithTierForUpdateRow struct { - ID uuid.UUID - CreatedAt time.Time - IsBlocked bool - Name string - Tier string - Email string - IsBanned bool - BlockedReason *string - ClusterID *uuid.UUID - SandboxSchedulingLabels []string - Slug string - IsDefault bool - ID_2 uuid.UUID - MaxLengthHours int64 - ConcurrentSandboxes int32 - ConcurrentTemplateBuilds int32 - MaxVcpu int32 - MaxRamMb int32 - DiskMb int32 -} - -func (q *Queries) GetTeamsWithUsersTeamsWithTierForUpdate(ctx context.Context, userID uuid.UUID) ([]GetTeamsWithUsersTeamsWithTierForUpdateRow, error) { - rows, err := q.db.Query(ctx, getTeamsWithUsersTeamsWithTierForUpdate, userID) - if err != nil { - return nil, err - } - defer rows.Close() - var items []GetTeamsWithUsersTeamsWithTierForUpdateRow - for rows.Next() { - var i GetTeamsWithUsersTeamsWithTierForUpdateRow - if err := rows.Scan( - &i.ID, - &i.CreatedAt, - &i.IsBlocked, - &i.Name, - &i.Tier, - &i.Email, - &i.IsBanned, - &i.BlockedReason, - &i.ClusterID, - &i.SandboxSchedulingLabels, - &i.Slug, - &i.IsDefault, - &i.ID_2, - &i.MaxLengthHours, - &i.ConcurrentSandboxes, - &i.ConcurrentTemplateBuilds, - &i.MaxVcpu, - &i.MaxRamMb, - &i.DiskMb, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - -const lockPublicUserForUpdate = `-- name: LockPublicUserForUpdate :one -SELECT id -FROM public.users -WHERE id = $1::uuid -FOR UPDATE -` - -func (q *Queries) LockPublicUserForUpdate(ctx context.Context, id uuid.UUID) (uuid.UUID, error) { - row := q.db.QueryRow(ctx, lockPublicUserForUpdate, id) - err := row.Scan(&id) - return id, err -} diff --git a/packages/db/queries/team_lifecycle.sql.go b/packages/db/queries/team_lifecycle.sql.go deleted file mode 100644 index 815d5f3dd5..0000000000 --- a/packages/db/queries/team_lifecycle.sql.go +++ /dev/null @@ -1,172 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.29.0 -// source: team_lifecycle.sql - -package queries - -import ( - "context" - "time" - - "github.com/google/uuid" -) - -const createTeam = `-- name: CreateTeam :one -INSERT INTO public.teams (name, tier, email, is_blocked, blocked_reason) -VALUES ( - $1::text, - $2::text, - $3::text, - $4::boolean, - $5::text -) -RETURNING - id, - created_at, - is_blocked, - name, - tier, - email, - is_banned, - blocked_reason, - cluster_id, - sandbox_scheduling_labels, - slug -` - -type CreateTeamParams struct { - Name string - Tier string - Email string - IsBlocked bool - BlockedReason *string -} - -type CreateTeamRow struct { - ID uuid.UUID - CreatedAt time.Time - IsBlocked bool - Name string - Tier string - Email string - IsBanned bool - BlockedReason *string - ClusterID *uuid.UUID - SandboxSchedulingLabels []string - Slug string -} - -func (q *Queries) CreateTeam(ctx context.Context, arg CreateTeamParams) (CreateTeamRow, error) { - row := q.db.QueryRow(ctx, createTeam, - arg.Name, - arg.Tier, - arg.Email, - arg.IsBlocked, - arg.BlockedReason, - ) - var i CreateTeamRow - err := row.Scan( - &i.ID, - &i.CreatedAt, - &i.IsBlocked, - &i.Name, - &i.Tier, - &i.Email, - &i.IsBanned, - &i.BlockedReason, - &i.ClusterID, - &i.SandboxSchedulingLabels, - &i.Slug, - ) - return i, err -} - -const createTeamMembership = `-- name: CreateTeamMembership :exec -INSERT INTO public.users_teams (user_id, team_id, is_default, added_by) -VALUES ( - $1::uuid, - $2::uuid, - $3::boolean, - $4::uuid -) -` - -type CreateTeamMembershipParams struct { - UserID uuid.UUID - TeamID uuid.UUID - IsDefault bool - AddedBy *uuid.UUID -} - -func (q *Queries) CreateTeamMembership(ctx context.Context, arg CreateTeamMembershipParams) error { - _, err := q.db.Exec(ctx, createTeamMembership, - arg.UserID, - arg.TeamID, - arg.IsDefault, - arg.AddedBy, - ) - return err -} - -const deleteTeamByID = `-- name: DeleteTeamByID :exec -DELETE FROM public.teams -WHERE id = $1::uuid -` - -func (q *Queries) DeleteTeamByID(ctx context.Context, id uuid.UUID) error { - _, err := q.db.Exec(ctx, deleteTeamByID, id) - return err -} - -const getDefaultTeamByUserID = `-- name: GetDefaultTeamByUserID :one -SELECT - t.id, - t.created_at, - t.is_blocked, - t.name, - t.tier, - t.email, - t.is_banned, - t.blocked_reason, - t.cluster_id, - t.sandbox_scheduling_labels, - t.slug -FROM public.teams t -JOIN public.users_teams ut ON ut.team_id = t.id -WHERE ut.user_id = $1::uuid - AND ut.is_default = true -` - -type GetDefaultTeamByUserIDRow struct { - ID uuid.UUID - CreatedAt time.Time - IsBlocked bool - Name string - Tier string - Email string - IsBanned bool - BlockedReason *string - ClusterID *uuid.UUID - SandboxSchedulingLabels []string - Slug string -} - -func (q *Queries) GetDefaultTeamByUserID(ctx context.Context, userID uuid.UUID) (GetDefaultTeamByUserIDRow, error) { - row := q.db.QueryRow(ctx, getDefaultTeamByUserID, userID) - var i GetDefaultTeamByUserIDRow - err := row.Scan( - &i.ID, - &i.CreatedAt, - &i.IsBlocked, - &i.Name, - &i.Tier, - &i.Email, - &i.IsBanned, - &i.BlockedReason, - &i.ClusterID, - &i.SandboxSchedulingLabels, - &i.Slug, - ) - return i, err -} diff --git a/packages/db/queries/teams/team_creation_guard.sql b/packages/db/queries/teams/team_creation_guard.sql deleted file mode 100644 index cb0c2a7c0f..0000000000 --- a/packages/db/queries/teams/team_creation_guard.sql +++ /dev/null @@ -1,32 +0,0 @@ --- name: LockPublicUserForUpdate :one -SELECT id -FROM public.users -WHERE id = sqlc.arg(id)::uuid -FOR UPDATE; - --- name: GetTeamsWithUsersTeamsWithTierForUpdate :many -SELECT - t.id, - t.created_at, - t.is_blocked, - t.name, - t.tier, - t.email, - t.is_banned, - t.blocked_reason, - t.cluster_id, - t.sandbox_scheduling_labels, - t.slug, - ut.is_default, - tl.id, - tl.max_length_hours, - tl.concurrent_sandboxes, - tl.concurrent_template_builds, - tl.max_vcpu, - tl.max_ram_mb, - tl.disk_mb -FROM public.teams t -JOIN public.users_teams ut ON ut.team_id = t.id -JOIN public.team_limits tl ON tl.id = t.id -WHERE ut.user_id = sqlc.arg(user_id)::uuid -FOR UPDATE OF ut; diff --git a/packages/db/queries/teams/team_lifecycle.sql b/packages/db/queries/teams/team_lifecycle.sql deleted file mode 100644 index c385a5c66a..0000000000 --- a/packages/db/queries/teams/team_lifecycle.sql +++ /dev/null @@ -1,52 +0,0 @@ --- name: CreateTeam :one -INSERT INTO public.teams (name, tier, email, is_blocked, blocked_reason) -VALUES ( - sqlc.arg(name)::text, - sqlc.arg(tier)::text, - sqlc.arg(email)::text, - sqlc.arg(is_blocked)::boolean, - sqlc.narg(blocked_reason)::text -) -RETURNING - id, - created_at, - is_blocked, - name, - tier, - email, - is_banned, - blocked_reason, - cluster_id, - sandbox_scheduling_labels, - slug; - --- name: CreateTeamMembership :exec -INSERT INTO public.users_teams (user_id, team_id, is_default, added_by) -VALUES ( - sqlc.arg(user_id)::uuid, - sqlc.arg(team_id)::uuid, - sqlc.arg(is_default)::boolean, - sqlc.narg(added_by)::uuid -); - --- name: GetDefaultTeamByUserID :one -SELECT - t.id, - t.created_at, - t.is_blocked, - t.name, - t.tier, - t.email, - t.is_banned, - t.blocked_reason, - t.cluster_id, - t.sandbox_scheduling_labels, - t.slug -FROM public.teams t -JOIN public.users_teams ut ON ut.team_id = t.id -WHERE ut.user_id = sqlc.arg(user_id)::uuid - AND ut.is_default = true; - --- name: DeleteTeamByID :exec -DELETE FROM public.teams -WHERE id = sqlc.arg(id)::uuid; diff --git a/packages/db/queries/upsert_public_user.sql.go b/packages/db/queries/upsert_public_user.sql.go deleted file mode 100644 index dc0fd6eba3..0000000000 --- a/packages/db/queries/upsert_public_user.sql.go +++ /dev/null @@ -1,31 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.29.0 -// source: upsert_public_user.sql - -package queries - -import ( - "context" - - "github.com/google/uuid" -) - -const upsertPublicUser = `-- name: UpsertPublicUser :exec -INSERT INTO public.users (id, email) -VALUES ($1::uuid, $2::text) -ON CONFLICT (id) -DO UPDATE SET - email = EXCLUDED.email, - updated_at = now() -` - -type UpsertPublicUserParams struct { - ID uuid.UUID - Email string -} - -func (q *Queries) UpsertPublicUser(ctx context.Context, arg UpsertPublicUserParams) error { - _, err := q.db.Exec(ctx, upsertPublicUser, arg.ID, arg.Email) - return err -} diff --git a/packages/db/queries/users/delete_public_user.sql b/packages/db/queries/users/delete_public_user.sql deleted file mode 100644 index 492f1051cd..0000000000 --- a/packages/db/queries/users/delete_public_user.sql +++ /dev/null @@ -1,3 +0,0 @@ --- name: DeletePublicUser :exec -DELETE FROM public.users -WHERE id = sqlc.arg(id)::uuid; diff --git a/packages/db/queries/users/upsert_public_user.sql b/packages/db/queries/users/upsert_public_user.sql deleted file mode 100644 index ebabd969e6..0000000000 --- a/packages/db/queries/users/upsert_public_user.sql +++ /dev/null @@ -1,7 +0,0 @@ --- name: UpsertPublicUser :exec -INSERT INTO public.users (id, email) -VALUES (sqlc.arg(id)::uuid, sqlc.arg(email)::text) -ON CONFLICT (id) -DO UPDATE SET - email = EXCLUDED.email, - updated_at = now(); From 83f66e78bdc1f10321588e8a548255c95fcbf688 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Tue, 14 Apr 2026 14:19:33 -0700 Subject: [PATCH 90/92] Add sqlc auth queries for teams and users --- .../auth/queries/delete_public_user.sql.go | 22 +++ .../auth/queries/team_creation_guard.sql.go | 103 +++++++++++++ .../db/pkg/auth/queries/team_lifecycle.sql.go | 143 ++++++++++++++++++ .../auth/queries/upsert_public_user.sql.go | 31 ++++ .../sql_queries/teams/team_creation_guard.sql | 26 ++++ .../auth/sql_queries/teams/team_lifecycle.sql | 52 +++++++ .../sql_queries/users/delete_public_user.sql | 3 + .../sql_queries/users/upsert_public_user.sql | 7 + 8 files changed, 387 insertions(+) create mode 100644 packages/db/pkg/auth/queries/delete_public_user.sql.go create mode 100644 packages/db/pkg/auth/queries/team_creation_guard.sql.go create mode 100644 packages/db/pkg/auth/queries/team_lifecycle.sql.go create mode 100644 packages/db/pkg/auth/queries/upsert_public_user.sql.go create mode 100644 packages/db/pkg/auth/sql_queries/teams/team_creation_guard.sql create mode 100644 packages/db/pkg/auth/sql_queries/teams/team_lifecycle.sql create mode 100644 packages/db/pkg/auth/sql_queries/users/delete_public_user.sql create mode 100644 packages/db/pkg/auth/sql_queries/users/upsert_public_user.sql diff --git a/packages/db/pkg/auth/queries/delete_public_user.sql.go b/packages/db/pkg/auth/queries/delete_public_user.sql.go new file mode 100644 index 0000000000..b7a3c975a3 --- /dev/null +++ b/packages/db/pkg/auth/queries/delete_public_user.sql.go @@ -0,0 +1,22 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: delete_public_user.sql + +package authqueries + +import ( + "context" + + "github.com/google/uuid" +) + +const deletePublicUser = `-- name: DeletePublicUser :exec +DELETE FROM public.users +WHERE id = $1::uuid +` + +func (q *Queries) DeletePublicUser(ctx context.Context, id uuid.UUID) error { + _, err := q.db.Exec(ctx, deletePublicUser, id) + return err +} diff --git a/packages/db/pkg/auth/queries/team_creation_guard.sql.go b/packages/db/pkg/auth/queries/team_creation_guard.sql.go new file mode 100644 index 0000000000..f4abb68baf --- /dev/null +++ b/packages/db/pkg/auth/queries/team_creation_guard.sql.go @@ -0,0 +1,103 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: team_creation_guard.sql + +package authqueries + +import ( + "context" + "time" + + "github.com/google/uuid" +) + +const getTeamsWithUsersTeamsWithTierForUpdate = `-- name: GetTeamsWithUsersTeamsWithTierForUpdate :many +SELECT + t.id, + t.created_at, + t.is_blocked, + t.name, + t.tier, + t.email, + t.is_banned, + t.blocked_reason, + t.cluster_id, + t.sandbox_scheduling_labels, + t.slug, + ut.is_default, + tl.id, + tl.max_length_hours, + tl.concurrent_sandboxes, + tl.concurrent_template_builds, + tl.max_vcpu, + tl.max_ram_mb, + tl.disk_mb +FROM public.teams t +JOIN public.users_teams ut ON ut.team_id = t.id +JOIN public.team_limits tl ON tl.id = t.id +WHERE ut.user_id = $1::uuid +FOR UPDATE OF ut +` + +type GetTeamsWithUsersTeamsWithTierForUpdateRow struct { + ID uuid.UUID + CreatedAt time.Time + IsBlocked bool + Name string + Tier string + Email string + IsBanned bool + BlockedReason *string + ClusterID *uuid.UUID + SandboxSchedulingLabels []string + Slug string + IsDefault bool + ID_2 uuid.UUID + MaxLengthHours int64 + ConcurrentSandboxes int32 + ConcurrentTemplateBuilds int32 + MaxVcpu int32 + MaxRamMb int32 + DiskMb int32 +} + +func (q *Queries) GetTeamsWithUsersTeamsWithTierForUpdate(ctx context.Context, userID uuid.UUID) ([]GetTeamsWithUsersTeamsWithTierForUpdateRow, error) { + rows, err := q.db.Query(ctx, getTeamsWithUsersTeamsWithTierForUpdate, userID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetTeamsWithUsersTeamsWithTierForUpdateRow + for rows.Next() { + var i GetTeamsWithUsersTeamsWithTierForUpdateRow + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.IsBlocked, + &i.Name, + &i.Tier, + &i.Email, + &i.IsBanned, + &i.BlockedReason, + &i.ClusterID, + &i.SandboxSchedulingLabels, + &i.Slug, + &i.IsDefault, + &i.ID_2, + &i.MaxLengthHours, + &i.ConcurrentSandboxes, + &i.ConcurrentTemplateBuilds, + &i.MaxVcpu, + &i.MaxRamMb, + &i.DiskMb, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/packages/db/pkg/auth/queries/team_lifecycle.sql.go b/packages/db/pkg/auth/queries/team_lifecycle.sql.go new file mode 100644 index 0000000000..f9328f2af2 --- /dev/null +++ b/packages/db/pkg/auth/queries/team_lifecycle.sql.go @@ -0,0 +1,143 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: team_lifecycle.sql + +package authqueries + +import ( + "context" + + "github.com/google/uuid" +) + +const createTeam = `-- name: CreateTeam :one +INSERT INTO public.teams (name, tier, email, is_blocked, blocked_reason) +VALUES ( + $1::text, + $2::text, + $3::text, + $4::boolean, + $5::text +) +RETURNING + id, + created_at, + is_blocked, + name, + tier, + email, + is_banned, + blocked_reason, + cluster_id, + sandbox_scheduling_labels, + slug +` + +type CreateTeamParams struct { + Name string + Tier string + Email string + IsBlocked bool + BlockedReason *string +} + +func (q *Queries) CreateTeam(ctx context.Context, arg CreateTeamParams) (Team, error) { + row := q.db.QueryRow(ctx, createTeam, + arg.Name, + arg.Tier, + arg.Email, + arg.IsBlocked, + arg.BlockedReason, + ) + var i Team + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.IsBlocked, + &i.Name, + &i.Tier, + &i.Email, + &i.IsBanned, + &i.BlockedReason, + &i.ClusterID, + &i.SandboxSchedulingLabels, + &i.Slug, + ) + return i, err +} + +const createTeamMembership = `-- name: CreateTeamMembership :exec +INSERT INTO public.users_teams (user_id, team_id, is_default, added_by) +VALUES ( + $1::uuid, + $2::uuid, + $3::boolean, + $4::uuid +) +` + +type CreateTeamMembershipParams struct { + UserID uuid.UUID + TeamID uuid.UUID + IsDefault bool + AddedBy *uuid.UUID +} + +func (q *Queries) CreateTeamMembership(ctx context.Context, arg CreateTeamMembershipParams) error { + _, err := q.db.Exec(ctx, createTeamMembership, + arg.UserID, + arg.TeamID, + arg.IsDefault, + arg.AddedBy, + ) + return err +} + +const deleteTeamByID = `-- name: DeleteTeamByID :exec +DELETE FROM public.teams +WHERE id = $1::uuid +` + +func (q *Queries) DeleteTeamByID(ctx context.Context, id uuid.UUID) error { + _, err := q.db.Exec(ctx, deleteTeamByID, id) + return err +} + +const getDefaultTeamByUserID = `-- name: GetDefaultTeamByUserID :one +SELECT + t.id, + t.created_at, + t.is_blocked, + t.name, + t.tier, + t.email, + t.is_banned, + t.blocked_reason, + t.cluster_id, + t.sandbox_scheduling_labels, + t.slug +FROM public.teams t +JOIN public.users_teams ut ON ut.team_id = t.id +WHERE ut.user_id = $1::uuid + AND ut.is_default = true +` + +func (q *Queries) GetDefaultTeamByUserID(ctx context.Context, userID uuid.UUID) (Team, error) { + row := q.db.QueryRow(ctx, getDefaultTeamByUserID, userID) + var i Team + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.IsBlocked, + &i.Name, + &i.Tier, + &i.Email, + &i.IsBanned, + &i.BlockedReason, + &i.ClusterID, + &i.SandboxSchedulingLabels, + &i.Slug, + ) + return i, err +} diff --git a/packages/db/pkg/auth/queries/upsert_public_user.sql.go b/packages/db/pkg/auth/queries/upsert_public_user.sql.go new file mode 100644 index 0000000000..6c0f05ec5a --- /dev/null +++ b/packages/db/pkg/auth/queries/upsert_public_user.sql.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: upsert_public_user.sql + +package authqueries + +import ( + "context" + + "github.com/google/uuid" +) + +const upsertPublicUser = `-- name: UpsertPublicUser :exec +INSERT INTO public.users (id, email) +VALUES ($1::uuid, $2::text) +ON CONFLICT (id) +DO UPDATE SET + email = EXCLUDED.email, + updated_at = now() +` + +type UpsertPublicUserParams struct { + ID uuid.UUID + Email string +} + +func (q *Queries) UpsertPublicUser(ctx context.Context, arg UpsertPublicUserParams) error { + _, err := q.db.Exec(ctx, upsertPublicUser, arg.ID, arg.Email) + return err +} diff --git a/packages/db/pkg/auth/sql_queries/teams/team_creation_guard.sql b/packages/db/pkg/auth/sql_queries/teams/team_creation_guard.sql new file mode 100644 index 0000000000..8a027583d1 --- /dev/null +++ b/packages/db/pkg/auth/sql_queries/teams/team_creation_guard.sql @@ -0,0 +1,26 @@ +-- name: GetTeamsWithUsersTeamsWithTierForUpdate :many +SELECT + t.id, + t.created_at, + t.is_blocked, + t.name, + t.tier, + t.email, + t.is_banned, + t.blocked_reason, + t.cluster_id, + t.sandbox_scheduling_labels, + t.slug, + ut.is_default, + tl.id, + tl.max_length_hours, + tl.concurrent_sandboxes, + tl.concurrent_template_builds, + tl.max_vcpu, + tl.max_ram_mb, + tl.disk_mb +FROM public.teams t +JOIN public.users_teams ut ON ut.team_id = t.id +JOIN public.team_limits tl ON tl.id = t.id +WHERE ut.user_id = sqlc.arg(user_id)::uuid +FOR UPDATE OF ut; diff --git a/packages/db/pkg/auth/sql_queries/teams/team_lifecycle.sql b/packages/db/pkg/auth/sql_queries/teams/team_lifecycle.sql new file mode 100644 index 0000000000..26f7094156 --- /dev/null +++ b/packages/db/pkg/auth/sql_queries/teams/team_lifecycle.sql @@ -0,0 +1,52 @@ +-- name: CreateTeam :one +INSERT INTO public.teams (name, tier, email, is_blocked, blocked_reason) +VALUES ( + sqlc.arg(name)::text, + sqlc.arg(tier)::text, + sqlc.arg(email)::text, + sqlc.arg(is_blocked)::boolean, + sqlc.narg(blocked_reason)::text +) +RETURNING + id, + created_at, + is_blocked, + name, + tier, + email, + is_banned, + blocked_reason, + cluster_id, + sandbox_scheduling_labels, + slug; + +-- name: CreateTeamMembership :exec +INSERT INTO public.users_teams (user_id, team_id, is_default, added_by) +VALUES ( + sqlc.arg(user_id)::uuid, + sqlc.arg(team_id)::uuid, + sqlc.arg(is_default)::boolean, + sqlc.narg(added_by)::uuid +); + +-- name: GetDefaultTeamByUserID :one +SELECT + t.id, + t.created_at, + t.is_blocked, + t.name, + t.tier, + t.email, + t.is_banned, + t.blocked_reason, + t.cluster_id, + t.sandbox_scheduling_labels, + t.slug +FROM public.teams t +JOIN public.users_teams ut ON ut.team_id = t.id +WHERE ut.user_id = sqlc.arg(user_id)::uuid + AND ut.is_default = true; + +-- name: DeleteTeamByID :exec +DELETE FROM public.teams +WHERE id = sqlc.arg(id)::uuid; diff --git a/packages/db/pkg/auth/sql_queries/users/delete_public_user.sql b/packages/db/pkg/auth/sql_queries/users/delete_public_user.sql new file mode 100644 index 0000000000..492f1051cd --- /dev/null +++ b/packages/db/pkg/auth/sql_queries/users/delete_public_user.sql @@ -0,0 +1,3 @@ +-- name: DeletePublicUser :exec +DELETE FROM public.users +WHERE id = sqlc.arg(id)::uuid; diff --git a/packages/db/pkg/auth/sql_queries/users/upsert_public_user.sql b/packages/db/pkg/auth/sql_queries/users/upsert_public_user.sql new file mode 100644 index 0000000000..ebabd969e6 --- /dev/null +++ b/packages/db/pkg/auth/sql_queries/users/upsert_public_user.sql @@ -0,0 +1,7 @@ +-- name: UpsertPublicUser :exec +INSERT INTO public.users (id, email) +VALUES (sqlc.arg(id)::uuid, sqlc.arg(email)::text) +ON CONFLICT (id) +DO UPDATE SET + email = EXCLUDED.email, + updated_at = now(); From 449b14acddeaf87ceb24c41665ab50f1e9aa7727 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Tue, 14 Apr 2026 14:36:43 -0700 Subject: [PATCH 91/92] Remove unused IsBadRequest and net/http import --- packages/dashboard-api/internal/teamprovision/sink.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/dashboard-api/internal/teamprovision/sink.go b/packages/dashboard-api/internal/teamprovision/sink.go index 1a75bcea63..7fe61002bb 100644 --- a/packages/dashboard-api/internal/teamprovision/sink.go +++ b/packages/dashboard-api/internal/teamprovision/sink.go @@ -3,7 +3,6 @@ package teamprovision import ( "context" "fmt" - "net/http" "go.opentelemetry.io/otel/attribute" "go.uber.org/zap" @@ -44,10 +43,6 @@ func (e *ProvisionError) Unwrap() error { return e.Err } -func (e *ProvisionError) IsBadRequest() bool { - return e.StatusCode == http.StatusBadRequest -} - func provisionLogFields(req sharedteamprovision.TeamBillingProvisionRequestedV1, sink string) []zap.Field { return []zap.Field{ logger.WithTeamID(req.TeamID.String()), From b8c3bca3f1afee2ba465dffe1e307e0868ea82bb Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Tue, 14 Apr 2026 18:41:47 -0700 Subject: [PATCH 92/92] Do not block teams for NULL user.created_at Allow missing Supabase created_at to be treated as unknown so new-user team block isn't applied. Change AuthUser.CreatedAt to *time.Time and make DB schema created_at nullable. Update teamBlockPolicy to accept a *time.Time and treat nil as not-new. Add tests and helper to cover nil created_at and refactor test helper. --- .../internal/handlers/team_handlers_test.go | 36 ++++++++++++++++++- .../internal/handlers/team_provisioning.go | 5 +-- packages/db/pkg/supabase/queries/models.go | 2 +- .../supabase/schema/auth_users_override.sql | 2 +- 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/packages/dashboard-api/internal/handlers/team_handlers_test.go b/packages/dashboard-api/internal/handlers/team_handlers_test.go index 4958f9c847..7fd9e7709d 100644 --- a/packages/dashboard-api/internal/handlers/team_handlers_test.go +++ b/packages/dashboard-api/internal/handlers/team_handlers_test.go @@ -259,12 +259,20 @@ func TestDeleteTeamsTeamIDMembersUserId_RechecksDefaultAfterLock(t *testing.T) { func createHandlerTestUser(t *testing.T, db *testutils.Database) uuid.UUID { t.Helper() - return createHandlerTestUserAt(t, db, time.Now().Add(-newUserNewTeamRequireBillingMethodThreshold-time.Hour)) + createdAt := time.Now().Add(-newUserNewTeamRequireBillingMethodThreshold - time.Hour) + + return createHandlerTestUserWithCreatedAt(t, db, &createdAt) } func createHandlerTestUserAt(t *testing.T, db *testutils.Database, createdAt time.Time) uuid.UUID { t.Helper() + return createHandlerTestUserWithCreatedAt(t, db, &createdAt) +} + +func createHandlerTestUserWithCreatedAt(t *testing.T, db *testutils.Database, createdAt *time.Time) uuid.UUID { + t.Helper() + userID := uuid.New() email := handlerTestUserEmail(userID) @@ -530,6 +538,32 @@ func TestCreateTeam_RecentUserCreatesBlockedTeam(t *testing.T) { } } +func TestCreateTeam_NullCreatedAtLeavesTeamUnblocked(t *testing.T) { + t.Parallel() + + testDB := testutils.SetupDatabase(t) + ctx := t.Context() + userID := createHandlerTestUserWithCreatedAt(t, testDB, nil) + + store := &APIStore{ + db: testDB.SqlcClient, + authDB: testDB.AuthDB, + supabaseDB: testDB.SupabaseDB, + teamProvisionSink: &fakeTeamProvisionSink{}, + } + + team, err := store.createTeam(ctx, userID, "Acme") + if err != nil { + t.Fatalf("expected team creation to succeed with nil created_at, got %v", err) + } + if team.IsBlocked { + t.Fatal("expected nil created_at team to remain unblocked") + } + if team.BlockedReason != nil { + t.Fatalf("expected nil blocked reason, got %v", team.BlockedReason) + } +} + func TestPostTeams_LocalPolicyDeniedReturnsBadRequestWithoutCreatingTeam(t *testing.T) { t.Parallel() diff --git a/packages/dashboard-api/internal/handlers/team_provisioning.go b/packages/dashboard-api/internal/handlers/team_provisioning.go index 5c5cca1a5e..70e1f51e45 100644 --- a/packages/dashboard-api/internal/handlers/team_provisioning.go +++ b/packages/dashboard-api/internal/handlers/team_provisioning.go @@ -317,8 +317,9 @@ func validateTeamCreationAllowed(ctx context.Context, authTxDB *authqueries.Quer return nil } -func teamBlockPolicy(userCreatedAt, now time.Time) (bool, *string) { - if userCreatedAt.After(now.Add(-newUserNewTeamRequireBillingMethodThreshold)) { +func teamBlockPolicy(userCreatedAt *time.Time, now time.Time) (bool, *string) { + // Some Supabase users have a NULL created_at; unknown age should not trigger the new-user block. + if userCreatedAt != nil && userCreatedAt.After(now.Add(-newUserNewTeamRequireBillingMethodThreshold)) { reason := blockedReasonMissingPayment return true, &reason diff --git a/packages/db/pkg/supabase/queries/models.go b/packages/db/pkg/supabase/queries/models.go index 11384b0cad..8e9daed53f 100644 --- a/packages/db/pkg/supabase/queries/models.go +++ b/packages/db/pkg/supabase/queries/models.go @@ -13,5 +13,5 @@ import ( type AuthUser struct { ID uuid.UUID Email string - CreatedAt time.Time + CreatedAt *time.Time } diff --git a/packages/db/pkg/supabase/schema/auth_users_override.sql b/packages/db/pkg/supabase/schema/auth_users_override.sql index 7522d1902d..3974df32e7 100644 --- a/packages/db/pkg/supabase/schema/auth_users_override.sql +++ b/packages/db/pkg/supabase/schema/auth_users_override.sql @@ -14,6 +14,6 @@ GRANT EXECUTE ON FUNCTION auth.uid() TO postgres; CREATE TABLE auth.users ( id uuid NOT NULL DEFAULT gen_random_uuid(), email text NOT NULL, - created_at timestamptz NOT NULL DEFAULT now(), + created_at timestamptz DEFAULT now(), PRIMARY KEY (id) );