From 633f8598345fe724afd90709e48eb9fc02bab931 Mon Sep 17 00:00:00 2001 From: Cheese Date: Sat, 18 Jul 2026 20:22:25 +0800 Subject: [PATCH 1/3] feat: wait for Starter cluster activation --- AGENTS.md | 12 +- README.md | 22 +-- .../done/0006-starter-db-cluster-lifecycle.md | 27 ++- e2e/cli_test.go | 2 + e2e/live_test.go | 13 +- internal/cli/commands.go | 6 + internal/cli/root_test.go | 8 +- internal/db/cluster.go | 144 ++++++++++++++-- internal/db/cluster_test.go | 158 ++++++++++++++++++ 9 files changed, 354 insertions(+), 38 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d9300c1..9a8b5bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -195,9 +195,10 @@ the profile for both focused and complete live targets. Live e2e must strictly cover every implemented interface and command for the current project stage, including real create/update/delete flows when those commands are implemented. For Starter DB clusters, the live suite creates a -uniquely named `tdc-e2e-*` cluster without a spending limit or explicit -`--project-id`, verifies its project label matches the configured default, and -deletes only that cluster. For Starter DB branches, the live suite creates, reads, lists, +uniquely named `tdc-e2e-*` cluster with `--wait-until-active`, without a +spending limit or explicit `--project-id`, verifies the returned state is +`ACTIVE` and its project label matches the configured default, and deletes only +that cluster. For Starter DB branches, the live suite creates, reads, lists, and deletes only a `tdc-e2e-branch-*` branch on the cluster created by the same test run. For Starter DB SQL access, the live suite prepares tdc-managed read-only, read-write, and admin SQL users on the temporary cluster, verifies @@ -317,6 +318,10 @@ Follow these rules unless `docs/priciples.md` is updated: - Mutating control-plane commands support `--dry-run`. - `--dry-run` must validate local config, credentials, provider, and region before reporting a planned mutation. +- `tdc db create-db-cluster --wait-until-active` waits up to 12 minutes for the + created cluster to reach `ACTIVE`. It must never delete the cluster on + timeout, cancellation, a polling failure, or a terminal state; errors must + retain the created cluster ID and provide an inspection command. - Read-only commands reject `--dry-run`. - Apply `--query` after command execution and before rendering. - Users provide cloud placement as one canonical `region_code`, never as @@ -374,6 +379,7 @@ Implemented command behavior: - `tdc organization list-projects --query 'projects[0].id'` - `tdc organization list-projects --output text` - `tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter` +- `tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter --wait-until-active` - `tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter --dry-run` - `tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter --project-id ` - `tdc db list-db-clusters` diff --git a/README.md b/README.md index 8a460ef..b5a7986 100644 --- a/README.md +++ b/README.md @@ -33,29 +33,23 @@ echo "Hello Sandbox Workspace!" >> /path_to_workspace/hello.txt tdc fs unmount-file-system --mount-path /path_to_workspace --region ``` -### Always-on, zero infrastructure MySQL — The 4-Command Superpower +### Always-on, zero infrastructure MySQL — The 3-Command Superpower -An agent can go from zero to live HTAP SQL (Hybrid Transaction / Analytical Processing) in four commands: +An agent can go from zero to live HTAP SQL (Hybrid Transaction / Analytical Processing) in three commands: -1. Start provisioning a serverless MySQL-compatible cluster and capture its ID +1. Provision a serverless MySQL-compatible cluster, wait until it is active, and capture its ID ```shell -export CLUSTER_ID="$(tdc db create-db-cluster --db-cluster-type starter --db-cluster-name my-app-db --query id --output text)" +export CLUSTER_ID="$(tdc db create-db-cluster --db-cluster-type starter --db-cluster-name my-app-db --wait-until-active --query id --output text)" ``` -2. Wait for the cluster to become active (typically around 30 seconds) - -```shell -until [ "$(tdc db describe-db-cluster --db-cluster-id "$CLUSTER_ID" --query state --output text)" = "ACTIVE" ]; do sleep 2; done -``` - -3. Create the SQL users it needs to connect +2. Create the SQL users it needs to connect ```shell tdc db create-db-sql-users --db-cluster-id "$CLUSTER_ID" ``` -4. Retrieve the database connection string for your agent and share it across sandboxes as needed +3. Retrieve the database connection string for your agent and share it across sandboxes as needed ```shell export DATABASE_URL="$(tdc db format-db-connection-string --db-cluster-id "$CLUSTER_ID" --read-write --query connection_string --output text)" @@ -149,11 +143,13 @@ tdc fs mount-file-system --file-system-name agent-workspace --mount-path /path_t ### TiDB Cloud Starter ```shell -tdc db create-db-cluster --db-cluster-name my-distributed-mysql --db-cluster-type starter +tdc db create-db-cluster --db-cluster-name my-distributed-mysql --db-cluster-type starter --wait-until-active ``` Cluster creation uses the configured `project_id` by default. Use optional `--project-id ` to create in another accessible project. An explicit empty `--project-id` is rejected instead of falling back to the profile. +Without `--wait-until-active`, cluster creation returns as soon as TiDB Cloud accepts the asynchronous create request. With the flag, tdc waits up to 12 minutes and returns the final `ACTIVE` cluster. A timeout or interruption leaves the created cluster intact and reports its ID for inspection. + ### Organization Projects ```shell diff --git a/docs/spec/done/0006-starter-db-cluster-lifecycle.md b/docs/spec/done/0006-starter-db-cluster-lifecycle.md index f92677b..b782fd0 100644 --- a/docs/spec/done/0006-starter-db-cluster-lifecycle.md +++ b/docs/spec/done/0006-starter-db-cluster-lifecycle.md @@ -18,6 +18,7 @@ Primary create shape: ```bash tdc db create-db-cluster --db-cluster-name --db-cluster-type starter +tdc db create-db-cluster --db-cluster-name --db-cluster-type starter --wait-until-active ``` ## Behavior @@ -28,6 +29,12 @@ tdc db create-db-cluster --db-cluster-name --db-cluster-type starter - Use long flags only. - Mutating commands support `--dry-run`. - Commands must not prompt. +- Create returns after the API accepts the asynchronous request by default. + Optional `--wait-until-active` polls the created cluster until it reaches + `ACTIVE`, for up to 12 minutes, and returns the final cluster representation. +- Waiting never deletes the created resource. Timeout, cancellation, polling + failure, or a terminal `DELETING`, `DELETED`, or `INACTIVE` state returns an + actionable error that includes the cluster ID and an inspection command. - Delete must be non-interactive. It reads the remote cluster first, validates Starter-only behavior when plan metadata is available, and then deletes by cluster ID; it must never prompt. @@ -50,6 +57,9 @@ tdc db create-db-cluster --db-cluster-name --db-cluster-type starter fields internally. - Create and update return the remote resource representation or operation status returned by the API. +- Create with `--wait-until-active` returns a cluster whose state is `ACTIVE`. + If waiting fails after creation, the error states that the cluster remains + allocated and includes its ID. - Delete returns a structured confirmation or operation status. - Errors should distinguish validation failures, authentication failures, permission failures, not found, quota/capacity issues, and backend API errors. @@ -64,6 +74,7 @@ and region: ```bash tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter --dry-run tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter +tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter --wait-until-active tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter --project-id tdc db list-db-clusters --query 'clusters[].id' tdc db describe-db-cluster --db-cluster-id @@ -108,6 +119,10 @@ Command mapping: and set label `tidb.cloud/project`. 3. Call `POST /v1beta1/clusters` with `displayName`, project label, region name such as `regions/aws-us-east-1`, and only other confirmed fields. + 4. When `--wait-until-active` is set and the create response is not already + `ACTIVE`, call `GET /v1beta1/clusters/{clusterId}` every two seconds until + `ACTIVE`, a terminal state, cancellation, polling failure, or the + 12-minute deadline. - `tdc db describe-db-cluster` 1. Call `GET /v1beta1/clusters/{clusterId}` with optional `view`. 2. If the returned cluster exposes `clusterPlan` and it is not `STARTER`, @@ -145,14 +160,18 @@ Available but not part of this lifecycle MVP: ## Acceptance Criteria - Mock API tests cover create/list/describe/update/delete. +- Mock API tests cover wait success, an immediately active create response, + terminal state, polling failure, timeout, and default non-wait behavior. - Tests cover required `--db-cluster-type starter` on create. -- Tests cover dry-run request validation without sending mutating requests. +- Tests cover dry-run request validation and wait-plan reporting without + sending mutating requests. - Tests cover stable JSON output and `--query`. - Tests cover delete safety behavior without prompts. - `make live-e2e` covers the real cluster lifecycle: create a uniquely named - `tdc-e2e-*` Starter cluster without a spending limit, read it, update it, - read it again, delete it, and verify the cluster becomes deleted or not - found. The cleanup path only deletes the cluster created by that test run. + `tdc-e2e-*` Starter cluster with `--wait-until-active` and without a spending + limit, verify the create result is `ACTIVE`, read it, update it, read it + again, delete it, and verify the cluster becomes deleted or not found. The + cleanup path only deletes the cluster created by that test run. ## Out Of Scope diff --git a/e2e/cli_test.go b/e2e/cli_test.go index 65101c8..a245f4b 100644 --- a/e2e/cli_test.go +++ b/e2e/cli_test.go @@ -163,6 +163,7 @@ func TestOutputQueryAndDryRun(t *testing.T) { dryRun.wantExitCode(0) dryRun.wantStdoutContains(`"dry_run": true`) dryRun.wantStdoutContains(`"would_send_request": true`) + dryRun.wantStdoutContains(`"post_create_wait"`) regionOverride := runTDCWithInput(t, bin, "", []string{ "TDC_PUBLIC_KEY=e2e-public", @@ -198,6 +199,7 @@ func createClusterDryRunArgs() []string { "--db-cluster-name", "demo-cluster", "--db-cluster-type", "starter", "--project-id", "project-1", + "--wait-until-active", "--dry-run", } } diff --git a/e2e/live_test.go b/e2e/live_test.go index 8de7fe6..417e9f5 100644 --- a/e2e/live_test.go +++ b/e2e/live_test.go @@ -1458,6 +1458,7 @@ func TestLiveDBClusterLifecycle(t *testing.T) { "db", "create-db-cluster", "--db-cluster-name", clusterName, "--db-cluster-type", "starter", + "--wait-until-active", ) create.wantExitCode(0) created := decodeLiveCluster(t, create) @@ -1467,9 +1468,15 @@ func TestLiveDBClusterLifecycle(t *testing.T) { clusterID = created.ID defer cleanupLiveSQLCredentials(t, clusterID) - described := waitLiveCluster(t, bin, profileName, clusterID, func(cluster liveCluster) bool { - return cluster.ID == clusterID && cluster.DisplayName == clusterName && cluster.State == "ACTIVE" - }, 12*time.Minute, "become ACTIVE after create") + if created.State != "ACTIVE" { + t.Fatalf("--wait-until-active returned cluster in state %q: %#v", created.State, created) + } + describe := runTDC(t, bin, "--profile", profileName, "db", "describe-db-cluster", "--db-cluster-id", clusterID, "--view", "FULL") + describe.wantExitCode(0) + described := decodeLiveCluster(t, describe) + if described.ID != clusterID || described.DisplayName != clusterName || described.State != "ACTIVE" { + t.Fatalf("unexpected described cluster: %#v\n%s", described, describe.stdout) + } if described.ClusterPlan != "" && described.ClusterPlan != "STARTER" { t.Fatalf("expected STARTER cluster, got %#v", described) } diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 669bac4..81b3538 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -212,6 +212,7 @@ func newDBCreateClusterCommand(info version.Info) *cobra.Command { cmd.Flags().String("db-cluster-type", "", "DB cluster type; must be starter") cmd.Flags().String("project-id", "", "TiDB Cloud project id") cmd.Flags().Int32("monthly-spending-limit-usd-cents", -1, "monthly spending limit in USD cents; omit to use the API default") + cmd.Flags().Bool("wait-until-active", false, "wait until the created cluster becomes ACTIVE before returning") markUsageRequired(cmd, "db-cluster-name", "db-cluster-type") return cmd } @@ -654,6 +655,10 @@ func createClusterOptions(ctx commandContext, profile *config.Profile) (db.Creat if err != nil { return db.CreateClusterOptions{}, err } + waitUntilActive, err := ctx.BoolFlag("wait-until-active") + if err != nil { + return db.CreateClusterOptions{}, err + } return db.CreateClusterOptions{ Profile: profile, DisplayName: name, @@ -661,6 +666,7 @@ func createClusterOptions(ctx commandContext, profile *config.Profile) (db.Creat ProjectID: projectID, ProjectIDExplicit: ctx.FlagChanged("project-id"), MonthlySpendingLimitUSDCents: spendingLimit, + WaitUntilActive: waitUntilActive, }, nil } diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 8e20df7..a4be0cb 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -227,6 +227,9 @@ func TestHelpUsageShowsRequiredFirstAndOptionalBracketed(t *testing.T) { if !strings.Contains(stdout, " [--project-id ]") { t.Fatalf("expected --project-id to be optional, got:\n%s", stdout) } + if !strings.Contains(stdout, " [--wait-until-active]") { + t.Fatalf("expected --wait-until-active to be optional, got:\n%s", stdout) + } stdout, _, err = executeForTest("update", "help") if err != nil { @@ -565,7 +568,7 @@ func TestControlPlaneCommandSpecUsesCustomDryRun(t *testing.T) { func TestMutatingControlPlaneDryRunRendersJSON(t *testing.T) { withConfigEnv(t) - stdout, _, err := executeForTest("db", "create-db-cluster", "--db-cluster-name", "demo-cluster", "--db-cluster-type", "starter", "--project-id", "project-1", "--dry-run") + stdout, _, err := executeForTest("db", "create-db-cluster", "--db-cluster-name", "demo-cluster", "--db-cluster-type", "starter", "--project-id", "project-1", "--wait-until-active", "--dry-run") if err != nil { t.Fatalf("expected dry-run to succeed, got %v", err) } @@ -591,6 +594,9 @@ func TestMutatingControlPlaneDryRunRendersJSON(t *testing.T) { if !strings.Contains(stdout, "permission_requirement") || !strings.Contains(stdout, string(authz.StarterClusterCreate)) { t.Fatalf("dry-run output did not include permission requirement:\n%s", stdout) } + if !strings.Contains(stdout, "post_create_wait") || !strings.Contains(stdout, "ACTIVE") { + t.Fatalf("dry-run output did not include the requested wait:\n%s", stdout) + } } func TestRegionOverrideWinsOverEnvironmentCredentials(t *testing.T) { diff --git a/internal/db/cluster.go b/internal/db/cluster.go index 7d3c0e1..b6d6d80 100644 --- a/internal/db/cluster.go +++ b/internal/db/cluster.go @@ -2,6 +2,7 @@ package db import ( "context" + "errors" "fmt" "io" "net/http" @@ -19,18 +20,24 @@ import ( "github.com/tidbcloud/tdc/internal/dryrun" ) -const monthlySpendingLimitUnset int32 = -1 +const ( + monthlySpendingLimitUnset int32 = -1 + defaultClusterWaitTimeout = 12 * time.Minute + defaultClusterWaitPollInterval = 2 * time.Second +) type Service struct { - Resolver endpoints.Resolver - HTTPClient *http.Client - Transport http.RoundTripper - Timeout time.Duration - Debug bool - DebugWriter io.Writer - HomeDir string - SQLHTTPBaseURL string - MySQLDriverName string + Resolver endpoints.Resolver + HTTPClient *http.Client + Transport http.RoundTripper + Timeout time.Duration + ClusterWaitTimeout time.Duration + ClusterWaitPollInterval time.Duration + Debug bool + DebugWriter io.Writer + HomeDir string + SQLHTTPBaseURL string + MySQLDriverName string } type ListClustersOptions struct { @@ -49,6 +56,7 @@ type CreateClusterOptions struct { ProjectID string ProjectIDExplicit bool MonthlySpendingLimitUSDCents int32 + WaitUntilActive bool } type DescribeClusterOptions struct { @@ -120,6 +128,12 @@ func (s Service) CreateCluster(ctx context.Context, opts CreateClusterOptions) ( if err := ensureStarterCluster(cluster); err != nil { return ClusterResult{}, err } + if opts.WaitUntilActive { + cluster, err = s.waitUntilClusterActive(ctx, client, cluster) + if err != nil { + return ClusterResult{}, err + } + } return ClusterResult{Cluster: cluster}, nil } @@ -199,6 +213,19 @@ func (s Service) DryRunCreateCluster(ctx context.Context, commandPath string, op if err != nil { return dryrun.Result{}, err } + checks := []dryrun.Check{ + {Name: "config_and_credentials", Status: "passed", Message: fmt.Sprintf("profile %q loaded", profileName(opts.Profile))}, + {Name: "endpoint_selection", Status: "passed", Message: fmt.Sprintf("%s %s", endpoint.Provider, endpoint.RegionCode)}, + {Name: "permission_requirement", Status: "passed", Message: string(authz.StarterClusterCreate)}, + {Name: "cluster_type", Status: "passed", Message: validate.ClusterTypeStarter}, + } + if opts.WaitUntilActive { + checks = append(checks, dryrun.Check{ + Name: "post_create_wait", + Status: "passed", + Message: fmt.Sprintf("normal execution waits up to %s for state ACTIVE", s.clusterWaitTimeout()), + }) + } return dryrun.New( commandPath, "create_db_cluster", @@ -216,13 +243,102 @@ func (s Service) DryRunCreateCluster(ctx context.Context, commandPath string, op "spendingLimit": request.SpendingLimit, }, }, - dryrun.Check{Name: "config_and_credentials", Status: "passed", Message: fmt.Sprintf("profile %q loaded", profileName(opts.Profile))}, - dryrun.Check{Name: "endpoint_selection", Status: "passed", Message: fmt.Sprintf("%s %s", endpoint.Provider, endpoint.RegionCode)}, - dryrun.Check{Name: "permission_requirement", Status: "passed", Message: string(authz.StarterClusterCreate)}, - dryrun.Check{Name: "cluster_type", Status: "passed", Message: validate.ClusterTypeStarter}, + checks..., ), nil } +func (s Service) waitUntilClusterActive(ctx context.Context, client *apistarter.Client, cluster apistarter.Cluster) (apistarter.Cluster, error) { + if cluster.State == "ACTIVE" { + return cluster, nil + } + if strings.TrimSpace(cluster.ID) == "" { + return apistarter.Cluster{}, apperr.New( + "db.cluster_wait_missing_id", + "api", + 1, + "Starter cluster creation was accepted but the response did not include a cluster ID; list DB clusters before retrying", + ) + } + + timeout := s.clusterWaitTimeout() + waitCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + ticker := time.NewTicker(s.clusterWaitPollInterval()) + defer ticker.Stop() + + for { + current, err := client.GetCluster(waitCtx, cluster.ID, apistarter.GetClusterOptions{}) + if err != nil { + if waitErr := clusterWaitContextError(ctx, waitCtx, cluster.ID, timeout); waitErr != nil { + return apistarter.Cluster{}, waitErr + } + return apistarter.Cluster{}, apperr.Wrap( + "db.cluster_wait_read_failed", + "api", + 1, + fmt.Sprintf("DB cluster %q was created but tdc could not read its state while waiting for ACTIVE; the cluster was not deleted; inspect it with `tdc db describe-db-cluster --db-cluster-id %s`", cluster.ID, cluster.ID), + err, + ) + } + if err := ensureStarterCluster(current); err != nil { + return apistarter.Cluster{}, err + } + switch current.State { + case "ACTIVE": + return current, nil + case "DELETING", "DELETED", "INACTIVE": + return apistarter.Cluster{}, apperr.New( + "db.cluster_wait_terminal_state", + "api", + 1, + fmt.Sprintf("DB cluster %q was created but entered state %q before becoming ACTIVE; the cluster was not deleted; inspect it with `tdc db describe-db-cluster --db-cluster-id %s`", cluster.ID, current.State, cluster.ID), + ) + } + + select { + case <-waitCtx.Done(): + return apistarter.Cluster{}, clusterWaitContextError(ctx, waitCtx, cluster.ID, timeout) + case <-ticker.C: + } + } +} + +func clusterWaitContextError(parent, waitCtx context.Context, clusterID string, timeout time.Duration) error { + if parent.Err() != nil { + return apperr.Wrap( + "db.cluster_wait_canceled", + "runtime", + 1, + fmt.Sprintf("waiting for DB cluster %q to become ACTIVE was canceled; the cluster was not deleted", clusterID), + parent.Err(), + ) + } + if errors.Is(waitCtx.Err(), context.DeadlineExceeded) { + return apperr.New( + "db.cluster_wait_timeout", + "api", + 1, + fmt.Sprintf("DB cluster %q was created but did not become ACTIVE within %s; the cluster was not deleted; inspect it with `tdc db describe-db-cluster --db-cluster-id %s`", clusterID, timeout, clusterID), + ) + } + return nil +} + +func (s Service) clusterWaitTimeout() time.Duration { + if s.ClusterWaitTimeout > 0 { + return s.ClusterWaitTimeout + } + return defaultClusterWaitTimeout +} + +func (s Service) clusterWaitPollInterval() time.Duration { + if s.ClusterWaitPollInterval > 0 { + return s.ClusterWaitPollInterval + } + return defaultClusterWaitPollInterval +} + func (s Service) DryRunUpdateCluster(ctx context.Context, commandPath string, opts UpdateClusterOptions) (dryrun.Result, error) { clusterID, request, endpoint, err := s.updateRequestAndEndpoint(opts) if err != nil { diff --git a/internal/db/cluster_test.go b/internal/db/cluster_test.go index 471771b..6586b4f 100644 --- a/internal/db/cluster_test.go +++ b/internal/db/cluster_test.go @@ -3,10 +3,12 @@ package db import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" "testing" + "time" "github.com/tidbcloud/tdc/internal/api/endpoints" "github.com/tidbcloud/tdc/internal/apperr" @@ -52,6 +54,152 @@ func TestCreateCluster(t *testing.T) { } } +func TestCreateClusterWaitsUntilActive(t *testing.T) { + requests := make([]string, 0, 3) + gets := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.Method) + switch r.Method { + case http.MethodPost: + _, _ = w.Write([]byte(`{"clusterId":"cluster-1","displayName":"demo-cluster","clusterPlan":"STARTER","state":"CREATING"}`)) + case http.MethodGet: + if r.URL.Path != "/v1beta1/clusters/cluster-1" { + t.Fatalf("unexpected GET path %s", r.URL.Path) + } + gets++ + state := "CREATING" + if gets == 2 { + state = "ACTIVE" + } + _, _ = fmt.Fprintf(w, `{"clusterId":"cluster-1","displayName":"demo-cluster","clusterPlan":"STARTER","state":%q}`, state) + default: + t.Fatalf("unexpected method %s", r.Method) + } + })) + defer server.Close() + + service := testService(server.URL) + service.ClusterWaitTimeout = time.Second + service.ClusterWaitPollInterval = time.Millisecond + result, err := service.CreateCluster(context.Background(), CreateClusterOptions{ + Profile: testProfile(), + DisplayName: "demo-cluster", + ClusterType: "starter", + ProjectID: "project-1", + MonthlySpendingLimitUSDCents: -1, + WaitUntilActive: true, + }) + if err != nil { + t.Fatalf("CreateCluster failed: %v", err) + } + if result.ID != "cluster-1" || result.State != "ACTIVE" { + t.Fatalf("unexpected result: %#v", result) + } + if got := strings.Join(requests, ","); got != "POST,GET,GET" { + t.Fatalf("unexpected requests %q", got) + } +} + +func TestCreateClusterWaitReturnsImmediatelyWhenCreateIsActive(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if r.Method != http.MethodPost { + t.Fatalf("unexpected method %s", r.Method) + } + _, _ = w.Write([]byte(`{"clusterId":"cluster-1","displayName":"demo-cluster","clusterPlan":"STARTER","state":"ACTIVE"}`)) + })) + defer server.Close() + + result, err := testService(server.URL).CreateCluster(context.Background(), CreateClusterOptions{ + Profile: testProfile(), + DisplayName: "demo-cluster", + ClusterType: "starter", + ProjectID: "project-1", + MonthlySpendingLimitUSDCents: -1, + WaitUntilActive: true, + }) + if err != nil { + t.Fatalf("CreateCluster failed: %v", err) + } + if result.State != "ACTIVE" || requests != 1 { + t.Fatalf("unexpected result %#v or request count %d", result, requests) + } +} + +func TestCreateClusterWaitErrorsPreserveCreatedCluster(t *testing.T) { + tests := []struct { + name string + getResponse func(http.ResponseWriter) + timeout time.Duration + wantCode string + wantText string + }{ + { + name: "terminal state", + getResponse: func(w http.ResponseWriter) { + _, _ = w.Write([]byte(`{"clusterId":"cluster-1","displayName":"demo-cluster","clusterPlan":"STARTER","state":"DELETED"}`)) + }, + timeout: time.Second, + wantCode: "db.cluster_wait_terminal_state", + wantText: "DELETED", + }, + { + name: "read failure", + getResponse: func(w http.ResponseWriter) { + http.Error(w, `{"message":"backend unavailable"}`, http.StatusInternalServerError) + }, + timeout: time.Second, + wantCode: "db.cluster_wait_read_failed", + wantText: "could not read its state", + }, + { + name: "timeout", + getResponse: func(w http.ResponseWriter) { + _, _ = w.Write([]byte(`{"clusterId":"cluster-1","displayName":"demo-cluster","clusterPlan":"STARTER","state":"CREATING"}`)) + }, + timeout: 10 * time.Millisecond, + wantCode: "db.cluster_wait_timeout", + wantText: "was not deleted", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodPost: + _, _ = w.Write([]byte(`{"clusterId":"cluster-1","displayName":"demo-cluster","clusterPlan":"STARTER","state":"CREATING"}`)) + case http.MethodGet: + tt.getResponse(w) + default: + t.Fatalf("unexpected method %s", r.Method) + } + })) + defer server.Close() + + service := testService(server.URL) + service.ClusterWaitTimeout = tt.timeout + service.ClusterWaitPollInterval = time.Millisecond + _, err := service.CreateCluster(context.Background(), CreateClusterOptions{ + Profile: testProfile(), + DisplayName: "demo-cluster", + ClusterType: "starter", + ProjectID: "project-1", + MonthlySpendingLimitUSDCents: -1, + WaitUntilActive: true, + }) + if apperr.CodeFor(err) != tt.wantCode { + t.Fatalf("error code = %q, want %q: %v", apperr.CodeFor(err), tt.wantCode, err) + } + message := apperr.MessageFor(err) + if !strings.Contains(message, "cluster-1") || !strings.Contains(message, tt.wantText) { + t.Fatalf("error should preserve cluster identity and context, got %q", message) + } + }) + } +} + func TestCreateClusterUsesProfileProjectAndAllowsExplicitOverride(t *testing.T) { var projectIDs []string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -230,6 +378,7 @@ func TestDryRunCreateClusterDoesNotSendRequest(t *testing.T) { ClusterType: "starter", ProjectID: "project-1", MonthlySpendingLimitUSDCents: -1, + WaitUntilActive: true, }) if err != nil { t.Fatalf("DryRunCreateCluster failed: %v", err) @@ -240,6 +389,15 @@ func TestDryRunCreateClusterDoesNotSendRequest(t *testing.T) { if called { t.Fatal("dry-run should not send a request") } + foundWait := false + for _, check := range result.Checks { + if check.Name == "post_create_wait" && strings.Contains(check.Message, "12m0s") { + foundWait = true + } + } + if !foundWait { + t.Fatalf("dry-run should describe the post-create wait: %#v", result.Checks) + } } func TestCreateRequiresStarterType(t *testing.T) { From 979a93062484cefaf9ce8c787e32094f3ba28577 Mon Sep 17 00:00:00 2001 From: Cheese Date: Sat, 18 Jul 2026 22:34:55 +0800 Subject: [PATCH 2/3] feat: add synchronous lifecycle waits --- AGENTS.md | 23 ++- README.md | 23 ++- docs/pingcap-docs/docs | 2 +- docs/pingcap-docs/docs-cn | 2 +- docs/present.md | 15 +- .../done/0006-starter-db-cluster-lifecycle.md | 35 +++-- .../done/0007-starter-db-branch-lifecycle.md | 23 ++- ...015-drive9-companion-wrapper-for-tdc-fs.md | 16 +- .../done/0016-profile-fs-resource-registry.md | 7 +- e2e/cli_test.go | 9 +- e2e/live_test.go | 95 ++++-------- e2e/testdata/fake-drive9.go | 4 + internal/cli/commands.go | 37 ++++- internal/cli/root_test.go | 30 +++- internal/db/branch.go | 123 +++++++++++++++- internal/db/branch_test.go | 130 +++++++++++++++- internal/db/cluster.go | 114 +++++++++++++- internal/db/cluster_test.go | 118 +++++++++++++++ internal/fs/control.go | 24 ++- internal/fs/drive9_companion.go | 139 +++++++++++++++++- internal/fs/drive9_companion_test.go | 75 +++++++++- 21 files changed, 897 insertions(+), 147 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9a8b5bf..af82138 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -195,12 +195,13 @@ the profile for both focused and complete live targets. Live e2e must strictly cover every implemented interface and command for the current project stage, including real create/update/delete flows when those commands are implemented. For Starter DB clusters, the live suite creates a -uniquely named `tdc-e2e-*` cluster with `--wait-until-active`, without a +uniquely named `tdc-e2e-*` cluster with `--wait`, without a spending limit or explicit `--project-id`, verifies the returned state is `ACTIVE` and its project label matches the configured default, and deletes only that cluster. For Starter DB branches, the live suite creates, reads, lists, and deletes only a `tdc-e2e-branch-*` branch on the cluster created by the same -test run. For Starter DB SQL access, the live suite prepares tdc-managed +test run. Branch creation must use `--wait`; cluster deletion must +use `--wait`. For Starter DB SQL access, the live suite prepares tdc-managed read-only, read-write, and admin SQL users on the temporary cluster, verifies connection string output, and executes the HTTPS SQL API with all three access modes. @@ -318,10 +319,21 @@ Follow these rules unless `docs/priciples.md` is updated: - Mutating control-plane commands support `--dry-run`. - `--dry-run` must validate local config, credentials, provider, and region before reporting a planned mutation. -- `tdc db create-db-cluster --wait-until-active` waits up to 12 minutes for the +- `tdc db create-db-cluster --wait` waits up to 12 minutes for the created cluster to reach `ACTIVE`. It must never delete the cluster on timeout, cancellation, a polling failure, or a terminal state; errors must retain the created cluster ID and provide an inspection command. +- `tdc db create-db-cluster-branch --wait` waits up to 5 minutes + for the created branch to reach `ACTIVE`. A failed wait must not delete or + recreate the accepted branch. +- `tdc db delete-db-cluster --wait` waits up to 12 minutes and + succeeds when the API reports `DELETED` or the deleted cluster is no longer + accessible. A failed wait must state that deletion may still be in progress. +- `tdc fs create-file-system --wait` waits up to 10 minutes for the + root to become readable through the public Drive9 CLI. It must retain the + resource and local credentials when waiting fails. +- `tdc fs delete-file-system` is asynchronous. After Drive9 accepts deletion, + output status is `deleting`, not `deleted`. - Read-only commands reject `--dry-run`. - Apply `--query` after command execution and before rendering. - Users provide cloud placement as one canonical `region_code`, never as @@ -379,7 +391,7 @@ Implemented command behavior: - `tdc organization list-projects --query 'projects[0].id'` - `tdc organization list-projects --output text` - `tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter` -- `tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter --wait-until-active` +- `tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter --wait` - `tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter --dry-run` - `tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter --project-id ` - `tdc db list-db-clusters` @@ -388,8 +400,10 @@ Implemented command behavior: - `tdc db update-db-cluster --db-cluster-id --db-cluster-name new-name` - `tdc db update-db-cluster --db-cluster-id --monthly-spending-limit-usd-cents 1000 --dry-run` - `tdc db delete-db-cluster --db-cluster-id ` +- `tdc db delete-db-cluster --db-cluster-id --wait` - `tdc db delete-db-cluster --db-cluster-id --dry-run` - `tdc db create-db-cluster-branch --db-cluster-id --db-cluster-branch-name dev` +- `tdc db create-db-cluster-branch --db-cluster-id --db-cluster-branch-name dev --wait` - `tdc db create-db-cluster-branch --db-cluster-id --db-cluster-branch-name dev --dry-run` - `tdc db list-db-cluster-branches --db-cluster-id ` - `tdc db list-db-cluster-branches --db-cluster-id --query 'branches[].id'` @@ -410,6 +424,7 @@ Implemented command behavior: - `tdc db execute-sql-statement --db-cluster-id --transport https --sql "select 1"` - `tdc db execute-sql-statement --db-cluster-id --transport mysql --sql "select 1"` - `tdc fs create-file-system --file-system-name workspace` +- `tdc fs create-file-system --file-system-name workspace --wait` - `tdc fs create-file-system --file-system-name workspace --dry-run` - `tdc fs create-file-system --file-system-name scratch --set-default` - `tdc fs delete-file-system --file-system-name workspace --confirm-file-system-name workspace` diff --git a/README.md b/README.md index b5a7986..bd2ae22 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ An agent can persist state between sessions, share files across sandboxes, snaps 1. Create a filesystem resource and get the returning token (one-time, out of the sandbox) ```shell -export TDC_FS_TOKEN="$(tdc fs create-file-system --file-system-name agent-workspace --region --query fs_token --output text)" +export TDC_FS_TOKEN="$(tdc fs create-file-system --file-system-name agent-workspace --region --wait --query fs_token --output text)" ``` 2. Mount the filesystem and use just like any regular POSIX-compliant filesystem (inside the sandbox environment) @@ -40,7 +40,7 @@ An agent can go from zero to live HTAP SQL (Hybrid Transaction / Analytical Proc 1. Provision a serverless MySQL-compatible cluster, wait until it is active, and capture its ID ```shell -export CLUSTER_ID="$(tdc db create-db-cluster --db-cluster-type starter --db-cluster-name my-app-db --wait-until-active --query id --output text)" +export CLUSTER_ID="$(tdc db create-db-cluster --db-cluster-type starter --db-cluster-name my-app-db --wait --query id --output text)" ``` 2. Create the SQL users it needs to connect @@ -113,7 +113,7 @@ Supported regions: `aws-us-east-1` and `aws-ap-southeast-1`. ```shell mkdir ~/my-workspace -tdc fs create-file-system --file-system-name my-workspace +tdc fs create-file-system --file-system-name my-workspace --wait tdc fs mount-file-system --mount-path ~/my-workspace ``` @@ -130,9 +130,11 @@ tdc fs describe-file-system --file-system-name scratch `create-file-system` returns an file system token (`fs_token`) in its JSON result. This is the file system owner credential and should be handled as a secret. A configured machine can provision a file system and capture the token without printing the full result: ```shell -export TDC_FS_TOKEN="$(tdc fs create-file-system --file-system-name agent-workspace --query fs_token --output text)" +export TDC_FS_TOKEN="$(tdc fs create-file-system --file-system-name agent-workspace --wait --query fs_token --output text)" ``` +Without `--wait`, file system creation returns after Drive9 accepts provisioning. With the flag, tdc waits up to 10 minutes until the file system root is readable through the public Drive9 data-plane CLI. A timeout or interruption leaves the file system and its locally stored credentials intact. + An agent sandbox can then use that existing file system without running `tdc configure` or providing TiDB Cloud API keys: ```shell @@ -143,12 +145,21 @@ tdc fs mount-file-system --file-system-name agent-workspace --mount-path /path_t ### TiDB Cloud Starter ```shell -tdc db create-db-cluster --db-cluster-name my-distributed-mysql --db-cluster-type starter --wait-until-active +tdc db create-db-cluster --db-cluster-name my-distributed-mysql --db-cluster-type starter --wait ``` Cluster creation uses the configured `project_id` by default. Use optional `--project-id ` to create in another accessible project. An explicit empty `--project-id` is rejected instead of falling back to the profile. -Without `--wait-until-active`, cluster creation returns as soon as TiDB Cloud accepts the asynchronous create request. With the flag, tdc waits up to 12 minutes and returns the final `ACTIVE` cluster. A timeout or interruption leaves the created cluster intact and reports its ID for inspection. +Without `--wait`, cluster creation returns as soon as TiDB Cloud accepts the asynchronous create request. With the flag, tdc waits up to 12 minutes and returns the final `ACTIVE` cluster. A timeout or interruption leaves the created cluster intact and reports its ID for inspection. + +Branch creation and cluster deletion have equivalent explicit wait modes: + +```shell +tdc db create-db-cluster-branch --db-cluster-id --db-cluster-branch-name development --wait +tdc db delete-db-cluster --db-cluster-id --wait +``` + +Branch waiting lasts up to 5 minutes. Cluster deletion waiting lasts up to 12 minutes and succeeds when the API reports `DELETED` or the deleted cluster is no longer accessible. ### Organization Projects diff --git a/docs/pingcap-docs/docs b/docs/pingcap-docs/docs index 388e0b6..d9fcf59 160000 --- a/docs/pingcap-docs/docs +++ b/docs/pingcap-docs/docs @@ -1 +1 @@ -Subproject commit 388e0b6e0a0e57a4fc231d3ded5c70dbc95a0037 +Subproject commit d9fcf59d0b0ee3efc81b1ad093b75be42bdffac9 diff --git a/docs/pingcap-docs/docs-cn b/docs/pingcap-docs/docs-cn index cd88bf5..090b8bb 160000 --- a/docs/pingcap-docs/docs-cn +++ b/docs/pingcap-docs/docs-cn @@ -1 +1 @@ -Subproject commit cd88bf5a7b24b4d6752346b37a40dba12747d5f7 +Subproject commit 090b8bba9ca134cf94380df7a5a4a3bbe51e7b50 diff --git a/docs/present.md b/docs/present.md index daeaa49..8e9453a 100644 --- a/docs/present.md +++ b/docs/present.md @@ -56,7 +56,8 @@ bin/tdc db create-db-cluster \ bin/tdc db create-db-cluster \ --db-cluster-name "$CLUSTER_NAME" \ - --db-cluster-type starter + --db-cluster-type starter \ + --wait ``` 从 JSON 结果中找到 cluster ID： @@ -67,7 +68,7 @@ export CLUSTER_ID="$(bin/tdc db list-db-clusters | jq -r --arg name "$CLUSTER_NA bin/tdc db describe-db-cluster --db-cluster-id "$CLUSTER_ID" --output text ``` -等待状态变为 `ACTIVE`，然后更新名称并重新读取： +创建命令返回 `ACTIVE` 后，更新名称并重新读取： ```bash export CLUSTER_RENAMED="${CLUSTER_NAME}-renamed" @@ -84,7 +85,8 @@ bin/tdc db describe-db-cluster --db-cluster-id "$CLUSTER_ID" --output text ```bash bin/tdc db create-db-cluster-branch \ --db-cluster-id "$CLUSTER_ID" \ - --db-cluster-branch-name demo-branch + --db-cluster-branch-name demo-branch \ + --wait bin/tdc db list-db-cluster-branches --db-cluster-id "$CLUSTER_ID" --output text ``` @@ -153,7 +155,8 @@ bin/tdc fs create-file-system \ --dry-run bin/tdc fs create-file-system \ - --file-system-name tdc-demo-workspace + --file-system-name tdc-demo-workspace \ + --wait bin/tdc fs list-file-systems --output text bin/tdc fs describe-file-system \ @@ -391,7 +394,9 @@ bin/tdc fs delete-file-system \ 删除演示 cluster： ```bash -bin/tdc db delete-db-cluster --db-cluster-id "$CLUSTER_ID" +bin/tdc db delete-db-cluster \ + --db-cluster-id "$CLUSTER_ID" \ + --wait ``` ## 讲解重点 diff --git a/docs/spec/done/0006-starter-db-cluster-lifecycle.md b/docs/spec/done/0006-starter-db-cluster-lifecycle.md index b782fd0..f320e86 100644 --- a/docs/spec/done/0006-starter-db-cluster-lifecycle.md +++ b/docs/spec/done/0006-starter-db-cluster-lifecycle.md @@ -18,7 +18,7 @@ Primary create shape: ```bash tdc db create-db-cluster --db-cluster-name --db-cluster-type starter -tdc db create-db-cluster --db-cluster-name --db-cluster-type starter --wait-until-active +tdc db create-db-cluster --db-cluster-name --db-cluster-type starter --wait ``` ## Behavior @@ -30,7 +30,7 @@ tdc db create-db-cluster --db-cluster-name --db-cluster-type starter --wa - Mutating commands support `--dry-run`. - Commands must not prompt. - Create returns after the API accepts the asynchronous request by default. - Optional `--wait-until-active` polls the created cluster until it reaches + Optional `--wait` polls the created cluster until it reaches `ACTIVE`, for up to 12 minutes, and returns the final cluster representation. - Waiting never deletes the created resource. Timeout, cancellation, polling failure, or a terminal `DELETING`, `DELETED`, or `INACTIVE` state returns an @@ -38,6 +38,13 @@ tdc db create-db-cluster --db-cluster-name --db-cluster-type starter --wa - Delete must be non-interactive. It reads the remote cluster first, validates Starter-only behavior when plan metadata is available, and then deletes by cluster ID; it must never prompt. +- Delete returns after the API accepts the asynchronous request by default. + Optional `--wait` polls for up to 12 minutes and succeeds when + the cluster reaches `DELETED` or is no longer accessible after the accepted + delete request. +- A delete wait timeout, cancellation, or polling failure must not submit + another delete request. The error states that deletion may still be in + progress and identifies the cluster. ## Inputs And Config @@ -57,10 +64,12 @@ tdc db create-db-cluster --db-cluster-name --db-cluster-type starter --wa fields internally. - Create and update return the remote resource representation or operation status returned by the API. -- Create with `--wait-until-active` returns a cluster whose state is `ACTIVE`. +- Create with `--wait` returns a cluster whose state is `ACTIVE`. If waiting fails after creation, the error states that the cluster remains allocated and includes its ID. -- Delete returns a structured confirmation or operation status. +- Delete returns a structured confirmation or operation status. With + `--wait`, it returns a cluster representation with state + `DELETED`. - Errors should distinguish validation failures, authentication failures, permission failures, not found, quota/capacity issues, and backend API errors. - Permission errors must name the required permission, such as @@ -74,11 +83,12 @@ and region: ```bash tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter --dry-run tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter -tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter --wait-until-active +tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter --wait tdc db create-db-cluster --db-cluster-name demo --db-cluster-type starter --project-id tdc db list-db-clusters --query 'clusters[].id' tdc db describe-db-cluster --db-cluster-id tdc db delete-db-cluster --db-cluster-id +tdc db delete-db-cluster --db-cluster-id --wait ``` No command asks for a server URL. The active profile's `cloud_provider` and @@ -119,7 +129,7 @@ Command mapping: and set label `tidb.cloud/project`. 3. Call `POST /v1beta1/clusters` with `displayName`, project label, region name such as `regions/aws-us-east-1`, and only other confirmed fields. - 4. When `--wait-until-active` is set and the create response is not already + 4. When `--wait` is set and the create response is not already `ACTIVE`, call `GET /v1beta1/clusters/{clusterId}` every two seconds until `ACTIVE`, a terminal state, cancellation, polling failure, or the 12-minute deadline. @@ -138,6 +148,10 @@ Command mapping: 2. Call `GET /v1beta1/clusters/{clusterId}`. 3. If `clusterPlan` is present, verify it is `STARTER`. 4. Call `DELETE /v1beta1/clusters/{clusterId}`. + 5. When `--wait` is set, call + `GET /v1beta1/clusters/{clusterId}` every two seconds until the state is + `DELETED`, the resource is no longer accessible, cancellation, polling + failure, or the 12-minute deadline. Available but not part of this lifecycle MVP: @@ -162,16 +176,19 @@ Available but not part of this lifecycle MVP: - Mock API tests cover create/list/describe/update/delete. - Mock API tests cover wait success, an immediately active create response, terminal state, polling failure, timeout, and default non-wait behavior. +- Delete wait tests cover `DELETING` to `DELETED`, inaccessible-after-delete, + timeout, and dry-run wait-plan reporting. - Tests cover required `--db-cluster-type starter` on create. - Tests cover dry-run request validation and wait-plan reporting without sending mutating requests. - Tests cover stable JSON output and `--query`. - Tests cover delete safety behavior without prompts. - `make live-e2e` covers the real cluster lifecycle: create a uniquely named - `tdc-e2e-*` Starter cluster with `--wait-until-active` and without a spending + `tdc-e2e-*` Starter cluster with `--wait` and without a spending limit, verify the create result is `ACTIVE`, read it, update it, read it - again, delete it, and verify the cluster becomes deleted or not found. The - cleanup path only deletes the cluster created by that test run. + again, delete it with `--wait`, and verify the returned state + is `DELETED`. The cleanup path only deletes the cluster created by that test + run and also uses the wait flag. ## Out Of Scope diff --git a/docs/spec/done/0007-starter-db-branch-lifecycle.md b/docs/spec/done/0007-starter-db-branch-lifecycle.md index 4ef5b20..23df740 100644 --- a/docs/spec/done/0007-starter-db-branch-lifecycle.md +++ b/docs/spec/done/0007-starter-db-branch-lifecycle.md @@ -20,6 +20,12 @@ Initial command set: - Mutating branch commands support `--dry-run`. - Commands must not prompt. - Branch operations require explicit cluster identification. +- Create returns after the API accepts the asynchronous request by default. + Optional `--wait` polls the created branch for up to five + minutes and returns the final `ACTIVE` branch. +- A branch wait timeout, cancellation, polling failure, or terminal `DELETED` + state leaves the accepted branch unchanged and returns an actionable error + containing both cluster and branch IDs. - Delete is non-interactive and deletes by explicit cluster ID plus branch ID after reading the remote branch. @@ -36,7 +42,8 @@ Use only fields supported by the Starter branch API. ## Output And Errors - JSON is the default output. -- Branch create returns the branch resource or operation status. +- Branch create returns the branch resource or operation status. With + `--wait`, it returns a branch whose state is `ACTIVE`. - Not-found errors should identify whether the missing object is the cluster or the branch. - Permission errors must name the required permission, such as @@ -48,6 +55,7 @@ Users can manage Starter branch workflows without adding another command level: ```bash tdc db create-db-cluster-branch --db-cluster-id --db-cluster-branch-name dev --dry-run +tdc db create-db-cluster-branch --db-cluster-id --db-cluster-branch-name dev --wait tdc db list-db-cluster-branches --db-cluster-id tdc db describe-db-cluster-branch --db-cluster-id --db-cluster-branch-id tdc db delete-db-cluster-branch --db-cluster-id --db-cluster-branch-id @@ -84,6 +92,10 @@ Command mapping: 1. Validate `--db-cluster-id` and API-backed branch fields such as `--db-cluster-branch-name`. 2. Call `POST /v1beta1/clusters/{clusterId}/branches`. + 3. When `--wait` is set, call + `GET /v1beta1/clusters/{clusterId}/branches/{branchId}` every two seconds + until `ACTIVE`, terminal `DELETED`, cancellation, polling failure, or the + five-minute deadline. - `tdc db describe-db-cluster-branch` 1. Call `GET /v1beta1/clusters/{clusterId}/branches/{branchId}`. - `tdc db delete-db-cluster-branch` @@ -110,13 +122,16 @@ Available but out of scope for this spec: ## Acceptance Criteria - Mock API tests cover create/list/describe/delete. +- Mock API tests cover branch wait success, terminal state, polling failure, + timeout, and dry-run wait-plan reporting. - Tests cover required cluster identification. - Tests cover dry-run for create and delete. - Tests cover branch not-found errors. - Tests cover `--query` over list output. -- `make live-e2e` creates a temporary `tdc-e2e-branch-*` branch on the - temporary `tdc-e2e-*` Starter cluster, then lists, describes, deletes, and - verifies deletion for that branch without touching pre-existing branches. +- `make live-e2e` creates a temporary `tdc-e2e-branch-*` branch with + `--wait` on the temporary `tdc-e2e-*` Starter cluster, asserts + the returned state is `ACTIVE`, then lists, describes, deletes, and verifies + deletion without touching pre-existing branches. ## Out Of Scope diff --git a/docs/spec/done/0015-drive9-companion-wrapper-for-tdc-fs.md b/docs/spec/done/0015-drive9-companion-wrapper-for-tdc-fs.md index f95839b..c1d561a 100644 --- a/docs/spec/done/0015-drive9-companion-wrapper-for-tdc-fs.md +++ b/docs/spec/done/0015-drive9-companion-wrapper-for-tdc-fs.md @@ -115,8 +115,9 @@ The tdc command list must match Drive9's public external CLI surface, not Drive9 - tdc keeps `~/.tdc/` as the user-facing source of truth. Users must not be asked to edit `~/.drive9`. - The wrapper translates tdc profile, region, filesystem resource, and credential state into the environment and flags expected by Drive9. - If the companion is missing or incompatible, fs commands fail with an actionable tdc error. They must not silently fall back to a tdc-owned HTTP/FUSE/WebDAV implementation. -- `tdc fs create-file-system` provisions the resource through the companion and persists the returned tdc fs resource metadata and API key back into the active tdc profile. -- `tdc fs delete-file-system` deletes only the active tdc-managed resource and removes the active profile's fs metadata and `fs_api_key` after successful deletion. +- `tdc fs create-file-system` provisions the resource through the companion and persists the returned tdc fs resource metadata and API key back into the active tdc profile. Optional `--wait` invokes the public Drive9 `fs stat --output json :/` command until the root becomes readable or the ten-minute deadline expires. +- `tdc fs delete-file-system` submits deletion for only the selected tdc-managed resource and removes that resource's local registry metadata and API key after Drive9 accepts the request. Because remote deletion is asynchronous, the structured result reports `status: "deleting"` and `remote_deletion_state: "deleting"`, not `deleted`. +- A failed readiness wait never deletes the provisioned resource or removes its local credentials. The error identifies the file system and states that it remains registered for inspection or retry. - `tdc fs check-file-system` verifies that the active profile has fs metadata and that the Drive9 companion can reach the resource. - Data-plane and mount commands stream through Drive9. tdc must not buffer arbitrary file contents just to normalize output. - `tdc fs create-directory --mode` remains accepted for tdc CLI compatibility and must validate the octal value, but Drive9's public `mkdir` command does not currently apply directory modes. Do not emulate directory chmod through a non-public backend call. @@ -262,6 +263,7 @@ Provisioning: 4. Drive9 calls its `/v1/provision` backend with TiDB Cloud keys and region parameters. 5. Drive9 returns resource metadata and a tdc fs data-plane API key. 6. tdc writes non-secret fs metadata to `~/.tdc/config` and `fs_api_key` to `~/.tdc/credentials`. +7. When `--wait` is set, tdc invokes `tdc-drive9 fs stat --output json :/` with the stored resource credentials every five seconds until it succeeds or ten minutes elapse. Data-plane command: @@ -296,7 +298,9 @@ Unit and e2e coverage must focus on tdc wrapper behavior: - tdc profile to Drive9 environment mapping. - Argument translation for every fs, fs-git, fs-journal, and fs-vault command. - `create-file-system` JSON parsing and all-or-nothing tdc config writes. -- `delete-file-system` cleanup behavior. +- `create-file-system --wait` success, transient retry, timeout, + cancellation, and preservation of locally stored credentials on failure. +- `delete-file-system` cleanup behavior and asynchronous `deleting` output. - Raw streaming behavior for `read-file`. - `--query` rejection on raw commands and `--query` support on captured JSON commands. - No tests should be retained for tdc commands that are not part of the Drive9 public CLI surface. Low-level layer entry/object/event commands, low-level Git workspace/tree/state/object-pack/overlay commands, and legacy vault token commands should not appear in command-surface, e2e, or README examples. @@ -304,7 +308,7 @@ Unit and e2e coverage must focus on tdc wrapper behavior: Live e2e must run real commands through the wrapper: -- create a temporary tdc fs resource +- create a temporary tdc fs resource with `--wait` - check it - copy/read/list/describe/move/delete files - create directories and recursive copies @@ -325,7 +329,7 @@ Drive9 owns deep FUSE correctness tests. tdc must still carry regression tests t Users still use tdc commands: ```bash -tdc fs create-file-system --file-system-name workspace +tdc fs create-file-system --file-system-name workspace --wait tdc fs cp --from-local ./README.md --to-remote /README.md tdc fs cat --path /README.md tdc fs mount-file-system --mount-path ./mnt --remote-path / @@ -346,6 +350,8 @@ The user-facing experience remains tdc, but filesystem correctness and advanced - `tdc update --check` reports companion compatibility. - `tdc update` updates the companion for direct-installer installs. - `tdc fs create-file-system` provisions through the companion and writes only tdc-owned state under `~/.tdc`. +- `tdc fs create-file-system --wait` returns `status: "ready"` only after the public Drive9 root stat succeeds; failures retain the resource and credentials. +- `tdc fs delete-file-system` reports asynchronous remote state as `deleting`. - `tdc fs check-file-system` succeeds for a configured profile without requiring user-visible `~/.drive9` setup. - All retained public `tdc fs` and Unix-alias commands route through the companion. - `tdc fs mount-file-system` defaults to Drive9 FUSE where available and supports explicit WebDAV when Drive9 supports it. diff --git a/docs/spec/done/0016-profile-fs-resource-registry.md b/docs/spec/done/0016-profile-fs-resource-registry.md index 980ed97..bb62d1c 100644 --- a/docs/spec/done/0016-profile-fs-resource-registry.md +++ b/docs/spec/done/0016-profile-fs-resource-registry.md @@ -82,11 +82,16 @@ tdc [ERROR]: tdc fs is not configured for profile "default"; run `tdc fs create- - If the selected profile already has resources, the new resource is stored but does not become default unless `--set-default` is passed. - Re-running with the same name is idempotent when the resource already exists locally and has complete credentials. - Re-running with a name that exists locally but incomplete credentials returns an actionable repair error. +- `--wait` is optional. It waits up to ten minutes for the selected + resource root to become readable through the public Drive9 CLI and returns + `status: "ready"`; wait failure preserves the registry entry and credential + file. `tdc fs delete-file-system` behavior: - Deletes only the named resource. - Removes only that resource's local registry entry and credential file. +- Reports the accepted asynchronous remote deletion state as `deleting`. - If the deleted resource was the default, clear `fs_default_file_system_name`. If exactly one resource remains, tdc may set that remaining resource as default; otherwise leave default empty and force explicit selection. - Does not delete or mutate any other resource under the profile. @@ -95,7 +100,7 @@ tdc [ERROR]: tdc fs is not configured for profile "default"; run `tdc fs create- New or changed commands: ```bash -tdc fs create-file-system --file-system-name [--set-default] [--dry-run] +tdc fs create-file-system --file-system-name [--set-default] [--wait] [--dry-run] tdc fs delete-file-system --file-system-name --confirm-file-system-name [--dry-run] tdc fs list-file-systems tdc fs describe-file-system --file-system-name diff --git a/e2e/cli_test.go b/e2e/cli_test.go index a245f4b..bc6fe32 100644 --- a/e2e/cli_test.go +++ b/e2e/cli_test.go @@ -199,7 +199,7 @@ func createClusterDryRunArgs() []string { "--db-cluster-name", "demo-cluster", "--db-cluster-type", "starter", "--project-id", "project-1", - "--wait-until-active", + "--wait", "--dry-run", } } @@ -333,12 +333,14 @@ func TestFSResourceRegistrySelectionAcrossCommandFamilies(t *testing.T) { ), "configure", "--profile", "stage", "--non-interactive") configured.wantExitCode(0) - createWorkspace := runTDCWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "create-file-system", "--file-system-name", "workspace") + createWorkspace := runTDCWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "create-file-system", "--file-system-name", "workspace", "--wait") createWorkspace.wantExitCode(0) + createWorkspace.wantStdoutContains(`"status": "ready"`) createWorkspace.wantStdoutContains(`"credentials_stored": true`) createWorkspace.wantStdoutContains(`"fs_token": "key-workspace"`) - createScratch := runTDCWithInput(t, bin, "", baseEnv, "--profile", "stage", "--region", "aws-us-west-2", "fs", "create-file-system", "--file-system-name", "scratch") + createScratch := runTDCWithInput(t, bin, "", baseEnv, "--profile", "stage", "--region", "aws-us-west-2", "fs", "create-file-system", "--file-system-name", "scratch", "--wait") createScratch.wantExitCode(0) + createScratch.wantStdoutContains(`"status": "ready"`) createScratch.wantStdoutContains(`"credentials_stored": true`) list := runTDCWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "list-file-systems") @@ -400,6 +402,7 @@ func TestFSResourceRegistrySelectionAcrossCommandFamilies(t *testing.T) { deleteScratch := runTDCWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "delete-file-system", "--file-system-name", "scratch", "--confirm-file-system-name", "scratch") deleteScratch.wantExitCode(0) + deleteScratch.wantStdoutContains(`"status": "deleting"`) afterDelete := runTDCWithInput(t, bin, "", baseEnv, "--profile", "stage", "fs", "list-file-systems") afterDelete.wantExitCode(0) afterDelete.wantStdoutContains(`"file_system_name": "workspace"`) diff --git a/e2e/live_test.go b/e2e/live_test.go index 417e9f5..88fdfe6 100644 --- a/e2e/live_test.go +++ b/e2e/live_test.go @@ -110,7 +110,7 @@ func TestLiveFSResourceRegistryLifecycle(t *testing.T) { }() for i, name := range names { - create := runTDC(t, bin, "--profile", profileName, "fs", "create-file-system", "--file-system-name", name) + create := runTDC(t, bin, "--profile", profileName, "fs", "create-file-system", "--file-system-name", name, "--wait") if create.exitCode != 0 { if isLiveFSQuotaError(create.stderr) { if i == 0 { @@ -130,8 +130,7 @@ func TestLiveFSResourceRegistryLifecycle(t *testing.T) { created[name] = true stateMutated = true create.wantStdoutContains(`"credentials_stored": true`) - selected := resolveLiveFSResource(t, profile, name) - waitLiveFSReady(t, bin, profileName, selected, 10*time.Minute) + create.wantStdoutContains(`"status": "ready"`) } list := runTDC(t, bin, "--profile", profileName, "fs", "list-file-systems") @@ -151,6 +150,8 @@ func TestLiveFSResourceRegistryLifecycle(t *testing.T) { deleteFirst := runTDC(t, bin, "--profile", profileName, "fs", "delete-file-system", "--file-system-name", names[0], "--confirm-file-system-name", names[0]) deleteFirst.wantExitCode(0) + deleteFirst.wantStdoutContains(`"status": "deleting"`) + deleteFirst.wantStdoutContains(`"remote_deletion_state": "deleting"`) created[names[0]] = false remaining := runTDC(t, bin, "--profile", profileName, "fs", "describe-file-system", "--file-system-name", names[1]) remaining.wantExitCode(0) @@ -158,6 +159,8 @@ func TestLiveFSResourceRegistryLifecycle(t *testing.T) { deleteSecond := runTDC(t, bin, "--profile", profileName, "fs", "delete-file-system", "--file-system-name", names[1], "--confirm-file-system-name", names[1]) deleteSecond.wantExitCode(0) + deleteSecond.wantStdoutContains(`"status": "deleting"`) + deleteSecond.wantStdoutContains(`"remote_deletion_state": "deleting"`) created[names[1]] = false } @@ -220,7 +223,8 @@ func TestLiveDBCommandSurface(t *testing.T) { {"db", "list-db-clusters", "help"}, }) testLiveMutatingDryRuns(t, bin, profileName, [][]string{ - {"db", "create-db-cluster-branch", "--db-cluster-id", "cluster-1", "--db-cluster-branch-name", "dev"}, + {"db", "create-db-cluster-branch", "--db-cluster-id", "cluster-1", "--db-cluster-branch-name", "dev", "--wait"}, + {"db", "delete-db-cluster", "--db-cluster-id", "cluster-1", "--wait"}, {"db", "delete-db-cluster-branch", "--db-cluster-id", "cluster-1", "--db-cluster-branch-id", "branch-1"}, {"db", "create-db-sql-users", "--db-cluster-id", "cluster-1"}, }, "remote_mutation") @@ -286,7 +290,7 @@ func TestLiveFSCommandSurface(t *testing.T) { {"fs", "mount", "help"}, {"fs", "drain", "help"}, {"fs", "umount", "help"}, }) testLiveMutatingDryRuns(t, bin, profileName, [][]string{ - {"fs", "create-file-system", "--file-system-name", fileSystemName}, + {"fs", "create-file-system", "--file-system-name", fileSystemName, "--wait"}, {"fs", "delete-file-system", "--file-system-name", fileSystemName, "--confirm-file-system-name", fileSystemName}, {"fs", "create-layer", "--layer-id", "layer-1", "--base-root-path", "/workspace", "--layer-name", "dev"}, {"fs", "create-layer-checkpoint", "--layer-id", "layer-1", "--checkpoint-id", "cp-1"}, @@ -1229,7 +1233,7 @@ func TestLiveFSConfigurationFreeAccess(t *testing.T) { profileName := liveProfileName(t) suffix := fmt.Sprintf("%s-%d", time.Now().UTC().Format("20060102150405"), os.Getpid()) fileSystemName := "tdc-e2e-token-" + suffix - create := runTDC(t, bin, "--profile", profileName, "fs", "create-file-system", "--file-system-name", fileSystemName) + create := runTDC(t, bin, "--profile", profileName, "fs", "create-file-system", "--file-system-name", fileSystemName, "--wait") create.wantExitCode(0) var created struct { FileSystemName string `json:"file_system_name"` @@ -1247,6 +1251,9 @@ func TestLiveFSConfigurationFreeAccess(t *testing.T) { if created.Status == "exists" { t.Fatalf("generated configuration-free FS resource name unexpectedly existed") } + if created.Status != "ready" { + t.Fatalf("--wait returned tdc fs resource in status %q", created.Status) + } deletedResource := false defer func() { @@ -1264,8 +1271,6 @@ func TestLiveFSConfigurationFreeAccess(t *testing.T) { if selected.FSAPIKey != created.FSToken || selected.FSPlacementRegionCode != created.RegionCode { t.Fatal("stored FS resource credentials or placement differ from create output") } - waitLiveFSReady(t, bin, profileName, selected, 10*time.Minute) - cleanHome := t.TempDir() authEnv := liveFSTokenEnv(selected, cleanHome) remoteRoot := "/tdc-e2e-token-" + suffix @@ -1360,6 +1365,8 @@ func TestLiveFSConfigurationFreeAccess(t *testing.T) { remoteDeleted = true deleteResource := runTDC(t, bin, "--profile", profileName, "fs", "delete-file-system", "--file-system-name", fileSystemName, "--confirm-file-system-name", fileSystemName) deleteResource.wantExitCode(0) + deleteResource.wantStdoutContains(`"status": "deleting"`) + deleteResource.wantStdoutContains(`"remote_deletion_state": "deleting"`) deletedResource = true } @@ -1445,7 +1452,7 @@ func TestLiveDBClusterLifecycle(t *testing.T) { if clusterID == "" || deleted { return } - cleanup := runTDC(t, bin, "--profile", profileName, "db", "delete-db-cluster", "--db-cluster-id", clusterID) + cleanup := runTDC(t, bin, "--profile", profileName, "db", "delete-db-cluster", "--db-cluster-id", clusterID, "--wait") if cleanup.exitCode != 0 && cleanup.exitCode != 5 { t.Logf("cleanup delete failed for cluster %s: exit=%d stdout=%s stderr=%s", clusterID, cleanup.exitCode, cleanup.stdout, cleanup.stderr) } @@ -1458,7 +1465,7 @@ func TestLiveDBClusterLifecycle(t *testing.T) { "db", "create-db-cluster", "--db-cluster-name", clusterName, "--db-cluster-type", "starter", - "--wait-until-active", + "--wait", ) create.wantExitCode(0) created := decodeLiveCluster(t, create) @@ -1469,7 +1476,7 @@ func TestLiveDBClusterLifecycle(t *testing.T) { defer cleanupLiveSQLCredentials(t, clusterID) if created.State != "ACTIVE" { - t.Fatalf("--wait-until-active returned cluster in state %q: %#v", created.State, created) + t.Fatalf("--wait returned cluster in state %q: %#v", created.State, created) } describe := runTDC(t, bin, "--profile", profileName, "db", "describe-db-cluster", "--db-cluster-id", clusterID, "--view", "FULL") describe.wantExitCode(0) @@ -1537,6 +1544,7 @@ func TestLiveDBClusterLifecycle(t *testing.T) { "db", "create-db-cluster-branch", "--db-cluster-id", clusterID, "--db-cluster-branch-name", branchName, + "--wait", ) branchCreate.wantExitCode(0) createdBranch := decodeLiveBranch(t, branchCreate) @@ -1544,10 +1552,9 @@ func TestLiveDBClusterLifecycle(t *testing.T) { t.Fatalf("unexpected created branch: %#v\n%s", createdBranch, branchCreate.stdout) } branchID = createdBranch.ID - - waitLiveBranch(t, bin, profileName, clusterID, branchID, func(branch liveBranch) bool { - return branch.ID == branchID && branch.DisplayName == branchName && branch.State == "ACTIVE" - }, 5*time.Minute, "become ACTIVE after create") + if createdBranch.State != "ACTIVE" { + t.Fatalf("--wait returned branch in state %q: %#v", createdBranch.State, createdBranch) + } branches := runTDC(t, bin, "--profile", profileName, "db", "list-db-cluster-branches", "--db-cluster-id", clusterID, "--page-size", "100") branches.wantExitCode(0) @@ -1610,15 +1617,17 @@ func TestLiveDBClusterLifecycle(t *testing.T) { "--profile", profileName, "db", "delete-db-cluster", "--db-cluster-id", clusterID, + "--wait", ) remove.wantExitCode(0) removed := decodeLiveCluster(t, remove) if removed.ID != clusterID { t.Fatalf("delete response did not reference created cluster %s:\n%s", clusterID, remove.stdout) } + if removed.State != "DELETED" { + t.Fatalf("--wait returned cluster in state %q: %#v", removed.State, removed) + } deleted = true - - waitLiveClusterDeleted(t, bin, profileName, clusterID, 12*time.Minute) } func requireLive(t *testing.T) { @@ -1721,54 +1730,6 @@ func waitLiveCluster(t *testing.T, bin, profileName, clusterID string, ready fun } } -func waitLiveBranch(t *testing.T, bin, profileName, clusterID, branchID string, ready func(liveBranch) bool, timeout time.Duration, description string) liveBranch { - t.Helper() - deadline := time.Now().Add(timeout) - var last liveBranch - for { - describe := runTDC(t, bin, "--profile", profileName, "db", "describe-db-cluster-branch", "--db-cluster-id", clusterID, "--db-cluster-branch-id", branchID, "--view", "FULL") - describe.wantExitCode(0) - last = decodeLiveBranch(t, describe) - if ready(last) { - return last - } - if time.Now().After(deadline) { - t.Fatalf("timed out waiting for branch %s to %s; last=%#v", branchID, description, last) - } - time.Sleep(5 * time.Second) - } -} - -func waitLiveClusterDeleted(t *testing.T, bin, profileName, clusterID string, timeout time.Duration) { - t.Helper() - deadline := time.Now().Add(timeout) - for { - describe := runTDC(t, bin, "--profile", profileName, "db", "describe-db-cluster", "--db-cluster-id", clusterID) - switch describe.exitCode { - case 0: - cluster := decodeLiveCluster(t, describe) - if cluster.ID != clusterID { - t.Fatalf("post-delete read returned a different cluster: %#v", cluster) - } - if cluster.State == "DELETED" { - return - } - case 5: - return - case 4: - // TiDB Cloud can return 403 for a just-deleted cluster that was - // readable before the successful DELETE. - return - default: - describe.fail("post-delete read should return deleted cluster state, not found, or no longer readable; got exit code %d", describe.exitCode) - } - if time.Now().After(deadline) { - t.Fatalf("timed out waiting for cluster %s to be deleted", clusterID) - } - time.Sleep(10 * time.Second) - } -} - func waitLiveSQL(t *testing.T, bin, profileName, clusterID string, modeArgs []string, description string) { t.Helper() deadline := time.Now().Add(5 * time.Minute) @@ -1950,9 +1911,10 @@ func ensureLiveFSResource(t *testing.T, bin, profileName string) *config.Profile t.Fatalf("resolve live fs resource %q: %v", name, err) } - create := runTDC(t, bin, "--profile", profileName, "fs", "create-file-system", "--file-system-name", name) + create := runTDC(t, bin, "--profile", profileName, "fs", "create-file-system", "--file-system-name", name, "--wait") create.wantExitCode(0) create.wantStdoutContains(`"credentials_stored": true`) + create.wantStdoutContains(`"status": "ready"`) liveFSResourceAutoCreated = true profile = liveProfile(t) @@ -1960,7 +1922,6 @@ func ensureLiveFSResource(t *testing.T, bin, profileName string) *config.Profile if err != nil { t.Fatalf("tdc fs resource %q was created but is not in profile %q registry: %v", name, profileName, err) } - waitLiveFSReady(t, bin, profileName, selected, 10*time.Minute) return selected } diff --git a/e2e/testdata/fake-drive9.go b/e2e/testdata/fake-drive9.go index 7966df5..7450bab 100644 --- a/e2e/testdata/fake-drive9.go +++ b/e2e/testdata/fake-drive9.go @@ -52,6 +52,10 @@ func main() { }) return } + if len(args) >= 1 && args[0] == "delete" { + fmt.Println(`{"status":"deleting"}`) + return + } if len(args) >= 2 && args[0] == "vault" && args[1] == "ls" { fmt.Println(`{"secrets":[]}`) return diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 81b3538..4dc5dd9 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -212,7 +212,7 @@ func newDBCreateClusterCommand(info version.Info) *cobra.Command { cmd.Flags().String("db-cluster-type", "", "DB cluster type; must be starter") cmd.Flags().String("project-id", "", "TiDB Cloud project id") cmd.Flags().Int32("monthly-spending-limit-usd-cents", -1, "monthly spending limit in USD cents; omit to use the API default") - cmd.Flags().Bool("wait-until-active", false, "wait until the created cluster becomes ACTIVE before returning") + cmd.Flags().Bool("wait", false, "wait until the created cluster becomes ACTIVE before returning") markUsageRequired(cmd, "db-cluster-name", "db-cluster-type") return cmd } @@ -364,6 +364,7 @@ func newDBDeleteClusterCommand(info version.Info) *cobra.Command { }, }, info) cmd.Flags().String("db-cluster-id", "", "Starter DB cluster id") + cmd.Flags().Bool("wait", false, "wait until the deleted cluster reaches DELETED or is no longer accessible") markUsageRequired(cmd, "db-cluster-id") return cmd } @@ -399,6 +400,7 @@ func newDBCreateBranchCommand(info version.Info) *cobra.Command { }, info) cmd.Flags().String("db-cluster-id", "", "Starter DB cluster id") cmd.Flags().String("db-cluster-branch-name", "", "Starter DB cluster branch display name") + cmd.Flags().Bool("wait", false, "wait until the created branch becomes ACTIVE before returning") markUsageRequired(cmd, "db-cluster-id", "db-cluster-branch-name") return cmd } @@ -655,7 +657,7 @@ func createClusterOptions(ctx commandContext, profile *config.Profile) (db.Creat if err != nil { return db.CreateClusterOptions{}, err } - waitUntilActive, err := ctx.BoolFlag("wait-until-active") + waitUntilActive, err := ctx.BoolFlag("wait") if err != nil { return db.CreateClusterOptions{}, err } @@ -696,9 +698,14 @@ func deleteClusterOptions(ctx commandContext, profile *config.Profile) (db.Delet if err != nil { return db.DeleteClusterOptions{}, err } + waitUntilDeleted, err := ctx.BoolFlag("wait") + if err != nil { + return db.DeleteClusterOptions{}, err + } return db.DeleteClusterOptions{ - Profile: profile, - ClusterID: clusterID, + Profile: profile, + ClusterID: clusterID, + WaitUntilDeleted: waitUntilDeleted, }, nil } @@ -711,10 +718,15 @@ func createBranchOptions(ctx commandContext, profile *config.Profile) (db.Create if err != nil { return db.CreateBranchOptions{}, err } + waitUntilActive, err := ctx.BoolFlag("wait") + if err != nil { + return db.CreateBranchOptions{}, err + } return db.CreateBranchOptions{ - Profile: profile, - ClusterID: clusterID, - DisplayName: name, + Profile: profile, + ClusterID: clusterID, + DisplayName: name, + WaitUntilActive: waitUntilActive, }, nil } @@ -933,10 +945,15 @@ func newFSCreateFileSystemCommand(info version.Info) *cobra.Command { if err != nil { return nil, err } + waitUntilReady, err := ctx.BoolFlag("wait") + if err != nil { + return nil, err + } return service.CreateFileSystem(ctx.cmd.Context(), tdcfs.CreateFileSystemOptions{ Profile: profile, FileSystemName: name, SetDefault: setDefault, + WaitUntilReady: waitUntilReady, }) }, DryRun: func(ctx commandContext) (dryrun.Result, error) { @@ -952,15 +969,21 @@ func newFSCreateFileSystemCommand(info version.Info) *cobra.Command { if err != nil { return dryrun.Result{}, err } + waitUntilReady, err := ctx.BoolFlag("wait") + if err != nil { + return dryrun.Result{}, err + } return service.DryRunCreateFileSystem(ctx.cmd.Context(), ctx.CommandPath(), tdcfs.CreateFileSystemOptions{ Profile: profile, FileSystemName: name, SetDefault: setDefault, + WaitUntilReady: waitUntilReady, }) }, }, info) cmd.Flags().String("file-system-name", "", "tdc fs resource name") cmd.Flags().Bool("set-default", false, "make the created file system the profile default") + cmd.Flags().Bool("wait", false, "wait until the created file system data plane is ready") markUsageRequired(cmd, "file-system-name") return cmd } diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index a4be0cb..fc0be9e 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -227,8 +227,32 @@ func TestHelpUsageShowsRequiredFirstAndOptionalBracketed(t *testing.T) { if !strings.Contains(stdout, " [--project-id ]") { t.Fatalf("expected --project-id to be optional, got:\n%s", stdout) } - if !strings.Contains(stdout, " [--wait-until-active]") { - t.Fatalf("expected --wait-until-active to be optional, got:\n%s", stdout) + if !strings.Contains(stdout, " [--wait]") { + t.Fatalf("expected --wait to be optional, got:\n%s", stdout) + } + + stdout, _, err = executeForTest("db", "delete-db-cluster", "help") + if err != nil { + t.Fatalf("expected db delete help to succeed, got %v", err) + } + if !strings.Contains(stdout, " [--wait]") { + t.Fatalf("expected --wait to be optional, got:\n%s", stdout) + } + + stdout, _, err = executeForTest("db", "create-db-cluster-branch", "help") + if err != nil { + t.Fatalf("expected db branch create help to succeed, got %v", err) + } + if !strings.Contains(stdout, " [--wait]") { + t.Fatalf("expected branch --wait to be optional, got:\n%s", stdout) + } + + stdout, _, err = executeForTest("fs", "create-file-system", "help") + if err != nil { + t.Fatalf("expected fs create help to succeed, got %v", err) + } + if !strings.Contains(stdout, " [--wait]") { + t.Fatalf("expected --wait to be optional, got:\n%s", stdout) } stdout, _, err = executeForTest("update", "help") @@ -568,7 +592,7 @@ func TestControlPlaneCommandSpecUsesCustomDryRun(t *testing.T) { func TestMutatingControlPlaneDryRunRendersJSON(t *testing.T) { withConfigEnv(t) - stdout, _, err := executeForTest("db", "create-db-cluster", "--db-cluster-name", "demo-cluster", "--db-cluster-type", "starter", "--project-id", "project-1", "--wait-until-active", "--dry-run") + stdout, _, err := executeForTest("db", "create-db-cluster", "--db-cluster-name", "demo-cluster", "--db-cluster-type", "starter", "--project-id", "project-1", "--wait", "--dry-run") if err != nil { t.Fatalf("expected dry-run to succeed, got %v", err) } diff --git a/internal/db/branch.go b/internal/db/branch.go index c5b8564..4e7eb20 100644 --- a/internal/db/branch.go +++ b/internal/db/branch.go @@ -2,13 +2,16 @@ package db import ( "context" + "errors" "fmt" "net/http" "strings" "text/tabwriter" + "time" "github.com/tidbcloud/tdc/internal/api/endpoints" apistarter "github.com/tidbcloud/tdc/internal/api/starter" + "github.com/tidbcloud/tdc/internal/apperr" "github.com/tidbcloud/tdc/internal/authz" "github.com/tidbcloud/tdc/internal/config" "github.com/tidbcloud/tdc/internal/db/validate" @@ -23,9 +26,10 @@ type ListBranchesOptions struct { } type CreateBranchOptions struct { - Profile *config.Profile - ClusterID string - DisplayName string + Profile *config.Profile + ClusterID string + DisplayName string + WaitUntilActive bool } type DescribeBranchOptions struct { @@ -87,6 +91,12 @@ func (s Service) CreateBranch(ctx context.Context, opts CreateBranchOptions) (Br if err != nil { return BranchResult{}, err } + if opts.WaitUntilActive { + branch, err = s.waitUntilBranchActive(ctx, client, clusterID, branch) + if err != nil { + return BranchResult{}, err + } + } return BranchResult{Branch: branch}, nil } @@ -134,6 +144,19 @@ func (s Service) DryRunCreateBranch(ctx context.Context, commandPath string, opt if err != nil { return dryrun.Result{}, err } + checks := []dryrun.Check{ + {Name: "config_and_credentials", Status: "passed", Message: fmt.Sprintf("profile %q loaded", profileName(opts.Profile))}, + {Name: "endpoint_selection", Status: "passed", Message: fmt.Sprintf("%s %s", endpoint.Provider, endpoint.RegionCode)}, + {Name: "permission_requirement", Status: "passed", Message: string(authz.StarterBranchCreate)}, + {Name: "cluster_id", Status: "passed", Message: clusterID}, + } + if opts.WaitUntilActive { + checks = append(checks, dryrun.Check{ + Name: "post_create_wait", + Status: "passed", + Message: fmt.Sprintf("normal execution waits up to %s for state ACTIVE", s.branchWaitTimeout()), + }) + } return dryrun.New( commandPath, "create_db_cluster_branch", @@ -144,13 +167,99 @@ func (s Service) DryRunCreateBranch(ctx context.Context, commandPath string, opt "displayName": request.DisplayName, }, }, - dryrun.Check{Name: "config_and_credentials", Status: "passed", Message: fmt.Sprintf("profile %q loaded", profileName(opts.Profile))}, - dryrun.Check{Name: "endpoint_selection", Status: "passed", Message: fmt.Sprintf("%s %s", endpoint.Provider, endpoint.RegionCode)}, - dryrun.Check{Name: "permission_requirement", Status: "passed", Message: string(authz.StarterBranchCreate)}, - dryrun.Check{Name: "cluster_id", Status: "passed", Message: clusterID}, + checks..., ), nil } +func (s Service) waitUntilBranchActive(ctx context.Context, client *apistarter.Client, clusterID string, branch apistarter.Branch) (apistarter.Branch, error) { + if branch.State == "ACTIVE" { + return branch, nil + } + if strings.TrimSpace(branch.ID) == "" { + return apistarter.Branch{}, apperr.New( + "db.branch_wait_missing_id", + "api", + 1, + fmt.Sprintf("Starter branch creation in cluster %q was accepted but the response did not include a branch ID; list DB cluster branches before retrying", clusterID), + ) + } + + timeout := s.branchWaitTimeout() + waitCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + ticker := time.NewTicker(s.branchWaitPollInterval()) + defer ticker.Stop() + + for { + current, err := client.GetBranch(waitCtx, clusterID, branch.ID, apistarter.GetBranchOptions{}) + if err != nil { + if waitErr := branchWaitContextError(ctx, waitCtx, clusterID, branch.ID, timeout); waitErr != nil { + return apistarter.Branch{}, waitErr + } + return apistarter.Branch{}, apperr.Wrap( + "db.branch_wait_read_failed", + "api", + 1, + fmt.Sprintf("DB branch %q was created in cluster %q but tdc could not read its state while waiting for ACTIVE; the branch was not deleted", branch.ID, clusterID), + err, + ) + } + switch current.State { + case "ACTIVE": + return current, nil + case "DELETED": + return apistarter.Branch{}, apperr.New( + "db.branch_wait_terminal_state", + "api", + 1, + fmt.Sprintf("DB branch %q in cluster %q was created but entered state DELETED before becoming ACTIVE; the branch was not recreated", branch.ID, clusterID), + ) + } + + select { + case <-waitCtx.Done(): + return apistarter.Branch{}, branchWaitContextError(ctx, waitCtx, clusterID, branch.ID, timeout) + case <-ticker.C: + } + } +} + +func branchWaitContextError(parent, waitCtx context.Context, clusterID, branchID string, timeout time.Duration) error { + if parent.Err() != nil { + return apperr.Wrap( + "db.branch_wait_canceled", + "runtime", + 1, + fmt.Sprintf("waiting for DB branch %q in cluster %q to become ACTIVE was canceled; the branch was not deleted", branchID, clusterID), + parent.Err(), + ) + } + if errors.Is(waitCtx.Err(), context.DeadlineExceeded) { + return apperr.New( + "db.branch_wait_timeout", + "api", + 1, + fmt.Sprintf("DB branch %q in cluster %q did not become ACTIVE within %s; the branch was not deleted", branchID, clusterID, timeout), + ) + } + return nil +} + +func (s Service) branchWaitTimeout() time.Duration { + if s.BranchWaitTimeout > 0 { + return s.BranchWaitTimeout + } + return defaultBranchWaitTimeout +} + +func (s Service) branchWaitPollInterval() time.Duration { + if s.BranchWaitPollInterval > 0 { + return s.BranchWaitPollInterval + } + return defaultBranchWaitPollInterval +} + func (s Service) DryRunDeleteBranch(ctx context.Context, commandPath string, opts DeleteBranchOptions) (dryrun.Result, error) { clusterID, branchID, endpoint, err := s.deleteBranchRequestAndEndpoint(opts) if err != nil { diff --git a/internal/db/branch_test.go b/internal/db/branch_test.go index b9a1be5..6ec7ef5 100644 --- a/internal/db/branch_test.go +++ b/internal/db/branch_test.go @@ -3,10 +3,12 @@ package db import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" "testing" + "time" "github.com/tidbcloud/tdc/internal/apperr" ) @@ -40,6 +42,118 @@ func TestCreateBranch(t *testing.T) { } } +func TestCreateBranchWaitsUntilActive(t *testing.T) { + requests := make([]string, 0, 3) + gets := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.Method) + switch r.Method { + case http.MethodPost: + _, _ = w.Write([]byte(`{"branchId":"branch-1","displayName":"dev","clusterId":"cluster-1","state":"CREATING"}`)) + case http.MethodGet: + gets++ + state := "CREATING" + if gets == 2 { + state = "ACTIVE" + } + _, _ = fmt.Fprintf(w, `{"branchId":"branch-1","displayName":"dev","clusterId":"cluster-1","state":%q}`, state) + default: + t.Fatalf("unexpected method %s", r.Method) + } + })) + defer server.Close() + + service := testService(server.URL) + service.BranchWaitTimeout = time.Second + service.BranchWaitPollInterval = time.Millisecond + result, err := service.CreateBranch(context.Background(), CreateBranchOptions{ + Profile: testProfile(), + ClusterID: "cluster-1", + DisplayName: "dev", + WaitUntilActive: true, + }) + if err != nil { + t.Fatalf("CreateBranch failed: %v", err) + } + if result.ID != "branch-1" || result.State != "ACTIVE" { + t.Fatalf("unexpected result: %#v", result) + } + if got := strings.Join(requests, ","); got != "POST,GET,GET" { + t.Fatalf("unexpected requests %q", got) + } +} + +func TestCreateBranchWaitErrorsPreserveCreatedBranch(t *testing.T) { + tests := []struct { + name string + getResponse func(http.ResponseWriter) + timeout time.Duration + wantCode string + wantText string + }{ + { + name: "terminal state", + getResponse: func(w http.ResponseWriter) { + _, _ = w.Write([]byte(`{"branchId":"branch-1","displayName":"dev","clusterId":"cluster-1","state":"DELETED"}`)) + }, + timeout: time.Second, + wantCode: "db.branch_wait_terminal_state", + wantText: "DELETED", + }, + { + name: "read failure", + getResponse: func(w http.ResponseWriter) { + http.Error(w, `{"message":"backend unavailable"}`, http.StatusInternalServerError) + }, + timeout: time.Second, + wantCode: "db.branch_wait_read_failed", + wantText: "could not read its state", + }, + { + name: "timeout", + getResponse: func(w http.ResponseWriter) { + _, _ = w.Write([]byte(`{"branchId":"branch-1","displayName":"dev","clusterId":"cluster-1","state":"CREATING"}`)) + }, + timeout: 10 * time.Millisecond, + wantCode: "db.branch_wait_timeout", + wantText: "was not deleted", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodPost: + _, _ = w.Write([]byte(`{"branchId":"branch-1","displayName":"dev","clusterId":"cluster-1","state":"CREATING"}`)) + case http.MethodGet: + tt.getResponse(w) + default: + t.Fatalf("unexpected method %s", r.Method) + } + })) + defer server.Close() + + service := testService(server.URL) + service.BranchWaitTimeout = tt.timeout + service.BranchWaitPollInterval = time.Millisecond + _, err := service.CreateBranch(context.Background(), CreateBranchOptions{ + Profile: testProfile(), + ClusterID: "cluster-1", + DisplayName: "dev", + WaitUntilActive: true, + }) + if apperr.CodeFor(err) != tt.wantCode { + t.Fatalf("error code = %q, want %q: %v", apperr.CodeFor(err), tt.wantCode, err) + } + message := apperr.MessageFor(err) + if !strings.Contains(message, "branch-1") || !strings.Contains(message, "cluster-1") || !strings.Contains(message, tt.wantText) { + t.Fatalf("error should preserve branch identity and context, got %q", message) + } + }) + } +} + func TestListBranches(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Query().Get("pageSize") != "1" { @@ -134,9 +248,10 @@ func TestDryRunCreateBranchDoesNotSendRequest(t *testing.T) { defer server.Close() result, err := testService(server.URL).DryRunCreateBranch(context.Background(), "tdc db create-db-cluster-branch", CreateBranchOptions{ - Profile: testProfile(), - ClusterID: "cluster-1", - DisplayName: "dev", + Profile: testProfile(), + ClusterID: "cluster-1", + DisplayName: "dev", + WaitUntilActive: true, }) if err != nil { t.Fatalf("DryRunCreateBranch failed: %v", err) @@ -147,6 +262,15 @@ func TestDryRunCreateBranchDoesNotSendRequest(t *testing.T) { if called { t.Fatal("dry-run should not send a request") } + foundWait := false + for _, check := range result.Checks { + if check.Name == "post_create_wait" && strings.Contains(check.Message, "5m0s") { + foundWait = true + } + } + if !foundWait { + t.Fatalf("dry-run should describe the post-create wait: %#v", result.Checks) + } } func TestDryRunDeleteBranchDoesNotSendRequest(t *testing.T) { diff --git a/internal/db/cluster.go b/internal/db/cluster.go index b6d6d80..b343a05 100644 --- a/internal/db/cluster.go +++ b/internal/db/cluster.go @@ -24,6 +24,8 @@ const ( monthlySpendingLimitUnset int32 = -1 defaultClusterWaitTimeout = 12 * time.Minute defaultClusterWaitPollInterval = 2 * time.Second + defaultBranchWaitTimeout = 5 * time.Minute + defaultBranchWaitPollInterval = 2 * time.Second ) type Service struct { @@ -33,6 +35,8 @@ type Service struct { Timeout time.Duration ClusterWaitTimeout time.Duration ClusterWaitPollInterval time.Duration + BranchWaitTimeout time.Duration + BranchWaitPollInterval time.Duration Debug bool DebugWriter io.Writer HomeDir string @@ -73,8 +77,9 @@ type UpdateClusterOptions struct { } type DeleteClusterOptions struct { - Profile *config.Profile - ClusterID string + Profile *config.Profile + ClusterID string + WaitUntilDeleted bool } type ListClustersResult struct { @@ -205,6 +210,12 @@ func (s Service) DeleteCluster(ctx context.Context, opts DeleteClusterOptions) ( if err != nil { return ClusterResult{}, err } + if opts.WaitUntilDeleted { + cluster, err = s.waitUntilClusterDeleted(ctx, client, cluster) + if err != nil { + return ClusterResult{}, err + } + } return ClusterResult{Cluster: cluster}, nil } @@ -374,6 +385,18 @@ func (s Service) DryRunDeleteCluster(ctx context.Context, commandPath string, op if err != nil { return dryrun.Result{}, err } + checks := []dryrun.Check{ + {Name: "config_and_credentials", Status: "passed", Message: fmt.Sprintf("profile %q loaded", profileName(opts.Profile))}, + {Name: "endpoint_selection", Status: "passed", Message: fmt.Sprintf("%s %s", endpoint.Provider, endpoint.RegionCode)}, + {Name: "permission_requirement", Status: "passed", Message: string(authz.StarterClusterDelete)}, + } + if opts.WaitUntilDeleted { + checks = append(checks, dryrun.Check{ + Name: "post_delete_wait", + Status: "passed", + Message: fmt.Sprintf("normal execution waits up to %s for state DELETED or for the cluster to become inaccessible after deletion", s.clusterWaitTimeout()), + }) + } return dryrun.New( commandPath, "delete_db_cluster", @@ -382,12 +405,93 @@ func (s Service) DryRunDeleteCluster(ctx context.Context, commandPath string, op Path: "/v1beta1/clusters/" + clusterID, Description: "normal execution first reads the cluster and verifies it is a Starter cluster before deleting", }, - dryrun.Check{Name: "config_and_credentials", Status: "passed", Message: fmt.Sprintf("profile %q loaded", profileName(opts.Profile))}, - dryrun.Check{Name: "endpoint_selection", Status: "passed", Message: fmt.Sprintf("%s %s", endpoint.Provider, endpoint.RegionCode)}, - dryrun.Check{Name: "permission_requirement", Status: "passed", Message: string(authz.StarterClusterDelete)}, + checks..., ), nil } +func (s Service) waitUntilClusterDeleted(ctx context.Context, client *apistarter.Client, cluster apistarter.Cluster) (apistarter.Cluster, error) { + if cluster.State == "DELETED" { + return cluster, nil + } + if strings.TrimSpace(cluster.ID) == "" { + return apistarter.Cluster{}, apperr.New( + "db.cluster_delete_wait_missing_id", + "api", + 1, + "Starter cluster deletion was accepted but the response did not include a cluster ID; list DB clusters before retrying", + ) + } + + timeout := s.clusterWaitTimeout() + waitCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + ticker := time.NewTicker(s.clusterWaitPollInterval()) + defer ticker.Stop() + + for { + current, err := client.GetCluster(waitCtx, cluster.ID, apistarter.GetClusterOptions{}) + if err != nil { + if waitErr := clusterDeleteWaitContextError(ctx, waitCtx, cluster.ID, timeout); waitErr != nil { + return apistarter.Cluster{}, waitErr + } + if isDeletedClusterReadError(err) { + cluster.State = "DELETED" + return cluster, nil + } + return apistarter.Cluster{}, apperr.Wrap( + "db.cluster_delete_wait_read_failed", + "api", + 1, + fmt.Sprintf("DB cluster %q deletion was accepted but tdc could not confirm completion; deletion may still be in progress", cluster.ID), + err, + ) + } + if err := ensureStarterCluster(current); err != nil { + return apistarter.Cluster{}, err + } + if current.State == "DELETED" { + return current, nil + } + + select { + case <-waitCtx.Done(): + return apistarter.Cluster{}, clusterDeleteWaitContextError(ctx, waitCtx, cluster.ID, timeout) + case <-ticker.C: + } + } +} + +func isDeletedClusterReadError(err error) bool { + switch apperr.CodeFor(err) { + case "api.not_found", "authz.permission_denied": + return true + default: + return false + } +} + +func clusterDeleteWaitContextError(parent, waitCtx context.Context, clusterID string, timeout time.Duration) error { + if parent.Err() != nil { + return apperr.Wrap( + "db.cluster_delete_wait_canceled", + "runtime", + 1, + fmt.Sprintf("waiting for DB cluster %q deletion was canceled; deletion may still be in progress", clusterID), + parent.Err(), + ) + } + if errors.Is(waitCtx.Err(), context.DeadlineExceeded) { + return apperr.New( + "db.cluster_delete_wait_timeout", + "api", + 1, + fmt.Sprintf("DB cluster %q did not become DELETED within %s; deletion may still be in progress", clusterID, timeout), + ) + } + return nil +} + func (s Service) createRequest(opts CreateClusterOptions) (apistarter.CreateClusterRequest, error) { request, _, err := s.createRequestAndEndpoint(opts) return request, err diff --git a/internal/db/cluster_test.go b/internal/db/cluster_test.go index 6586b4f..ed3ee94 100644 --- a/internal/db/cluster_test.go +++ b/internal/db/cluster_test.go @@ -365,6 +365,104 @@ func TestDeleteClusterReadsBeforeDelete(t *testing.T) { } } +func TestDeleteClusterWaitsUntilDeleted(t *testing.T) { + requests := make([]string, 0, 4) + gets := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.Method) + switch r.Method { + case http.MethodGet: + gets++ + state := "ACTIVE" + if gets == 2 { + state = "DELETING" + } + if gets == 3 { + state = "DELETED" + } + _, _ = fmt.Fprintf(w, `{"clusterId":"cluster-1","displayName":"demo-cluster","clusterPlan":"STARTER","state":%q}`, state) + case http.MethodDelete: + _, _ = w.Write([]byte(`{"clusterId":"cluster-1","displayName":"demo-cluster","clusterPlan":"STARTER","state":"DELETING"}`)) + default: + t.Fatalf("unexpected method %s", r.Method) + } + })) + defer server.Close() + + service := testService(server.URL) + service.ClusterWaitTimeout = time.Second + service.ClusterWaitPollInterval = time.Millisecond + result, err := service.DeleteCluster(context.Background(), DeleteClusterOptions{ + Profile: testProfile(), + ClusterID: "cluster-1", + WaitUntilDeleted: true, + }) + if err != nil { + t.Fatalf("DeleteCluster failed: %v", err) + } + if result.ID != "cluster-1" || result.State != "DELETED" { + t.Fatalf("unexpected result: %#v", result) + } + if got := strings.Join(requests, ","); got != "GET,DELETE,GET,GET" { + t.Fatalf("unexpected requests %q", got) + } +} + +func TestDeleteClusterWaitTreatsPostDeleteNotFoundAsDeleted(t *testing.T) { + gets := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + gets++ + if gets == 1 { + _, _ = w.Write([]byte(`{"clusterId":"cluster-1","displayName":"demo-cluster","clusterPlan":"STARTER","state":"ACTIVE"}`)) + return + } + http.NotFound(w, r) + case http.MethodDelete: + _, _ = w.Write([]byte(`{"clusterId":"cluster-1","displayName":"demo-cluster","clusterPlan":"STARTER","state":"DELETING"}`)) + default: + t.Fatalf("unexpected method %s", r.Method) + } + })) + defer server.Close() + + result, err := testService(server.URL).DeleteCluster(context.Background(), DeleteClusterOptions{ + Profile: testProfile(), + ClusterID: "cluster-1", + WaitUntilDeleted: true, + }) + if err != nil { + t.Fatalf("DeleteCluster failed: %v", err) + } + if result.ID != "cluster-1" || result.State != "DELETED" { + t.Fatalf("unexpected result: %#v", result) + } +} + +func TestDeleteClusterWaitTimeoutPreservesAcceptedDeletion(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + state := "ACTIVE" + if r.Method == http.MethodDelete { + state = "DELETING" + } + _, _ = fmt.Fprintf(w, `{"clusterId":"cluster-1","displayName":"demo-cluster","clusterPlan":"STARTER","state":%q}`, state) + })) + defer server.Close() + + service := testService(server.URL) + service.ClusterWaitTimeout = 10 * time.Millisecond + service.ClusterWaitPollInterval = time.Millisecond + _, err := service.DeleteCluster(context.Background(), DeleteClusterOptions{ + Profile: testProfile(), + ClusterID: "cluster-1", + WaitUntilDeleted: true, + }) + if apperr.CodeFor(err) != "db.cluster_delete_wait_timeout" || !strings.Contains(apperr.MessageFor(err), "may still be in progress") { + t.Fatalf("unexpected error: %v", err) + } +} + func TestDryRunCreateClusterDoesNotSendRequest(t *testing.T) { called := false server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -400,6 +498,26 @@ func TestDryRunCreateClusterDoesNotSendRequest(t *testing.T) { } } +func TestDryRunDeleteClusterDescribesWait(t *testing.T) { + result, err := testService("https://starter.test").DryRunDeleteCluster(context.Background(), "tdc db delete-db-cluster", DeleteClusterOptions{ + Profile: testProfile(), + ClusterID: "cluster-1", + WaitUntilDeleted: true, + }) + if err != nil { + t.Fatalf("DryRunDeleteCluster failed: %v", err) + } + found := false + for _, check := range result.Checks { + if check.Name == "post_delete_wait" && strings.Contains(check.Message, "12m0s") { + found = true + } + } + if !found { + t.Fatalf("dry-run should describe the post-delete wait: %#v", result.Checks) + } +} + func TestCreateRequiresStarterType(t *testing.T) { _, err := Service{}.DryRunCreateCluster(context.Background(), "tdc db create-db-cluster", CreateClusterOptions{ Profile: testProfile(), diff --git a/internal/fs/control.go b/internal/fs/control.go index b84f1e4..ea9016f 100644 --- a/internal/fs/control.go +++ b/internal/fs/control.go @@ -23,13 +23,15 @@ import ( ) type Service struct { - Resolver endpoints.Resolver - HTTPClient *http.Client - Transport http.RoundTripper - Timeout time.Duration - Debug bool - DebugWriter io.Writer - HomeDir string + Resolver endpoints.Resolver + HTTPClient *http.Client + Transport http.RoundTripper + Timeout time.Duration + FSReadyWaitTimeout time.Duration + FSReadyWaitPollInterval time.Duration + Debug bool + DebugWriter io.Writer + HomeDir string CompanionPath string Stdin io.Reader @@ -41,6 +43,7 @@ type CreateFileSystemOptions struct { Profile *config.Profile FileSystemName string SetDefault bool + WaitUntilReady bool } type DeleteFileSystemOptions struct { @@ -174,6 +177,13 @@ func (s Service) DryRunCreateFileSystem(ctx context.Context, commandPath string, {Name: "file_system_name", Status: "passed", Message: name}, } checks = append(checks, endpointDryRunCheck(endpoint, endpointErr)) + if opts.WaitUntilReady { + checks = append(checks, dryrun.Check{ + Name: "post_create_wait", + Status: "passed", + Message: fmt.Sprintf("normal execution waits up to %s for the Drive9 data plane root to become readable", s.fsReadyWaitTimeout()), + }) + } return dryrun.New( commandPath, "create_file_system", diff --git a/internal/fs/drive9_companion.go b/internal/fs/drive9_companion.go index ba2505b..6875917 100644 --- a/internal/fs/drive9_companion.go +++ b/internal/fs/drive9_companion.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -22,6 +23,11 @@ import ( "github.com/tidbcloud/tdc/internal/fswrap" ) +const ( + defaultFSReadyWaitTimeout = 10 * time.Minute + defaultFSReadyWaitPollInterval = 5 * time.Second +) + type drive9CreateOutput struct { Context string `json:"context"` TenantID string `json:"tenant_id"` @@ -35,6 +41,11 @@ type drive9CreateOutput struct { Config string `json:"config"` } +type drive9DeleteOutput struct { + Status string `json:"status"` + Server string `json:"server,omitempty"` +} + type drive9StatMetadata struct { Path string `json:"path,omitempty"` Size int64 `json:"size"` @@ -154,7 +165,7 @@ func (s Service) drive9CreateFileSystem(ctx context.Context, opts CreateFileSyst return FileSystemResult{}, err } } - return FileSystemResult{ + fileSystem := FileSystemResult{ FileSystemName: existing.Name, TenantID: existing.TenantID, CloudProvider: existing.CloudProvider, @@ -162,7 +173,14 @@ func (s Service) drive9CreateFileSystem(ctx context.Context, opts CreateFileSyst FSToken: existing.APIKey, Status: "exists", CredentialsStored: true, - }, nil + } + if opts.WaitUntilReady { + if err := s.waitUntilFileSystemReady(ctx, homeDir, opts.Profile, name); err != nil { + return FileSystemResult{}, err + } + fileSystem.Status = "ready" + } + return fileSystem, nil } else if apperr.CodeFor(getErr) != "fs.resource_not_found" { return FileSystemResult{}, getErr } @@ -200,7 +218,7 @@ func (s Service) drive9CreateFileSystem(ctx context.Context, opts CreateFileSyst if err := fscred.Store(homeDir, opts.Profile, name, out.TenantID, cloudProvider, regionCode, out.APIKey, opts.SetDefault); err != nil { return FileSystemResult{}, err } - return FileSystemResult{ + fileSystem := FileSystemResult{ FileSystemName: name, TenantID: out.TenantID, CloudProvider: cloudProvider, @@ -208,7 +226,14 @@ func (s Service) drive9CreateFileSystem(ctx context.Context, opts CreateFileSyst FSToken: out.APIKey, Status: status, CredentialsStored: true, - }, nil + } + if opts.WaitUntilReady { + if err := s.waitUntilFileSystemReady(ctx, homeDir, opts.Profile, name); err != nil { + return FileSystemResult{}, err + } + fileSystem.Status = "ready" + } + return fileSystem, nil } func (s Service) drive9DeleteFileSystem(ctx context.Context, opts DeleteFileSystemOptions) (DeleteResult, error) { @@ -218,7 +243,7 @@ func (s Service) drive9DeleteFileSystem(ctx context.Context, opts DeleteFileSyst } resource := fscred.FromProfile(opts.Profile) args := []string{"delete", "--json", "--yes"} - _, err = s.drive9Runner().Run(ctx, fswrap.RunOptions{ + result, err := s.drive9Runner().Run(ctx, fswrap.RunOptions{ Profile: opts.Profile, Args: args, CaptureStdout: true, @@ -228,6 +253,14 @@ func (s Service) drive9DeleteFileSystem(ctx context.Context, opts DeleteFileSyst if err != nil { return DeleteResult{}, err } + var out drive9DeleteOutput + if err := json.Unmarshal(result.Stdout, &out); err != nil { + return DeleteResult{}, apperr.Wrap("fs.companion_decode", "runtime", 1, "decode tdc fs deletion response", err) + } + status := strings.TrimSpace(out.Status) + if status == "" { + status = "deleting" + } homeDir, err := s.homeDir() if err != nil { return DeleteResult{}, err @@ -241,12 +274,104 @@ func (s Service) drive9DeleteFileSystem(ctx context.Context, opts DeleteFileSyst return DeleteResult{ FileSystemName: name, TenantID: resource.TenantID, - Status: "deleted", + Status: status, CredentialsRemoved: true, - RemoteDeletionState: "deleted", + RemoteDeletionState: status, }, nil } +func (s Service) waitUntilFileSystemReady(ctx context.Context, homeDir string, profile *config.Profile, name string) error { + selected, _, err := fscred.Resolve(homeDir, profile, name, true, nil) + if err != nil { + return err + } + timeout := s.fsReadyWaitTimeout() + waitCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + ticker := time.NewTicker(s.fsReadyWaitPollInterval()) + defer ticker.Stop() + + for { + _, err := s.drive9Run(waitCtx, selected, []string{"fs", "stat", "--output", "json", ":/"}, true) + if err == nil { + return nil + } + if waitErr := fsReadyWaitContextError(ctx, waitCtx, name, timeout); waitErr != nil { + return waitErr + } + if !isDrive9ReadinessError(err) { + return apperr.Wrap( + "fs.ready_wait_failed", + "runtime", + 1, + fmt.Sprintf("tdc fs resource %q was provisioned and its credentials were stored, but its Drive9 data plane readiness check failed", name), + err, + ) + } + + select { + case <-waitCtx.Done(): + return fsReadyWaitContextError(ctx, waitCtx, name, timeout) + case <-ticker.C: + } + } +} + +func fsReadyWaitContextError(parent, waitCtx context.Context, name string, timeout time.Duration) error { + if parent.Err() != nil { + return apperr.Wrap( + "fs.ready_wait_canceled", + "runtime", + 1, + fmt.Sprintf("waiting for tdc fs resource %q to become ready was canceled; the resource and its local credentials were not deleted", name), + parent.Err(), + ) + } + if errors.Is(waitCtx.Err(), context.DeadlineExceeded) { + return apperr.New( + "fs.ready_wait_timeout", + "runtime", + 1, + fmt.Sprintf("tdc fs resource %q did not become ready within %s; the resource and its local credentials were not deleted", name, timeout), + ) + } + return nil +} + +func (s Service) fsReadyWaitTimeout() time.Duration { + if s.FSReadyWaitTimeout > 0 { + return s.FSReadyWaitTimeout + } + return defaultFSReadyWaitTimeout +} + +func (s Service) fsReadyWaitPollInterval() time.Duration { + if s.FSReadyWaitPollInterval > 0 { + return s.FSReadyWaitPollInterval + } + return defaultFSReadyWaitPollInterval +} + +func isDrive9ReadinessError(err error) bool { + if err == nil { + return false + } + message := strings.ToLower(err.Error()) + return strings.Contains(message, "storage backend unavailable") || + strings.Contains(message, "http 503") || + strings.Contains(message, "provision") || + strings.Contains(message, "service unavailable") || + strings.Contains(message, "connection refused") || + strings.Contains(message, "connection reset") || + strings.Contains(message, "no such host") || + strings.Contains(message, "network connectivity") || + strings.Contains(message, ": eof") || + strings.Contains(message, "i/o timeout") || + strings.Contains(message, "timeout awaiting response headers") || + strings.Contains(message, "unexpected eof") +} + func (s Service) drive9CheckFileSystem(ctx context.Context, opts CheckFileSystemOptions) (CheckResult, error) { if opts.Profile == nil { return CheckResult{}, apperr.New("fs.missing_profile", "config", 2, "active profile is required") diff --git a/internal/fs/drive9_companion_test.go b/internal/fs/drive9_companion_test.go index 7c5b475..4e5fdd2 100644 --- a/internal/fs/drive9_companion_test.go +++ b/internal/fs/drive9_companion_test.go @@ -90,6 +90,62 @@ func TestDrive9CreateFileSystemStoresRegistryCredentialsAndUsesCanonicalRegion(t } } +func TestDrive9CreateFileSystemWaitsUntilReady(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TDC_FAKE_DRIVE9_RECORD", recordPath) + t.Setenv("TDC_FAKE_DRIVE9_STAT_FAILURE_SEQUENCE", filepath.Join(t.TempDir(), "stat-attempted")) + + service := testCompanionService(home, companion) + service.FSReadyWaitTimeout = time.Second + service.FSReadyWaitPollInterval = time.Millisecond + result, err := service.CreateFileSystem(context.Background(), CreateFileSystemOptions{ + Profile: testProfile(), + FileSystemName: "workspace", + WaitUntilReady: true, + }) + if err != nil { + t.Fatalf("CreateFileSystem failed: %v", err) + } + if result.Status != "ready" || !result.CredentialsStored { + t.Fatalf("unexpected result: %#v", result) + } + statCalls := 0 + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) >= 2 && call.Args[0] == "fs" && call.Args[1] == "stat" { + statCalls++ + if call.Env["DRIVE9_API_KEY"] != "fs-secret" { + t.Fatalf("readiness stat used wrong credentials: %#v", call.Env) + } + } + } + if statCalls != 2 { + t.Fatalf("readiness stat calls = %d, want 2", statCalls) + } +} + +func TestDrive9CreateFileSystemReadyTimeoutPreservesCredentials(t *testing.T) { + home := t.TempDir() + companion, _ := buildFakeDrive9(t) + t.Setenv("TDC_FAKE_DRIVE9_STAT_ALWAYS_FAIL", "1") + + service := testCompanionService(home, companion) + service.FSReadyWaitTimeout = 10 * time.Millisecond + service.FSReadyWaitPollInterval = time.Millisecond + _, err := service.CreateFileSystem(context.Background(), CreateFileSystemOptions{ + Profile: testProfile(), + FileSystemName: "workspace", + WaitUntilReady: true, + }) + if apperr.CodeFor(err) != "fs.ready_wait_timeout" { + t.Fatalf("unexpected error: %v", err) + } + resource, getErr := fscred.Get(home, "stage", "workspace") + if getErr != nil || resource.APIKey != "fs-secret" { + t.Fatalf("readiness timeout removed stored credentials: resource=%#v err=%v", resource, getErr) + } +} + func TestDrive9CreateFileSystemFromEnvironmentProfileStoresDefaultProfile(t *testing.T) { home := t.TempDir() companion, recordPath := buildFakeDrive9(t) @@ -208,7 +264,7 @@ func TestDrive9DeleteFileSystemDeletesOnlySelectedRegistryResource(t *testing.T) if err != nil { t.Fatalf("DeleteFileSystem failed: %v", err) } - if !result.CredentialsRemoved || result.Status != "deleted" { + if !result.CredentialsRemoved || result.Status != "deleting" || result.RemoteDeletionState != "deleting" { t.Fatalf("unexpected delete result: %#v", result) } deleteCall := requireFakeDrive9Call(t, recordPath, "delete") @@ -501,6 +557,7 @@ func TestDryRunCreateFileSystemUsesRedactedProvisionShape(t *testing.T) { result, err := Service{Resolver: supportedFSManifestResolver("https://fs.test")}.DryRunCreateFileSystem(context.Background(), "tdc fs create-file-system", CreateFileSystemOptions{ Profile: profile, FileSystemName: "workspace", + WaitUntilReady: true, }) if err != nil { t.Fatalf("DryRunCreateFileSystem failed: %v", err) @@ -525,6 +582,9 @@ func TestDryRunCreateFileSystemUsesRedactedProvisionShape(t *testing.T) { if !hasDryRunCheck(result.Checks, "endpoint_selection", "passed") { t.Fatalf("expected endpoint dry-run check: %#v", result.Checks) } + if !hasDryRunCheck(result.Checks, "post_create_wait", "passed") { + t.Fatalf("expected readiness wait dry-run check: %#v", result.Checks) + } } func TestDryRunDeleteFileSystemReportsRegistryFiles(t *testing.T) { @@ -616,7 +676,7 @@ func main() { "server": os.Getenv("DRIVE9_SERVER"), }) case args[0] == "delete": - _ = json.NewEncoder(os.Stdout).Encode(map[string]string{"status": "deleted"}) + _ = json.NewEncoder(os.Stdout).Encode(map[string]string{"status": "deleting"}) case len(args) >= 2 && args[0] == "fs" && args[1] == "cat": fmt.Fprint(os.Stdout, "file bytes") case len(args) >= 2 && args[0] == "fs" && args[1] == "cp" && os.Getenv("TDC_FAKE_DRIVE9_CP_FAILURE_SEQUENCE") != "": @@ -629,6 +689,17 @@ func main() { fmt.Fprintln(os.Stderr, "fs cp: remote resource not found") os.Exit(1) case len(args) >= 2 && args[0] == "fs" && args[1] == "stat": + if os.Getenv("TDC_FAKE_DRIVE9_STAT_ALWAYS_FAIL") == "1" { + fmt.Fprintln(os.Stderr, "fs stat: storage backend unavailable; resource is still provisioning") + os.Exit(1) + } + if sequencePath := os.Getenv("TDC_FAKE_DRIVE9_STAT_FAILURE_SEQUENCE"); sequencePath != "" { + if _, err := os.Stat(sequencePath); os.IsNotExist(err) { + _ = os.WriteFile(sequencePath, []byte("attempted"), 0600) + fmt.Fprintln(os.Stderr, "fs stat: storage backend unavailable; resource is still provisioning") + os.Exit(1) + } + } _ = json.NewEncoder(os.Stdout).Encode(map[string]any{"path": ":/", "size": 12, "isdir": false, "revision": 3}) default: return From fb0f2b4581e762c30f2fac6d579e3aa02b7818ea Mon Sep 17 00:00:00 2001 From: Cheese Date: Sat, 18 Jul 2026 22:58:06 +0800 Subject: [PATCH 3/3] chore: keep docs updates in their repositories --- docs/pingcap-docs/docs | 2 +- docs/pingcap-docs/docs-cn | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/pingcap-docs/docs b/docs/pingcap-docs/docs index d9fcf59..388e0b6 160000 --- a/docs/pingcap-docs/docs +++ b/docs/pingcap-docs/docs @@ -1 +1 @@ -Subproject commit d9fcf59d0b0ee3efc81b1ad093b75be42bdffac9 +Subproject commit 388e0b6e0a0e57a4fc231d3ded5c70dbc95a0037 diff --git a/docs/pingcap-docs/docs-cn b/docs/pingcap-docs/docs-cn index 090b8bb..cd88bf5 160000 --- a/docs/pingcap-docs/docs-cn +++ b/docs/pingcap-docs/docs-cn @@ -1 +1 @@ -Subproject commit 090b8bba9ca134cf94380df7a5a4a3bbe51e7b50 +Subproject commit cd88bf5a7b24b4d6752346b37a40dba12747d5f7