diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4af1e238..dfdb6ef8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,39 +13,52 @@ on: permissions: contents: read - pull-requests: read jobs: # --------------------------------------------------------------------------- - # BUILD AND UNIT TESTS (special case - Gazelle + build + unit tests) + # LINT (gofmt via Go SDK, yamlfmt via go run) # --------------------------------------------------------------------------- - build-and-unit-test: - name: Build and Unit Test + lint: + name: Lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup + - name: Run linters + run: make lint + + # --------------------------------------------------------------------------- + # TIDY (module files + BUILD files in sync) + # --------------------------------------------------------------------------- + tidy: + name: Tidy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup + + - name: Check module files are tidy + run: make check-tidy + - name: Check BUILD files are up to date - run: | - echo "Running Gazelle to check BUILD files..." >&2 - make gazelle - if ! git diff --quiet; then - echo "BUILD files are out of date!" >&2 - echo "" >&2 - echo "The following files were modified by Gazelle:" >&2 - git diff --name-only >&2 - echo "" >&2 - echo "Please run 'make gazelle' locally and commit the changes." >&2 - exit 1 - fi - echo "BUILD files are up to date" >&2 + run: make check-gazelle + + # --------------------------------------------------------------------------- + # BUILD AND UNIT TESTS + # --------------------------------------------------------------------------- + build-and-unit-test: + name: Build and Unit Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup - name: Build project run: make build - name: Run unit tests - run: make test || echo "No unit tests found" + run: make test # --------------------------------------------------------------------------- # INTEGRATION TESTS (e2e, gateway, orchestrator) @@ -133,6 +146,8 @@ jobs: name: Required Checks runs-on: ubuntu-latest needs: + - lint + - tidy - build-and-unit-test - e2e - gateway-integration-test diff --git a/.yamlfmt b/.yamlfmt new file mode 100644 index 00000000..ce51ba5c --- /dev/null +++ b/.yamlfmt @@ -0,0 +1,5 @@ +formatter: + type: basic + indent: 2 + retain_line_breaks_single: true + include_document_start: false diff --git a/CLAUDE.md b/CLAUDE.md index 07064c15..87c13559 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,9 +48,10 @@ submitqueue/ │ ├── counter/ # Sequential number generation (interface + mysql/) │ ├── queue/ # Messaging queue abstraction (interface + sql/) │ └── storage/ # Storage abstraction (interface + mysql/) -├── core/ # Shared infrastructure packages reused across services +├── core/ # Shared infrastructure packages reused across services │ ├── consumer/ # Queue consumption framework (lifecycle, ack/nack, routing) │ └── errs/ # Error classification framework (user vs infra, retryability) +├── tool/ # Development and CI tooling ├── example/server/ # Runnable servers with Docker Compose ├── test/ │ ├── e2e/ # End-to-end tests (full stack) @@ -157,10 +158,15 @@ integration-test: build-all-linux ## Run all integration tests (auto-builds bina ```bash make build # Build all services make test # Run unit tests +make lint # Run all linters (fmt + YAML) +make fmt # Format Go and YAML code +make check-tidy # Check go.mod and MODULE.bazel are tidy +make check-gazelle # Check BUILD.bazel files are up to date +make tidy # Run go mod tidy + bazel mod tidy +make gazelle # Update BUILD.bazel files make integration-test # Run all integration tests (Docker-based) make e2e-test # Run end-to-end tests make proto # Regenerate proto files -make gazelle # Update BUILD.bazel files make local-start # Start full stack with Docker Compose make local-ps # Show running containers and ports make local-logs # View logs from all services @@ -256,6 +262,15 @@ deps = [ See [doc/howto/TESTING.md](doc/howto/TESTING.md) for full testing guide. +### CI and Validation + +CI runs on every PR and enforces all checks via a `required-checks` gate. **Before committing, validate locally:** + +1. `make fmt` — format Go and YAML code (CI will reject unformatted code) +2. `make lint` — run all linters (formatting check) +3. `make check-tidy` — ensure `go.mod` and `MODULE.bazel` are tidy +4. `make check-gazelle` — ensure `BUILD.bazel` files are up to date + ### Code Style 1. **Structured logging** — `zap.SugaredLogger` with `Debugw`/`Infow`/`Errorw(msg, key, val, ...)`. Never unstructured methods. diff --git a/Makefile b/Makefile index 77ad1f94..d339b664 100644 --- a/Makefile +++ b/Makefile @@ -10,10 +10,27 @@ ORCHESTRATOR_COMPOSE_FILE = example/server/orchestrator/docker-compose.yml # Fixed project name for local manual testing (tests use unique random names) LOCAL_PROJECT = submitqueue +# yamlfmt version for YAML formatting (override with: make fmt YAMLFMT_VERSION=v0.16.0) +YAMLFMT_VERSION ?= v0.16.0 + +# goimports version for Go formatting + import fixing +GOIMPORTS_VERSION ?= v0.33.0 + # Set REPO_ROOT for docker-compose export REPO_ROOT := $(shell pwd) -.PHONY: build build-all-linux build-gateway-linux build-orchestrator-linux clean clean-proto deps e2e-test gazelle integration-test integration-test-consumer integration-test-extensions integration-test-gateway integration-test-orchestrator license-fix lint lint-license local-clean local-gateway-start local-gateway-stop local-init-schemas local-logs local-orchestrator-start local-orchestrator-stop local-ps local-restart local-start local-stop proto query-deps query-targets run-client-gateway run-client-orchestrator run-queue-admin test test-no-cache help +# Fails if git working tree is dirty. Usage: $(call assert_clean,fix command) +define assert_clean + @if ! git diff --quiet; then \ + echo "The following files need updating:" >&2; \ + git diff --name-only >&2; \ + echo "" >&2; \ + echo "Please run '$(1)' locally and commit the changes." >&2; \ + exit 1; \ + fi +endef + +.PHONY: build build-all-linux build-gateway-linux build-orchestrator-linux check-gazelle check-tidy clean clean-proto deps e2e-test fmt gazelle integration-test integration-test-consumer integration-test-extensions integration-test-gateway integration-test-orchestrator license-fix lint lint-fmt lint-license local-clean local-gateway-start local-gateway-stop local-init-schemas local-logs local-orchestrator-start local-orchestrator-stop local-ps local-restart local-start local-stop proto query-deps query-targets run-client-gateway run-client-orchestrator run-queue-admin test test-no-cache tidy tidy-bazel tidy-go help build: ## Build all services and examples @@ -41,6 +58,16 @@ build-orchestrator-linux: ## Build Orchestrator Linux binary for Docker cp -f bazel-bin/example/server/orchestrator/orchestrator .docker-bin/orchestrator @echo "Orchestrator Linux binary ready at .docker-bin/orchestrator" +check-gazelle: ## Check BUILD.bazel files are up to date + @echo "Running Gazelle to check BUILD files..." + @$(BAZEL) run //:gazelle + $(call assert_clean,make gazelle) + @echo "BUILD files are up to date." + +check-tidy: tidy ## Check that go.mod and MODULE.bazel are tidy + $(call assert_clean,make tidy) + @echo "Module files are up to date." + clean: ## Clean generated files and binaries @echo "Cleaning with Bazel..." @$(BAZEL) clean @@ -53,16 +80,20 @@ clean-proto: ## Clean generated proto files @rm -rf orchestrator/protopb/*.pb.go @echo "Proto clean complete!" -deps: ## Install Go dependencies - @echo "Installing Go dependencies..." - @go mod download - @go mod tidy +deps: tidy-go ## Download and tidy Go dependencies @echo "Dependencies installed!" e2e-test: build-all-linux ## Run end-to-end tests (hermetic, auto-builds binaries) @echo "Running end-to-end tests..." @$(BAZEL) test //test/e2e:e2e_test --test_output=streamed +fmt: ## Format Go and YAML code + @echo "Formatting Go code..." + @$(BAZEL) run @rules_go//go -- run golang.org/x/tools/cmd/goimports@$(GOIMPORTS_VERSION) -w . + @echo "Formatting YAML files..." + @$(BAZEL) run @rules_go//go -- run github.com/google/yamlfmt/cmd/yamlfmt@$(YAMLFMT_VERSION) + @echo "Formatting complete!" + gazelle: ## Update BUILD.bazel files @echo "Running Gazelle to update BUILD files..." @$(BAZEL) run //:gazelle @@ -90,7 +121,12 @@ integration-test-orchestrator: build-orchestrator-linux ## Run Orchestrator inte license-fix: ## Add missing license headers to source files @$(BAZEL) run //tool/linter/licenseheader -- --fix -lint: lint-license ## Run all linters +lint: lint-fmt lint-license ## Run all linters + @echo "All lint checks passed." + +lint-fmt: fmt ## Check code formatting (fails if unformatted) + $(call assert_clean,make fmt) + @echo "All code is properly formatted." lint-license: ## Check license headers on all source files @$(BAZEL) run //tool/linter/licenseheader -- --check @@ -241,6 +277,16 @@ test-no-cache: ## Run unit tests without cache (force re-run) @echo "Running unit tests (no cache)..." @$(BAZEL) test //... --test_tag_filters=-manual,-integration --nocache_test_results +tidy: tidy-go tidy-bazel ## Run go mod tidy and bazel mod tidy + +tidy-bazel: ## Run bazel mod tidy + @echo "Running bazel mod tidy..." + @$(BAZEL) mod tidy + +tidy-go: ## Run go mod tidy + @echo "Running go mod tidy..." + @$(BAZEL) run @rules_go//go -- mod tidy -e + help: ## Show this help message @echo "Available targets:" @echo "" diff --git a/core/consumer/registry.go b/core/consumer/registry.go index c99a71d3..83f0d943 100644 --- a/core/consumer/registry.go +++ b/core/consumer/registry.go @@ -71,8 +71,8 @@ type TopicConfig struct { // TopicRegistry provides queue, topic name, and subscription config for topics. // Each topic can have a different queue backend and topic name. type TopicRegistry struct { - queues map[TopicKey]queue.Queue - topicNames map[TopicKey]string + queues map[TopicKey]queue.Queue + topicNames map[TopicKey]string subscriptionConfigs map[topicGroup]queue.SubscriptionConfig } diff --git a/core/consumer/registry_test.go b/core/consumer/registry_test.go index a84eb913..6fbe64a9 100644 --- a/core/consumer/registry_test.go +++ b/core/consumer/registry_test.go @@ -32,8 +32,8 @@ func TestNewTopicRegistry(t *testing.T) { registry, err := consumer.NewTopicRegistry( []consumer.TopicConfig{ { - Key: consumer.TopicKeyRequest, - Name: "request", + Key: consumer.TopicKeyRequest, + Name: "request", Queue: mockQ, Subscription: extqueue.DefaultSubscriptionConfig( "worker-1", "group-a", @@ -58,7 +58,7 @@ func TestNewTopicRegistry(t *testing.T) { func TestNewTopicRegistry_InvalidTopicName(t *testing.T) { tests := []struct { - name string + name string topicName string }{ { diff --git a/core/errs/errs.go b/core/errs/errs.go index 8da5cef2..c18e0319 100644 --- a/core/errs/errs.go +++ b/core/errs/errs.go @@ -15,8 +15,8 @@ package errs import ( - "errors" "context" + "errors" ) // userError represents an error caused by invalid user input or actions. diff --git a/entity/build.go b/entity/build.go index 3354bbb4..f5956d16 100644 --- a/entity/build.go +++ b/entity/build.go @@ -53,7 +53,6 @@ func (s BuildStatus) IsTerminal() bool { return s == BuildStatusPassed || s == BuildStatusFailed || s == BuildStatusCancelled } - // SpeculationPathInfo represents the base and head commits of a speculation path used in a build. type SpeculationPathInfo struct { // Base is a list of batchIDs(in order) that form the base of this speculation path. diff --git a/entity/queue/message_test.go b/entity/queue/message_test.go index a66cffd2..c0e815c5 100644 --- a/entity/queue/message_test.go +++ b/entity/queue/message_test.go @@ -76,4 +76,3 @@ func TestMessage_Fields(t *testing.T) { assert.Equal(t, msg.PublishedAt, copied.PublishedAt) assert.Equal(t, msg.Metadata, copied.Metadata) } - diff --git a/entity/request.go b/entity/request.go index e4e8d39f..70060f92 100644 --- a/entity/request.go +++ b/entity/request.go @@ -16,7 +16,6 @@ package entity import "encoding/json" - // RequestLandStrategy defines the possible source control integration methods. type RequestLandStrategy string diff --git a/example/server/docker-compose.yml b/example/server/docker-compose.yml index 77ed1896..16877d95 100644 --- a/example/server/docker-compose.yml +++ b/example/server/docker-compose.yml @@ -16,7 +16,7 @@ services: MYSQL_ROOT_PASSWORD: root MYSQL_DATABASE: submitqueue ports: - - "3306" # Random ephemeral port to avoid conflicts + - "3306" # Random ephemeral port to avoid conflicts healthcheck: # Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats # "localhost" as a socket connection, which can be ready before the TCP @@ -34,7 +34,7 @@ services: MYSQL_ROOT_PASSWORD: root MYSQL_DATABASE: submitqueue ports: - - "3306" # Random ephemeral port to avoid conflicts + - "3306" # Random ephemeral port to avoid conflicts healthcheck: test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"] interval: 5s @@ -46,7 +46,7 @@ services: context: ${REPO_ROOT} dockerfile: example/server/gateway/Dockerfile ports: - - "8080" # Random ephemeral port to avoid conflicts + - "8080" # Random ephemeral port to avoid conflicts environment: - PORT=:8080 # Application database connection @@ -64,7 +64,7 @@ services: context: ${REPO_ROOT} dockerfile: example/server/orchestrator/Dockerfile ports: - - "8080" # Random ephemeral port to avoid conflicts + - "8080" # Random ephemeral port to avoid conflicts environment: - PORT=:8080 # Application database connection (for request state, batches, etc.) diff --git a/example/server/gateway/docker-compose.yml b/example/server/gateway/docker-compose.yml index 7c9e84ec..b5fc783d 100644 --- a/example/server/gateway/docker-compose.yml +++ b/example/server/gateway/docker-compose.yml @@ -16,7 +16,7 @@ services: MYSQL_ROOT_PASSWORD: root MYSQL_DATABASE: submitqueue ports: - - "3306" # Random ephemeral port to avoid conflicts + - "3306" # Random ephemeral port to avoid conflicts healthcheck: # Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats # "localhost" as a socket connection, which can be ready before the TCP @@ -34,7 +34,7 @@ services: MYSQL_ROOT_PASSWORD: root MYSQL_DATABASE: submitqueue ports: - - "3306" # Random ephemeral port to avoid conflicts + - "3306" # Random ephemeral port to avoid conflicts healthcheck: test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"] interval: 5s @@ -46,7 +46,7 @@ services: context: ${REPO_ROOT} dockerfile: example/server/gateway/Dockerfile ports: - - "8080" # Random ephemeral port to avoid conflicts + - "8080" # Random ephemeral port to avoid conflicts environment: - PORT=:8080 # Application database connection diff --git a/example/server/orchestrator/docker-compose.yml b/example/server/orchestrator/docker-compose.yml index 0f5b3505..8707abfe 100644 --- a/example/server/orchestrator/docker-compose.yml +++ b/example/server/orchestrator/docker-compose.yml @@ -16,7 +16,7 @@ services: MYSQL_ROOT_PASSWORD: root MYSQL_DATABASE: submitqueue ports: - - "3306" # Random ephemeral port to avoid conflicts + - "3306" # Random ephemeral port to avoid conflicts healthcheck: # Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats # "localhost" as a socket connection, which can be ready before the TCP @@ -34,7 +34,7 @@ services: MYSQL_ROOT_PASSWORD: root MYSQL_DATABASE: submitqueue ports: - - "3306" # Random ephemeral port to avoid conflicts + - "3306" # Random ephemeral port to avoid conflicts healthcheck: test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"] interval: 5s @@ -46,7 +46,7 @@ services: context: ${REPO_ROOT} dockerfile: example/server/orchestrator/Dockerfile ports: - - "8080" # Random ephemeral port to avoid conflicts + - "8080" # Random ephemeral port to avoid conflicts environment: - PORT=:8080 # Application database connection (for request state, batches, etc.) diff --git a/example/server/orchestrator/main.go b/example/server/orchestrator/main.go index a8644fe2..dfa964d4 100644 --- a/example/server/orchestrator/main.go +++ b/example/server/orchestrator/main.go @@ -36,14 +36,14 @@ import ( githubchecker "github.com/uber/submitqueue/extension/mergechecker/github" extqueue "github.com/uber/submitqueue/extension/queue" queueMySQL "github.com/uber/submitqueue/extension/queue/mysql" - mysqlstorage "github.com/uber/submitqueue/extension/storage/mysql" "github.com/uber/submitqueue/extension/storage" + mysqlstorage "github.com/uber/submitqueue/extension/storage/mysql" "github.com/uber/submitqueue/orchestrator/controller" "github.com/uber/submitqueue/orchestrator/controller/batch" "github.com/uber/submitqueue/orchestrator/controller/build" + "github.com/uber/submitqueue/orchestrator/controller/buildsignal" "github.com/uber/submitqueue/orchestrator/controller/conclude" "github.com/uber/submitqueue/orchestrator/controller/merge" - "github.com/uber/submitqueue/orchestrator/controller/buildsignal" "github.com/uber/submitqueue/orchestrator/controller/request" "github.com/uber/submitqueue/orchestrator/controller/score" "github.com/uber/submitqueue/orchestrator/controller/speculate" @@ -264,7 +264,7 @@ func run() error { } // Stop consumers with 30s timeout, by this time the context should be cancelled and the processing threads may already be exiting; recollect them - errStop := c.Stop(30000); + errStop := c.Stop(30000) if errStop != nil { errStop = fmt.Errorf("failed to stop consumers: %w", errStop) } diff --git a/extension/queue/mysql/message_store.go b/extension/queue/mysql/message_store.go index bf66bbcc..8e5374fb 100644 --- a/extension/queue/mysql/message_store.go +++ b/extension/queue/mysql/message_store.go @@ -27,7 +27,6 @@ import ( "github.com/uber/submitqueue/entity/queue" ) - // sqlmessageStore is the SQL implementation of messageStore type sqlmessageStore struct { db *sql.DB @@ -37,8 +36,8 @@ type sqlmessageStore struct { // Metric names for message store const ( - metricInsertErrors = "insert.errors" - metricFetchErrors = "fetch.errors" + metricInsertErrors = "insert.errors" + metricFetchErrors = "fetch.errors" metricMoveToDLQErrors = "move_to_dlq.errors" ) diff --git a/extension/queue/mysql/offset_store.go b/extension/queue/mysql/offset_store.go index 531e1872..07adc788 100644 --- a/extension/queue/mysql/offset_store.go +++ b/extension/queue/mysql/offset_store.go @@ -24,7 +24,6 @@ import ( "go.uber.org/zap" ) - // sqloffsetStore is the SQL implementation of offsetStore type sqloffsetStore struct { db *sql.DB diff --git a/extension/queue/mysql/partition_lease_store.go b/extension/queue/mysql/partition_lease_store.go index 65c83d79..3144586f 100644 --- a/extension/queue/mysql/partition_lease_store.go +++ b/extension/queue/mysql/partition_lease_store.go @@ -24,7 +24,6 @@ import ( "go.uber.org/zap" ) - // sqlpartitionLeaseStore is the SQL implementation of partitionLeaseStore type sqlpartitionLeaseStore struct { db *sql.DB @@ -34,10 +33,10 @@ type sqlpartitionLeaseStore struct { // Metric names for partition lease store const ( - metricTryAcquireLeaseErrors = "try_acquire_lease.errors" - metricRenewLeaseErrors = "renew_lease.errors" - metricGetLeasedPartitionsErrors = "get_leased_partitions.errors" - metricDiscoverAndAcquireErrors = "discover_and_acquire.errors" + metricTryAcquireLeaseErrors = "try_acquire_lease.errors" + metricRenewLeaseErrors = "renew_lease.errors" + metricGetLeasedPartitionsErrors = "get_leased_partitions.errors" + metricDiscoverAndAcquireErrors = "discover_and_acquire.errors" ) // newPartitionLeaseStore creates a new SQL partition lease store diff --git a/extension/queue/mysql/subscriber_test.go b/extension/queue/mysql/subscriber_test.go index bf146d57..cd909424 100644 --- a/extension/queue/mysql/subscriber_test.go +++ b/extension/queue/mysql/subscriber_test.go @@ -94,14 +94,14 @@ func TestSubscriber_Subscribe(t *testing.T) { func TestSQLDelivery_Reject(t *testing.T) { tests := []struct { - name string - dlqEnabled bool - alreadyAcked bool - moveToDLQErr error - ackMessageErr error - expectErr bool - expectMoveDLQ bool - expectAck bool + name string + dlqEnabled bool + alreadyAcked bool + moveToDLQErr error + ackMessageErr error + expectErr bool + expectMoveDLQ bool + expectAck bool }{ { name: "DLQ enabled moves message to DLQ", diff --git a/extension/queue/subscription_config.go b/extension/queue/subscription_config.go index 6eecc39a..0adf1c5c 100644 --- a/extension/queue/subscription_config.go +++ b/extension/queue/subscription_config.go @@ -83,11 +83,11 @@ func DefaultSubscriptionConfig(subscriberName, consumerGroup string) Subscriptio return SubscriptionConfig{ SubscriberName: subscriberName, ConsumerGroup: consumerGroup, - PollIntervalMs: 100, // 100ms + PollIntervalMs: 100, // 100ms BatchSize: 10, - VisibilityTimeoutMs: 60000, // 60s - LeaseRenewalIntervalMs: 10000, // 10s - LeaseDurationMs: 30000, // 30s + VisibilityTimeoutMs: 60000, // 60s + LeaseRenewalIntervalMs: 10000, // 10s + LeaseDurationMs: 30000, // 30s Retry: RetryConfig{ MaxAttempts: 3, InitialBackoffMs: 1000, // 1s diff --git a/extension/scorer/heuristic/scorer_test.go b/extension/scorer/heuristic/scorer_test.go index ba9a87db..d7fbccbf 100644 --- a/extension/scorer/heuristic/scorer_test.go +++ b/extension/scorer/heuristic/scorer_test.go @@ -33,11 +33,11 @@ func staticValue(value int) ValueFunc { func TestScorer_Score(t *testing.T) { tests := []struct { - name string - buckets []Bucket + name string + buckets []Bucket valueFunc ValueFunc - want float64 - wantErr bool + want float64 + wantErr bool }{ { name: "single bucket covering all values", @@ -45,7 +45,7 @@ func TestScorer_Score(t *testing.T) { {Min: 0, Max: 1000, Score: 0.9}, }, valueFunc: staticValue(5), - want: 0.9, + want: 0.9, }, { name: "multiple buckets with different ranges", @@ -55,7 +55,7 @@ func TestScorer_Score(t *testing.T) { {Min: 21, Max: 100, Score: 0.5}, }, valueFunc: staticValue(10), - want: 0.75, + want: 0.75, }, { name: "exact min boundary", @@ -64,7 +64,7 @@ func TestScorer_Score(t *testing.T) { {Min: 6, Max: 20, Score: 0.75}, }, valueFunc: staticValue(6), - want: 0.75, + want: 0.75, }, { name: "exact max boundary", @@ -91,7 +91,7 @@ func TestScorer_Score(t *testing.T) { {Min: 1, Max: 100, Score: 0.8}, }, valueFunc: staticValue(0), - want: 1.0, + want: 1.0, }, { name: "first matching bucket wins", diff --git a/extension/storage/mysql/change_provider_store.go b/extension/storage/mysql/change_provider_store.go index e0e62dc0..93329e28 100644 --- a/extension/storage/mysql/change_provider_store.go +++ b/extension/storage/mysql/change_provider_store.go @@ -45,7 +45,6 @@ func NewChangeProviderStore(db *sql.DB, scope tally.Scope) storage.ChangeProvide // to be the same as the request to which it belongs. The caller is repsonsible // for inspecting and mapping the result of this function to the // order of changes within the original request. -// func (s *changeProviderStore) Get(ctx context.Context, requestID string) (ret []entity.ChangeProvider, retErr error) { op := metrics.Begin(s.scope, "get") defer func() { op.Complete(retErr) }() diff --git a/extension/storage/mysql/storage.go b/extension/storage/mysql/storage.go index 2c6df29e..3c0f1121 100644 --- a/extension/storage/mysql/storage.go +++ b/extension/storage/mysql/storage.go @@ -24,27 +24,27 @@ import ( ) type mysqlStorage struct { - db *sql.DB - requestStore storage.RequestStore - changeProviderStore storage.ChangeProviderStore - batchStore storage.BatchStore - batchDependentStore storage.BatchDependentStore - buildStore storage.BuildStore - speculationTreeStore storage.SpeculationTreeStore - requestLogStore storage.RequestLogStore + db *sql.DB + requestStore storage.RequestStore + changeProviderStore storage.ChangeProviderStore + batchStore storage.BatchStore + batchDependentStore storage.BatchDependentStore + buildStore storage.BuildStore + speculationTreeStore storage.SpeculationTreeStore + requestLogStore storage.RequestLogStore } // NewStorage creates a new MySQL storage. func NewStorage(db *sql.DB, scope tally.Scope) (storage.Storage, error) { return &mysqlStorage{ - db: db, - requestStore: NewRequestStore(db, scope.SubScope("request_store")), - changeProviderStore: NewChangeProviderStore(db, scope.SubScope("change_provider_store")), - batchStore: NewBatchStore(db, scope.SubScope("batch_store")), - batchDependentStore: NewBatchDependentStore(db, scope.SubScope("batch_dependent_store")), - buildStore: NewBuildStore(db, scope.SubScope("build_store")), - speculationTreeStore: NewSpeculationTreeStore(db, scope.SubScope("speculation_tree_store")), - requestLogStore: NewRequestLogStore(db, scope.SubScope("request_log_store")), + db: db, + requestStore: NewRequestStore(db, scope.SubScope("request_store")), + changeProviderStore: NewChangeProviderStore(db, scope.SubScope("change_provider_store")), + batchStore: NewBatchStore(db, scope.SubScope("batch_store")), + batchDependentStore: NewBatchDependentStore(db, scope.SubScope("batch_dependent_store")), + buildStore: NewBuildStore(db, scope.SubScope("build_store")), + speculationTreeStore: NewSpeculationTreeStore(db, scope.SubScope("speculation_tree_store")), + requestLogStore: NewRequestLogStore(db, scope.SubScope("request_log_store")), }, nil } diff --git a/extension/storage/storage.go b/extension/storage/storage.go index bc7be743..75bcb84a 100644 --- a/extension/storage/storage.go +++ b/extension/storage/storage.go @@ -16,8 +16,10 @@ package storage //go:generate mockgen -source=storage.go -destination=mock/storage.go -package=mock -import "errors" -import "fmt" +import ( + "errors" + "fmt" +) // ErrNotFound is returned by storage implementations when the requested record is not found in the database. var ErrNotFound = errors.New("record not found") diff --git a/gateway/protopb/gateway.pb.go b/gateway/protopb/gateway.pb.go index 0b35ad43..ee85ac9e 100644 --- a/gateway/protopb/gateway.pb.go +++ b/gateway/protopb/gateway.pb.go @@ -7,11 +7,12 @@ package protopb import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" unsafe "unsafe" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" ) const ( diff --git a/gateway/protopb/gateway_grpc.pb.go b/gateway/protopb/gateway_grpc.pb.go index 46128f3f..6ab82364 100644 --- a/gateway/protopb/gateway_grpc.pb.go +++ b/gateway/protopb/gateway_grpc.pb.go @@ -8,6 +8,7 @@ package protopb import ( context "context" + grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" diff --git a/orchestrator/protopb/orchestrator.pb.go b/orchestrator/protopb/orchestrator.pb.go index 70914dfa..de3d688f 100644 --- a/orchestrator/protopb/orchestrator.pb.go +++ b/orchestrator/protopb/orchestrator.pb.go @@ -7,11 +7,12 @@ package protopb import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" unsafe "unsafe" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" ) const ( diff --git a/orchestrator/protopb/orchestrator_grpc.pb.go b/orchestrator/protopb/orchestrator_grpc.pb.go index 22d8ea92..12c6af09 100644 --- a/orchestrator/protopb/orchestrator_grpc.pb.go +++ b/orchestrator/protopb/orchestrator_grpc.pb.go @@ -8,6 +8,7 @@ package protopb import ( context "context" + grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 0306bde3..17a74ed4 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -44,8 +44,8 @@ type E2EIntegrationSuite struct { stack *testutil.ComposeStack gatewayClient gatewaypb.SubmitQueueGatewayClient orchestratorClient orchestratorpb.SubmitQueueOrchestratorClient - db *sql.DB // App database - queueDB *sql.DB // Queue database + db *sql.DB // App database + queueDB *sql.DB // Queue database } func TestE2EIntegration(t *testing.T) { diff --git a/test/integration/core/consumer/consumer_test.go b/test/integration/core/consumer/consumer_test.go index 828b1150..53a3f117 100644 --- a/test/integration/core/consumer/consumer_test.go +++ b/test/integration/core/consumer/consumer_test.go @@ -162,8 +162,8 @@ func (s *ConsumerIntegrationSuite) TestConsumerPerPartitionIsolation() { publisher := q.Publisher() // Channels for synchronizing the test with the controller - partAStarted := make(chan struct{}) // signals partition-a processing began - partAUnblock := make(chan struct{}) // unblocks partition-a processing + partAStarted := make(chan struct{}) // signals partition-a processing began + partAUnblock := make(chan struct{}) // unblocks partition-a processing partBProcessed := make(chan struct{}) // signals partition-b was processed ctrl := &testController{ diff --git a/test/integration/extension/counter/mysql/docker-compose.yml b/test/integration/extension/counter/mysql/docker-compose.yml index 2f129aeb..9b38b659 100644 --- a/test/integration/extension/counter/mysql/docker-compose.yml +++ b/test/integration/extension/counter/mysql/docker-compose.yml @@ -9,7 +9,7 @@ services: MYSQL_ROOT_PASSWORD: root MYSQL_DATABASE: submitqueue ports: - - "3306" # Random ephemeral port to avoid conflicts + - "3306" # Random ephemeral port to avoid conflicts healthcheck: # Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats # "localhost" as a socket connection, which can be ready before the TCP diff --git a/test/integration/extension/queue/mysql/docker-compose.yml b/test/integration/extension/queue/mysql/docker-compose.yml index 1b17364b..4ae98dc7 100644 --- a/test/integration/extension/queue/mysql/docker-compose.yml +++ b/test/integration/extension/queue/mysql/docker-compose.yml @@ -9,7 +9,7 @@ services: MYSQL_ROOT_PASSWORD: root MYSQL_DATABASE: submitqueue ports: - - "3306" # Random ephemeral port to avoid conflicts + - "3306" # Random ephemeral port to avoid conflicts healthcheck: # Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats # "localhost" as a socket connection, which can be ready before the TCP diff --git a/test/integration/extension/queue/mysql/queue_test.go b/test/integration/extension/queue/mysql/queue_test.go index b54fba2d..ae15ea8a 100644 --- a/test/integration/extension/queue/mysql/queue_test.go +++ b/test/integration/extension/queue/mysql/queue_test.go @@ -376,8 +376,8 @@ func (s *SQLQueueIntegrationSuite) TestVisibilityTimeoutAndRetry() { // Use short visibility timeout for faster test subConfig := extqueue.DefaultSubscriptionConfig("worker-1", "retry-consumer") - subConfig.VisibilityTimeoutMs = 2000 // 2 seconds - subConfig.PollIntervalMs = 100 // 100 milliseconds + subConfig.VisibilityTimeoutMs = 2000 // 2 seconds + subConfig.PollIntervalMs = 100 // 100 milliseconds // Subscribe deliveryChan, err := subscriber.Subscribe(s.ctx, topic, subConfig) @@ -617,10 +617,10 @@ func (s *SQLQueueIntegrationSuite) TestCrashRecovery() { // Use short timeouts for faster test subConfig := extqueue.DefaultSubscriptionConfig("worker-1", "crash-consumer") - subConfig.VisibilityTimeoutMs = 2000 // 2 seconds - subConfig.PollIntervalMs = 100 // 100 milliseconds - subConfig.LeaseDurationMs = 3000 // 3 seconds - short lease for testing crash recovery - subConfig.LeaseRenewalIntervalMs = 1000 // 1 second - must be less than LeaseDuration + subConfig.VisibilityTimeoutMs = 2000 // 2 seconds + subConfig.PollIntervalMs = 100 // 100 milliseconds + subConfig.LeaseDurationMs = 3000 // 3 seconds - short lease for testing crash recovery + subConfig.LeaseRenewalIntervalMs = 1000 // 1 second - must be less than LeaseDuration // Subscribe with first worker deliveryChan1, err := subscriber1.Subscribe(s.ctx, topic, subConfig) @@ -656,10 +656,10 @@ func (s *SQLQueueIntegrationSuite) TestCrashRecovery() { subscriber2 := q2.Subscriber() subConfig2 := extqueue.DefaultSubscriptionConfig("worker-2", "crash-consumer") - subConfig2.VisibilityTimeoutMs = 2000 // 2 seconds - subConfig2.PollIntervalMs = 100 // 100 milliseconds - subConfig2.LeaseDurationMs = 3000 // 3 seconds - subConfig2.LeaseRenewalIntervalMs = 1000 // 1 second + subConfig2.VisibilityTimeoutMs = 2000 // 2 seconds + subConfig2.PollIntervalMs = 100 // 100 milliseconds + subConfig2.LeaseDurationMs = 3000 // 3 seconds + subConfig2.LeaseRenewalIntervalMs = 1000 // 1 second deliveryChan2, err := subscriber2.Subscribe(s.ctx, topic, subConfig2) require.NoError(t, err) diff --git a/test/integration/extension/storage/mysql/docker-compose.yml b/test/integration/extension/storage/mysql/docker-compose.yml index 9a6c3baa..9ccb3fcc 100644 --- a/test/integration/extension/storage/mysql/docker-compose.yml +++ b/test/integration/extension/storage/mysql/docker-compose.yml @@ -9,7 +9,7 @@ services: MYSQL_ROOT_PASSWORD: root MYSQL_DATABASE: submitqueue ports: - - "3306" # Random ephemeral port to avoid conflicts + - "3306" # Random ephemeral port to avoid conflicts healthcheck: # Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats # "localhost" as a socket connection, which can be ready before the TCP diff --git a/test/integration/gateway/suite_test.go b/test/integration/gateway/suite_test.go index 8fa706e4..e50c48e6 100644 --- a/test/integration/gateway/suite_test.go +++ b/test/integration/gateway/suite_test.go @@ -45,8 +45,8 @@ type GatewayIntegrationSuite struct { log *testutil.TestLogger stack *testutil.ComposeStack client pb.SubmitQueueGatewayClient - db *sql.DB // App database - queueDB *sql.DB // Queue database + db *sql.DB // App database + queueDB *sql.DB // Queue database } func TestGatewayIntegration(t *testing.T) { diff --git a/test/integration/orchestrator/suite_test.go b/test/integration/orchestrator/suite_test.go index 94e0e50b..8266036c 100644 --- a/test/integration/orchestrator/suite_test.go +++ b/test/integration/orchestrator/suite_test.go @@ -45,8 +45,8 @@ type OrchestratorIntegrationSuite struct { log *testutil.TestLogger stack *testutil.ComposeStack client pb.SubmitQueueOrchestratorClient - db *sql.DB // App database - queueDB *sql.DB // Queue database + db *sql.DB // App database + queueDB *sql.DB // Queue database } func TestOrchestratorIntegration(t *testing.T) { @@ -118,4 +118,3 @@ func (s *OrchestratorIntegrationSuite) TestPingAPI() { s.log.Logf("Orchestrator Ping test passed: %s", resp.Message) } - diff --git a/test/testutil/compose.go b/test/testutil/compose.go index 257fc833..6ca22ad6 100644 --- a/test/testutil/compose.go +++ b/test/testutil/compose.go @@ -38,8 +38,8 @@ type ComposeStack struct { t *testing.T log *TestLogger ctx context.Context - composeCmd []string // docker-compose command (either ["docker-compose"] or ["docker", "compose"]) - logCmd *exec.Cmd // background "docker compose logs -f" process + composeCmd []string // docker-compose command (either ["docker-compose"] or ["docker", "compose"]) + logCmd *exec.Cmd // background "docker compose logs -f" process } // getDockerComposeCommand returns the docker-compose command to use.