feat(ci): add lint, tidy, and formatting CI jobs

Split the monolithic build-and-unit-test job into three parallel jobs:
- lint: gofmt, yamllint, jsonlint, shellcheck
- tidy: go mod tidy, bazel mod tidy, gazelle
- build-and-unit-test: bazel build + unit tests

Add corresponding Makefile targets (check-fmt, check-gazelle, check-json,
check-shell, check-tidy, check-yaml, fmt, lint) so CI and local dev use
the same validation logic.

Author: behinddwalls

.github/workflows/ci.yml

@@ -17,29 +17,57 @@ permissions:
 
 jobs:
   # ---------------------------------------------------------------------------
-  # BUILD AND UNIT TESTS (special case - Gazelle + build + unit tests)
+  # LINT (gofmt from Bazel Go SDK + yamllint + shellcheck + jq)
   # ---------------------------------------------------------------------------
-  build-and-unit-test:
-    name: Build and Unit Test
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Install linters
+        run: pip install yamllint
+
+      - name: Run linters
+        run: make lint
+
+  # ---------------------------------------------------------------------------
+  # TIDY (module files + BUILD files in sync)
+  # ---------------------------------------------------------------------------
+  tidy:
+    name: Tidy
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup
 
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+          cache: false
+
+      - name: Check module files are tidy
+        run: make check-tidy
+
       - name: Check BUILD files are up to date
-        run: |
-          echo "Running Gazelle to check BUILD files..." >&2
-          make gazelle
-          if ! git diff --quiet; then
-            echo "BUILD files are out of date!" >&2
-            echo "" >&2
-            echo "The following files were modified by Gazelle:" >&2
-            git diff --name-only >&2
-            echo "" >&2
-            echo "Please run 'make gazelle' locally and commit the changes." >&2
-            exit 1
-          fi
-          echo "BUILD files are up to date" >&2
+        run: make check-gazelle
+
+  # ---------------------------------------------------------------------------
+  # BUILD AND UNIT TESTS
+  # ---------------------------------------------------------------------------
+  build-and-unit-test:
+    name: Build and Unit Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
 
       - name: Build project
         run: make build
@@ -120,6 +148,8 @@ jobs:
     name: Required Checks
     runs-on: ubuntu-latest
     needs:
+      - lint
+      - tidy
       - build-and-unit-test
       - e2e
       - gateway-integration-test

.yamllint.yml

@@ -0,0 +1,7 @@
+extends: default
+
+rules:
+  document-start: disable
+  line-length: disable
+  truthy:
+    check-keys: false

Makefile

@@ -13,7 +13,7 @@ LOCAL_PROJECT = submitqueue
 # Set REPO_ROOT for docker-compose
 export REPO_ROOT := $(shell pwd)
 
-.PHONY: build build-all-linux build-gateway-linux build-orchestrator-linux clean clean-proto deps e2e-test gazelle integration-test integration-test-extensions integration-test-gateway integration-test-orchestrator local-clean local-gateway-start local-gateway-stop local-init-schemas local-logs local-orchestrator-start local-orchestrator-stop local-ps local-restart local-start local-stop proto query-deps query-targets run-client-gateway run-client-orchestrator run-queue-admin test test-no-cache help
+.PHONY: build build-all-linux build-gateway-linux build-orchestrator-linux check-gazelle check-tidy clean clean-proto deps e2e-test fmt gazelle integration-test integration-test-extensions integration-test-gateway integration-test-orchestrator lint lint-fmt lint-json lint-shell lint-yaml local-clean local-gateway-start local-gateway-stop local-init-schemas local-logs local-orchestrator-start local-orchestrator-stop local-ps local-restart local-start local-stop proto query-deps query-targets run-client-gateway run-client-orchestrator run-queue-admin test test-no-cache help
 
 
 build: ## Build all services and examples
@@ -41,6 +41,36 @@ build-orchestrator-linux: ## Build Orchestrator Linux binary for Docker
 	 cp -f bazel-bin/example/server/orchestrator/orchestrator .docker-bin/orchestrator
 	@echo "Orchestrator Linux binary ready at .docker-bin/orchestrator"
 
+check-gazelle: ## Check BUILD.bazel files are up to date
+	@echo "Running Gazelle to check BUILD files..."
+	@$(BAZEL) run //:gazelle
+	@if ! git diff --quiet; then \
+		echo "BUILD files are out of date!" >&2; \
+		echo "" >&2; \
+		echo "The following files were modified by Gazelle:" >&2; \
+		git diff --name-only >&2; \
+		echo "" >&2; \
+		echo "Please run 'make gazelle' locally and commit the changes." >&2; \
+		exit 1; \
+	fi
+	@echo "BUILD files are up to date."
+
+check-tidy: ## Check that go.mod and MODULE.bazel are tidy
+	@echo "Running go mod tidy..."
+	@go mod tidy -e
+	@echo "Running bazel mod tidy..."
+	@$(BAZEL) mod tidy
+	@if ! git diff --quiet; then \
+		echo "Module files are out of date!" >&2; \
+		echo "" >&2; \
+		echo "The following files were modified:" >&2; \
+		git diff --name-only >&2; \
+		echo "" >&2; \
+		echo "Please run 'go mod tidy' and 'bazel mod tidy' locally and commit the changes." >&2; \
+		exit 1; \
+	fi
+	@echo "Module files are up to date."
+
 clean: ## Clean generated files and binaries
 	@echo "Cleaning with Bazel..."
 	@$(BAZEL) clean
@@ -63,6 +93,12 @@ e2e-test: build-all-linux ## Run end-to-end tests (hermetic, auto-builds binarie
 	@echo "Running end-to-end tests..."
 	@$(BAZEL) test //test/e2e:e2e_test --test_output=streamed
 
+fmt: ## Format Go code
+	@GOFMT=$$($(BAZEL) run @rules_go//go -- env GOROOT 2>/dev/null | tail -1)/bin/gofmt; \
+	echo "Formatting Go code (using $$GOFMT)..."; \
+	$$GOFMT -w .; \
+	echo "Formatting complete!"
+
 gazelle: ## Update BUILD.bazel files
 	@echo "Running Gazelle to update BUILD files..."
 	@$(BAZEL) run //:gazelle
@@ -83,6 +119,47 @@ integration-test-orchestrator: build-orchestrator-linux ## Run Orchestrator inte
 	@echo "Running Orchestrator integration tests..."
 	@$(BAZEL) test //test/integration/orchestrator:orchestrator_test --test_output=streamed
 
+lint: lint-fmt lint-json lint-shell lint-yaml ## Run all linters
+	@echo "All lint checks passed."
+
+lint-fmt: ## Check Go code formatting (fails if unformatted)
+	@GOFMT=$$($(BAZEL) run @rules_go//go -- env GOROOT 2>/dev/null | tail -1)/bin/gofmt; \
+	echo "Checking Go code formatting (using $$GOFMT)..."; \
+	unformatted=$$($$GOFMT -l .); \
+	if [ -n "$$unformatted" ]; then \
+		echo "The following files are not formatted:" >&2; \
+		echo "$$unformatted" >&2; \
+		echo "" >&2; \
+		echo "Please run 'make fmt' and commit the changes." >&2; \
+		exit 1; \
+	fi
+	@echo "All Go files are properly formatted."
+
+lint-json: ## Check JSON files are valid
+	@echo "Checking JSON files..."
+	@invalid=0; \
+	for f in $$(find . -name '*.json' -not -path './bazel-*' -not -path './.git/*' 2>/dev/null); do \
+		if ! jq . "$$f" > /dev/null 2>&1; then \
+			echo "Invalid JSON: $$f" >&2; \
+			invalid=1; \
+		fi; \
+	done; \
+	if [ "$$invalid" -eq 1 ]; then exit 1; fi
+	@echo "All JSON files are valid."
+
+lint-shell: ## Check shell scripts with shellcheck
+	@echo "Checking shell scripts..."
+	@scripts=$$(find . -name '*.sh' -not -path './bazel-*' -not -path './.git/*' 2>/dev/null); \
+	if [ -n "$$scripts" ]; then \
+		shellcheck $$scripts; \
+	fi
+	@echo "All shell scripts are valid."
+
+lint-yaml: ## Check YAML files with yamllint
+	@echo "Checking YAML files..."
+	@yamllint -c .yamllint.yml .
+	@echo "All YAML files are valid."
+
 local-clean: ## Stop and remove all local services, volumes, and images
 	@echo "Cleaning all services and data..."
 	@$(COMPOSE) -f $(COMPOSE_FILE) -p $(LOCAL_PROJECT) down -v --rmi local

core/consumer/registry.go

@@ -57,8 +57,8 @@ type TopicConfig struct {
 // TopicRegistry provides queue, topic name, and subscription config for topics.
 // Each topic can have a different queue backend and topic name.
 type TopicRegistry struct {
-	queues     map[TopicKey]queue.Queue
-	topicNames map[TopicKey]string
+	queues              map[TopicKey]queue.Queue
+	topicNames          map[TopicKey]string
 	subscriptionConfigs map[topicGroup]queue.SubscriptionConfig
 }
 

core/consumer/registry_test.go

@@ -18,8 +18,8 @@ func TestNewTopicRegistry(t *testing.T) {
 	registry, err := consumer.NewTopicRegistry(
 		[]consumer.TopicConfig{
 			{
-				Key:  consumer.TopicKeyRequest,
-				Name: "request",
+				Key:   consumer.TopicKeyRequest,
+				Name:  "request",
 				Queue: mockQ,
 				Subscription: extqueue.DefaultSubscriptionConfig(
 					"worker-1", "group-a",
@@ -44,7 +44,7 @@ func TestNewTopicRegistry(t *testing.T) {
 
 func TestNewTopicRegistry_InvalidTopicName(t *testing.T) {
 	tests := []struct {
-		name string
+		name      string
 		topicName string
 	}{
 		{

entity/build.go

@@ -39,7 +39,6 @@ func (s BuildStatus) IsTerminal() bool {
 	return s == BuildStatusPassed || s == BuildStatusFailed || s == BuildStatusCancelled
 }
 
-
 // SpeculationPathInfo represents the base and head commits of a speculation path used in a build.
 type SpeculationPathInfo struct {
 	// Base is a list of batchIDs(in order) that form the base of this speculation path.

entity/queue/message_test.go

@@ -62,4 +62,3 @@ func TestMessage_Fields(t *testing.T) {
 	assert.Equal(t, msg.PublishedAt, copied.PublishedAt)
 	assert.Equal(t, msg.Metadata, copied.Metadata)
 }
-

entity/request.go

@@ -2,7 +2,6 @@ package entity
 
 import "encoding/json"
 
-
 // RequestLandStrategy defines the possible source control integration methods.
 type RequestLandStrategy string
 

example/server/orchestrator/main.go

@@ -21,14 +21,14 @@ import (
 	githubchecker "github.com/uber/submitqueue/extension/mergechecker/github"
 	extqueue "github.com/uber/submitqueue/extension/queue"
 	queueMySQL "github.com/uber/submitqueue/extension/queue/mysql"
-	mysqlstorage "github.com/uber/submitqueue/extension/storage/mysql"
 	"github.com/uber/submitqueue/extension/storage"
+	mysqlstorage "github.com/uber/submitqueue/extension/storage/mysql"
 	"github.com/uber/submitqueue/orchestrator/controller"
 	"github.com/uber/submitqueue/orchestrator/controller/batch"
 	"github.com/uber/submitqueue/orchestrator/controller/build"
+	"github.com/uber/submitqueue/orchestrator/controller/buildsignal"
 	"github.com/uber/submitqueue/orchestrator/controller/conclude"
 	"github.com/uber/submitqueue/orchestrator/controller/merge"
-	"github.com/uber/submitqueue/orchestrator/controller/buildsignal"
 	"github.com/uber/submitqueue/orchestrator/controller/request"
 	"github.com/uber/submitqueue/orchestrator/controller/score"
 	"github.com/uber/submitqueue/orchestrator/controller/speculate"

extension/queue/mysql/message_store.go

@@ -13,7 +13,6 @@ import (
 	"github.com/uber/submitqueue/entity/queue"
 )
 
-
 // sqlmessageStore is the SQL implementation of messageStore
 type sqlmessageStore struct {
 	db      *sql.DB
@@ -23,8 +22,8 @@ type sqlmessageStore struct {
 
 // Metric names for message store
 const (
-	metricInsertErrors   = "insert.errors"
-	metricFetchErrors    = "fetch.errors"
+	metricInsertErrors    = "insert.errors"
+	metricFetchErrors     = "fetch.errors"
 	metricMoveToDLQErrors = "move_to_dlq.errors"
 )