diff --git a/BUILD.bazel b/BUILD.bazel index 052c08d9..8ed1687d 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -5,7 +5,6 @@ load("@gazelle//:def.bzl", "gazelle") # Resolve protobuf import ambiguities - use the actual protopb packages, not the proto aliases # gazelle:resolve go github.com/uber/submitqueue/gateway/protopb //gateway/protopb # gazelle:resolve go github.com/uber/submitqueue/orchestrator/protopb //orchestrator/protopb -# gazelle:resolve go github.com/uber/submitqueue/speculator/protopb //speculator/protopb # Export marker files for test data dependencies (used by FindRepoRoot in tests) exports_files( diff --git a/Makefile b/Makefile index e2181f6e..98162f98 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ LOCAL_PROJECT = submitqueue # Set REPO_ROOT for docker-compose export REPO_ROOT := $(shell pwd) -.PHONY: build build-all-linux build-gateway-linux build-orchestrator-linux clean clean-proto deps e2e-test gazelle integration-test integration-test-extensions integration-test-gateway integration-test-orchestrator local-clean local-gateway-start local-gateway-stop local-init-schemas local-logs local-orchestrator-start local-orchestrator-stop local-ps local-restart local-start local-stop proto query-deps query-targets run-client-gateway run-client-orchestrator run-client-speculator run-queue-admin test test-no-cache help +.PHONY: build build-all-linux build-gateway-linux build-orchestrator-linux clean clean-proto deps e2e-test gazelle integration-test integration-test-extensions integration-test-gateway integration-test-orchestrator local-clean local-gateway-start local-gateway-stop local-init-schemas local-logs local-orchestrator-start local-orchestrator-stop local-ps local-restart local-start local-stop proto query-deps query-targets run-client-gateway run-client-orchestrator run-queue-admin test test-no-cache help build: ## Build all services and examples @@ -51,7 +51,6 @@ clean-proto: ## Clean generated proto files @echo "Cleaning generated proto files..." @rm -rf gateway/protopb/*.pb.go @rm -rf orchestrator/protopb/*.pb.go - @rm -rf speculator/protopb/*.pb.go @echo "Proto clean complete!" deps: ## Install Go dependencies @@ -202,10 +201,6 @@ proto: ## Generate protobuf files from .proto definitions --go-grpc_out=orchestrator/protopb --go-grpc_opt=paths=source_relative \ --yarpc-go_out=orchestrator/protopb --yarpc-go_opt=paths=source_relative \ --proto_path=orchestrator/proto orchestrator/proto/orchestrator.proto - @protoc --go_out=speculator/protopb --go_opt=paths=source_relative \ - --go-grpc_out=speculator/protopb --go-grpc_opt=paths=source_relative \ - --yarpc-go_out=speculator/protopb --yarpc-go_opt=paths=source_relative \ - --proto_path=speculator/proto speculator/proto/speculator.proto @echo "Protobuf files generated successfully!" # Bazel query helpers @@ -223,10 +218,6 @@ run-client-gateway: run-client-orchestrator: @$(BAZEL) run //example/client/orchestrator:orchestrator -- -addr $(or $(SERVER_ADDR),localhost:8082) -message "$(or $(MESSAGE),ping)" -# Run speculator client (connects to any running speculator service) -run-client-speculator: - @$(BAZEL) run //example/client/speculator:speculator -- -addr $(or $(SERVER_ADDR),localhost:8083) -message "$(or $(MESSAGE),ping)" - run-queue-admin: ## Run queue-admin CLI (use ARGS to pass arguments, e.g. make run-queue-admin ARGS="list-topics") @$(BAZEL) run //extension/queue/mysql/ctl -- $(ARGS) diff --git a/README.md b/README.md index c72e5c80..4045bfd6 100644 --- a/README.md +++ b/README.md @@ -2,18 +2,16 @@ ## Services -Submit Queue consists of three main services: +Submit Queue consists of two main services: - **Gateway**: Entry point for external requests (port 8081) - **Orchestrator**: Coordinates job execution (port 8082) -- **Speculator**: Performs speculative builds (port 8083) ## gRPC API Each service has its own proto definitions and exposes its own gRPC API: -- **SubmitQueueGateway**: Gateway API with Ping method (port 8081) +- **SubmitQueueGateway**: Gateway API with Ping and Land methods (port 8081) - **SubmitQueueOrchestrator**: Orchestrator API with Ping method (port 8082) -- **SubmitQueueSpeculator**: Speculator API with Ping method (port 8083) ### Quick Start @@ -185,12 +183,10 @@ make proto # Run servers make run-gateway # Gateway on port 8081 make run-orchestrator # Orchestrator on port 8082 -make run-speculator # Speculator on port 8083 # Run clients make run-client-gateway MESSAGE="hello" make run-client-orchestrator MESSAGE="hello" -make run-client-speculator MESSAGE="hello" # Clean binaries make clean @@ -211,12 +207,10 @@ go build ./... # Build example servers go build -o bin/gateway_server ./example/server/gateway/ go build -o bin/orchestrator_server ./example/server/orchestrator/ -go build -o bin/speculator_server ./example/server/speculator/ # Build clients go build -o bin/gateway_client ./example/client/gateway/ go build -o bin/orchestrator_client ./example/client/orchestrator/ -go build -o bin/speculator_client ./example/client/speculator/ # Run a server ./bin/gateway_server @@ -354,13 +348,9 @@ make run-gateway # Terminal 2: Start orchestrator make run-orchestrator -# Terminal 3: Start speculator -make run-speculator - -# Terminal 4: Test each service +# Terminal 3: Test each service make run-client-gateway MESSAGE="test gateway" make run-client-orchestrator MESSAGE="test orchestrator" -make run-client-speculator MESSAGE="test speculator" ``` #### Using Bazel for Testing diff --git a/doc/rfc/sql-queue-rfc.md b/doc/rfc/sql-queue-rfc.md index c490f813..18ba1905 100644 --- a/doc/rfc/sql-queue-rfc.md +++ b/doc/rfc/sql-queue-rfc.md @@ -18,8 +18,7 @@ MySQL-based distributed message queue with partition leasing, visibility timeout ### Motivation SubmitQueue needs a reliable message queue for coordinating asynchronous workflows: -- **Orchestrator** publishes merge jobs to workers -- **Speculator** publishes speculative build requests +- **Orchestrator** publishes merge jobs and speculative build requests to workers - **Workers** need distributed coordination without duplicate processing - **Crash recovery** must preserve exactly where processing stopped diff --git a/example/README.md b/example/README.md index 9c7bf896..2581de27 100644 --- a/example/README.md +++ b/example/README.md @@ -1,127 +1,78 @@ -# How to Run gRPC Servers +# Examples -This directory contains example gRPC servers and clients for each service in the submitqueue project. +Example gRPC servers and clients for running the submitqueue services locally. ## Services -Each service has its own proto definitions and gRPC API: +- **Gateway** (port 8081) — entry point for land requests. Exposes `Ping` and `Land` RPCs. +- **Orchestrator** (port 8082) — coordinates the pipeline. Exposes `Ping` RPC and consumes queue messages across 8 pipeline topics. -- **SubmitQueueGateway**: Gateway API (port 8081) -- **SubmitQueueOrchestrator**: Orchestrator API (port 8082) -- **SubmitQueueSpeculator**: Speculator API (port 8083) +Both services require MySQL (app database + queue database). Docker Compose handles this automatically. -## Building and Running +## Directory Structure -### Using Make (Recommended) +``` +example/ +├── server/ +│ ├── docker-compose.yml # Full stack (Gateway + Orchestrator + 2x MySQL) +│ ├── gateway/ +│ │ ├── main.go # Gateway server entry point +│ │ ├── Dockerfile +│ │ └── docker-compose.yml # Gateway-only stack +│ └── orchestrator/ +│ ├── main.go # Orchestrator server entry point +│ ├── Dockerfile +│ └── docker-compose.yml # Orchestrator-only stack +└── client/ + ├── gateway/main.go # Gateway ping client + └── orchestrator/main.go # Orchestrator ping client +``` + +## Running -The easiest way to build and run the servers: +### Docker Compose (Recommended) ```bash -# Build everything (including proto generation if needed) -make build +# Start full stack (Gateway + Orchestrator + MySQL) +make local-start -# Run specific servers -make run-gateway # Runs on port 8081 -make run-orchestrator # Runs on port 8082 -make run-speculator # Runs on port 8083 +# Start individual services +make local-gateway-start +make local-orchestrator-start -# Run clients -make run-client-gateway MESSAGE="hello" -make run-client-orchestrator MESSAGE="hello" -make run-client-speculator MESSAGE="hello" +# View logs and status +make local-logs +make local-ps -# Clean everything -make clean +# Stop +make local-stop ``` -### Using Bazel +### Bazel -Build a specific server: ```bash -# Build gateway server +# Build servers bazel build //example/server/gateway:gateway - -# Build orchestrator server bazel build //example/server/orchestrator:orchestrator -# Build speculator server -bazel build //example/server/speculator:speculator - # Build clients bazel build //example/client/gateway:gateway bazel build //example/client/orchestrator:orchestrator -bazel build //example/client/speculator:speculator ``` -Run a specific server: -```bash -# Run gateway server -bazel run //example/server/gateway:gateway - -# Run orchestrator server -bazel run //example/server/orchestrator:orchestrator - -# Run speculator server -bazel run //example/server/speculator:speculator - -# Or use '.' from the directory -cd example/server/gateway && bazel run . -cd example/server/orchestrator && bazel run . -cd example/server/speculator && bazel run . -``` +### Go -Run clients: ```bash -# Run gateway client -bazel run //example/client/gateway:gateway -- -message "hello" - -# Run orchestrator client -bazel run //example/client/orchestrator:orchestrator -- -message "hello" - -# Run speculator client -bazel run //example/client/speculator:speculator -- -message "hello" - -# Or use '.' from the directory -cd example/client/gateway && bazel run . -- -message "hello" -cd example/client/orchestrator && bazel run . -- -message "hello" -cd example/client/speculator && bazel run . -- -message "hello" -``` - -### Using Go directly - -You can also run the servers directly with Go (from the repository root): - -```bash -# Run gateway server go run example/server/gateway/main.go - -# Run orchestrator server go run example/server/orchestrator/main.go - -# Run speculator server -go run example/server/speculator/main.go - -# Run clients -go run example/client/gateway/main.go -message "hello" -go run example/client/orchestrator/main.go -message "hello" -go run example/client/speculator/main.go -message "hello" ``` -## Testing the Services +## Testing with Clients -### Using the Go Clients - -Build and run the clients: ```bash -# Using Make -make run-client-gateway MESSAGE="test from gateway client" -make run-client-orchestrator MESSAGE="test from orchestrator client" -make run-client-speculator MESSAGE="test from speculator client" - -# Using Go +# Go clients go run example/client/gateway/main.go -addr localhost:8081 -message "hello" go run example/client/orchestrator/main.go -addr localhost:8082 -message "hello" -go run example/client/speculator/main.go -addr localhost:8083 -message "hello" ``` Client flags: @@ -129,7 +80,7 @@ Client flags: - `-message`: Message to send in the ping request - `-timeout`: Request timeout (default: 5s) -### Using grpcurl +### grpcurl Install grpcurl if you don't have it: ```bash @@ -138,119 +89,37 @@ brew install grpcurl # macOS go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest ``` -Test the ping service: ```bash -# Test gateway (port 8081) +# Ping grpcurl -plaintext -d '{"message": "hello"}' localhost:8081 uber.devexp.submitqueue.gateway.SubmitQueueGateway/Ping - -# Test orchestrator (port 8082) grpcurl -plaintext -d '{"message": "hello"}' localhost:8082 uber.devexp.submitqueue.orchestrator.SubmitQueueOrchestrator/Ping -# Test speculator (port 8083) -grpcurl -plaintext -d '{"message": "hello"}' localhost:8083 uber.devexp.submitqueue.speculator.SubmitQueueSpeculator/Ping -``` - -List available services: -```bash +# List services grpcurl -plaintext localhost:8081 list grpcurl -plaintext localhost:8082 list -grpcurl -plaintext localhost:8083 list -``` -Describe a service: -```bash +# Describe a service grpcurl -plaintext localhost:8081 describe uber.devexp.submitqueue.gateway.SubmitQueueGateway grpcurl -plaintext localhost:8082 describe uber.devexp.submitqueue.orchestrator.SubmitQueueOrchestrator -grpcurl -plaintext localhost:8083 describe uber.devexp.submitqueue.speculator.SubmitQueueSpeculator ``` ## API Reference -### Service Interfaces - -Each service exposes a Ping method with the same request/response structure but different service names: - -#### Gateway Service +### Gateway Service **Service**: `uber.devexp.submitqueue.gateway.SubmitQueueGateway` **Proto**: `gateway/proto/gateway.proto` -#### Orchestrator Service +| Method | Description | +|--------|-------------| +| `Ping` | Health check, returns service name and timestamp | +| `Land` | Submit a land request for code changes | + +### Orchestrator Service **Service**: `uber.devexp.submitqueue.orchestrator.SubmitQueueOrchestrator` **Proto**: `orchestrator/proto/orchestrator.proto` -#### Speculator Service - -**Service**: `uber.devexp.submitqueue.speculator.SubmitQueueSpeculator` -**Proto**: `speculator/proto/speculator.proto` - -### Ping Method - -**Request:** -```protobuf -message PingRequest { - string message = 1; // Optional message to include in the ping -} -``` - -**Response:** -```protobuf -message PingResponse { - string message = 1; // The response message - string service_name = 2; // The service name that handled the request - int64 timestamp = 3; // Timestamp of when the ping was received -} -``` - -**Example:** -```bash -grpcurl -plaintext -d '{"message": "test"}' localhost:8081 uber.devexp.submitqueue.gateway.SubmitQueueGateway/Ping -``` - -Expected response: -```json -{ - "message": "echo: test", - "service_name": "gateway", - "timestamp": 1705234567 -} -``` - -## Project Structure - -Each service directory (in the repository root) contains: -- `proto/`: Protocol buffer definitions - - `.proto`: Service API definition -- `protopb/`: Generated Go code (committed to repository) - - `*.pb.go`, `*_grpc.pb.go`, `*.pb.yarpc.go` -- `core/controller/`: Service implementation - - `ping.go`: Ping service implementation -- `BUILD.bazel`: Bazel build rules - -The `example/` directory contains: -- `server/`: Example server implementations - - `gateway/`, `orchestrator/`, `speculator/`: Server examples -- `client/`: Example client implementations - - `gateway/`, `orchestrator/`, `speculator/`: Client examples - -## Using the Services as a Library - -The proto packages are designed to be consumed as a library. To use them in your own code: - -```go -import ( - gatewaypb "github.com/uber/submitqueue/gateway/protopb" - orchestratorpb "github.com/uber/submitqueue/orchestrator/protopb" - speculatorpb "github.com/uber/submitqueue/speculator/protopb" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -// Use the clients -conn, _ := grpc.NewClient("localhost:8081", grpc.WithTransportCredentials(insecure.NewCredentials())) -client := gatewaypb.NewSubmitQueueGatewayClient(conn) -resp, _ := client.Ping(context.Background(), &gatewaypb.PingRequest{Message: "hello"}) -``` - -The generated proto files are committed to the repository, so you can import and use them directly without needing to regenerate them. +| Method | Description | +|--------|-------------| +| `Ping` | Health check, returns service name and timestamp |