Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ load("@gazelle//:def.bzl", "gazelle")
# Resolve protobuf import ambiguities - use the actual protopb packages, not the proto aliases
# gazelle:resolve go github.com/uber/submitqueue/gateway/protopb //gateway/protopb
# gazelle:resolve go github.com/uber/submitqueue/orchestrator/protopb //orchestrator/protopb
# gazelle:resolve go github.com/uber/submitqueue/stovepipe/protopb //stovepipe/protopb

# Export marker files for test data dependencies (used by FindRepoRoot in tests)
exports_files(
Expand Down
41 changes: 39 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ COMPOSE = docker-compose
COMPOSE_FILE = example/server/docker-compose.yml
GATEWAY_COMPOSE_FILE = example/server/gateway/docker-compose.yml
ORCHESTRATOR_COMPOSE_FILE = example/server/orchestrator/docker-compose.yml
STOVEPIPE_COMPOSE_FILE = example/server/stovepipe/docker-compose.yml

# Fixed project name for local manual testing (tests use unique random names)
LOCAL_PROJECT = submitqueue
Expand All @@ -30,7 +31,7 @@ define assert_clean
fi
endef

.PHONY: build build-all-linux build-gateway-linux build-orchestrator-linux check-gazelle check-mocks check-tidy clean clean-proto deps e2e-test fmt gazelle integration-test integration-test-consumer integration-test-extensions integration-test-gateway integration-test-orchestrator license-fix lint lint-fmt lint-license local-clean local-gateway-start local-gateway-stop local-init-schemas local-logs local-orchestrator-start local-orchestrator-stop local-ps local-restart local-start local-stop mocks proto query-deps query-targets run-client-gateway run-client-orchestrator run-queue-admin test test-no-cache tidy tidy-bazel tidy-go help
.PHONY: build build-all-linux build-gateway-linux build-orchestrator-linux build-stovepipe-linux check-gazelle check-mocks check-tidy clean clean-proto deps e2e-test fmt gazelle integration-test integration-test-consumer integration-test-extensions integration-test-gateway integration-test-orchestrator integration-test-stovepipe license-fix lint lint-fmt lint-license local-clean local-gateway-start local-gateway-stop local-init-schemas local-logs local-orchestrator-start local-orchestrator-stop local-ps local-restart local-start local-stop local-stovepipe-start local-stovepipe-stop mocks proto query-deps query-targets run-client-gateway run-client-orchestrator run-client-stovepipe run-queue-admin test test-no-cache tidy tidy-bazel tidy-go help


build: ## Build all services and examples
Expand All @@ -39,7 +40,7 @@ build: ## Build all services and examples
@echo "Build complete!"

# Build Linux binaries required for Docker containers
build-all-linux: build-gateway-linux build-orchestrator-linux ## Build all Linux binaries for Docker
build-all-linux: build-gateway-linux build-orchestrator-linux build-stovepipe-linux ## Build all Linux binaries for Docker
@echo "All Linux binaries ready for Docker"

build-gateway-linux: ## Build Gateway Linux binary for Docker
Expand All @@ -58,6 +59,14 @@ build-orchestrator-linux: ## Build Orchestrator Linux binary for Docker
cp -f bazel-bin/example/server/orchestrator/orchestrator .docker-bin/orchestrator
@echo "Orchestrator Linux binary ready at .docker-bin/orchestrator"

build-stovepipe-linux: ## Build Stovepipe Linux binary for Docker
@echo "Building Stovepipe Linux binary for Docker..."
@$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //example/server/stovepipe:stovepipe
@mkdir -p .docker-bin
@cp -f bazel-bin/example/server/stovepipe/stovepipe_/stovepipe .docker-bin/stovepipe 2>/dev/null || \
cp -f bazel-bin/example/server/stovepipe/stovepipe .docker-bin/stovepipe
@echo "Stovepipe Linux binary ready at .docker-bin/stovepipe"

check-gazelle: ## Check BUILD.bazel files are up to date
@echo "Running Gazelle to check BUILD files..."
@$(BAZEL) run //:gazelle
Expand All @@ -82,6 +91,7 @@ clean-proto: ## Clean generated proto files
@echo "Cleaning generated proto files..."
@rm -rf gateway/protopb/*.pb.go
@rm -rf orchestrator/protopb/*.pb.go
@rm -rf stovepipe/protopb/*.pb.go
@echo "Proto clean complete!"

deps: tidy-go ## Download and tidy Go dependencies
Expand Down Expand Up @@ -122,6 +132,10 @@ integration-test-orchestrator: build-orchestrator-linux ## Run Orchestrator inte
@echo "Running Orchestrator integration tests..."
@$(BAZEL) test //test/integration/orchestrator:orchestrator_test --test_output=streamed

integration-test-stovepipe: build-stovepipe-linux ## Run Stovepipe integration tests (auto-builds binary)
@echo "Running Stovepipe integration tests..."
@$(BAZEL) test //test/integration/stovepipe:stovepipe_test --test_output=streamed

license-fix: ## Add missing license headers to source files
@$(BAZEL) run //tool/linter/licenseheader -- --fix

Expand Down Expand Up @@ -243,6 +257,21 @@ local-stop: ## Stop all services (keep data)
@$(COMPOSE) -f $(COMPOSE_FILE) -p $(LOCAL_PROJECT) down
@echo "Services stopped. Data volumes preserved."

local-stovepipe-start: build-stovepipe-linux ## Start Stovepipe service locally (stateless, no databases)
@echo "Starting Stovepipe with docker-compose..."
@$(COMPOSE) -f $(STOVEPIPE_COMPOSE_FILE) -p $(LOCAL_PROJECT) up -d --build --wait
@echo ""
@echo "✅ Stovepipe is running!"
@echo ""
@$(COMPOSE) -f $(STOVEPIPE_COMPOSE_FILE) -p $(LOCAL_PROJECT) ps
@echo ""
@echo "Stovepipe gRPC port: $$(docker port $(LOCAL_PROJECT)-stovepipe-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')"

local-stovepipe-stop: ## Stop Stovepipe service
@echo "Stopping Stovepipe services..."
@$(COMPOSE) -f $(STOVEPIPE_COMPOSE_FILE) -p $(LOCAL_PROJECT) down
@echo "Stovepipe services stopped."

mocks: ## Generate mock files using mockgen
@echo "Generating mocks..."
@$(BAZEL) run @rules_go//go -- generate ./extension/storage/... ./extension/counter/... ./extension/queue/... ./extension/mergechecker/... ./extension/pusher/... ./extension/scorer/... ./core/consumer/...
Expand All @@ -258,6 +287,10 @@ proto: ## Generate protobuf files from .proto definitions
--go-grpc_out=orchestrator/protopb --go-grpc_opt=paths=source_relative \
--yarpc-go_out=orchestrator/protopb --yarpc-go_opt=paths=source_relative \
--proto_path=orchestrator/proto orchestrator/proto/orchestrator.proto
@protoc --go_out=stovepipe/protopb --go_opt=paths=source_relative \
--go-grpc_out=stovepipe/protopb --go-grpc_opt=paths=source_relative \
--yarpc-go_out=stovepipe/protopb --yarpc-go_opt=paths=source_relative \
--proto_path=stovepipe/proto stovepipe/proto/stovepipe.proto
@echo "Protobuf files generated successfully!"

# Bazel query helpers
Expand All @@ -275,6 +308,10 @@ run-client-gateway:
run-client-orchestrator:
@$(BAZEL) run //example/client/orchestrator:orchestrator -- -addr $(or $(SERVER_ADDR),localhost:8082) -message "$(or $(MESSAGE),ping)"

# Run stovepipe client (connects to any running stovepipe service)
run-client-stovepipe:
@$(BAZEL) run //example/client/stovepipe:stovepipe -- -addr $(or $(SERVER_ADDR),localhost:8083) -message "$(or $(MESSAGE),ping)"

run-queue-admin: ## Run queue-admin CLI (use ARGS to pass arguments, e.g. make run-queue-admin ARGS="list-topics")
@$(BAZEL) run //extension/queue/mysql/ctl -- $(ARGS)

Expand Down
19 changes: 19 additions & 0 deletions example/client/stovepipe/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
load("@rules_go//go:def.bzl", "go_binary", "go_library")

go_library(
name = "stovepipe_lib",
srcs = ["main.go"],
importpath = "github.com/uber/submitqueue/example/client/stovepipe",
visibility = ["//visibility:private"],
deps = [
"//stovepipe/protopb",
"@org_golang_google_grpc//:grpc",
"@org_golang_google_grpc//credentials/insecure",
],
)

go_binary(
name = "stovepipe",
embed = [":stovepipe_lib"],
visibility = ["//visibility:public"],
)
78 changes: 78 additions & 0 deletions example/client/stovepipe/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright (c) 2025 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"context"
"flag"
"fmt"
"os"
"time"

pb "github.com/uber/submitqueue/stovepipe/protopb"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)

func main() {
addr := flag.String("addr", "localhost:8083", "stovepipe server address")
message := flag.String("message", "", "message to send in ping request")
timeout := flag.Duration("timeout", 5*time.Second, "request timeout")
flag.Parse()

if err := run(*addr, *message, *timeout); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
}

func run(addr, message string, timeout time.Duration) error {
// Create a gRPC connection
conn, err := grpc.NewClient(
addr,
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
if err != nil {
return fmt.Errorf("failed to connect: %w", err)
}
defer conn.Close()

// Create a client
client := pb.NewSubmitQueueStovepipeClient(conn)

// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()

// Make the ping request
req := &pb.PingRequest{
Message: message,
}

fmt.Printf("Sending ping to stovepipe at %s...\n", addr)
resp, err := client.Ping(ctx, req)
if err != nil {
return fmt.Errorf("ping failed: %w", err)
}

// Print the response
fmt.Printf("\nResponse:\n")
fmt.Printf(" Message: %s\n", resp.Message)
fmt.Printf(" Service Name: %s\n", resp.ServiceName)
fmt.Printf(" Timestamp: %d (%s)\n", resp.Timestamp, time.Unix(resp.Timestamp, 0))
fmt.Printf(" Hostname: %s\n", resp.Hostname)

return nil
}
27 changes: 27 additions & 0 deletions example/server/stovepipe/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
load("@rules_go//go:def.bzl", "go_binary", "go_library")

exports_files(
["docker-compose.yml"],
visibility = ["//visibility:public"],
)

go_library(
name = "stovepipe_lib",
srcs = ["main.go"],
importpath = "github.com/uber/submitqueue/example/server/stovepipe",
visibility = ["//visibility:private"],
deps = [
"//stovepipe/controller",
"//stovepipe/protopb",
"@com_github_uber_go_tally_v4//:tally",
"@org_golang_google_grpc//:grpc",
"@org_golang_google_grpc//reflection",
"@org_uber_go_zap//:zap",
],
)

go_binary(
name = "stovepipe",
embed = [":stovepipe_lib"],
visibility = ["//visibility:public"],
)
12 changes: 12 additions & 0 deletions example/server/stovepipe/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
FROM debian:bookworm-slim

RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/*
WORKDIR /root/

# Copy pre-built Linux binary
# Built via: make build-stovepipe-linux
COPY .docker-bin/stovepipe ./stovepipe

EXPOSE 8080

CMD ["./stovepipe"]
19 changes: 19 additions & 0 deletions example/server/stovepipe/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Docker Compose for Stovepipe Manual Testing
#
# IMPORTANT: Before running docker-compose, build the Linux binary:
# make build-stovepipe-linux
# OR
# bazel build --platforms=@rules_go//go/toolchain:linux_amd64 //example/server/stovepipe
#
# Quick start:
# make local-stovepipe-start

services:
stovepipe-service:
build:
context: ${REPO_ROOT}
dockerfile: example/server/stovepipe/Dockerfile
ports:
- "8080" # Random ephemeral port to avoid conflicts
environment:
- PORT=:8080
Loading
Loading