feat(queueconfig): add YAML-backed Store and wire gateway Land validation

MODULE.bazel

@@ -36,6 +36,7 @@ use_repo(
     "com_github_spf13_cobra",
     "com_github_stretchr_testify",
     "com_github_uber_go_tally_v4",
+    "in_gopkg_yaml_v3",
     "org_golang_google_grpc",
     "org_golang_google_protobuf",
     "org_golang_x_oauth2",

Makefile

@@ -274,7 +274,7 @@ local-stovepipe-stop: ## Stop Stovepipe service
 
 mocks: ## Generate mock files using mockgen
 	@echo "Generating mocks..."
-	@$(BAZEL) run @rules_go//go -- generate ./extension/storage/... ./extension/changestore/... ./extension/counter/... ./extension/queue/... ./extension/mergechecker/... ./extension/pusher/... ./extension/scorer/... ./extension/conflict/... ./core/consumer/...
+	@$(BAZEL) run @rules_go//go -- generate ./extension/storage/... ./extension/changestore/... ./extension/counter/... ./extension/queue/... ./extension/queueconfig/... ./extension/mergechecker/... ./extension/pusher/... ./extension/scorer/... ./extension/conflict/... ./core/consumer/...
 	@echo "Mocks generated successfully!"
 
 proto: ## Generate protobuf files from .proto definitions

entity/queue_config.go

@@ -42,7 +42,7 @@ type QueueConfig struct {
 	Target string `json:"target" yaml:"target"`
 
 	// BuildRunner identifies the CI pipeline or job that runs builds for this queue.
-	// Opaque to the system; meaningful only to the build extension implementation.
+	// Opaque to the system; meaningful only to the build runner extension implementation.
 	// Examples:
 	//   - Buildkite: "buildkite.com/uber/submitqueue-ci"
 	//   - Jenkins: "jenkins.example.com/job/submitqueue-verify"

example/server/docker-compose.yml

@@ -53,6 +53,8 @@ services:
       - MYSQL_DSN=root:root@tcp(mysql-app:3306)/submitqueue?parseTime=true
       # Queue infrastructure connection (separate database)
       - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true
+      # Path to YAML queue configuration baked into the image
+      - QUEUE_CONFIG_PATH=/root/queues.yaml
     depends_on:
       mysql-app:
         condition: service_healthy

example/server/gateway/BUILD.bazel

@@ -14,6 +14,7 @@ go_library(
         "//core/consumer",
         "//extension/counter/mysql",
         "//extension/queue/mysql",
+        "//extension/queueconfig/yaml",
         "//extension/storage/mysql",
         "//gateway/controller",
         "//gateway/protopb",

example/server/gateway/Dockerfile

@@ -7,6 +7,10 @@ WORKDIR /root/
 # Built via: make build-gateway-linux
 COPY .docker-bin/gateway ./gateway
 
+# Sample queue configuration; the gateway reads it on startup via
+# QUEUE_CONFIG_PATH (set in docker-compose.yml).
+COPY example/server/gateway/queues.yaml ./queues.yaml
+
 EXPOSE 8080
 
 CMD ["./gateway"]

example/server/gateway/docker-compose.yml

@@ -53,6 +53,8 @@ services:
       - MYSQL_DSN=root:root@tcp(mysql-app:3306)/submitqueue?parseTime=true
       # Queue infrastructure connection (separate database)
       - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true
+      # Path to YAML queue configuration baked into the image
+      - QUEUE_CONFIG_PATH=/root/queues.yaml
     depends_on:
       mysql-app:
         condition: service_healthy

example/server/gateway/main.go

@@ -31,6 +31,7 @@ import (
 	"github.com/uber/submitqueue/core/consumer"
 	mysqlcounter "github.com/uber/submitqueue/extension/counter/mysql"
 	queueMySQL "github.com/uber/submitqueue/extension/queue/mysql"
+	yamlqueueconfig "github.com/uber/submitqueue/extension/queueconfig/yaml"
 	mysqlstorage "github.com/uber/submitqueue/extension/storage/mysql"
 	"github.com/uber/submitqueue/gateway/controller"
 	pb "github.com/uber/submitqueue/gateway/protopb"
@@ -175,9 +176,20 @@ func run() error {
 	// Initialize request log store from shared app database connection
 	requestLogStore := mysqlstorage.NewRequestLogStore(appDB, scope.SubScope("request_log_store"))
 
+	// Load queue configurations from YAML. Path is required so the gateway
+	// can reject requests for unknown queues at the edge.
+	queueConfigPath := os.Getenv("QUEUE_CONFIG_PATH")
+	if queueConfigPath == "" {
+		return fmt.Errorf("QUEUE_CONFIG_PATH environment variable is required")
+	}
+	queueConfigs, err := yamlqueueconfig.NewStore(queueConfigPath)
+	if err != nil {
+		return fmt.Errorf("failed to load queue configs: %w", err)
+	}
+
 	// Create controllers and wrap them for gRPC
 	pingController := controller.NewPingController(logger, scope)
-	landController := controller.NewLandController(logger.Sugar(), scope, cnt, requestLogStore, registry)
+	landController := controller.NewLandController(logger.Sugar(), scope, cnt, requestLogStore, queueConfigs, registry)
 	gatewayServer := &GatewayServer{
 		pingController: pingController,
 		landController: landController,

example/server/gateway/queues.yaml

@@ -0,0 +1,22 @@
+# Example queue configurations consumed by the gateway YAML store.
+# The gateway loads this file at startup; QUEUE_CONFIG_PATH must point
+# at it. Each entry maps a queue name to the VCS repository + target
+# branch and selects the extension implementations used downstream.
+queues:
+  - name: test-queue
+    vcs_type: git
+    vcs_address: git@github.com:uber/submitqueue.git
+    target: main
+    build_runner: buildkite.com/uber/submitqueue-ci
+    change_provider: github
+    merge_checker: github
+    land_provider: github
+
+  - name: e2e-test-queue
+    vcs_type: git
+    vcs_address: git@github.com:uber/submitqueue.git
+    target: main
+    build_runner: buildkite.com/uber/submitqueue-ci
+    change_provider: github
+    merge_checker: github
+    land_provider: github

extension/queueconfig/mock/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "mock",
+    srcs = ["queueconfig_mock.go"],
+    importpath = "github.com/uber/submitqueue/extension/queueconfig/mock",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//entity",
+        "@org_uber_go_mock//gomock",
+    ],
+)

extension/queueconfig/queueconfig.go

@@ -14,6 +14,8 @@
 
 package queueconfig
 
+//go:generate mockgen -source=queueconfig.go -destination=mock/queueconfig_mock.go -package=mock
+
 import (
 	"context"
 	"errors"

extension/queueconfig/yaml/BUILD.bazel

@@ -0,0 +1,24 @@
+load("@rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "yaml",
+    srcs = ["yaml.go"],
+    importpath = "github.com/uber/submitqueue/extension/queueconfig/yaml",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//entity",
+        "//extension/queueconfig",
+        "@in_gopkg_yaml_v3//:yaml_v3",
+    ],
+)
+
+go_test(
+    name = "yaml_test",
+    srcs = ["yaml_test.go"],
+    embed = [":yaml"],
+    deps = [
+        "//extension/queueconfig",
+        "@com_github_stretchr_testify//assert",
+        "@com_github_stretchr_testify//require",
+    ],
+)

extension/queueconfig/yaml/yaml.go

@@ -0,0 +1,90 @@
+// Copyright (c) 2025 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package yaml provides a YAML-file-backed implementation of
+// queueconfig.Store. The file is read once at construction time and held
+// in memory; the file is not watched for changes.
+package yaml
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/uber/submitqueue/entity"
+	"github.com/uber/submitqueue/extension/queueconfig"
+	yamlv3 "gopkg.in/yaml.v3"
+)
+
+// fileContents is the top-level YAML schema. A configuration file is a
+// single document with a "queues" key holding a list of QueueConfig.
+type fileContents struct {
+	Queues []entity.QueueConfig `yaml:"queues"`
+}
+
+// Store is a queueconfig.Store backed by an in-memory snapshot of a YAML
+// file. Construct via NewStore. Safe for concurrent reads.
+type Store struct {
+	byName map[string]entity.QueueConfig
+	all    []entity.QueueConfig
+}
+
+// NewStore reads queue configurations from the YAML file at path and
+// returns a Store. If the file omits the top-level "queues" key, it is
+// treated as an empty queue list.
+// Returns an error if the file is unreadable, malformed, contains a queue
+// with an empty name, or contains duplicate queue names.
+func NewStore(path string) (Store, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return Store{}, fmt.Errorf("failed to read queue config file %q: %w", path, err)
+	}
+
+	var contents fileContents
+	if err := yamlv3.Unmarshal(data, &contents); err != nil {
+		return Store{}, fmt.Errorf("failed to parse queue config file %q: %w", path, err)
+	}
+
+	byName := make(map[string]entity.QueueConfig, len(contents.Queues))
+	for _, q := range contents.Queues {
+		if q.Name == "" {
+			return Store{}, fmt.Errorf("queue config in %q has empty name", path)
+		}
+		if _, dup := byName[q.Name]; dup {
+			return Store{}, fmt.Errorf("queue config in %q has duplicate name %q", path, q.Name)
+		}
+		byName[q.Name] = q
+	}
+
+	all := make([]entity.QueueConfig, len(contents.Queues))
+	copy(all, contents.Queues)
+
+	return Store{byName: byName, all: all}, nil
+}
+
+// Get returns the configuration for the named queue, or queueconfig.ErrNotFound.
+func (s Store) Get(_ context.Context, name string) (entity.QueueConfig, error) {
+	cfg, ok := s.byName[name]
+	if !ok {
+		return entity.QueueConfig{}, queueconfig.ErrNotFound
+	}
+	return cfg, nil
+}
+
+// List returns a copy of all configured queues in file order.
+func (s Store) List(_ context.Context) ([]entity.QueueConfig, error) {
+	out := make([]entity.QueueConfig, len(s.all))
+	copy(out, s.all)
+	return out, nil
+}