feat(entity,storage): rework speculation tree model and store

Reshape the speculation tree data model around the Base/Head path that the build stage actually consumes, and align its store with the speculation design.

entity: SpeculationPath{Base, Head} becomes the unit; SpeculationPathInfo carries the path plus its Score, controller-owned Status (candidate/selected/prioritized/building/passed/failed/cancelling/cancelled), and BuildID — Path is immutable once persisted, Score/Status/BuildID are updateable by the controller. Score is scorer-computed and controller-persisted dynamic state — recomputed on every respeculate, not set at enumeration. Selected/Prioritized and Cancelling/Cancelled split desire vs budget-cleared and cancel-intent vs terminal, so the persisted status carries the cross-stage contract. Adds SpeculationPathAction (Promote/Cancel) and SpeculationPathDecision for the selector and prioritizer seams. SpeculationTree.Speculations becomes Paths, and the tree gains a Version for optimistic locking (starts at 1, incremented per change). The Build entity uses the shared SpeculationPath (Head = the batch under verification) and drops its own Score, which was never populated or read.

storage: SpeculationTreeStore.UpdateSpeculations(batchID, []SpeculationInfo) becomes a pure conditional write — Update(ctx, batchID, oldVersion, newVersion, paths) — returning ErrVersionMismatch when the persisted version does not match, per the repo's optimistic-locking pattern (version arithmetic owned by the controller). The MySQL impl treats rowsAffected == 1 as success, 0 as version mismatch, and anything else as a generic error. The MySQL column speculations is renamed paths and the schema gains a version column. MySQL impl, mock, and storage integration coverage (create/get round-trip, duplicate, versioned whole-tree overwrite, stale-write rejection, not-found) added/updated.

Author: behinddwalls

submitqueue/entity/build.go

@@ -53,12 +53,6 @@ func (s BuildStatus) IsTerminal() bool {
 	return s == BuildStatusSucceeded || s == BuildStatusFailed || s == BuildStatusCancelled
 }
 
-// SpeculationPathInfo represents the base and head commits of a speculation path used in a build.
-type SpeculationPathInfo struct {
-	// Base is a list of batchIDs(in order) that form the base of this speculation path.
-	Base []string
-}
-
 // Build represents a build scheduled for a batch along a specific speculation path.
 // All fields except the Status are immutable after creation.
 type Build struct {
@@ -69,10 +63,9 @@ type Build struct {
 	BatchID string
 	// SpeculationPath is the speculation path that represents this build. For
 	// a given batch this path is crafted from the graph that is generated from the
-	// dependencies of this batch.
-	SpeculationPath SpeculationPathInfo
-	// Score represents the build prediction score for this speculation path.
-	Score float32
+	// dependencies of this batch. Its Head is the batch being verified (equal to
+	// BatchID) and its Base is the assumed-good prefix of predecessor batches.
+	SpeculationPath SpeculationPath
 	// Status represents the state of the build lifecycle this build is in.
 	Status BuildStatus
 }

submitqueue/entity/build_test.go

@@ -70,10 +70,10 @@ func TestBuild_ToBytes(t *testing.T) {
 	build := Build{
 		ID:      "build-1",
 		BatchID: "batch-1",
-		SpeculationPath: SpeculationPathInfo{
+		SpeculationPath: SpeculationPath{
 			Base: []string{"batch-0", "batch-prev"},
+			Head: "batch-1",
 		},
-		Score:  0.85,
 		Status: BuildStatusAccepted,
 	}
 
@@ -92,10 +92,10 @@ func TestBuildFromBytes(t *testing.T) {
 	original := Build{
 		ID:      "build-42",
 		BatchID: "batch-7",
-		SpeculationPath: SpeculationPathInfo{
+		SpeculationPath: SpeculationPath{
 			Base: []string{"batch-5", "batch-6"},
+			Head: "batch-7",
 		},
-		Score:  0.92,
 		Status: BuildStatusAccepted,
 	}
 
@@ -111,7 +111,6 @@ func TestBuildFromBytes(t *testing.T) {
 	assert.Equal(t, original.ID, deserialized.ID)
 	assert.Equal(t, original.BatchID, deserialized.BatchID)
 	assert.Equal(t, original.SpeculationPath.Base, deserialized.SpeculationPath.Base)
-	assert.Equal(t, original.Score, deserialized.Score)
 	assert.Equal(t, original.Status, deserialized.Status)
 }
 
@@ -132,7 +131,6 @@ func TestBuildFromBytes_EmptyData(t *testing.T) {
 	assert.Empty(t, build.ID)
 	assert.Empty(t, build.BatchID)
 	assert.Equal(t, BuildStatusUnknown, build.Status)
-	assert.Equal(t, float32(0), build.Score)
 }
 
 func TestBuild_SerializationRoundTrip(t *testing.T) {
@@ -145,10 +143,10 @@ func TestBuild_SerializationRoundTrip(t *testing.T) {
 			build: Build{
 				ID:      "build-100",
 				BatchID: "batch-50",
-				SpeculationPath: SpeculationPathInfo{
+				SpeculationPath: SpeculationPath{
 					Base: []string{"batch-48", "batch-49"},
+					Head: "batch-50",
 				},
-				Score:  0.75,
 				Status: BuildStatusAccepted,
 			},
 		},
@@ -157,19 +155,18 @@ func TestBuild_SerializationRoundTrip(t *testing.T) {
 			build: Build{
 				ID:      "build-200",
 				BatchID: "batch-60",
-				Score:   1.0,
 				Status:  BuildStatusSucceeded,
 			},
 		},
 		{
-			name: "failed build with zero score",
+			name: "failed build",
 			build: Build{
 				ID:      "build-300",
 				BatchID: "batch-70",
-				SpeculationPath: SpeculationPathInfo{
+				SpeculationPath: SpeculationPath{
 					Base: []string{"batch-65"},
+					Head: "batch-70",
 				},
-				Score:  0,
 				Status: BuildStatusFailed,
 			},
 		},

submitqueue/entity/speculation_tree.go

@@ -14,38 +14,151 @@
 
 package entity
 
-// SpeculationPathAction defines the possible actions for a speculation path.
+// SpeculationPath is a single speculation path: an assumed-good prefix of
+// predecessor batches (Base) on top of which the batch under verification
+// (Head) is built and validated.
+//
+// This is the unit the build stage consumes: Base maps to the build runner's
+// base changes (an assumed-good prefix to apply) and Head maps to the changes
+// being validated.
+type SpeculationPath struct {
+	// Base is the ordered list of predecessor batch IDs assumed to have passed.
+	// Empty means the path builds the head batch directly on the target branch.
+	Base []string
+	// Head is the batch ID being verified by this path.
+	Head string
+}
+
+// SpeculationPathStatus is the observed lifecycle state of a speculation path.
+// It is written only by the orchestrator's speculate controller (into the
+// speculation tree store) and read by the decision seams (selector, prioritizer)
+// as input; the seams never write it.
+type SpeculationPathStatus string
+
+const (
+	// SpeculationPathStatusUnknown is the unreachable zero value, set by default
+	// on init. A persisted path always carries a real status (candidate onward),
+	// so this should never be seen in the store.
+	SpeculationPathStatusUnknown SpeculationPathStatus = ""
+	// SpeculationPathStatusCandidate is a freshly enumerated path the controller
+	// has persisted but not yet acted on.
+	SpeculationPathStatusCandidate SpeculationPathStatus = "candidate"
+	// SpeculationPathStatusSelected is a path the selector has promoted — a
+	// per-batch desire to build it — that the queue-wide prioritizer has not yet
+	// cleared. It is not sent to build while Selected: it waits on the build
+	// budget until it is prioritized (or dropped).
+	SpeculationPathStatusSelected SpeculationPathStatus = "selected"
+	// SpeculationPathStatusPrioritized is a path the prioritizer has admitted
+	// under the queue's build budget; it is cleared to run. The build effector
+	// triggers only Prioritized paths.
+	SpeculationPathStatusPrioritized SpeculationPathStatus = "prioritized"
+	// SpeculationPathStatusBuilding is a path a build signal has confirmed is in
+	// flight; its BuildID is known.
+	SpeculationPathStatusBuilding SpeculationPathStatus = "building"
+	// SpeculationPathStatusPassed is a path whose build succeeded.
+	SpeculationPathStatusPassed SpeculationPathStatus = "passed"
+	// SpeculationPathStatusFailed is a path whose build failed.
+	SpeculationPathStatusFailed SpeculationPathStatus = "failed"
+	// SpeculationPathStatusCancelling is a path whose in-flight build the
+	// controller has asked to stop but whose cancellation is not yet confirmed. It
+	// mirrors the batch-level cancelling intent (BatchStateCancelling): the cancel
+	// decision is recorded here and the effector drives it to terminal Cancelled.
+	// A path with no build in flight is dropped straight to Cancelled instead.
+	SpeculationPathStatusCancelling SpeculationPathStatus = "cancelling"
+	// SpeculationPathStatusCancelled is the terminal state for a path that is no
+	// longer pursued — its in-flight build was confirmed stopped, or the path was
+	// abandoned before a build started.
+	SpeculationPathStatusCancelled SpeculationPathStatus = "cancelled"
+)
+
+// SpeculationPathAction is the decision a seam (the selector or the prioritizer)
+// asks the controller to take for a path. It names the decision, not its effect:
+// it is ephemeral (recomputed every time a seam runs) and never persisted. The
+// controller maps it to the corresponding SpeculationPathStatus transition —
+// applied under the tree's optimistic lock (Version) — and records the result;
+// the seams never write status themselves.
 type SpeculationPathAction string
 
 const (
-	// SpeculationPathActionUnknown is the default zero value for SpeculationPathAction.
+	// SpeculationPathActionUnknown is the unreachable zero value. A seam
+	// expresses "leave this path as-is" by omitting the path from its decisions,
+	// not by returning this.
 	SpeculationPathActionUnknown SpeculationPathAction = ""
-	// TODO: Add comprehensive list of actions
+	// SpeculationPathActionPromote asks the controller to advance this path one
+	// stage toward running. The target status depends on which seam decided it:
+	// the selector's promote moves a path to Selected; the prioritizer's promote
+	// moves it to Prioritized (cleared to build).
+	SpeculationPathActionPromote SpeculationPathAction = "promote"
+	// SpeculationPathActionCancel asks the controller to stop pursuing this path:
+	// it moves to Cancelling if a build is in flight (the effector then confirms
+	// the stop and drives it to terminal Cancelled), or straight to Cancelled if
+	// no build has started.
+	SpeculationPathActionCancel SpeculationPathAction = "cancel"
 )
 
-// SpeculationInfo represents metadata about a single speculation path, including the path through the dependency graph, its current state, and the predicted build score.
-type SpeculationInfo struct {
-	// Path represents the speculation path; which is an ordered list of batches.
-	Path []string
-	// Action is a state that this path is in.
-	Action SpeculationPathAction
-	// Score is score for this speculation path.
+// SpeculationPathInfo is the per-path entry in a speculation tree: a path, its
+// latest predicted-success score, its controller-owned status, and a link to
+// the build dispatched for it (if any). Path is immutable once the entry is
+// persisted; Score, Status, and BuildID are updateable, written only by the
+// controller under the tree's Version optimistic lock.
+type SpeculationPathInfo struct {
+	// Path is the Base/Head split this entry covers. Immutable: it identifies
+	// the entry and never changes after the path is first persisted.
+	Path SpeculationPath
+	// Score is the path's predicted-success score. Updateable: it is computed by
+	// the scorer and persisted by the controller, not set at enumeration — the
+	// enumerator produces structure only. It is dynamic: the controller re-runs
+	// the scorer on every respeculate (as dependencies land, dependency builds
+	// pass, or sibling paths fail), so the value tracks the latest state rather
+	// than a figure frozen when the path was first enumerated (~0 until the
+	// first pass).
 	Score float32
+	// Status is the observed lifecycle state of the path. Updateable: written
+	// only by the controller; read by the decision seams (scorer, selector,
+	// prioritizer).
+	Status SpeculationPathStatus
+	// BuildID links this path to its build. Updateable: empty until a build
+	// signal confirms the build and the controller records it (Prioritized ->
+	// Building); the controller never knows the ID at send time.
+	BuildID string
+}
+
+// SpeculationPathDecision is a seam's decision for a single path: the action the
+// controller should take for it. It is the output of both the selector (per
+// batch) and the prioritizer (queue-wide), and is not persisted. A seam returns
+// a decision only for the paths it wants to act on; omitted paths are left
+// as-is.
+type SpeculationPathDecision struct {
+	// Path identifies the speculation path the action applies to.
+	Path SpeculationPath
+	// Action is what the controller should do for the path.
+	Action SpeculationPathAction
 }
 
-// SpeculationTree represents the set of speculation paths constructed for a batch based on its dependency graph.
+// SpeculationTree is the set of candidate speculation paths for a batch, built
+// from its dependency graph. BatchID is immutable; Paths is updateable (the
+// controller overwrites it wholesale on every respeculate), guarded by Version.
 type SpeculationTree struct {
 	// BatchID is the batch for which this speculation tree is constructed.
+	// Immutable: it identifies the tree.
 	BatchID string
-	// Speculations is a list of speculation paths for this batch based on a graph of its
-	// dependents.
+	// Paths is the candidate speculation paths for this batch, derived from a
+	// graph of its dependencies. Each entry's per-path dynamic state (Score,
+	// Status, BuildID) is documented on SpeculationPathInfo.
 	//
 	// For e.g - Consider batches - queueA/batch/1, queueA/batch/2, queueA/batch/3
-	// such that - queueA/batch/2 and queueA/batch/3 depend on queueA/batch/1
+	// such that - queueA/batch/2 and queueA/batch/3 depend on queueA/batch/1.
+	// Each dependent batch gets two paths: build alone, or build on the
+	// assumed-good predecessor. Just after enumeration every path is a candidate:
 	//
-	// Speculations for queueA/batch/1 - [{Path: []string{"queueA/batch/1"}, State: "scheduled", Score: 0.1}]
-	// Speculations for queueA/batch/2 - [{Path: []string{"queueA/batch/2"}, State: "scheduled", Score: 0.9}, {Path: []string{"queueA/batch/1", "queueA/batch/2"}, State: "scheduled", Score: 0.3}]
-	// Speculations for queueA/batch/3 - [{Path: []string{"queueA/batch/3"}, State: "scheduled", Score: 0.9}, {Path: []string{"queueA/batch/1", "queueA/batch/3"}, State: "scheduled", Score: 0.3}]
+	// Paths for queueA/batch/2 - [{Path: {Base: [], Head: "queueA/batch/2"}, Status: "candidate"}, {Path: {Base: ["queueA/batch/1"], Head: "queueA/batch/2"}, Status: "candidate"}]
+	// Paths for queueA/batch/3 - [{Path: {Base: [], Head: "queueA/batch/3"}, Status: "candidate"}, {Path: {Base: ["queueA/batch/1"], Head: "queueA/batch/3"}, Status: "candidate"}]
 	//
-	Speculations []SpeculationInfo
+	Paths []SpeculationPathInfo
+	// Version is the version of the object. It is used for optimistic locking:
+	// updates are conditional on the persisted version matching the caller's
+	// expected version. Versioning starts at 1 and is incremented for each
+	// change to the object; version arithmetic is owned by the controller, the
+	// store performs a pure conditional write.
+	Version int32
 }

submitqueue/extension/storage/mock/speculation_tree_store_mock.go

@@ -48,9 +48,9 @@ func (s *buildStore) Get(ctx context.Context, id string) (ret entity.Build, retE
 	var speculationPathJSON []byte
 
 	err := s.db.QueryRowContext(ctx,
-		"SELECT id, batch_id, speculation_path, score, status FROM build WHERE id = ?",
+		"SELECT id, batch_id, speculation_path, status FROM build WHERE id = ?",
 		id,
-	).Scan(&build.ID, &build.BatchID, &speculationPathJSON, &build.Score, &build.Status)
+	).Scan(&build.ID, &build.BatchID, &speculationPathJSON, &build.Status)
 
 	if errors.Is(err, sql.ErrNoRows) {
 		return entity.Build{}, storage.WrapNotFound(err)
@@ -77,8 +77,8 @@ func (s *buildStore) Create(ctx context.Context, build entity.Build) (retErr err
 	}
 
 	_, err = s.db.ExecContext(ctx,
-		"INSERT INTO build (id, batch_id, speculation_path, score, status) VALUES (?, ?, ?, ?, ?)",
-		build.ID, build.BatchID, speculationPathJSON, build.Score, build.Status,
+		"INSERT INTO build (id, batch_id, speculation_path, status) VALUES (?, ?, ?, ?)",
+		build.ID, build.BatchID, speculationPathJSON, build.Status,
 	)
 	if err != nil {
 		var mysqlErr *mysql.MySQLError

submitqueue/extension/storage/mysql/build_store.go

@@ -2,7 +2,6 @@ CREATE TABLE IF NOT EXISTS build (
     id VARCHAR(255) NOT NULL,
     batch_id VARCHAR(255) NOT NULL,
     speculation_path JSON NOT NULL,
-    score FLOAT NOT NULL,
     status VARCHAR(64) NOT NULL,
     PRIMARY KEY (id)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

submitqueue/extension/storage/mysql/schema/build.sql

@@ -1,5 +1,6 @@
 CREATE TABLE IF NOT EXISTS speculation_tree (
     batch_id VARCHAR(255) NOT NULL,
-    speculations JSON NOT NULL,
+    paths JSON NOT NULL,
+    version INT NOT NULL,
     PRIMARY KEY (batch_id)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;