From 1f156142e4b29b2c6d63037d6ccc57cbd4bd2553 Mon Sep 17 00:00:00 2001 From: joshua-spacetime Date: Fri, 23 Jan 2026 14:32:04 -0800 Subject: [PATCH 1/3] [2.0 Breaking]: Remove sender field from [Reducer|View|Procedure]Context (#4101) # Description of Changes Makes the `sender` field on `ViewContext` private and exposes a `sender()` method. Does the same for `ReducerContext` and `ProcedureContext`. The purpose of this change: So that the host can determine if/when a view invokes or reads the `sender`. Currently, because `sender` is a field, the host assumes that it is always read. This means views must be materialized per client, even if the view doesn't actually depend on `sender`, resulting in data duplication. The initial solution for this problem was `AnonymousViewContext` which doesn't have a `sender` field. The better solution is to make `sender` a method so that it can call into the host and record when it's actually invoked. Note, this patch only updates the module API, so the current implementation does not change. `ViewContext` views are still duplicated across clients. Changing this requires a new host syscall and for `sender()` to invoke that syscall. This however is backwards compatible and can be done anytime after the module APIs for the other languages (C#, TypeScript, C++) are updated. Also note that `ReducerContext` and `ProcedureContext` were updated purely for consistency. There are currently no plans to track reads of `sender` in these contexts. # API and ABI breaking changes Breaks the rust module api. # Expected complexity level and risk 1 # Testing N/A --- .../test-react-router-app/server/src/lib.rs | 12 +- crates/bindings/README.md | 2 +- crates/bindings/src/lib.rs | 30 +- demo/Blackholio/server-rust/src/lib.rs | 28 +- .../00400-key-architecture.md | 2 +- .../00300-tutorials/00100-chat-app.md | 12 +- .../00300-unity-tutorial/00300-part-2.md | 4 +- .../00300-unity-tutorial/00400-part-3.md | 8 +- .../00300-unity-tutorial/00500-part-4.md | 4 +- .../00400-unreal-tutorial/00300-part-2.md | 4 +- .../00400-unreal-tutorial/00400-part-3.md | 14 +- .../00400-unreal-tutorial/00500-part-4.md | 4 +- .../00100-databases/00500-cheat-sheet.md | 4 +- .../00300-incremental-migrations.md | 16 +- .../00300-reducers/00400-reducer-context.md | 4 +- .../00300-reducers/00500-lifecycle.md | 8 +- .../00200-functions/00500-views.md | 12 +- docs/docs/00200-core-concepts/00300-tables.md | 2 +- .../00300-tables/00210-file-storage.md | 2 +- .../00300-tables/00400-access-permissions.md | 10 +- .../00100-how-to/00300-logging.md | 4 +- docs/llms/docs-benchmark-analysis.md | 472 ++--- docs/llms/docs-benchmark-comment.md | 16 +- docs/llms/docs-benchmark-details.json | 1793 ++++++++--------- docs/llms/docs-benchmark-summary.json | 52 +- modules/module-test/src/lib.rs | 6 +- .../sdk-test-connect-disconnect/src/lib.rs | 4 +- modules/sdk-test-view/src/lib.rs | 10 +- modules/sdk-test/src/lib.rs | 10 +- smoketests/tests/rls.py | 4 +- smoketests/tests/views.py | 4 +- smoketests/tests/zz_docker.py | 4 +- .../chat-console-rs/spacetimedb/src/lib.rs | 16 +- 33 files changed, 1250 insertions(+), 1327 deletions(-) diff --git a/crates/bindings-typescript/test-react-router-app/server/src/lib.rs b/crates/bindings-typescript/test-react-router-app/server/src/lib.rs index 520e68eeaec..496acfd090f 100644 --- a/crates/bindings-typescript/test-react-router-app/server/src/lib.rs +++ b/crates/bindings-typescript/test-react-router-app/server/src/lib.rs @@ -22,23 +22,23 @@ fn init(ctx: &ReducerContext) { #[reducer(client_connected)] fn client_connected(ctx: &ReducerContext) { - let existing_user = ctx.db.offline_user().identity().find(ctx.sender); + let existing_user = ctx.db.offline_user().identity().find(ctx.sender()); if let Some(user) = existing_user { ctx.db.user().insert(user); - ctx.db.offline_user().identity().delete(ctx.sender); + ctx.db.offline_user().identity().delete(ctx.sender()); return; } ctx.db.offline_user().insert(User { - identity: ctx.sender, + identity: ctx.sender(), has_incremented_count: 0, }); } #[reducer(client_disconnected)] fn client_disconnected(ctx: &ReducerContext) -> Result<(), String> { - let existing_user = ctx.db.user().identity().find(ctx.sender).ok_or("User not found")?; + let existing_user = ctx.db.user().identity().find(ctx.sender()).ok_or("User not found")?; ctx.db.offline_user().insert(existing_user); - ctx.db.user().identity().delete(ctx.sender); + ctx.db.user().identity().delete(ctx.sender()); Ok(()) } @@ -48,7 +48,7 @@ fn increment_counter(ctx: &ReducerContext) -> Result<(), String> { counter.count += 1; ctx.db.counter().id().update(counter); - let mut user = ctx.db.user().identity().find(ctx.sender).ok_or("User not found")?; + let mut user = ctx.db.user().identity().find(ctx.sender()).ok_or("User not found")?; user.has_incremented_count += 1; ctx.db.user().identity().update(user); diff --git a/crates/bindings/README.md b/crates/bindings/README.md index a02a8ff0328..6374d9f79e9 100644 --- a/crates/bindings/README.md +++ b/crates/bindings/README.md @@ -608,7 +608,7 @@ struct PlayerAndLevel { // At-most-one row: return Option #[view(name = my_player, public)] fn my_player(ctx: &ViewContext) -> Option { - ctx.db.player().identity().find(ctx.sender) + ctx.db.player().identity().find(ctx.sender()) } // Multiple rows: return Vec diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index b8f59220776..ebe5f7075b3 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -662,7 +662,7 @@ pub use spacetimedb_bindings_macro::table; /// /// #[reducer] /// fn scheduled(ctx: &ReducerContext, args: ScheduledArgs) -> Result<(), String> { -/// if ctx.sender != ctx.identity() { +/// if ctx.sender() != ctx.identity() { /// return Err("Reducer `scheduled` may not be invoked by clients, only via scheduling.".into()); /// } /// // Reducer body... @@ -707,7 +707,7 @@ pub use spacetimedb_bindings_macro::reducer; /// #[procedure] /// fn return_value(ctx: &mut ProcedureContext, arg: MyArgument) -> MyReturnValue { /// MyReturnValue { -/// a: format!("Hello, {}", ctx.sender), +/// a: format!("Hello, {}", ctx.sender()), /// b: ctx.timestamp, /// } /// } @@ -816,13 +816,13 @@ pub use spacetimedb_bindings_macro::procedure; /// // A view that selects at most one row from a table /// #[view(name = my_player, public)] /// fn my_player(ctx: &ViewContext) -> Option { -/// ctx.db.player().identity().find(ctx.sender) +/// ctx.db.player().identity().find(ctx.sender()) /// } /// /// // An example of column projection /// #[view(name = my_player_id, public)] /// fn my_player_id(ctx: &ViewContext) -> Option { -/// ctx.db.player().identity().find(ctx.sender).map(|Player { id, .. }| PlayerId { id }) +/// ctx.db.player().identity().find(ctx.sender()).map(|Player { id, .. }| PlayerId { id }) /// } /// /// // An example that is analogous to a semijoin in sql @@ -894,7 +894,7 @@ impl Default for AnonymousViewContext { /// The other is [`AnonymousViewContext`]. /// Use this type if the view depends on the caller's identity. pub struct ViewContext { - pub sender: Identity, + sender: Identity, pub db: LocalReadOnly, pub from: QueryBuilder, } @@ -907,6 +907,11 @@ impl ViewContext { from: QueryBuilder {}, } } + + /// The `Identity` of the client that invoked the view. + pub fn sender(&self) -> Identity { + self.sender + } } /// The context that any reducer is provided with. @@ -924,7 +929,7 @@ impl ViewContext { #[non_exhaustive] pub struct ReducerContext { /// The `Identity` of the client that invoked the reducer. - pub sender: Identity, + sender: Identity, /// The time at which the reducer was started. pub timestamp: Timestamp, @@ -1014,6 +1019,11 @@ impl ReducerContext { } } + /// The `Identity` of the client that invoked the reducer. + pub fn sender(&self) -> Identity { + self.sender + } + /// Returns the authorization information for the caller of this reducer. pub fn sender_auth(&self) -> &AuthCtx { &self.sender_auth @@ -1124,7 +1134,7 @@ impl Deref for TxContext { #[cfg(feature = "unstable")] pub struct ProcedureContext { /// The `Identity` of the client that invoked the procedure. - pub sender: Identity, + sender: Identity, /// The time at which the procedure was started. pub timestamp: Timestamp, @@ -1162,6 +1172,12 @@ impl ProcedureContext { counter_uuid: Cell::new(0), } } + + /// The `Identity` of the client that invoked the procedure. + pub fn sender(&self) -> Identity { + self.sender + } + /// Read the current module's [`Identity`]. pub fn identity(&self) -> Identity { // Hypothetically, we *could* read the module identity out of the system tables. diff --git a/demo/Blackholio/server-rust/src/lib.rs b/demo/Blackholio/server-rust/src/lib.rs index 08a70a7a767..f12d554cf3f 100644 --- a/demo/Blackholio/server-rust/src/lib.rs +++ b/demo/Blackholio/server-rust/src/lib.rs @@ -143,7 +143,7 @@ pub fn init(ctx: &ReducerContext) -> Result<(), String> { #[spacetimedb::reducer(client_connected)] pub fn connect(ctx: &ReducerContext) -> Result<(), String> { - if let Some(player) = ctx.db.logged_out_player().identity().find(&ctx.sender) { + if let Some(player) = ctx.db.logged_out_player().identity().find(&ctx.sender()) { ctx.db.player().insert(player.clone()); ctx.db.logged_out_player().identity().delete(&player.identity); @@ -157,7 +157,7 @@ pub fn connect(ctx: &ReducerContext) -> Result<(), String> { } } else { ctx.db.player().try_insert(Player { - identity: ctx.sender, + identity: ctx.sender(), player_id: 0, name: String::new(), })?; @@ -167,10 +167,15 @@ pub fn connect(ctx: &ReducerContext) -> Result<(), String> { #[spacetimedb::reducer(client_disconnected)] pub fn disconnect(ctx: &ReducerContext) -> Result<(), String> { - let player = ctx.db.player().identity().find(&ctx.sender).ok_or("Player not found")?; + let player = ctx + .db + .player() + .identity() + .find(&ctx.sender()) + .ok_or("Player not found")?; let player_id = player.player_id; ctx.db.logged_out_player().insert(player); - ctx.db.player().identity().delete(&ctx.sender); + ctx.db.player().identity().delete(&ctx.sender()); // Move any circles from the arena into logged out tables for circle in ctx.db.circle().player_id().filter(&player_id) { @@ -187,7 +192,7 @@ pub fn disconnect(ctx: &ReducerContext) -> Result<(), String> { #[spacetimedb::reducer] pub fn enter_game(ctx: &ReducerContext, name: String) -> Result<(), String> { log::info!("Creating player with name {}", name); - let mut player: Player = ctx.db.player().identity().find(ctx.sender).ok_or("")?; + let mut player: Player = ctx.db.player().identity().find(ctx.sender()).ok_or("")?; let player_id = player.player_id; player.name = name; ctx.db.player().identity().update(player); @@ -234,7 +239,7 @@ pub fn respawn(ctx: &ReducerContext) -> Result<(), String> { .db .player() .identity() - .find(&ctx.sender) + .find(&ctx.sender()) .ok_or("No such player found")?; spawn_player_initial_circle(ctx, player.player_id)?; @@ -248,7 +253,7 @@ pub fn suicide(ctx: &ReducerContext) -> Result<(), String> { .db .player() .identity() - .find(&ctx.sender) + .find(&ctx.sender()) .ok_or("No such player found")?; for circle in ctx.db.circle().player_id().filter(&player.player_id) { @@ -260,7 +265,12 @@ pub fn suicide(ctx: &ReducerContext) -> Result<(), String> { #[spacetimedb::reducer] pub fn update_player_input(ctx: &ReducerContext, direction: DbVector2) -> Result<(), String> { - let player = ctx.db.player().identity().find(&ctx.sender).ok_or("Player not found")?; + let player = ctx + .db + .player() + .identity() + .find(&ctx.sender()) + .ok_or("Player not found")?; for mut circle in ctx.db.circle().player_id().filter(&player.player_id) { circle.direction = direction.normalized(); circle.speed = direction.magnitude().clamp(0.0, 1.0); @@ -467,7 +477,7 @@ pub fn player_split(ctx: &ReducerContext) -> Result<(), String> { .db .player() .identity() - .find(&ctx.sender) + .find(&ctx.sender()) .ok_or("Sender has no player")?; let circles: Vec = ctx.db.circle().player_id().filter(&player.player_id).collect(); let mut circle_count = circles.len() as i32; diff --git a/docs/docs/00100-intro/00100-getting-started/00400-key-architecture.md b/docs/docs/00100-intro/00100-getting-started/00400-key-architecture.md index 46d77df1ba5..ce1c8a7c322 100644 --- a/docs/docs/00100-intro/00100-getting-started/00400-key-architecture.md +++ b/docs/docs/00100-intro/00100-getting-started/00400-key-architecture.md @@ -451,7 +451,7 @@ A view can be written in Rust like so: ```rust #[spacetimedb::view(name = my_player, public)] fn my_player(ctx: &spacetimedb::ViewContext) -> Option { - ctx.db.player().identity().find(ctx.sender) + ctx.db.player().identity().find(ctx.sender()) } ``` diff --git a/docs/docs/00100-intro/00300-tutorials/00100-chat-app.md b/docs/docs/00100-intro/00300-tutorials/00100-chat-app.md index d1e6b2c3aae..06ef0c039ce 100644 --- a/docs/docs/00100-intro/00300-tutorials/00100-chat-app.md +++ b/docs/docs/00100-intro/00300-tutorials/00100-chat-app.md @@ -348,7 +348,7 @@ Add to `spacetimedb/src/lib.rs`: #[reducer] pub fn set_name(ctx: &ReducerContext, name: String) -> Result<(), String> { let name = validate_name(name)?; - if let Some(user) = ctx.db.user().identity().find(ctx.sender) { + if let Some(user) = ctx.db.user().identity().find(ctx.sender()) { ctx.db.user().identity().update(User { name: Some(name), ..user }); Ok(()) } else { @@ -439,7 +439,7 @@ pub fn send_message(ctx: &ReducerContext, text: String) -> Result<(), String> { let text = validate_message(text)?; log::info!("{}", text); ctx.db.message().insert(Message { - sender: ctx.sender, + sender: ctx.sender(), text, sent: ctx.timestamp, }); @@ -547,12 +547,12 @@ Add to `spacetimedb/src/lib.rs`: ```rust server #[reducer(client_connected)] pub fn client_connected(ctx: &ReducerContext) { - if let Some(user) = ctx.db.user().identity().find(ctx.sender) { + if let Some(user) = ctx.db.user().identity().find(ctx.sender()) { ctx.db.user().identity().update(User { online: true, ..user }); } else { ctx.db.user().insert(User { name: None, - identity: ctx.sender, + identity: ctx.sender(), online: true, }); } @@ -560,10 +560,10 @@ pub fn client_connected(ctx: &ReducerContext) { #[reducer(client_disconnected)] pub fn identity_disconnected(ctx: &ReducerContext) { - if let Some(user) = ctx.db.user().identity().find(ctx.sender) { + if let Some(user) = ctx.db.user().identity().find(ctx.sender()) { ctx.db.user().identity().update(User { online: false, ..user }); } else { - log::warn!("Disconnect event for unknown user with identity {:?}", ctx.sender); + log::warn!("Disconnect event for unknown user with identity {:?}", ctx.sender()); } } ``` diff --git a/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00300-part-2.md b/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00300-part-2.md index 9a5f26694ec..9f06001aa8a 100644 --- a/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00300-part-2.md +++ b/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00300-part-2.md @@ -345,7 +345,7 @@ public static void Debug(ReducerContext ctx) ```rust #[spacetimedb::reducer] pub fn debug(ctx: &ReducerContext) -> Result<(), String> { - log::debug!("This reducer was called by {}.", ctx.sender); + log::debug!("This reducer was called by {}.", ctx.sender()); Ok(()) } ``` @@ -463,7 +463,7 @@ Next let's connect our client to our database. Let's start by modifying our `deb ```rust #[spacetimedb::reducer(client_connected)] pub fn connect(ctx: &ReducerContext) -> Result<(), String> { - log::debug!("{} just connected.", ctx.sender); + log::debug!("{} just connected.", ctx.sender()); Ok(()) } ``` diff --git a/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00400-part-3.md b/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00400-part-3.md index a00681a6c49..ab95e71294d 100644 --- a/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00400-part-3.md +++ b/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00400-part-3.md @@ -391,7 +391,7 @@ Next, modify your `connect` reducer and add a new `disconnect` reducer below it: ```rust #[spacetimedb::reducer(client_connected)] pub fn connect(ctx: &ReducerContext) -> Result<(), String> { - if let Some(player) = ctx.db.logged_out_player().identity().find(&ctx.sender) { + if let Some(player) = ctx.db.logged_out_player().identity().find(&ctx.sender()) { ctx.db.player().insert(player.clone()); ctx.db .logged_out_player() @@ -399,7 +399,7 @@ pub fn connect(ctx: &ReducerContext) -> Result<(), String> { .delete(&player.identity); } else { ctx.db.player().try_insert(Player { - identity: ctx.sender, + identity: ctx.sender(), player_id: 0, name: String::new(), })?; @@ -413,11 +413,11 @@ pub fn disconnect(ctx: &ReducerContext) -> Result<(), String> { .db .player() .identity() - .find(&ctx.sender) + .find(&ctx.sender()) .ok_or("Player not found")?; let player_id = player.player_id; ctx.db.logged_out_player().insert(player); - ctx.db.player().identity().delete(&ctx.sender); + ctx.db.player().identity().delete(&ctx.sender()); Ok(()) } diff --git a/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00500-part-4.md b/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00500-part-4.md index f2ad998a52b..45ad7ee8ea8 100644 --- a/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00500-part-4.md +++ b/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00500-part-4.md @@ -207,7 +207,7 @@ pub fn update_player_input(ctx: &ReducerContext, direction: DbVector2) -> Result .db .player() .identity() - .find(&ctx.sender) + .find(&ctx.sender()) .ok_or("Player not found")?; for mut circle in ctx.db.circle().player_id().filter(&player.player_id) { circle.direction = direction.normalized(); @@ -218,7 +218,7 @@ pub fn update_player_input(ctx: &ReducerContext, direction: DbVector2) -> Result } ``` -This is a simple reducer that takes the movement input from the client and applies them to all circles that that player controls. Note that it is not possible for a player to move another player's circles using this reducer, because the `ctx.sender` value is not set by the client. Instead `ctx.sender` is set by SpacetimeDB after it has authenticated that sender. You can rest assured that the caller has been authenticated as that player by the time this reducer is called. +This is a simple reducer that takes the movement input from the client and applies them to all circles that that player controls. Note that it is not possible for a player to move another player's circles using this reducer, because the `ctx.sender()` value is not set by the client. Instead `ctx.sender()` is set by SpacetimeDB after it has authenticated that sender. You can rest assured that the caller has been authenticated as that player by the time this reducer is called. diff --git a/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00300-part-2.md b/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00300-part-2.md index 0502ab9f87f..bde4659ba26 100644 --- a/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00300-part-2.md +++ b/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00300-part-2.md @@ -343,7 +343,7 @@ public static void Debug(ReducerContext ctx) ```rust #[spacetimedb::reducer] pub fn debug(ctx: &ReducerContext) -> Result<(), String> { - log::debug!("This reducer was called by {}.", ctx.sender); + log::debug!("This reducer was called by {}.", ctx.sender()); Ok(()) } ``` @@ -459,7 +459,7 @@ Next let's connect our client to our database. Let's start by modifying our `deb ```rust #[spacetimedb::reducer(client_connected)] pub fn connect(ctx: &ReducerContext) -> Result<(), String> { - log::debug!("{} just connected.", ctx.sender); + log::debug!("{} just connected.", ctx.sender()); Ok(()) } ``` diff --git a/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00400-part-3.md b/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00400-part-3.md index 859ab5157e9..192c7d3ebc8 100644 --- a/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00400-part-3.md +++ b/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00400-part-3.md @@ -384,7 +384,7 @@ Next, modify your `connect` reducer and add a new `disconnect` reducer below it: ```rust #[spacetimedb::reducer(client_connected)] pub fn connect(ctx: &ReducerContext) -> Result<(), String> { - if let Some(player) = ctx.db.logged_out_player().identity().find(&ctx.sender) { + if let Some(player) = ctx.db.logged_out_player().identity().find(&ctx.sender()) { ctx.db.player().insert(player.clone()); ctx.db .logged_out_player() @@ -392,7 +392,7 @@ pub fn connect(ctx: &ReducerContext) -> Result<(), String> { .delete(&player.identity); } else { ctx.db.player().try_insert(Player { - identity: ctx.sender, + identity: ctx.sender(), player_id: 0, name: String::new(), })?; @@ -406,11 +406,11 @@ pub fn disconnect(ctx: &ReducerContext) -> Result<(), String> { .db .player() .identity() - .find(&ctx.sender) + .find(&ctx.sender()) .ok_or("Player not found")?; let player_id = player.player_id; ctx.db.logged_out_player().insert(player); - ctx.db.player().identity().delete(&ctx.sender); + ctx.db.player().identity().delete(&ctx.sender()); Ok(()) } @@ -499,7 +499,7 @@ const START_PLAYER_MASS: i32 = 15; #[spacetimedb::reducer] pub fn enter_game(ctx: &ReducerContext, name: String) -> Result<(), String> { log::info!("Creating player with name {}", name); - let mut player: Player = ctx.db.player().identity().find(ctx.sender).ok_or("")?; + let mut player: Player = ctx.db.player().identity().find(ctx.sender()).ok_or("")?; let player_id = player.player_id; player.name = name; ctx.db.player().identity().update(player); @@ -589,11 +589,11 @@ pub fn disconnect(ctx: &ReducerContext) -> Result<(), String> { .db .player() .identity() - .find(&ctx.sender) + .find(&ctx.sender()) .ok_or("Player not found")?; let player_id = player.player_id; ctx.db.logged_out_player().insert(player); - ctx.db.player().identity().delete(&ctx.sender); + ctx.db.player().identity().delete(&ctx.sender()); // Remove any circles from the arena for circle in ctx.db.circle().player_id().filter(&player_id) { diff --git a/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00500-part-4.md b/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00500-part-4.md index 643256545c2..ae8eb620633 100644 --- a/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00500-part-4.md +++ b/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00500-part-4.md @@ -207,7 +207,7 @@ pub fn update_player_input(ctx: &ReducerContext, direction: DbVector2) -> Result .db .player() .identity() - .find(&ctx.sender) + .find(&ctx.sender()) .ok_or("Player not found")?; for mut circle in ctx.db.circle().player_id().filter(&player.player_id) { circle.direction = direction.normalized(); @@ -218,7 +218,7 @@ pub fn update_player_input(ctx: &ReducerContext, direction: DbVector2) -> Result } ``` -This is a simple reducer that takes the movement input from the client and applies them to all circles that that player controls. Note that it is not possible for a player to move another player's circles using this reducer, because the `ctx.sender` value is not set by the client. Instead `ctx.sender` is set by SpacetimeDB after it has authenticated that sender. You can rest assured that the caller has been authenticated as that player by the time this reducer is called. +This is a simple reducer that takes the movement input from the client and applies them to all circles that that player controls. Note that it is not possible for a player to move another player's circles using this reducer, because the `ctx.sender()` value is not set by the client. Instead `ctx.sender()` is set by SpacetimeDB after it has authenticated that sender. You can rest assured that the caller has been authenticated as that player by the time this reducer is called. diff --git a/docs/docs/00200-core-concepts/00100-databases/00500-cheat-sheet.md b/docs/docs/00200-core-concepts/00100-databases/00500-cheat-sheet.md index 5377a63cdd9..b49afcd4c0b 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00500-cheat-sheet.md +++ b/docs/docs/00200-core-concepts/00100-databases/00500-cheat-sheet.md @@ -460,7 +460,7 @@ use spacetimedb::{view, ViewContext}; // Return single row #[view(name = my_player, public)] fn my_player(ctx: &ViewContext) -> Option { - ctx.db.player().identity().find(ctx.sender) + ctx.db.player().identity().find(ctx.sender()) } // Return multiple rows @@ -505,7 +505,7 @@ ctx.Rng // Random number generator ```rust ctx.db // Database access -ctx.sender // Identity of caller +ctx.sender() // Identity of caller ctx.connection_id // Option ctx.timestamp // Timestamp ctx.identity() // Module's identity diff --git a/docs/docs/00200-core-concepts/00100-databases/00500-migrations/00300-incremental-migrations.md b/docs/docs/00200-core-concepts/00100-databases/00500-migrations/00300-incremental-migrations.md index e8a2bcdd63c..692df88e501 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00500-migrations/00300-incremental-migrations.md +++ b/docs/docs/00200-core-concepts/00100-databases/00500-migrations/00300-incremental-migrations.md @@ -46,7 +46,7 @@ fn create_character(ctx: &ReducerContext, class: Class, nickname: String) { "Creating new level 1 {class:?} named {nickname}", ); ctx.db.character().insert(Character { - player_id: ctx.sender, + player_id: ctx.sender(), nickname, level: 1, class, @@ -57,7 +57,7 @@ fn find_character_for_player(ctx: &ReducerContext) -> Character { ctx.db .character() .player_id() - .find(ctx.sender) + .find(ctx.sender()) .expect("Player has not created a character") } @@ -154,7 +154,7 @@ fn choose_alliance(ctx: &ReducerContext, alliance: Alliance) { "Setting {}'s alliance to {:?} for player {}", character.nickname, alliance, - ctx.sender, + ctx.sender(), ); update_character( ctx, @@ -194,18 +194,18 @@ When a new player creates a character, we'll make rows in both tables for them. fn create_character(ctx: &ReducerContext, class: Class, nickname: String) { log::info!( "Creating new level 1 {class:?} named {nickname} for player {}", - ctx.sender, + ctx.sender(), ); ctx.db.character().insert(Character { - player_id: ctx.sender, + player_id: ctx.sender(), nickname: nickname.clone(), level: 1, class, }); ctx.db.character_v2().insert(CharacterV2 { - player_id: ctx.sender, + player_id: ctx.sender(), nickname, level: 1, class, @@ -218,7 +218,7 @@ We'll update our helper functions so that they operate on `character_v2` rows. I ```rust fn find_character_for_player(ctx: &ReducerContext) -> CharacterV2 { - if let Some(character) = ctx.db.character_v2().player_id().find(ctx.sender) { + if let Some(character) = ctx.db.character_v2().player_id().find(ctx.sender()) { // Already migrated; just return the new player. return character; } @@ -228,7 +228,7 @@ fn find_character_for_player(ctx: &ReducerContext) -> CharacterV2 { .db .character() .player_id() - .find(ctx.sender) + .find(ctx.sender()) .expect("Player has not created a character"); ctx.db.character_v2().insert(CharacterV2 { diff --git a/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00400-reducer-context.md b/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00400-reducer-context.md index 32b96e5ed33..38f447125d4 100644 --- a/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00400-reducer-context.md +++ b/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00400-reducer-context.md @@ -174,7 +174,7 @@ pub struct Player { #[reducer] fn update_score(ctx: &ReducerContext, new_score: u32) { // Get the caller's identity - let caller = ctx.sender; + let caller = ctx.sender(); // Find and update their player record if let Some(mut player) = ctx.db.player().identity().find(caller) { @@ -290,7 +290,7 @@ pub struct ScheduledTask { #[reducer] fn send_reminder(ctx: &ReducerContext, task: ScheduledTask) { // Only allow the scheduler (module identity) to call this - if ctx.sender != ctx.identity() { + if ctx.sender() != ctx.identity() { panic!("This reducer can only be called by the scheduler"); } diff --git a/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00500-lifecycle.md b/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00500-lifecycle.md index 975f589b437..20f43bc8394 100644 --- a/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00500-lifecycle.md +++ b/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00500-lifecycle.md @@ -131,7 +131,7 @@ public static void OnConnect(ReducerContext ctx) ```rust #[reducer(client_connected)] pub fn on_connect(ctx: &ReducerContext) -> Result<(), String> { - log::info!("Client connected: {}", ctx.sender); + log::info!("Client connected: {}", ctx.sender()); // ctx.connection_id is guaranteed to be Some(...) let conn_id = ctx.connection_id.unwrap(); @@ -139,7 +139,7 @@ pub fn on_connect(ctx: &ReducerContext) -> Result<(), String> { // Initialize client session ctx.db.sessions().try_insert(Session { connection_id: conn_id, - identity: ctx.sender, + identity: ctx.sender(), connected_at: ctx.timestamp, })?; @@ -198,7 +198,7 @@ public static void OnDisconnect(ReducerContext ctx) ```rust #[reducer(client_disconnected)] pub fn on_disconnect(ctx: &ReducerContext) -> Result<(), String> { - log::info!("Client disconnected: {}", ctx.sender); + log::info!("Client disconnected: {}", ctx.sender()); // ctx.connection_id is guaranteed to be Some(...) let conn_id = ctx.connection_id.unwrap(); @@ -230,6 +230,6 @@ Reducers can be triggered at specific times using schedule tables. See [Schedule :::info Scheduled Reducer Context Scheduled reducer calls originate from SpacetimeDB itself, not from a client. Therefore: -- `ctx.sender` will be the module's own identity +- `ctx.sender()` will be the module's own identity - `ctx.connection_id` will be `None`/`null`/`undefined` ::: diff --git a/docs/docs/00200-core-concepts/00200-functions/00500-views.md b/docs/docs/00200-core-concepts/00200-functions/00500-views.md index 97ec7e2077f..0603fa97dad 100644 --- a/docs/docs/00200-core-concepts/00200-functions/00500-views.md +++ b/docs/docs/00200-core-concepts/00200-functions/00500-views.md @@ -55,7 +55,7 @@ spacetimedb.view( { name: 'my_player', public: true }, t.option(players.rowType), (ctx) => { - const row = ctx.db.players.identity.find(ctx.sender); + const row = ctx.db.players.identity.find(ctx.sender()); return row ?? undefined; } ); @@ -197,7 +197,7 @@ pub struct PlayerAndLevel { // At-most-one row: return Option #[view(name = my_player, public)] fn my_player(ctx: &ViewContext) -> Option { - ctx.db.player().identity().find(ctx.sender) + ctx.db.player().identity().find(ctx.sender()) } // Multiple rows: return Vec @@ -232,7 +232,7 @@ Views can return either `Option` for at-most-one row or `Vec` for multiple Views use one of two context types: -- **`ViewContext`**: Provides access to the caller's `Identity` through `ctx.sender`. Use this when the view depends on who is querying it. +- **`ViewContext`**: Provides access to the caller's `Identity` through `ctx.sender()`. Use this when the view depends on who is querying it. - **`AnonymousViewContext`**: Does not provide caller information. Use this when the view produces the same results regardless of who queries it. Both contexts provide read-only access to tables and indexes through `ctx.db`. @@ -243,7 +243,7 @@ The choice between `ViewContext` and `AnonymousViewContext` has significant perf **Anonymous views can be shared across all subscribers.** When a view uses `AnonymousViewContext`, SpacetimeDB knows the result is the same for every client. The database can materialize the view once and serve that same result to all subscribers. When the underlying data changes, it recomputes the view once and broadcasts the update to everyone. -**Per-user views require separate computation for each subscriber.** When a view uses `ViewContext`, the result depends on `ctx.sender`, so each client potentially sees different data. SpacetimeDB must compute and track the view separately for each subscriber. With 1,000 connected users, that's 1,000 separate view computations and 1,000 separate sets of change tracking. +**Per-user views require separate computation for each subscriber.** When a view uses `ViewContext` and invokes `ctx.sender()`, each client potentially sees different data. SpacetimeDB must compute and track the view separately for each subscriber. With 1,000 connected users, that's 1,000 separate view computations and 1,000 separate sets of change tracking. **Prefer `AnonymousViewContext` when possible.** Design your views to be caller-independent when the use case allows. For example: @@ -294,7 +294,7 @@ public static Player? MyPlayer(ViewContext ctx) // Per-user: each client sees their own player #[view(name = my_player, public)] fn my_player(ctx: &ViewContext) -> Option { - ctx.db.player().identity().find(&ctx.sender) + ctx.db.player().identity().find(&ctx.sender()) } ``` @@ -541,7 +541,7 @@ fn entities_in_origin_chunk(ctx: &AnonymousViewContext) -> Vec { // Per-user: returns entities in the chunk the player is currently in #[view(name = entities_in_my_chunk, public)] fn entities_in_my_chunk(ctx: &ViewContext) -> Vec { - let Some(player) = ctx.db.player().identity().find(&ctx.sender) else { + let Some(player) = ctx.db.player().identity().find(&ctx.sender()) else { return vec![]; }; let Some(chunk) = ctx.db.player_chunk().player_id().find(&player.id) else { diff --git a/docs/docs/00200-core-concepts/00300-tables.md b/docs/docs/00200-core-concepts/00300-tables.md index c9a3405a2bc..cb241bc1ae1 100644 --- a/docs/docs/00200-core-concepts/00300-tables.md +++ b/docs/docs/00200-core-concepts/00300-tables.md @@ -378,7 +378,7 @@ ctx.db.player().insert(Player { /* ... */ }); ctx.db.logged_out_player().insert(Player { /* ... */ }); // Move a row between tables -if let Some(player) = ctx.db.logged_out_player().identity().find(&ctx.sender) { +if let Some(player) = ctx.db.logged_out_player().identity().find(&ctx.sender()) { ctx.db.player().insert(player.clone()); ctx.db.logged_out_player().identity().delete(&player.identity); } diff --git a/docs/docs/00200-core-concepts/00300-tables/00210-file-storage.md b/docs/docs/00200-core-concepts/00300-tables/00210-file-storage.md index 0925140db83..0a102b8377f 100644 --- a/docs/docs/00200-core-concepts/00300-tables/00210-file-storage.md +++ b/docs/docs/00200-core-concepts/00300-tables/00210-file-storage.md @@ -263,7 +263,7 @@ pub fn register_document( ) { ctx.db.document().insert(Document { id: 0, // auto-increment - owner_id: ctx.sender, + owner_id: ctx.sender(), filename, mime_type, size_bytes, diff --git a/docs/docs/00200-core-concepts/00300-tables/00400-access-permissions.md b/docs/docs/00200-core-concepts/00300-tables/00400-access-permissions.md index 659ddca3cbc..27ae27f34a6 100644 --- a/docs/docs/00200-core-concepts/00300-tables/00400-access-permissions.md +++ b/docs/docs/00200-core-concepts/00300-tables/00400-access-permissions.md @@ -309,7 +309,7 @@ Views can only access table data through indexed lookups, not by scanning all ro ### Filtering Rows by Caller -Use views with `ViewContext` to return only the rows that belong to the caller. The view accesses the caller's identity through `ctx.sender` and uses it to look up rows via an index. +Use views with `ViewContext` to return only the rows that belong to the caller. The view accesses the caller's identity through `ctx.sender()` and uses it to look up rows via an index. @@ -405,8 +405,8 @@ pub struct Message { #[spacetimedb::view(name = my_messages, public)] fn my_messages(ctx: &ViewContext) -> Vec { // Look up messages by index where caller is sender or recipient - let sent: Vec<_> = ctx.db.message().sender().filter(&ctx.sender).collect(); - let received: Vec<_> = ctx.db.message().recipient().filter(&ctx.sender).collect(); + let sent: Vec<_> = ctx.db.message().sender().filter(&ctx.sender()).collect(); + let received: Vec<_> = ctx.db.message().recipient().filter(&ctx.sender()).collect(); sent.into_iter().chain(received).collect() } ``` @@ -553,7 +553,7 @@ pub struct PublicUserProfile { #[spacetimedb::view(name = my_profile, public)] fn my_profile(ctx: &ViewContext) -> Option { // Look up the caller's account by their identity (unique index) - let user = ctx.db.user_account().identity().find(&ctx.sender)?; + let user = ctx.db.user_account().identity().find(&ctx.sender())?; Some(PublicUserProfile { id: user.id, username: user.username, @@ -713,7 +713,7 @@ pub struct TeamMember { #[spacetimedb::view(name = my_team, public)] fn my_team(ctx: &ViewContext) -> Vec { // Find the caller's employee record by identity (unique index) - let Some(me) = ctx.db.employee().identity().find(&ctx.sender) else { + let Some(me) = ctx.db.employee().identity().find(&ctx.sender()) else { return vec![]; }; diff --git a/docs/docs/00300-resources/00100-how-to/00300-logging.md b/docs/docs/00300-resources/00100-how-to/00300-logging.md index 83dd011566f..1958b01634e 100644 --- a/docs/docs/00300-resources/00100-how-to/00300-logging.md +++ b/docs/docs/00300-resources/00100-how-to/00300-logging.md @@ -102,7 +102,7 @@ pub fn process_data(ctx: &ReducerContext, value: u32) -> Result<(), String> { return Err("Value cannot be zero".to_string()); } - log::debug!("Debug information: ctx.sender = {:?}", ctx.sender); + log::debug!("Debug information: ctx.sender = {:?}", ctx.sender()); Ok(()) } @@ -220,7 +220,7 @@ use spacetimedb::log; pub fn transfer_credits(ctx: &ReducerContext, to_user: u64, amount: u32) -> Result<(), String> { log::info!( "Credit transfer: from={:?}, to={}, amount={}", - ctx.sender, + ctx.sender(), to_user, amount ); diff --git a/docs/llms/docs-benchmark-analysis.md b/docs/llms/docs-benchmark-analysis.md index 44f3a564731..1ea6e1ad6b1 100644 --- a/docs/llms/docs-benchmark-analysis.md +++ b/docs/llms/docs-benchmark-analysis.md @@ -4,24 +4,20 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do ## Summary -- **Total failures analyzed**: 35 +- **Total failures analyzed**: 31 --- -# Analysis of SpacetimeDB Benchmark Failures +# Analysis of SpacetimeDB Benchmark Test Failures -Here's an overview of the SpacetimeDB benchmark failures organized by programming language and mode, providing actionable insights for documentation updates. +## Rust / rustdoc_json Failures -## Rust / rustdoc_json Failures (9 Total) +### Compile/Publish Errors (3 Failures) -### Compile/Publish Errors (3 failures) - -#### t_002_scheduled_table +#### 1. **t_002_scheduled_table** - **Generated Code**: ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; - - #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] + #[table(name = tick_timer, schedule(column = scheduled_at, reducer = tick))] pub struct TickTimer { #[primary_key] #[auto_inc] @@ -29,11 +25,9 @@ Here's an overview of the SpacetimeDB benchmark failures organized by programmin scheduled_at: ScheduleAt, } ``` + - **Golden Example**: ```rust - use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; - use std::time::Duration; - #[table(name = tick_timer, scheduled(tick))] pub struct TickTimer { #[primary_key] @@ -42,126 +36,210 @@ Here's an overview of the SpacetimeDB benchmark failures organized by programmin pub scheduled_at: ScheduleAt, } ``` + - **Error**: `publish_error: spacetime publish failed (exit=1)` -- **Difference**: The LLM used `schedule(reducer = tick, column = scheduled_at)` instead of `scheduled(tick)`. -- **Root Cause**: The reducer scheduling syntax was incorrectly formatted. -- **Recommendation**: Update the documentation to clarify the correct syntax for scheduled attributes. + +- **Explanation**: + The LLM used incorrect syntax for the `scheduled` attribute. It should be `scheduled(tick)` instead of `schedule(column = scheduled_at, reducer = tick)`. + +- **Root Cause**: The documentation may not clearly explain the syntax for the `scheduled` attribute. + +- **Recommendation**: Update documentation to emphasize that the `scheduled` attribute must be structured as `scheduled(reducer_name)`. --- -#### t_015_product_type_columns +#### 2. **t_003_struct_in_table** - **Generated Code**: ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType}; - - #[derive(SpacetimeType)] - pub struct Address { - street: String, - zip: i32, + #[spacetimedb::table(name = entity)] + pub struct Entity { + #[primary_key] + id: i32, + pos: Position, } ``` + - **Golden Example**: ```rust - use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table}; - - #[derive(SpacetimeType, Clone, Debug)] - pub struct Address { - pub street: String, - pub zip: i32, + #[table(name = entity)] + pub struct Entity { + #[primary_key] + pub id: i32, + pub pos: Position, } ``` + - **Error**: `publish_error: spacetime publish failed (exit=1)` -- **Difference**: Missing `pub` modifiers on struct fields and missing `Clone` and `Debug` traits. -- **Root Cause**: The documentation didn't clearly indicate the necessary visibility modifiers and traits for SpacetimeDB types. -- **Recommendation**: Enhance examples in the documentation to include field visibility and common traits for struct definitions. + +- **Explanation**: + The LLM did not use `pub` for struct fields which is required for visibility in SpacetimeDB. + +- **Root Cause**: The visibility rules for struct fields in Rust may need clearer explanation in the documentation. + +- **Recommendation**: Include specific examples indicating that all fields in SpacetimeDB tables should be public. --- -#### t_017_scheduled_columns +#### 3. **t_017_scheduled_columns** - **Generated Code**: ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; - - #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - scheduled_id: u64, - scheduled_at: ScheduleAt, + #[reducer(init)] + pub fn init(ctx: &ReducerContext) { + if ctx.db.tick_timer().count() == 0 { + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: ScheduleAt::repeat_micros(50_000), + }); + } } ``` + - **Golden Example**: ```rust - use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; - use std::time::Duration; - - #[table(name = tick_timer, scheduled(tick))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - pub scheduled_id: u64, - pub scheduled_at: ScheduleAt, + #[reducer(init)] + pub fn init(ctx: &ReducerContext) { + let every_50ms: ScheduleAt = Duration::from_millis(50).into(); + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: every_50ms, + }); } ``` + - **Error**: `publish_error: spacetime publish failed (exit=1)` -- **Difference**: Same scheduling error as in `t_002_scheduled_table`. -- **Root Cause**: Similar to above, incorrect usage of scheduling syntax. -- **Recommendation**: Provide explicit examples illustrating scheduled and unscheduled table definitions. + +- **Explanation**: The method for initializing `scheduled_at` is incorrect. Instead of using `repeat_micros()`, the code should convert a `Duration` to `ScheduleAt`. + +- **Root Cause**: Misunderstanding of the proper way to initialize scheduled columns could be reflected in lacking documentation details. + +- **Recommendation**: Clarify the documentation regarding initializing `ScheduleAt`, emphasizing conversion from `Duration`. --- -### Other Failures (6 failures) +### Other Failures (2 Failures) -#### t_003_struct_in_table +#### 4. **t_016_sum_type_columns** - **Generated Code**: ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType}; + #[spacetimedb::table(name = drawing)] + pub struct Drawing { + #[primary_key] + id: i32, + a: Shape, + b: Shape, + } + ``` - #[derive(SpacetimeType)] - pub struct Position { - x: i32, - y: i32, +- **Golden Example**: + ```rust + #[table(name = drawing)] + pub struct Drawing { + #[primary_key] + pub id: i32, + pub a: Shape, + pub b: Shape, } + ``` - #[table(name = entity)] +- **Error**: Errors regarding tables not found. + +- **Explanation**: Missing the `pub` attribute on struct fields results in failure to compile. + +- **Root Cause**: Lack of clarity on the use of visibility attributes (`pub`) in struct definitions. + +- **Recommendation**: Revise documentation to instruct that fields must be public to work within SpacetimeDB. + +--- + +#### 5. **t_020_ecs** +- **Generated Code**: + ```rust + #[spacetimedb::table(name = entity)] pub struct Entity { #[primary_key] id: i32, - pos: Position, } - #[reducer(init)] - pub fn init(_ctx: &ReducerContext) {} + #[spacetimedb::table(name = position)] + pub struct Position { + #[primary_key] + entity_id: i32, + x: i32, + y: i32, + } ``` + - **Golden Example**: ```rust - use spacetimedb::{table, SpacetimeType}; - - #[derive(SpacetimeType, Clone, Debug)] - pub struct Position { - pub x: i32, - pub y: i32, - } #[table(name = entity)] pub struct Entity { #[primary_key] pub id: i32, - pub pos: Position, + } + + #[table(name = position)] + pub struct Position { + #[primary_key] + pub entity_id: i32, + pub x: i32, + pub y: i32, } ``` -- **Error**: `schema_parity: reducers differ - expected [], got ["add_entity()", "init()"]` -- **Difference**: Missing `pub` modifiers led to visibility issues; `add_entity` reducer was not included. -- **Root Cause**: The documentation should indicate proper visibility for struct members. -- **Recommendation**: Include visibility guidance in struct and reducer examples to prevent visibility mismatches. + +- **Error**: Errors regarding tables not found. + +- **Explanation**: Missing the `pub` attribute leads to the struct not being properly registered with SpacetimeDB. + +- **Root Cause**: Similar to previous errors, the need for public access to struct fields is unclear. + +- **Recommendation**: Ensure documentation explicitly states that public access is necessary for all fields in SpacetimeDB structs. --- -#### t_004_insert +## Rust / docs Failures (22 total) + +### Timeout Issues (8 Failures) + +- **Failures**: Various tasks timed out, indicating potential performance or configuration issues. + +- **Root Cause**: Specifics of timeout settings and performance optimization strategies should be more explicit in the documentation. + +- **Recommendation**: Include guidelines on optimizing performance for long-running tasks or emphasize best practices for structuring queries and data handling. + +--- + +### Other Failures (14 Failures) + +#### 6. **t_000_empty_reducers** - **Generated Code**: ```rust - use spacetimedb::{table, reducer, ReducerContext, Table}; + #[spacetimedb::reducer] + pub fn empty_reducer_no_args(_ctx: &spacetimedb::ReducerContext) { + } + ``` + +- **Golden Example**: + ```rust + #[reducer] + pub fn empty_reducer_no_args(ctx: &ReducerContext) -> Result<(), String> { + Ok(()) + } + ``` - #[table(name = user)] +- **Error**: Schema-related errors due to missing return type and proper handling. + +- **Explanation**: Missing return type (`Result<(), String>`) was not implemented. + +- **Root Cause**: The documentation may not explicitly mention that reducers should return results. + +- **Recommendation**: Adjust the documentation to specify that reducer functions must include appropriate return types. + +--- + +#### 7. **t_001_basic_tables** +- **Generated Code**: + ```rust + #[spacetimedb::table(name = user)] pub struct User { #[primary_key] id: i32, @@ -169,16 +247,10 @@ Here's an overview of the SpacetimeDB benchmark failures organized by programmin age: i32, active: bool, } - - #[reducer] - pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) { - ctx.db.user().insert(User { id, name, age, active }); - } ``` + - **Golden Example**: ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; - #[table(name = user)] pub struct User { #[primary_key] @@ -187,189 +259,139 @@ Here's an overview of the SpacetimeDB benchmark failures organized by programmin pub age: i32, pub active: bool, } - - #[reducer] - pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> { - ctx.db.user().insert(User { id, name, age, active }); - Ok(()) - } ``` -- **Error**: `data_parity_insert_user: spacetime sql failed: Error: no such table: user` -- **Difference**: Missing `pub` modifiers on struct fields and the reducer function lacked error handling. -- **Root Cause**: The need for visibility on struct fields and proper error handling for database operations is not emphasized in documentation. -- **Recommendation**: Emphasize that all struct fields must be public and demonstrate typical error handling in reducers. + +- **Error**: Schema-related errors due to missing `pub` modifiers. + +- **Explanation**: Missing public access modifiers on struct fields prevented expected behavior. + +- **Root Cause**: Visibility rules may not have been adequately covered in the documentation. + +- **Recommendation**: Ensure the documentation includes examples with visibility modifiers. --- -## C# / docs Failures (4 Total) +### C# / docs Failures (4 total) -### Other Failures (4 failures) +#### Other Failures (4 Failures) -#### t_014_elementary_columns +#### 8. **t_014_elementary_columns** - **Generated Code**: ```csharp - using SpacetimeDB; - - public static partial class Module + [SpacetimeDB.Table(Name = "Primitive", Public = true)] + public partial struct Primitive { - [SpacetimeDB.Table(Name = "Primitive", Public = true)] - public partial struct Primitive - { - [SpacetimeDB.PrimaryKey] - public int Id; - public int Count; - public long Total; - public float Price; - public double Ratio; - public bool Active; - public string Name; - } - - [SpacetimeDB.Reducer] - public static void Seed(ReducerContext ctx) - { - ctx.Db.Primitive.Insert(new Primitive - { - Id = 1, - Count = 2, - Total = 3000000000L, - Price = 1.5f, - Ratio = 2.25, - Active = true, - Name = "Alice" - }); - } + [SpacetimeDB.PrimaryKey] + public int Id; + public int Count; + ... } ``` + - **Golden Example**: ```csharp - using SpacetimeDB; - - public static partial class Module + [Table(Name = "Primitive")] + public partial struct Primitive { - [Table(Name = "Primitive")] - public partial struct Primitive - { - [PrimaryKey] public int Id; - public int Count; - public long Total; - public float Price; - public double Ratio; - public bool Active; - public string Name; - } - - [Reducer] - public static void Seed(ReducerContext ctx) - { - ctx.Db.Primitive.Insert(new Primitive { - Id = 1, - Count = 2, - Total = 3000000000, - Price = 1.5f, - Ratio = 2.25, - Active = true, - Name = "Alice" - }); - } + [PrimaryKey] public int Id; + public int Count; + ... } ``` -- **Error**: `spacetime sql failed: Error: no such table: primitive` -- **Difference**: The `Public` attribute is used incorrectly, which may lead to schema mismatch or missing construct. -- **Root Cause**: The correct usage of attributes may not be clearly noted in the documentation. -- **Recommendation**: Validate the appropriate usage of attributes in table and struct definitions. + +- **Error**: Table not found during sql operations. + +- **Explanation**: The `Public` attribute's use was incorrect; it's not necessary in the struct definition. + +- **Root Cause**: Confusion over the purpose and necessity of attributes. + +- **Recommendation**: Update documentation to clarify attributes' roles in table definitions, removing unnecessary ones for struct exposure. --- -#### t_016_sum_type_columns +#### 9. **t_016_sum_type_columns** - **Generated Code**: ```csharp - using SpacetimeDB; - - public static partial class Module + [SpacetimeDB.Table(Name = "Drawing", Public = true)] + public partial struct Drawing { - [SpacetimeDB.Table(Name = "Drawing", Public = true)] - public partial struct Drawing - { - [SpacetimeDB.PrimaryKey] - public int Id; - public Shape A; - public Shape B; - } - - // ... additional definitions ... + [SpacetimeDB.PrimaryKey] + public int Id; } ``` + - **Golden Example**: ```csharp - using SpacetimeDB; - - public static partial class Module + [Table(Name = "Drawing")] + public partial struct Drawing { - [Table(Name = "Drawing")] - public partial struct Drawing - { - [PrimaryKey] public int Id; - public Shape A; - public Shape B; - } - - // ... additional definitions ... + [PrimaryKey] public int Id; } ``` -- **Error**: `spacetime sql failed: Error: no such table: drawings` -- **Difference**: Similar to the previous error, the `Public` attribute on a table may not be necessary and may lead to runtime issues. -- **Root Cause**: Documentation isn't explicit about when to use visibility modifiers and how they interact with SpacetimeDB configuration. -- **Recommendation**: Clarify the usage of attribute visibility and provide clear examples. + +- **Error**: Table not found during sql operations. + +- **Explanation**: Similar to the previous failure, the `Public` attribute was misapplied. + +- **Root Cause**: Misalignment between understood attribute requirements and actual usage. + +- **Recommendation**: Further clarification of when and where to apply attributes in C# constructs related to SpacetimeDB. --- -#### t_017_scheduled_columns +#### 10. **t_017_scheduled_columns** - **Generated Code**: ```csharp - using System; - using SpacetimeDB; + [Table(Name = "TickTimer", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))] + public partial struct TickTimer + { + [PrimaryKey, AutoInc] + public ulong ScheduledId; + public ScheduleAt ScheduledAt; + } + ``` - public static partial class Module +- **Golden Example**: + ```csharp + [Table(Name = "TickTimer", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))] + public partial struct TickTimer { - [Table(Name = "TickTimer", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))] - public partial struct TickTimer - { - [PrimaryKey, AutoInc] - public ulong ScheduledId; - public ScheduleAt ScheduledAt; - } + [PrimaryKey, AutoInc] public ulong ScheduledId; + public ScheduleAt ScheduledAt; + } + ``` - [Reducer] - public static void Tick(ReducerContext ctx, TickTimer _timer) { } +- **Error**: Table not found during sql operations. - // ... init reducer ... - } +- **Explanation**: The field definitions and relationships were incorrectly configured. + +- **Root Cause**: Possible gaps in documentation regarding definitions of scheduled columns and expected real structures. + +- **Recommendation**: Revise documentation to ensure clear expectations about table configuration and proper struct setup. + +--- + +#### 11. **t_020_ecs** +- **Generated Code**: + ```csharp + [SpacetimeDB.Table(Name = "Entity", Public = true)] + public partial struct Entity { [SpacetimeDB.PrimaryKey] public int Id; } ``` + - **Golden Example**: ```csharp - using SpacetimeDB; + [Table(Name = "Entity")] + public partial struct Entity { [PrimaryKey] public int Id; } + ``` - public static partial class Module - { - [Table(Name = "TickTimer", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))] - public partial struct TickTimer - { - [PrimaryKey, AutoInc] public ulong ScheduledId; - public ScheduleAt ScheduledAt; - } +- **Error**: Errors related to missing tables. - [Reducer] - public static void Tick(ReducerContext ctx, TickTimer schedule) { } +- **Explanation**: Public attributes were misused in creating struct definitions for the tables. - // ... init reducer with more clarity ... - } - ``` -- **Error**: `spacetime sql failed: Error: no such table: tick_timer` -- **Difference**: Incorrect usage of parameter names in the reducer; `_timer` should just use a clear name. -- **Root Cause**: The documentation should illustrate effective naming conventions for function parameters. -- **Recommendation**: Include best practices for naming and structuring reducers alongside examples of scheduled fields. +- **Root Cause**: Attribute usage may be causing confusion in use cases. + +- **Recommendation**: Ensure documentation includes proper usage guidelines for attributes in defining entities. --- -This analytical report outlines the specific failures within SpacetimeDB benchmarks, pinpointing concrete documentation enhancements to improve understanding and usability. By implementing these changes, developers can reduce occurrence and severity of future benchmark failures. +By addressing the aforementioned discrepancies and gaps in documentation, developers can improve their implementation of SpacetimeDB, leading to smoother integrations and reduced error rates during execution. diff --git a/docs/llms/docs-benchmark-comment.md b/docs/llms/docs-benchmark-comment.md index e6e73436a0f..37272389e0d 100644 --- a/docs/llms/docs-benchmark-comment.md +++ b/docs/llms/docs-benchmark-comment.md @@ -2,16 +2,16 @@ | Language | Mode | Category | Tests Passed | Task Pass % | |----------|------|----------|--------------|-------------| -| Rust | rustdoc_json | basics | 22/27 | 74.3% ⬇️ -9.0% | -| Rust | rustdoc_json | schema | 23/34 | 65.3% | -| Rust | rustdoc_json | **total** | 45/61 | **70.2%** ⬇️ -4.9% | +| Rust | rustdoc_json | basics | 25/27 | 83.3% ⬆️ +9.0% | +| Rust | rustdoc_json | schema | 26/34 | 75.3% ⬆️ +10.0% | +| Rust | rustdoc_json | **total** | 51/61 | **79.7%** ⬆️ +9.5% | | Rust | docs | basics | 5/27 | 11.1% | -| Rust | docs | schema | 8/34 | 20.5% ⬆️ +5.0% | -| Rust | docs | **total** | 13/61 | **15.4%** ⬆️ +2.3% | +| Rust | docs | schema | 0/14 | 0.0% ⬇️ -20.5% | +| Rust | docs | **total** | 5/41 | **6.1%** ⬇️ -9.3% | | C# | docs | basics | 27/27 | 100.0% | -| C# | docs | schema | 25/34 | 73.7% ⬆️ +10.0% | -| C# | docs | **total** | 52/61 | **88.0%** ⬆️ +4.5% | +| C# | docs | schema | 25/34 | 73.7% | +| C# | docs | **total** | 52/61 | **88.0%** | _Compared against master branch baseline_ -Generated at: 2026-01-23T12:08:22.290Z +Generated at: 2026-01-23T20:21:59.670Z diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json index b6074d5fccf..2c8b8a16a11 100644 --- a/docs/llms/docs-benchmark-details.json +++ b/docs/llms/docs-benchmark-details.json @@ -5,21 +5,21 @@ "modes": [ { "mode": "rustdoc_json", - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_000_empty_reducers", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\nfn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\nfn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {\n}\n\n#[spacetimedb::reducer]\nfn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {\n}\n\n#[spacetimedb::reducer]\nfn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {\n}\n\n#[spacetimedb::reducer]\nfn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -35,18 +35,18 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:41115", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.595950645Z", - "finished_at": "2026-01-23T11:56:59.057829079Z" + "started_at": "2026-01-23T20:09:12.144740177Z", + "finished_at": "2026-01-23T20:09:46.065291337Z" }, "t_001_basic_tables": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_001_basic_tables", "lang": "rust", "golden_published": true, @@ -69,25 +69,25 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:41115", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.595256337Z", - "finished_at": "2026-01-23T11:57:01.112482005Z" + "started_at": "2026-01-23T20:09:14.464369753Z", + "finished_at": "2026-01-23T20:09:44.014749235Z" }, "t_002_scheduled_table": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_002_scheduled_table", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let t = ctx.db.tick_timer();\n for row in t.iter() {\n t.scheduled_id().delete(&row.scheduled_id);\n }\n t.insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext) {\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, ScheduleAt, Table};\n\n#[table(name = tick_timer, schedule(column = scheduled_at, reducer = tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext) {}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -99,24 +99,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling either v1.15.0\n Compiling zerocopy v0.8.33\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling bytes v1.11.0\n Compiling convert_case v0.4.0\n Compiling zmij v1.0.16\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling keccak v0.1.5\n Compiling bytemuck v1.24.0\n Compiling serde_json v1.0.149\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.4.2\n Compiling getrandom v0.2.17\n Compiling spacetimedb-lib v1.11.1\n Compiling itoa v1.0.17\n Compiling hex v0.4.3\n Compiling smallvec v1.15.1\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling second-stack v0.3.5\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling cc v1.2.54\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling blake3 v1.8.3\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling ethnum v1.5.2\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:18:14\n |\n18 | t.insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:20\n |\n14 | let t = ctx.db.tick_timer();\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:20:35\n |\n20 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 4 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling zerocopy v0.8.33\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling zmij v1.0.16\n Compiling keccak v0.1.5\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling bytes v1.11.0\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling itoa v1.0.17\n Compiling second-stack v0.3.5\n Compiling serde_json v1.0.149\n Compiling constant_time_eq v0.4.2\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling hex v0.4.3\n Compiling cc v1.2.54\n Compiling bytemuck v1.24.0\n Compiling arrayref v0.3.9\n Compiling spacetimedb-lib v1.11.1\n Compiling itertools v0.12.1\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling rand_core v0.6.4\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling blake3 v1.8.3\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(column = scheduled_at, reducer = tick))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:18:36\n |\n18 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:17:15\n |\n17 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:18:16\n |\n18 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:20:39\n |\n20 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.596087756Z", - "finished_at": "2026-01-23T11:57:51.917718856Z" + "started_at": "2026-01-23T20:09:35.992436277Z", + "finished_at": "2026-01-23T20:10:39.770857477Z" }, "t_003_struct_in_table": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_003_struct_in_table", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}\n\n#[reducer(init)]\npub fn init(_ctx: &ReducerContext) {}\n\n#[reducer]\npub fn add_entity(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n ctx.db.entity().insert(Entity { id, pos: Position { x, y } });\n}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[derive(spacetimedb::SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn add_entity(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n let pos = Position { x, y };\n ctx.db.entity().insert(Entity { id, pos });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -124,38 +124,27 @@ "work_dir_golden": "target/llm-runs/basics/t_003_struct_in_table/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_003_struct_in_table/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "golden_db": "basics-t-003-struct-in-table-golden", - "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", - "reducers_diff": { - "only_golden": [], - "only_llm": [ - "add_entity()", - "init()" - ] - }, - "reducers_equal": false, - "server": "http://127.0.0.1:43035", - "tables_diff": null, - "tables_equal": true + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling zerocopy v0.8.33\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling either v1.15.0\n Compiling find-msvc-tools v0.1.8\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling bytes v1.11.0\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling zmij v1.0.16\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling convert_case v0.4.0\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling second-stack v0.3.5\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling spacetimedb-lib v1.11.1\n Compiling itoa v1.0.17\n Compiling itertools v0.12.1\n Compiling constant_time_eq v0.4.2\n Compiling smallvec v1.15.1\n Compiling serde_json v1.0.149\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling rand_core v0.6.4\n Compiling scoped-tls v1.0.1\n Compiling cc v1.2.54\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_003_struct_in_table/rust/server/gpt-5/llm)\nerror[E0599]: no method named `insert` found for reference `&entity__TableHandle` in the current scope\n --> src/lib.rs:20:21\n |\n20 | ctx.db.entity().insert(Entity { id, pos });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n20 | ctx.db.entity().try_insert(Entity { id, pos });\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.595509989Z", - "finished_at": "2026-01-23T11:57:06.869986441Z" + "started_at": "2026-01-23T20:09:14.464130029Z", + "finished_at": "2026-01-23T20:09:54.314351498Z" }, "t_004_insert": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_004_insert", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, - "passed_tests": 1, + "passed_tests": 2, "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", @@ -164,6 +153,25 @@ "work_dir_golden": "target/llm-runs/basics/t_004_insert/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_004_insert/rust/server/gpt-5/llm", "scorer_details": { + "data_parity_insert_user": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + "Alice", + 30, + true + ], + "golden_db": "basics-t-004-insert-golden", + "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "insert_user", + "server": "http://127.0.0.1:41115" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -172,33 +180,25 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:41115", "tables_diff": null, "tables_equal": true } - }, - "data_parity_insert_user": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c2008f3693ac65719c539bc7c4992fe451a1bec99bb058273f581f5ee525695b/sql)\n", - "phase": "sql_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.595811655Z", - "finished_at": "2026-01-23T11:57:05.537662622Z" + "started_at": "2026-01-23T20:09:14.983974693Z", + "finished_at": "2026-01-23T20:09:35.992396086Z" }, "t_005_update": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_005_update", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let row = User { id, name, age, active };\n ctx.db.user().id().update(row);\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let row = User { id, name, age, active };\n ctx.db.user().id().update(row);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -206,19 +206,6 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-005-update-golden", - "llm_db": "basics-t-005-update-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:43035", - "tables_diff": null, - "tables_equal": true - } - }, "seed_users_row": { "pass": true, "partial": 1.0, @@ -242,23 +229,36 @@ "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT id, name, age, active FROM user WHERE id=1", "reducer": "update_user", - "server": "http://127.0.0.1:43035" + "server": "http://127.0.0.1:41115" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-005-update-golden", + "llm_db": "basics-t-005-update-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41115", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:59.057861578Z", - "finished_at": "2026-01-23T11:57:53.866612446Z" + "started_at": "2026-01-23T20:08:57.792213429Z", + "finished_at": "2026-01-23T20:09:53.091667873Z" }, "t_006_delete": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_006_delete", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -266,6 +266,15 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { + "delete_user_count_zero": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -274,7 +283,7 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:41115", "tables_diff": null, "tables_equal": true } @@ -285,30 +294,21 @@ "notes": { "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:02.840911142Z", - "finished_at": "2026-01-23T11:57:48.533997072Z" + "started_at": "2026-01-23T20:08:35.084889228Z", + "finished_at": "2026-01-23T20:09:14.643664761Z" }, "t_007_crud": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_007_crud", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n users.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n users.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n\n let _ = users.id().update(User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false });\n\n users.id().delete(&2);\n}", + "passed_tests": 4, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n let mut u1 = users.insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n let _u2 = users.insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n u1.name = \"Alice2\".to_string();\n u1.age = 31;\n u1.active = false;\n let _u1_updated = users.id().update(u1);\n\n users.id().delete(&2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -325,17 +325,18 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" } }, - "schema_parity": { + "crud_row_id1_parity": { "pass": true, "partial": 1.0, "notes": { + "args": [], "golden_db": "basics-t-007-crud-golden", + "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "llm_db": "basics-t-007-crud-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:43035", - "tables_diff": null, - "tables_equal": true + "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "crud", + "server": "http://127.0.0.1:41115" } }, "crud_total_count_one": { @@ -347,28 +348,33 @@ "sql": "SELECT COUNT(*) AS n FROM user" } }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c2009e8af0d449c17ae3da6d300c97be156e1eb9f1da87b7c6b9cefbcde67475/sql)\n", - "phase": "sql_golden" + "golden_db": "basics-t-007-crud-golden", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41115", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.596351347Z", - "finished_at": "2026-01-23T11:57:10.126845149Z" + "started_at": "2026-01-23T20:08:56.057273916Z", + "finished_at": "2026-01-23T20:09:48.569368820Z" }, "t_008_index_lookup": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_008_index_lookup", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\nfn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n let User { id, name, .. } = u;\n ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow { id, name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -383,19 +389,6 @@ "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:43035", - "tables_diff": null, - "tables_equal": true - } - }, "index_lookup_projection_parity": { "pass": true, "partial": 1.0, @@ -409,23 +402,36 @@ "llm_out": "id | name ----+--------- 1 | \"Alice\"", "query": "SELECT id, name FROM result WHERE id=1", "reducer": "lookup_user_name", - "server": "http://127.0.0.1:43035" + "server": "http://127.0.0.1:41115" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-008-index-lookup-golden", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41115", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:02.840644958Z", - "finished_at": "2026-01-23T11:57:40.599601481Z" + "started_at": "2026-01-23T20:09:14.643699808Z", + "finished_at": "2026-01-23T20:09:52.566672855Z" }, "t_009_init": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_009_init", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let user = ctx.db.user();\n user.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n user.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -433,6 +439,24 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { + "init_seed_bob": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + } + }, + "init_seed_alice": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -441,7 +465,7 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:41115", "tables_diff": null, "tables_equal": true } @@ -454,32 +478,14 @@ "expected": 2, "sql": "SELECT COUNT(*) AS n FROM user" } - }, - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:01.112523418Z", - "finished_at": "2026-01-23T11:57:24.446822222Z" + "started_at": "2026-01-23T20:08:57.046090650Z", + "finished_at": "2026-01-23T20:09:23.724177316Z" }, "t_010_connect": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_010_connect", "lang": "rust", "golden_published": true, @@ -502,25 +508,25 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:41115", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.596220949Z", - "finished_at": "2026-01-23T11:57:02.704314900Z" + "started_at": "2026-01-23T20:08:35.620982701Z", + "finished_at": "2026-01-23T20:09:12.144710803Z" }, "t_011_helper_function": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_011_helper_function", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 2, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", + "passed_tests": 3, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.result().insert(ResultRow { id, sum: add(a, b) });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -528,14 +534,6 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { - "helper_func_sum_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c20092cdd1c98eaef99ecfeb113a11c7cee6251c76ce243f1d56372113126f59/sql)\n", - "phase": "sql_golden" - } - }, "helper_func_sum_abs": { "pass": true, "partial": 1.0, @@ -545,6 +543,24 @@ "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" } }, + "helper_func_sum_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + 2, + 3 + ], + "golden_db": "basics-t-011-helper-function-golden", + "golden_out": "id | sum ----+----- 1 | 5", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "llm_out": "id | sum ----+----- 1 | 5", + "query": "SELECT id, sum FROM result WHERE id=1", + "reducer": "compute_sum", + "server": "http://127.0.0.1:41115" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -553,25 +569,25 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:41115", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.595665553Z", - "finished_at": "2026-01-23T11:57:02.840609277Z" + "started_at": "2026-01-23T20:09:23.724208808Z", + "finished_at": "2026-01-23T20:09:51.292524674Z" }, "t_012_spacetime_product_type": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_012_spacetime_product_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\nstruct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\nstruct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\nfn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -588,6 +604,19 @@ "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-012-spacetime-product-type-golden", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41115", + "tables_diff": null, + "tables_equal": true + } + }, "product_type_row_parity": { "pass": true, "partial": 1.0, @@ -603,36 +632,23 @@ "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", "query": "SELECT id, value FROM result WHERE id=1", "reducer": "set_score", - "server": "http://127.0.0.1:43035" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-012-spacetime-product-type-golden", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:43035", - "tables_diff": null, - "tables_equal": true + "server": "http://127.0.0.1:41115" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:40.599657560Z", - "finished_at": "2026-01-23T11:58:09.609768604Z" + "started_at": "2026-01-23T20:08:32.450266496Z", + "finished_at": "2026-01-23T20:09:14.310539584Z" }, "t_013_spacetime_sum_type": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_013_spacetime_sum_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\nfn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\nstruct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\nenum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\nstruct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\nfn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow { id, value: Shape::Circle(radius) });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -640,21 +656,13 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_row_parity": { + "sum_type_row_count": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - 10 - ], - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle", - "server": "http://127.0.0.1:43035" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } }, "schema_parity": { @@ -665,27 +673,35 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:41115", "tables_diff": null, "tables_equal": true } }, - "sum_type_row_count": { + "sum_type_row_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" + "args": [ + 1, + 10 + ], + "golden_db": "schema-t-013-spacetime-sum-type-golden", + "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", + "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", + "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", + "query": "SELECT id, value FROM result WHERE id=1", + "reducer": "set_circle", + "server": "http://127.0.0.1:41115" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:48.534039773Z", - "finished_at": "2026-01-23T11:58:41.485665974Z" + "started_at": "2026-01-23T20:07:57.743148689Z", + "finished_at": "2026-01-23T20:08:32.450224286Z" }, "t_014_elementary_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_014_elementary_columns", "lang": "rust", "golden_published": true, @@ -700,31 +716,31 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_parity": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [], "golden_db": "schema-t-014-elementary-columns-golden", - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed", - "server": "http://127.0.0.1:43035" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41115", + "tables_diff": null, + "tables_equal": true } }, - "schema_parity": { + "elementary_columns_row_parity": { "pass": true, "partial": 1.0, "notes": { + "args": [], "golden_db": "schema-t-014-elementary-columns-golden", + "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:43035", - "tables_diff": null, - "tables_equal": true + "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", + "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", + "reducer": "seed", + "server": "http://127.0.0.1:41115" } }, "elementary_columns_row_count": { @@ -738,18 +754,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:47.891667065Z", - "finished_at": "2026-01-23T11:58:20.627171437Z" + "started_at": "2026-01-23T20:08:30.379293605Z", + "finished_at": "2026-01-23T20:09:14.464089675Z" }, "t_015_product_type_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_015_product_type_columns", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.profile();\n for row in tbl.iter().collect::>() {\n tbl.delete(&row);\n }\n tbl.insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}", + "passed_tests": 3, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\nstruct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\nstruct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\nstruct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n let home = Address { street: \"1 Main\".to_string(), zip: 11111 };\n let work = Address { street: \"2 Broad\".to_string(), zip: 22222 };\n let pos = Position { x: 7, y: 9 };\n match ctx.db.profile().id().find(1) {\n Some(mut row) => {\n row.home = home;\n row.work = work;\n row.pos = pos;\n ctx.db.profile().id().update(row);\n }\n None => {\n ctx.db.profile().insert(Profile { id: 1, home, work, pos });\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -757,28 +773,56 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "product_type_columns_row_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling shlex v1.3.0\n Compiling either v1.15.0\n Compiling zerocopy v0.8.33\n Compiling find-msvc-tools v0.1.8\n Compiling serde v1.0.228\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling zmij v1.0.16\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling constant_time_eq v0.4.2\n Compiling serde_json v1.0.149\n Compiling arrayref v0.3.9\n Compiling itoa v1.0.17\n Compiling getrandom v0.2.17\n Compiling second-stack v0.3.5\n Compiling bytemuck v1.24.0\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling hex v0.4.3\n Compiling cc v1.2.54\n Compiling smallvec v1.15.1\n Compiling spacetimedb-lib v1.11.1\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling blake3 v1.8.3\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm)\nerror[E0308]: mismatched types\n --> src/lib.rs:29:20\n |\n 29 | tbl.delete(&row);\n | ------ ^^^^ expected `Profile`, found `&Profile`\n | |\n | arguments to this method are incorrect\n |\nnote: method defined here\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/table.rs:104:8\n |\n104 | fn delete(&self, row: Self::Row) -> bool {\n | ^^^^^^\nhelp: consider removing the borrow\n |\n 29 - tbl.delete(&row);\n 29 + tbl.delete(row);\n |\n\nFor more information about this error, try `rustc --explain E0308`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" + "args": [], + "golden_db": "schema-t-015-product-type-columns-golden", + "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", + "query": "SELECT id, home, work, pos FROM profile WHERE id=1", + "reducer": "seed", + "server": "http://127.0.0.1:41115" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-015-product-type-columns-golden", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41115", + "tables_diff": null, + "tables_equal": true + } + }, + "product_type_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:51.917727311Z", - "finished_at": "2026-01-23T11:58:32.335767995Z" + "started_at": "2026-01-23T20:07:57.743649083Z", + "finished_at": "2026-01-23T20:08:57.046048716Z" }, "t_016_sum_type_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_016_sum_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let row = Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n };\n ctx.db.drawing().insert(row);\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[derive(spacetimedb::SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(spacetimedb::SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n if ctx.db.drawing().id().find(1).is_none() {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -790,7 +834,7 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200fa0a7bdaa0b3fe32198acc7d7c51970587917bc93ce14e7d0e54f859979b/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41115/v1/database/c200be63a5c073cb57163fbe420fbf96d0ca1c24fb78e5d67bbed19841d5eeee/sql)\n", "phase": "sql_golden" } }, @@ -802,7 +846,7 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:41115", "tables_diff": null, "tables_equal": true } @@ -811,24 +855,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200b5079878ebf16faffc0bc609578eb070b7c5eb623e6f0ec9ca7c1e73e040/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41115/v1/database/c2001257ca4ff5c673696324b8a29b10931896a2877a4425230bec1856988aad/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:51.917991193Z", - "finished_at": "2026-01-23T11:58:22.938115353Z" + "started_at": "2026-01-23T20:07:57.742955241Z", + "finished_at": "2026-01-23T20:08:35.620943048Z" }, "t_017_scheduled_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_017_scheduled_columns", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n // Scheduled tick; no-op for this task.\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -840,24 +884,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.33\n Compiling find-msvc-tools v0.1.8\n Compiling anyhow v1.0.100\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling arrayvec v0.7.6\n Compiling zmij v1.0.16\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling heck v0.4.1\n Compiling bytes v1.11.0\n Compiling serde_json v1.0.149\n Compiling constant_time_eq v0.4.2\n Compiling second-stack v0.3.5\n Compiling spacetimedb-lib v1.11.1\n Compiling getrandom v0.2.17\n Compiling arrayref v0.3.9\n Compiling smallvec v1.15.1\n Compiling cc v1.2.54\n Compiling itoa v1.0.17\n Compiling bytemuck v1.24.0\n Compiling hex v0.4.3\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling blake3 v1.8.3\n Compiling decorum v0.3.1\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:12:11\n |\n12 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:13:42\n |\n13 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:20:36\n |\n20 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:19:15\n |\n19 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:20:16\n |\n20 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:22:39\n |\n22 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.33\n Compiling find-msvc-tools v0.1.8\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling arrayvec v0.7.6\n Compiling zmij v1.0.16\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling bytes v1.11.0\n Compiling keccak v0.1.5\n Compiling constant_time_eq v0.4.2\n Compiling bytemuck v1.24.0\n Compiling second-stack v0.3.5\n Compiling smallvec v1.15.1\n Compiling getrandom v0.2.17\n Compiling hex v0.4.3\n Compiling spacetimedb-lib v1.11.1\n Compiling itertools v0.12.1\n Compiling cc v1.2.54\n Compiling serde_json v1.0.149\n Compiling itoa v1.0.17\n Compiling arrayref v0.3.9\n Compiling rand_core v0.6.4\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling syn v2.0.114\n Compiling http v1.4.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling blake3 v1.8.3\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling ethnum v1.5.2\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:42\n |\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n 22 | #[reducer]\n | ---------- required by a bound introduced by this call\n 23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n22 | #[reducer]\n | ---------- required by a bound introduced by this call\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:31.320145539Z", - "finished_at": "2026-01-23T11:58:26.729342562Z" + "started_at": "2026-01-23T20:07:57.742610502Z", + "finished_at": "2026-01-23T20:08:56.057254318Z" }, "t_018_constraints": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_018_constraints", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.account();\n\n let ids: Vec = tbl.iter().map(|r| r.id).collect();\n for id in ids {\n tbl.id().delete(&id);\n }\n\n tbl.insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n tbl.insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db\n .account()\n .insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db\n .account()\n .insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -885,7 +929,7 @@ "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "query": "SELECT id, email, name FROM account WHERE id=1", "reducer": "seed", - "server": "http://127.0.0.1:43035" + "server": "http://127.0.0.1:41115" } }, "schema_parity": { @@ -896,25 +940,25 @@ "llm_db": "schema-t-018-constraints-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:41115", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:10.126874202Z", - "finished_at": "2026-01-23T11:57:47.891616379Z" + "started_at": "2026-01-23T20:07:57.742384971Z", + "finished_at": "2026-01-23T20:08:30.379256146Z" }, "t_019_many_to_many": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_019_many_to_many", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(&row.id);\n }\n for row in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(&row.user_id);\n }\n for row in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(&row.group_id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n // Clear existing rows to ensure exactly the requested rows exist after seeding\n let membership_ids: Vec = ctx.db.membership().iter().map(|m| m.id).collect();\n for id in membership_ids {\n ctx.db.membership().id().delete(&id);\n }\n\n let user_ids: Vec = ctx.db.user().iter().map(|u| u.user_id).collect();\n for id in user_ids {\n ctx.db.user().user_id().delete(&id);\n }\n\n let group_ids: Vec = ctx.db.group().iter().map(|g| g.group_id).collect();\n for id in group_ids {\n ctx.db.group().group_id().delete(&id);\n }\n\n // Insert users\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n // Insert groups\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n // Insert memberships\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -922,22 +966,35 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", "scorer_details": { - "m2m_has_2_20": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41115", + "tables_diff": null, + "tables_equal": true + } + }, + "m2m_has_1_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" } }, - "m2m_has_1_10": { + "m2m_has_2_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" } }, "memberships_three_rows": { @@ -949,42 +1006,29 @@ "sql": "SELECT COUNT(*) AS n FROM membership" } }, - "m2m_has_1_20": { + "m2m_has_1_10": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:43035", - "tables_diff": null, - "tables_equal": true + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:24.446860971Z", - "finished_at": "2026-01-23T11:58:08.134821703Z" + "started_at": "2026-01-23T20:07:57.743323315Z", + "finished_at": "2026-01-23T20:08:57.792175612Z" }, "t_020_ecs": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_020_ecs", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entities\n let entities = ctx.db.entity();\n if entities.id().find(1).is_none() {\n entities.insert(Entity { id: 1 });\n }\n if entities.id().find(2).is_none() {\n entities.insert(Entity { id: 2 });\n }\n\n // Positions\n let positions = ctx.db.position();\n if let Some(mut p) = positions.entity_id().find(1) {\n p.x = 0;\n p.y = 0;\n let _ = positions.entity_id().update(p);\n } else {\n positions.insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n if let Some(mut p) = positions.entity_id().find(2) {\n p.x = 10;\n p.y = 0;\n let _ = positions.entity_id().update(p);\n } else {\n positions.insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n\n // Velocities\n let velocities = ctx.db.velocity();\n if let Some(mut v) = velocities.entity_id().find(1) {\n v.vx = 1;\n v.vy = 0;\n let _ = velocities.entity_id().update(v);\n } else {\n velocities.insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n if let Some(mut v) = velocities.entity_id().find(2) {\n v.vx = -2;\n v.vy = 3;\n let _ = velocities.entity_id().update(v);\n } else {\n velocities.insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n let positions = ctx.db.position();\n let velocities = ctx.db.velocity();\n let next_positions = ctx.db.next_position();\n\n for pos in positions.iter() {\n if let Some(vel) = velocities.entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n\n if let Some(mut np) = next_positions.entity_id().find(pos.entity_id) {\n np.x = nx;\n np.y = ny;\n let _ = next_positions.entity_id().update(np);\n } else {\n next_positions.insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Clear and insert entities\n for &id in &[1, 2] {\n if ctx.db.entity().id().find(id).is_some() {\n ctx.db.entity().id().delete(&id);\n }\n ctx.db.entity().insert(Entity { id });\n }\n\n // Clear existing positions/velocities for these entities and insert new values\n // Entity 1: pos=(0,0), vel=(1,0)\n if ctx.db.position().entity_id().find(1).is_some() {\n ctx.db.position().entity_id().delete(&1);\n }\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n if ctx.db.velocity().entity_id().find(1).is_some() {\n ctx.db.velocity().entity_id().delete(&1);\n }\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n // Entity 2: pos=(10,0), vel=(-2,3)\n if ctx.db.position().entity_id().find(2).is_some() {\n ctx.db.position().entity_id().delete(&2);\n }\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n if ctx.db.velocity().entity_id().find(2).is_some() {\n ctx.db.velocity().entity_id().delete(&2);\n }\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n\n // Clean any existing next_position rows for these entities\n for &id in &[1, 2] {\n if ctx.db.next_position().entity_id().find(id).is_some() {\n ctx.db.next_position().entity_id().delete(&id);\n }\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n let next = NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n };\n\n if ctx.db.next_position().entity_id().find(pos.entity_id).is_some() {\n let _ = ctx.db.next_position().entity_id().update(next);\n } else {\n let _ = ctx.db.next_position().insert(next);\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -996,15 +1040,15 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200bfacda1977a80778696e02687f4296607c0e5dc2757b9dc0bd6ed69ac556/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41115/v1/database/c20022d8f2e2d547824e6b1c08b40372fd94143f227cebc3dd95d8debbce411d/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity2": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200bfacda1977a80778696e02687f4296607c0e5dc2757b9dc0bd6ed69ac556/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41115/v1/database/c20022d8f2e2d547824e6b1c08b40372fd94143f227cebc3dd95d8debbce411d/sql)\n", "phase": "sql" } }, @@ -1016,7 +1060,7 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:41115", "tables_diff": null, "tables_equal": true } @@ -1025,25 +1069,25 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200bfacda1977a80778696e02687f4296607c0e5dc2757b9dc0bd6ed69ac556/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41115/v1/database/c20022d8f2e2d547824e6b1c08b40372fd94143f227cebc3dd95d8debbce411d/sql)\n", "phase": "sql" } }, - "ecs_seed_positions_count": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200bfacda1977a80778696e02687f4296607c0e5dc2757b9dc0bd6ed69ac556/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41115/v1/database/c20022d8f2e2d547824e6b1c08b40372fd94143f227cebc3dd95d8debbce411d/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:05.537695688Z", - "finished_at": "2026-01-23T11:57:51.917639786Z" + "started_at": "2026-01-23T20:07:57.743495455Z", + "finished_at": "2026-01-23T20:09:14.983916389Z" }, "t_021_multi_column_index": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_021_multi_column_index", "lang": "rust", "golden_published": true, @@ -1058,6 +1102,15 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { + "mcindex_lookup_u7_d1": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -1066,7 +1119,7 @@ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:41115", "tables_diff": null, "tables_equal": true } @@ -1080,15 +1133,6 @@ "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" } }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" - } - }, "mcindex_seed_count": { "pass": true, "partial": 1.0, @@ -1100,8 +1144,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:06.870008743Z", - "finished_at": "2026-01-23T11:57:31.320114801Z" + "started_at": "2026-01-23T20:07:57.742772776Z", + "finished_at": "2026-01-23T20:08:35.084850930Z" } } } @@ -1109,21 +1153,21 @@ }, { "mode": "docs", - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_000_empty_reducers", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, count: i32, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, active: bool, ratio: f32, label: String) {}", + "llm_output": "#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &spacetimedb::ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &spacetimedb::ReducerContext, _count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &spacetimedb::ReducerContext, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &spacetimedb::ReducerContext, _count: i32, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &spacetimedb::ReducerContext, _active: bool, _ratio: f32, _label: String) {\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -1141,11 +1185,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.716223450Z", - "finished_at": "2026-01-23T11:59:49.748708946Z" + "started_at": "2026-01-23T20:12:10.018237719Z", + "finished_at": "2026-01-23T20:12:41.638729168Z" }, "t_001_basic_tables": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_001_basic_tables", "lang": "rust", "golden_published": true, @@ -1170,18 +1214,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.704113020Z", - "finished_at": "2026-01-23T11:59:42.429041793Z" + "started_at": "2026-01-23T20:12:46.725514922Z", + "finished_at": "2026-01-23T20:13:51.799945305Z" }, "t_002_scheduled_table": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_002_scheduled_table", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n let desired = ScheduleAt::Interval(Duration::from_micros(50_000).into());\n let count = ctx.db.tick_timer().count();\n if count == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: desired,\n });\n } else {\n let mut iter = ctx.db.tick_timer().iter();\n if let Some(mut first) = iter.next() {\n first.scheduled_at = desired;\n ctx.db.tick_timer().scheduled_id().update(first);\n }\n for extra in iter {\n ctx.db.tick_timer().scheduled_id().delete(&extra.scheduled_id);\n }\n }\n Ok(())\n}", + "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n // No-op scheduled tick.\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n Ok(())\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -1199,18 +1243,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.719410738Z", - "finished_at": "2026-01-23T11:59:57.789054932Z" + "started_at": "2026-01-23T20:13:28.197750258Z", + "finished_at": "2026-01-23T20:14:26.307958881Z" }, "t_003_struct_in_table": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_003_struct_in_table", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}", + "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -1228,11 +1272,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.706682369Z", - "finished_at": "2026-01-23T11:59:26.096561811Z" + "started_at": "2026-01-23T20:12:41.638758353Z", + "finished_at": "2026-01-23T20:13:47.479970679Z" }, "t_004_insert": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_004_insert", "lang": "rust", "golden_published": true, @@ -1247,29 +1291,29 @@ "work_dir_golden": "target/llm-runs/basics/t_004_insert/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_004_insert/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "data_parity_insert_user": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", + "phase": "call_reducer_golden" } }, - "data_parity_insert_user": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.712966773Z", - "finished_at": "2026-01-23T11:59:51.858461717Z" + "started_at": "2026-01-23T20:12:46.734575843Z", + "finished_at": "2026-01-23T20:13:46.613223334Z" }, "t_005_update": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_005_update", "lang": "rust", "golden_published": true, @@ -1292,30 +1336,30 @@ "phase": "describe_golden" } }, - "seed_users_row": { + "data_parity_update_user": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", + "phase": "call_reducer_golden" } }, - "data_parity_update_user": { + "seed_users_row": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "call_reducer_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", + "phase": "sql_golden", + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:26.096589408Z", - "finished_at": "2026-01-23T11:59:53.555166464Z" + "started_at": "2026-01-23T20:12:10.015338220Z", + "finished_at": "2026-01-23T20:13:14.713691221Z" }, "t_006_delete": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_006_delete", "lang": "rust", "golden_published": true, @@ -1330,13 +1374,12 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { - "seed_users_row": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", + "phase": "describe_golden" } }, "delete_user_count_zero": { @@ -1348,28 +1391,29 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" } }, - "schema_parity": { + "seed_users_row": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", - "phase": "describe_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", + "phase": "sql_golden", + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:43.249955612Z", - "finished_at": "2026-01-23T12:00:06.215097718Z" + "started_at": "2026-01-23T20:12:09.994528341Z", + "finished_at": "2026-01-23T20:12:46.205776277Z" }, "t_007_crud": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_007_crud", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n ctx.db.user().id().update(User { id: 1, name: \"Alice2\".into(), age: 31, active: false });\n ctx.db.user().id().delete(2);\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n // insert (id=1, name=\"Alice\", age=30, active=true)\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n // insert (id=2, name=\"Bob\", age=22, active=false)\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n // update (id=1, name=\"Alice2\", age=31, active=false)\n if let Some(mut u1) = ctx.db.user().id().find(1) {\n u1.name = \"Alice2\".to_string();\n u1.age = 31;\n u1.active = false;\n ctx.db.user().id().update(u1);\n }\n\n // delete id=2\n ctx.db.user().id().delete(2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -1377,6 +1421,14 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { + "crud_row_id1_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", + "phase": "call_reducer_golden" + } + }, "crud_row_id2_deleted": { "pass": true, "partial": 1.0, @@ -1386,45 +1438,37 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" } }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "describe_golden" - } - }, - "crud_row_id1_parity": { + "crud_total_count_one": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "call_reducer_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user" } }, - "crud_total_count_one": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.725929053Z", - "finished_at": "2026-01-23T11:59:43.249835351Z" + "started_at": "2026-01-23T20:12:10.004797508Z", + "finished_at": "2026-01-23T20:13:37.162002816Z" }, "t_008_index_lookup": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_008_index_lookup", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: user.id, name: user.name });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) -> Result<(), String> {\n if let Some(user) = ctx.db.user().id().find(id) {\n ctx.db\n .result()\n .try_insert(ResultRow { id: user.id, name: user.name.clone() })?;\n Ok(())\n } else {\n Err(\"User not found\".to_string())\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -1459,18 +1503,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:42.429072085Z", - "finished_at": "2026-01-23T12:00:06.978561278Z" + "started_at": "2026-01-23T20:12:46.729438058Z", + "finished_at": "2026-01-23T20:13:54.235945228Z" }, "t_009_init": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_009_init", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".into(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".into(),\n age: 22,\n active: false,\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n Ok(())\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -1515,11 +1559,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:40.767658962Z", - "finished_at": "2026-01-23T12:00:00.250974348Z" + "started_at": "2026-01-23T20:12:10.009828612Z", + "finished_at": "2026-01-23T20:12:45.617807272Z" }, "t_010_connect": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_010_connect", "lang": "rust", "golden_published": true, @@ -1544,18 +1588,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.722667055Z", - "finished_at": "2026-01-23T11:59:44.003413383Z" + "started_at": "2026-01-23T20:12:09.998960857Z", + "finished_at": "2026-01-23T20:13:28.197638116Z" }, "t_011_helper_function": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_011_helper_function", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let s = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum: s });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -1563,13 +1607,12 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { - "helper_func_sum_abs": { + "helper_func_sum_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n", + "phase": "call_reducer_golden" } }, "schema_parity": { @@ -1580,28 +1623,29 @@ "phase": "describe_golden" } }, - "helper_func_sum_parity": { + "helper_func_sum_abs": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n", - "phase": "call_reducer_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.709720362Z", - "finished_at": "2026-01-23T11:59:40.767569416Z" + "started_at": "2026-01-23T20:13:14.723917629Z", + "finished_at": "2026-01-23T20:14:00.225654958Z" }, "t_012_spacetime_product_type": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_012_spacetime_product_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[derive(spacetimedb::SpacetimeType, Clone, Debug)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -1609,6 +1653,15 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { + "product_type_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" + } + }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1624,76 +1677,50 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", "phase": "call_reducer_golden" } - }, - "product_type_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } } }, "vendor": "openai", - "started_at": "2026-01-23T12:00:00.251008970Z", - "finished_at": "2026-01-23T12:00:21.554582472Z" + "started_at": "2026-01-23T20:12:09.991277650Z", + "finished_at": "2026-01-23T20:12:46.725468970Z" }, "t_013_spacetime_sum_type": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_013_spacetime_sum_type", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 3, + "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "sum_type_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", - "phase": "call_reducer_golden" - } - }, - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", - "phase": "describe_golden" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:00:06.978584823Z", - "finished_at": "2026-01-23T12:00:33.966822730Z" + "started_at": "2026-01-23T20:14:26.308049242Z", + "finished_at": "2026-01-23T20:14:26.308049242Z" }, "t_014_elementary_columns": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_014_elementary_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5_f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".into(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -1701,14 +1728,6 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", - "phase": "call_reducer_golden" - } - }, "elementary_columns_row_count": { "pass": false, "partial": 0.0, @@ -1725,368 +1744,222 @@ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", "phase": "describe_golden" } - } - }, - "vendor": "openai", - "started_at": "2026-01-23T12:00:06.215125322Z", - "finished_at": "2026-01-23T12:00:33.902324216Z" - }, - "t_015_product_type_columns": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", - "task": "t_015_product_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".into(), zip: 11111 },\n work: Address { street: \"2 Broad\".into(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-015-product-type-columns-golden", - "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", - "phase": "describe_golden" - } }, - "product_type_columns_row_parity": { + "elementary_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", "phase": "call_reducer_golden" } - }, - "product_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } } }, "vendor": "openai", - "started_at": "2026-01-23T12:00:18.383198165Z", - "finished_at": "2026-01-23T12:01:34.378780677Z" + "started_at": "2026-01-23T20:12:09.987997099Z", + "finished_at": "2026-01-23T20:13:50.463947582Z" }, - "t_016_sum_type_columns": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", - "task": "t_016_sum_type_columns", + "t_015_product_type_columns": { + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "task": "t_015_product_type_columns", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 3, + "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", - "golden_db": "schema-t-016-sum-type-columns-golden", - "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "call_reducer_golden" - } - }, - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "describe_golden" + "error": "LLM call timed out", + "phase": "build_or_publish" } - }, - "sum_type_columns_row_count": { + } + }, + "vendor": "openai", + "started_at": "2026-01-23T20:14:26.308063959Z", + "finished_at": "2026-01-23T20:14:26.308063959Z" + }, + "t_016_sum_type_columns": { + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "task": "t_016_sum_type_columns", + "lang": "rust", + "golden_published": false, + "model_name": "GPT-5", + "total_tests": 1, + "passed_tests": 0, + "llm_output": null, + "category": "schema", + "route_api_model": "gpt-5", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, + "scorer_details": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40975/v1/database/c200b73d487f28a58ba0c1daee3f8e73c716071a8ecb6ca5512534eaaf3f94f4/sql)\n", - "phase": "sql" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:00:21.554609433Z", - "finished_at": "2026-01-23T12:00:50.848757403Z" + "started_at": "2026-01-23T20:14:26.308044663Z", + "finished_at": "2026-01-23T20:14:26.308044663Z" }, "t_017_scheduled_columns": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_017_scheduled_columns", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 2, - "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n Ok(())\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) -> Result<(), String> {\n Ok(())\n}", + "total_tests": 1, + "passed_tests": 0, + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", - "golden_db": "schema-t-017-scheduled-columns-golden", - "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" - } - }, - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n", - "phase": "describe_golden" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:57.789164943Z", - "finished_at": "2026-01-23T12:00:37.720475314Z" + "started_at": "2026-01-23T20:14:26.308033084Z", + "finished_at": "2026-01-23T20:14:26.308033084Z" }, "t_018_constraints": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_018_constraints", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 3, + "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let ids: Vec = ctx.db.account().iter().map(|a| a.id).collect();\n for id in ids {\n ctx.db.account().id().delete(id);\n }\n\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", - "golden_db": "schema-t-018-constraints-golden", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "constraints_row_parity_after_seed": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", - "phase": "call_reducer_golden" - } - }, - "constraints_seed_two_rows": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - }, - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", - "phase": "describe_golden" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:51.858540280Z", - "finished_at": "2026-01-23T12:00:18.383168342Z" + "started_at": "2026-01-23T20:14:26.308021447Z", + "finished_at": "2026-01-23T20:14:26.308021447Z" }, "t_019_many_to_many": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_019_many_to_many", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 5, - "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Ensure exactly these rows exist by clearing existing state.\n // Delete memberships first to avoid FK-like logical dependencies.\n let membership_ids: Vec = ctx.db.membership().iter().map(|m| m.id).collect();\n for id in membership_ids {\n ctx.db.membership().id().delete(&id);\n }\n\n let user_ids: Vec = ctx.db.user().iter().map(|u| u.user_id).collect();\n for id in user_ids {\n ctx.db.user().user_id().delete(&id);\n }\n\n let group_ids: Vec = ctx.db.group().iter().map(|g| g.group_id).collect();\n for id in group_ids {\n ctx.db.group().group_id().delete(&id);\n }\n\n // Insert users\n ctx.db.user().insert(User {\n user_id: 1,\n name: \"Alice\".to_string(),\n });\n ctx.db.user().insert(User {\n user_id: 2,\n name: \"Bob\".to_string(),\n });\n\n // Insert groups\n ctx.db.group().insert(Group {\n group_id: 10,\n title: \"Admin\".to_string(),\n });\n ctx.db.group().insert(Group {\n group_id: 20,\n title: \"Dev\".to_string(),\n });\n\n // Insert memberships\n ctx.db.membership().insert(Membership {\n id: 1,\n user_id: 1,\n group_id: 10,\n });\n ctx.db.membership().insert(Membership {\n id: 2,\n user_id: 1,\n group_id: 20,\n });\n ctx.db.membership().insert(Membership {\n id: 3,\n user_id: 2,\n group_id: 20,\n });\n}", + "total_tests": 1, + "passed_tests": 0, + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" - } - }, - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-019-many-to-many-golden`.\n", - "phase": "describe_golden" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:53.555195009Z", - "finished_at": "2026-01-23T12:00:37.548228808Z" + "started_at": "2026-01-23T20:14:26.308054994Z", + "finished_at": "2026-01-23T20:14:26.308054994Z" }, "t_020_ecs": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_020_ecs", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 5, + "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entity 1\n if ctx.db.entity().id().find(1).is_none() {\n ctx.db.entity().insert(Entity { id: 1 });\n }\n if ctx.db.position().entity_id().find(1).is_some() {\n ctx.db.position().entity_id().update(Position { entity_id: 1, x: 0, y: 0 });\n } else {\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n if ctx.db.velocity().entity_id().find(1).is_some() {\n ctx.db.velocity().entity_id().update(Velocity { entity_id: 1, vx: 1, vy: 0 });\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n\n // Entity 2\n if ctx.db.entity().id().find(2).is_none() {\n ctx.db.entity().insert(Entity { id: 2 });\n }\n if ctx.db.position().entity_id().find(2).is_some() {\n ctx.db.position().entity_id().update(Position { entity_id: 2, x: 10, y: 0 });\n } else {\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n if ctx.db.velocity().entity_id().find(2).is_some() {\n ctx.db.velocity().entity_id().update(Velocity { entity_id: 2, vx: -2, vy: 3 });\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for p in ctx.db.position().iter() {\n if let Some(v) = ctx.db.velocity().entity_id().find(p.entity_id) {\n let row = NextPosition {\n entity_id: p.entity_id,\n x: p.x + v.vx,\n y: p.y + v.vy,\n };\n if ctx.db.next_position().entity_id().find(p.entity_id).is_some() {\n ctx.db.next_position().entity_id().update(row);\n } else {\n ctx.db.next_position().insert(row);\n }\n }\n }\n}", + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", - "golden_db": "schema-t-020-ecs-golden", - "llm_db": "schema-t-020-ecs-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-020-ecs-golden`.\n", - "phase": "describe_golden" - } - }, - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40975/v1/database/c2001baf932156f49884b12b75839c4ef389d8ab4c55ac377157c4c2d21b7f10/sql)\n", - "phase": "sql" - } - }, - "ecs_step_next_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40975/v1/database/c2001baf932156f49884b12b75839c4ef389d8ab4c55ac377157c4c2d21b7f10/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity2": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40975/v1/database/c2001baf932156f49884b12b75839c4ef389d8ab4c55ac377157c4c2d21b7f10/sql)\n", - "phase": "sql" - } - }, - "ecs_seed_positions_count": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40975/v1/database/c2001baf932156f49884b12b75839c4ef389d8ab4c55ac377157c4c2d21b7f10/sql)\n", - "phase": "sql" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:44.003507320Z", - "finished_at": "2026-01-23T12:00:45.914588769Z" + "started_at": "2026-01-23T20:14:26.308059854Z", + "finished_at": "2026-01-23T20:14:26.308059854Z" }, "t_021_multi_column_index": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "task": "t_021_multi_column_index", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 4, - "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.log().iter() {\n ctx.db.log().id().delete(row.id);\n }\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}", + "total_tests": 1, + "passed_tests": 0, + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", - "golden_db": "schema-t-021-multi-column-index-golden", - "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - }, - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" - } - }, - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", - "phase": "describe_golden" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:49.748795415Z", - "finished_at": "2026-01-23T12:00:23.844832103Z" + "started_at": "2026-01-23T20:14:26.308040547Z", + "finished_at": "2026-01-23T20:14:26.308040547Z" } } } @@ -2277,14 +2150,14 @@ "modes": [ { "mode": "docs", - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_000_empty_reducers", "lang": "csharp", "golden_published": true, @@ -2307,18 +2180,18 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:04:18.656745307Z", - "finished_at": "2026-01-23T12:04:50.054718472Z" + "started_at": "2026-01-23T20:20:05.908071Z", + "finished_at": "2026-01-23T20:21:02.544054377Z" }, "t_001_basic_tables": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_001_basic_tables", "lang": "csharp", "golden_published": true, @@ -2341,25 +2214,25 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:03:14.882300132Z", - "finished_at": "2026-01-23T12:04:36.458730908Z" + "started_at": "2026-01-23T20:20:17.520008030Z", + "finished_at": "2026-01-23T20:21:03.652127542Z" }, "t_002_scheduled_table": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_002_scheduled_table", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n if (ctx.Db.TickTimer.Count == 0)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -2375,18 +2248,18 @@ "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:04:24.844091061Z", - "finished_at": "2026-01-23T12:05:24.876865213Z" + "started_at": "2026-01-23T20:21:05.396968262Z", + "finished_at": "2026-01-23T20:21:59.590160864Z" }, "t_003_struct_in_table": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_003_struct_in_table", "lang": "csharp", "golden_published": true, @@ -2409,25 +2282,25 @@ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:03:14.885907238Z", - "finished_at": "2026-01-23T12:04:30.187542427Z" + "started_at": "2026-01-23T20:20:12.029574819Z", + "finished_at": "2026-01-23T20:20:58.944028610Z" }, "t_004_insert": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_004_insert", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-004-insert-golden", @@ -2435,19 +2308,6 @@ "work_dir_golden": "target/llm-runs/basics/t_004_insert/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_004_insert/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-004-insert-golden", - "llm_db": "basics-t-004-insert-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true - } - }, "data_parity_insert_user": { "pass": true, "partial": 1.0, @@ -2464,23 +2324,36 @@ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "InsertUser", - "server": "http://127.0.0.1:35813" + "server": "http://127.0.0.1:41045" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-004-insert-golden", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41045", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:03:14.897716646Z", - "finished_at": "2026-01-23T12:04:18.656638441Z" + "started_at": "2026-01-23T20:21:02.544101204Z", + "finished_at": "2026-01-23T20:21:46.575966420Z" }, "t_005_update": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_005_update", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var existing = ctx.Db.User.Id.Find(id);\n if (existing == null)\n {\n throw new System.Exception(\"User not found\");\n }\n\n var updated = new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n };\n ctx.Db.User.Id.Update(updated);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Id.Update(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -2488,25 +2361,6 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/csharp/server/gpt-5/llm", "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_db": "basics-t-005-update-golden", - "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_db": "basics-t-005-update-gpt-5-llm", - "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "UpdateUser", - "server": "http://127.0.0.1:35813" - } - }, "seed_users_row": { "pass": true, "partial": 1.0, @@ -2522,25 +2376,44 @@ "llm_db": "basics-t-005-update-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } + }, + "data_parity_update_user": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + "Alice2", + 31, + false + ], + "golden_db": "basics-t-005-update-golden", + "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "llm_db": "basics-t-005-update-gpt-5-llm", + "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", + "reducer": "UpdateUser", + "server": "http://127.0.0.1:41045" + } } }, "vendor": "openai", - "started_at": "2026-01-23T12:04:50.054760665Z", - "finished_at": "2026-01-23T12:05:47.466386003Z" + "started_at": "2026-01-23T20:19:53.867992725Z", + "finished_at": "2026-01-23T20:21:05.391950708Z" }, "t_006_delete": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_006_delete", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -2548,13 +2421,6 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/csharp/server/gpt-5/llm", "scorer_details": { - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, "delete_user_count_zero": { "pass": true, "partial": 1.0, @@ -2572,25 +2438,32 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } + }, + "seed_users_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" + } } }, "vendor": "openai", - "started_at": "2026-01-23T12:05:24.880862690Z", - "finished_at": "2026-01-23T12:06:10.897631772Z" + "started_at": "2026-01-23T20:19:10.793697386Z", + "finished_at": "2026-01-23T20:19:53.867926470Z" }, "t_007_crud": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_007_crud", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User user1)\n {\n user1.Name = \"Alice2\";\n user1.Age = 31;\n user1.Active = false;\n ctx.Db.User.Id.Update(user1);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n var found = ctx.Db.User.Id.Find(1);\n if (found is User user1)\n {\n user1.Name = \"Alice2\";\n user1.Age = 31;\n user1.Active = false;\n ctx.Db.User.Id.Update(user1);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -2598,6 +2471,15 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm", "scorer_details": { + "crud_total_count_one": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM User" + } + }, "crud_row_id2_deleted": { "pass": true, "partial": 1.0, @@ -2615,7 +2497,7 @@ "llm_db": "basics-t-007-crud-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } @@ -2631,32 +2513,23 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "Crud", - "server": "http://127.0.0.1:35813" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User" + "server": "http://127.0.0.1:41045" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:04:36.458767943Z", - "finished_at": "2026-01-23T12:05:24.618417126Z" + "started_at": "2026-01-23T20:19:29.508011285Z", + "finished_at": "2026-01-23T20:20:17.519940667Z" }, "t_008_index_lookup": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_008_index_lookup", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n if (ctx.Db.Result.Id.Find(id) is Result existing)\n {\n existing.Name = user.Name;\n ctx.Db.Result.Id.Update(existing);\n }\n else\n {\n ctx.Db.Result.Insert(new Result { Id = user.Id, Name = user.Name });\n }\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial class User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name = \"\";\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial class Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name = \"\";\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User u)\n {\n ctx.Db.Result.Insert(new Result { Id = u.Id, Name = u.Name });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -2664,13 +2537,6 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm", "scorer_details": { - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2679,11 +2545,18 @@ "llm_db": "basics-t-008-index-lookup-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } }, + "seed_user_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" + } + }, "index_lookup_projection_parity": { "pass": true, "partial": 1.0, @@ -2697,23 +2570,23 @@ "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", "query": "SELECT Id, Name FROM Result WHERE Id=1", "reducer": "LookupUserName", - "server": "http://127.0.0.1:35813" + "server": "http://127.0.0.1:41045" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:05:24.876913581Z", - "finished_at": "2026-01-23T12:06:14.796358950Z" + "started_at": "2026-01-23T20:20:58.944081428Z", + "finished_at": "2026-01-23T20:21:57.704447793Z" }, "t_009_init": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_009_init", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"User\")]\n public partial struct User\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(SpacetimeDB.ReducerKind.Init)]\n public static void Init(SpacetimeDB.ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -2721,17 +2594,13 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM User" } }, "init_seed_alice": { @@ -2743,13 +2612,17 @@ "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" } }, - "init_total_two": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" + "golden_db": "basics-t-009-init-golden", + "llm_db": "basics-t-009-init-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41045", + "tables_diff": null, + "tables_equal": true } }, "init_seed_bob": { @@ -2763,18 +2636,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T12:05:18.162783655Z", - "finished_at": "2026-01-23T12:06:11.635711073Z" + "started_at": "2026-01-23T20:19:36.893890047Z", + "finished_at": "2026-01-23T20:20:12.029514649Z" }, "t_010_connect": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_010_connect", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\", Public = true)]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Kind = \"disconnected\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -2790,25 +2663,25 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:04:30.187575150Z", - "finished_at": "2026-01-23T12:05:18.162739983Z" + "started_at": "2026-01-23T20:19:23.574032624Z", + "finished_at": "2026-01-23T20:20:05.908027898Z" }, "t_011_helper_function": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_011_helper_function", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -2816,6 +2689,15 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/csharp/server/gpt-5/llm", "scorer_details": { + "helper_func_sum_abs": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2824,20 +2706,11 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" - } - }, "helper_func_sum_parity": { "pass": true, "partial": 1.0, @@ -2853,23 +2726,23 @@ "llm_out": "Id | Sum ----+----- 1 | 5", "query": "SELECT Id, Sum FROM Result WHERE Id=1", "reducer": "ComputeSum", - "server": "http://127.0.0.1:35813" + "server": "http://127.0.0.1:41045" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:03:14.890278760Z", - "finished_at": "2026-01-23T12:04:24.844053542Z" + "started_at": "2026-01-23T20:21:05.392014348Z", + "finished_at": "2026-01-23T20:21:51.189712882Z" }, "t_012_spacetime_product_type": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_012_spacetime_product_type", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score\n {\n Left = left,\n Right = right\n }\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right }\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -2877,6 +2750,19 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-012-spacetime-product-type-golden", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41045", + "tables_diff": null, + "tables_equal": true + } + }, "product_type_row_parity": { "pass": true, "partial": 1.0, @@ -2892,7 +2778,7 @@ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", "query": "SELECT Id, Value FROM Result WHERE Id=1", "reducer": "SetScore", - "server": "http://127.0.0.1:35813" + "server": "http://127.0.0.1:41045" } }, "product_type_row_count": { @@ -2903,34 +2789,21 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-012-spacetime-product-type-golden", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true - } } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:44.805001437Z", - "finished_at": "2026-01-23T12:07:16.896251109Z" + "started_at": "2026-01-23T20:18:54.161733902Z", + "finished_at": "2026-01-23T20:19:36.893826573Z" }, "t_013_spacetime_sum_type": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_013_spacetime_sum_type", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var existing = ctx.Db.Result.Id.Find(id);\n if (existing != null)\n {\n ctx.Db.Result.Id.Delete(id);\n }\n\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Circle { Radius = radius })\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var value = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.Result.Insert(new Result { Id = id, Value = value });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -2952,7 +2825,7 @@ "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", "query": "SELECT Id, Value FROM Result WHERE Id=1", "reducer": "SetCircle", - "server": "http://127.0.0.1:35813" + "server": "http://127.0.0.1:41045" } }, "schema_parity": { @@ -2963,7 +2836,7 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } @@ -2979,11 +2852,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T12:07:16.896327053Z", - "finished_at": "2026-01-23T12:08:07.936339455Z" + "started_at": "2026-01-23T20:17:49.057528012Z", + "finished_at": "2026-01-23T20:18:46.199910007Z" }, "t_014_elementary_columns": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_014_elementary_columns", "lang": "csharp", "golden_published": true, @@ -3006,34 +2879,34 @@ "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } }, - "elementary_columns_row_parity": { + "elementary_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c20099d4196f0168d509841ccbf52a41e879a5de183a360f652419198a45fa09/sql)\n", - "phase": "sql_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c200c583183a5879f79de7a06c7395893137fa8a834f830859f2ee7a4c9e10d4/sql)\n", + "phase": "sql" } }, - "elementary_columns_row_count": { + "elementary_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c20009038c3974b6dd9b27cd833b5bf2548c34fdbd6898aac40238455b2a0d9b/sql)\n", - "phase": "sql" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c2008dfd963859de34e7ba657c3416cbcee034a702128f54b0cb0b0b03aa914f/sql)\n", + "phase": "sql_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:54.360546232Z", - "finished_at": "2026-01-23T12:07:34.648602066Z" + "started_at": "2026-01-23T20:18:46.199983893Z", + "finished_at": "2026-01-23T20:19:29.507804243Z" }, "t_015_product_type_columns": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_015_product_type_columns", "lang": "csharp", "golden_published": true, @@ -3048,6 +2921,15 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm", "scorer_details": { + "product_type_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" + } + }, "product_type_columns_row_parity": { "pass": true, "partial": 1.0, @@ -3059,16 +2941,7 @@ "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:35813" - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" + "server": "http://127.0.0.1:41045" } }, "schema_parity": { @@ -3079,18 +2952,18 @@ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:07:16.899960622Z", - "finished_at": "2026-01-23T12:07:53.777092694Z" + "started_at": "2026-01-23T20:18:11.292039093Z", + "finished_at": "2026-01-23T20:18:54.161647622Z" }, "t_016_sum_type_columns": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_016_sum_type_columns", "lang": "csharp", "golden_published": true, @@ -3105,12 +2978,12 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm", "scorer_details": { - "sum_type_columns_row_parity": { + "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c200b0343bd80e3f58914be7679e885cb40098290c8573df5d5e85bf5f1d8191/sql)\n", - "phase": "sql_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c200b1fbaa5b44721798298318bab63a7fcd80d50f6e47060d881bfa066b8642/sql)\n", + "phase": "sql" } }, "schema_parity": { @@ -3121,33 +2994,33 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } }, - "sum_type_columns_row_count": { + "sum_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c200ad3cc29488a8fe57b5ab9ac0432417ab7d585d48e65d407028387050d856/sql)\n", - "phase": "sql" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c2003b0da13c17c6220fd590cdb5d23015a0e698b01b6c6ecbfd10b5aabbbf4c/sql)\n", + "phase": "sql_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:07:31.832777595Z", - "finished_at": "2026-01-23T12:08:22.217518757Z" + "started_at": "2026-01-23T20:16:40.802337426Z", + "finished_at": "2026-01-23T20:18:08.899029179Z" }, "t_017_scheduled_columns": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_017_scheduled_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromTicks(50_000 * 10))\n });\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n // Scheduled tick handler (no-op)\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n if (ctx.Db.TickTimer.Count == 0)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -3155,14 +3028,6 @@ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/gpt-5/llm", "scorer_details": { - "scheduled_seeded_one_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c200805f92c3324ddd49c14c46faf1bfa5b52a3ebae1e3664df720fbd9d4a487/sql)\n", - "phase": "sql" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3171,18 +3036,26 @@ "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } + }, + "scheduled_seeded_one_row": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c2000932373ef07be1f77db87f24b98a9819f6033c40da69c914e654c807ead2/sql)\n", + "phase": "sql" + } } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:40.441707268Z", - "finished_at": "2026-01-23T12:07:31.832707547Z" + "started_at": "2026-01-23T20:16:40.797164170Z", + "finished_at": "2026-01-23T20:18:08.735264338Z" }, "t_018_constraints": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_018_constraints", "lang": "csharp", "golden_published": true, @@ -3205,7 +3078,7 @@ "llm_db": "schema-t-018-constraints-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } @@ -3221,7 +3094,7 @@ "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:35813" + "server": "http://127.0.0.1:41045" } }, "constraints_seed_two_rows": { @@ -3235,18 +3108,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:11.639612541Z", - "finished_at": "2026-01-23T12:06:44.804960222Z" + "started_at": "2026-01-23T20:16:40.794071431Z", + "finished_at": "2026-01-23T20:18:11.291994106Z" }, "t_019_many_to_many": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_019_many_to_many", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\")]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(SpacetimeDB.ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\")]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -3254,24 +3127,6 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm", "scorer_details": { - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3280,7 +3135,7 @@ "llm_db": "schema-t-019-many-to-many-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } @@ -3294,6 +3149,15 @@ "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" } }, + "memberships_three_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Membership" + } + }, "m2m_has_1_20": { "pass": true, "partial": 1.0, @@ -3302,21 +3166,30 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" } + }, + "m2m_has_1_10": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" + } } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:14.796478395Z", - "finished_at": "2026-01-23T12:07:15.676945464Z" + "started_at": "2026-01-23T20:18:08.899074821Z", + "finished_at": "2026-01-23T20:19:10.793593915Z" }, "t_020_ecs": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_020_ecs", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Entity 1\n if (ctx.Db.Entity.Id.Find(1) is not Entity e1)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n }\n if (ctx.Db.Position.EntityId.Find(1) is not Position p1)\n {\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n }\n if (ctx.Db.Velocity.EntityId.Find(1) is not Velocity v1)\n {\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n }\n\n // Entity 2\n if (ctx.Db.Entity.Id.Find(2) is not Entity e2)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n }\n if (ctx.Db.Position.EntityId.Find(2) is not Position p2)\n {\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n }\n if (ctx.Db.Velocity.EntityId.Find(2) is not Velocity v2)\n {\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n if (ctx.Db.Velocity.EntityId.Find(pos.EntityId) is not Velocity vel)\n {\n continue;\n }\n\n int nx = pos.X + vel.VX;\n int ny = pos.Y + vel.VY;\n\n if (ctx.Db.NextPosition.EntityId.Find(pos.EntityId) is NextPosition np)\n {\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Entity 1\n if (ctx.Db.Entity.Id.Find(1) is not Entity)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n }\n if (ctx.Db.Position.EntityId.Find(1) is Position p1)\n {\n p1.X = 0;\n p1.Y = 0;\n ctx.Db.Position.EntityId.Update(p1);\n }\n else\n {\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n }\n if (ctx.Db.Velocity.EntityId.Find(1) is Velocity v1)\n {\n v1.VX = 1;\n v1.VY = 0;\n ctx.Db.Velocity.EntityId.Update(v1);\n }\n else\n {\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n }\n\n // Entity 2\n if (ctx.Db.Entity.Id.Find(2) is not Entity)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n }\n if (ctx.Db.Position.EntityId.Find(2) is Position p2)\n {\n p2.X = 10;\n p2.Y = 0;\n ctx.Db.Position.EntityId.Update(p2);\n }\n else\n {\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n }\n if (ctx.Db.Velocity.EntityId.Find(2) is Velocity v2)\n {\n v2.VX = -2;\n v2.VY = 3;\n ctx.Db.Velocity.EntityId.Update(v2);\n }\n else\n {\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n var vOpt = ctx.Db.Velocity.EntityId.Find(pos.EntityId);\n if (vOpt is Velocity vel)\n {\n int nx = pos.X + vel.VX;\n int ny = pos.Y + vel.VY;\n\n var npOpt = ctx.Db.NextPosition.EntityId.Find(pos.EntityId);\n if (npOpt is NextPosition np)\n {\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -3324,11 +3197,19 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm", "scorer_details": { + "ecs_next_pos_entity1": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c2005b7d5d9b2b6b33a4a581ebd8661a3b0380c077f59eee8e552ef3c67ff1b9/sql)\n", + "phase": "sql" + } + }, "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c2007f4d2ed8ff707a6d253e582ebc322ea6e6e956e5d4216cd59b8039172aaf/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c2005b7d5d9b2b6b33a4a581ebd8661a3b0380c077f59eee8e552ef3c67ff1b9/sql)\n", "phase": "sql" } }, @@ -3336,7 +3217,15 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c2007f4d2ed8ff707a6d253e582ebc322ea6e6e956e5d4216cd59b8039172aaf/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c2005b7d5d9b2b6b33a4a581ebd8661a3b0380c077f59eee8e552ef3c67ff1b9/sql)\n", + "phase": "sql" + } + }, + "ecs_step_next_positions_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c2005b7d5d9b2b6b33a4a581ebd8661a3b0380c077f59eee8e552ef3c67ff1b9/sql)\n", "phase": "sql" } }, @@ -3348,41 +3237,25 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } - }, - "ecs_step_next_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c2007f4d2ed8ff707a6d253e582ebc322ea6e6e956e5d4216cd59b8039172aaf/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c2007f4d2ed8ff707a6d253e582ebc322ea6e6e956e5d4216cd59b8039172aaf/sql)\n", - "phase": "sql" - } } }, "vendor": "openai", - "started_at": "2026-01-23T12:05:47.466492129Z", - "finished_at": "2026-01-23T12:06:40.441631392Z" + "started_at": "2026-01-23T20:18:08.902649901Z", + "finished_at": "2026-01-23T20:19:23.573927579Z" }, "t_021_multi_column_index": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "task": "t_021_multi_column_index", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { nameof(UserId), nameof(Day) })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Log.Iter())\n {\n ctx.Db.Log.Id.Delete(row.Id);\n }\n\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = [\"UserId\", \"Day\"])]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -3390,15 +3263,6 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm", "scorer_details": { - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3407,33 +3271,42 @@ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:41045", "tables_diff": null, "tables_equal": true } }, - "mcindex_seed_count": { + "mcindex_lookup_u7_d2": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" } }, - "mcindex_lookup_u7_d2": { + "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" + "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" + } + }, + "mcindex_seed_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Log" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:11.635827651Z", - "finished_at": "2026-01-23T12:06:54.360432909Z" + "started_at": "2026-01-23T20:16:40.799865241Z", + "finished_at": "2026-01-23T20:17:49.057471707Z" } } } diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json index f5405f61c51..8e7f8400b57 100644 --- a/docs/llms/docs-benchmark-summary.json +++ b/docs/llms/docs-benchmark-summary.json @@ -1,11 +1,11 @@ { "version": 1, - "generated_at": "2026-01-23T12:08:22.290Z", + "generated_at": "2026-01-23T20:21:59.670Z", "by_language": { "csharp": { "modes": { "docs": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", "models": { "GPT-5": { "categories": { @@ -42,7 +42,7 @@ "rust": { "modes": { "docs": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", "models": { "GPT-5": { "categories": { @@ -56,53 +56,53 @@ }, "schema": { "tasks": 10, - "total_tests": 34, - "passed_tests": 8, - "pass_pct": 23.529411, - "task_pass_equiv": 2.05, - "task_pass_pct": 20.5 + "total_tests": 14, + "passed_tests": 0, + "pass_pct": 0.0, + "task_pass_equiv": 0.0, + "task_pass_pct": 0.0 } }, "totals": { "tasks": 22, - "total_tests": 61, - "passed_tests": 13, - "pass_pct": 21.311476, - "task_pass_equiv": 3.3833334, - "task_pass_pct": 15.378788 + "total_tests": 41, + "passed_tests": 5, + "pass_pct": 12.195122, + "task_pass_equiv": 1.3333334, + "task_pass_pct": 6.060606 } } } }, "rustdoc_json": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "models": { "GPT-5": { "categories": { "basics": { "tasks": 12, "total_tests": 27, - "passed_tests": 22, - "pass_pct": 81.48148, - "task_pass_equiv": 8.916667, - "task_pass_pct": 74.30556 + "passed_tests": 25, + "pass_pct": 92.59259, + "task_pass_equiv": 10.0, + "task_pass_pct": 83.33333 }, "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 23, - "pass_pct": 67.64706, - "task_pass_equiv": 6.533333, - "task_pass_pct": 65.33333 + "passed_tests": 26, + "pass_pct": 76.47059, + "task_pass_equiv": 7.5333333, + "task_pass_pct": 75.333336 } }, "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 45, - "pass_pct": 73.77049, - "task_pass_equiv": 15.45, - "task_pass_pct": 70.22727 + "passed_tests": 51, + "pass_pct": 83.60656, + "task_pass_equiv": 17.533333, + "task_pass_pct": 79.69697 } } } diff --git a/modules/module-test/src/lib.rs b/modules/module-test/src/lib.rs index a5b59720cfd..433796e5b60 100644 --- a/modules/module-test/src/lib.rs +++ b/modules/module-test/src/lib.rs @@ -206,7 +206,7 @@ impl Foo<'_> { #[spacetimedb::view(name = my_player, public)] fn my_player(ctx: &ViewContext) -> Option { - ctx.db.player().identity().find(ctx.sender) + ctx.db.player().identity().find(ctx.sender()) } // ───────────────────────────────────────────────────────────────────────────── @@ -267,7 +267,7 @@ fn log_module_identity(ctx: &ReducerContext) { #[spacetimedb::reducer] pub fn test(ctx: &ReducerContext, arg: TestAlias, arg2: TestB, arg3: TestC, arg4: TestF) -> anyhow::Result<()> { log::info!("BEGIN"); - log::info!("sender: {:?}", ctx.sender); + log::info!("sender: {:?}", ctx.sender()); log::info!("timestamp: {:?}", ctx.timestamp); log::info!("bar: {:?}", arg2.foo); @@ -468,7 +468,7 @@ fn test_btree_index_args(ctx: &ReducerContext) { #[spacetimedb::reducer] fn assert_caller_identity_is_module_identity(ctx: &ReducerContext) { - let caller = ctx.sender; + let caller = ctx.sender(); let owner = ctx.identity(); if caller != owner { panic!("Caller {caller} is not the owner {owner}"); diff --git a/modules/sdk-test-connect-disconnect/src/lib.rs b/modules/sdk-test-connect-disconnect/src/lib.rs index c89b7af1f3c..4a733969c01 100644 --- a/modules/sdk-test-connect-disconnect/src/lib.rs +++ b/modules/sdk-test-connect-disconnect/src/lib.rs @@ -22,10 +22,10 @@ pub struct Disconnected { #[spacetimedb::reducer(client_connected)] pub fn identity_connected(ctx: &ReducerContext) { - ctx.db.connected().insert(Connected { identity: ctx.sender }); + ctx.db.connected().insert(Connected { identity: ctx.sender() }); } #[spacetimedb::reducer(client_disconnected)] pub fn identity_disconnected(ctx: &ReducerContext) { - ctx.db.disconnected().insert(Disconnected { identity: ctx.sender }); + ctx.db.disconnected().insert(Disconnected { identity: ctx.sender() }); } diff --git a/modules/sdk-test-view/src/lib.rs b/modules/sdk-test-view/src/lib.rs index 10b99a030e1..e00f64bdccc 100644 --- a/modules/sdk-test-view/src/lib.rs +++ b/modules/sdk-test-view/src/lib.rs @@ -52,10 +52,10 @@ fn delete_player(ctx: &ReducerContext, identity: Identity) { #[reducer] pub fn move_player(ctx: &ReducerContext, dx: i32, dy: i32) { - let my_player = ctx.db.player().identity().find(ctx.sender).unwrap_or_else(|| { + let my_player = ctx.db.player().identity().find(ctx.sender()).unwrap_or_else(|| { ctx.db.player().insert(Player { entity_id: 0, - identity: ctx.sender, + identity: ctx.sender(), }) }); match ctx.db.player_location().entity_id().find(my_player.entity_id) { @@ -82,7 +82,7 @@ pub fn move_player(ctx: &ReducerContext, dx: i32, dy: i32) { #[view(name = my_player, public)] fn my_player(ctx: &ViewContext) -> Option { - ctx.db.player().identity().find(ctx.sender) + ctx.db.player().identity().find(ctx.sender()) } #[view(name = my_player_and_level, public)] @@ -90,7 +90,7 @@ fn my_player_and_level(ctx: &ViewContext) -> Option { ctx.db .player() .identity() - .find(ctx.sender) + .find(ctx.sender()) .and_then(|Player { entity_id, identity }| { ctx.db .player_level() @@ -119,7 +119,7 @@ pub fn nearby_players(ctx: &ViewContext) -> Vec { ctx.db .player() .identity() - .find(ctx.sender) + .find(ctx.sender()) .and_then(|my_player| ctx.db.player_location().entity_id().find(my_player.entity_id)) .iter() .flat_map(|my_loc| { diff --git a/modules/sdk-test/src/lib.rs b/modules/sdk-test/src/lib.rs index 06778f733d5..0a6aed56358 100644 --- a/modules/sdk-test/src/lib.rs +++ b/modules/sdk-test/src/lib.rs @@ -637,25 +637,27 @@ fn delete_pk_u32_insert_pk_u32_two(ctx: &ReducerContext, n: u32, data: i32) -> a #[spacetimedb::reducer] fn insert_caller_one_identity(ctx: &ReducerContext) -> anyhow::Result<()> { - ctx.db.one_identity().insert(OneIdentity { i: ctx.sender }); + ctx.db.one_identity().insert(OneIdentity { i: ctx.sender() }); Ok(()) } #[spacetimedb::reducer] fn insert_caller_vec_identity(ctx: &ReducerContext) -> anyhow::Result<()> { - ctx.db.vec_identity().insert(VecIdentity { i: vec![ctx.sender] }); + ctx.db.vec_identity().insert(VecIdentity { i: vec![ctx.sender()] }); Ok(()) } #[spacetimedb::reducer] fn insert_caller_unique_identity(ctx: &ReducerContext, data: i32) -> anyhow::Result<()> { - ctx.db.unique_identity().insert(UniqueIdentity { i: ctx.sender, data }); + ctx.db + .unique_identity() + .insert(UniqueIdentity { i: ctx.sender(), data }); Ok(()) } #[spacetimedb::reducer] fn insert_caller_pk_identity(ctx: &ReducerContext, data: i32) -> anyhow::Result<()> { - ctx.db.pk_identity().insert(PkIdentity { i: ctx.sender, data }); + ctx.db.pk_identity().insert(PkIdentity { i: ctx.sender(), data }); Ok(()) } diff --git a/smoketests/tests/rls.py b/smoketests/tests/rls.py index 264f373c479..102c3378a9e 100644 --- a/smoketests/tests/rls.py +++ b/smoketests/tests/rls.py @@ -19,7 +19,7 @@ class Rls(Smoketest): #[spacetimedb::reducer] pub fn add_user(ctx: &ReducerContext, name: String) { - ctx.db.users().insert(Users { name, identity: ctx.sender }); + ctx.db.users().insert(Users { name, identity: ctx.sender() }); } """ @@ -88,7 +88,7 @@ class DisconnectRls(Smoketest): #[spacetimedb::reducer] pub fn add_user(ctx: &ReducerContext, name: String) { - ctx.db.users().insert(Users { name, identity: ctx.sender }); + ctx.db.users().insert(Users { name, identity: ctx.sender() }); } """ diff --git a/smoketests/tests/views.py b/smoketests/tests/views.py index e9d264ffbc7..23b29541789 100644 --- a/smoketests/tests/views.py +++ b/smoketests/tests/views.py @@ -609,12 +609,12 @@ class SubscribeViews(Smoketest): #[spacetimedb::view(name = my_player, public)] pub fn my_player(ctx: &ViewContext) -> Option { - ctx.db.player_state().identity().find(ctx.sender) + ctx.db.player_state().identity().find(ctx.sender()) } #[spacetimedb::reducer] pub fn insert_player(ctx: &ReducerContext, name: String) { - ctx.db.player_state().insert(PlayerState { name, identity: ctx.sender }); + ctx.db.player_state().insert(PlayerState { name, identity: ctx.sender() }); } """ diff --git a/smoketests/tests/zz_docker.py b/smoketests/tests/zz_docker.py index 86816383114..bcfdef16a73 100644 --- a/smoketests/tests/zz_docker.py +++ b/smoketests/tests/zz_docker.py @@ -112,14 +112,14 @@ class DockerRestartAutoDisconnect(Smoketest): #[spacetimedb::reducer(client_connected)] fn on_connect(ctx: &ReducerContext) { ctx.db.connected_client().insert(ConnectedClient { - identity: ctx.sender, + identity: ctx.sender(), connection_id: ctx.connection_id.expect("sender connection id unset"), }); } #[spacetimedb::reducer(client_disconnected)] fn on_disconnect(ctx: &ReducerContext) { - let sender_identity = &ctx.sender; + let sender_identity = &ctx.sender(); let sender_connection_id = ctx.connection_id.as_ref().expect("sender connection id unset"); let match_client = |row: &ConnectedClient| { &row.identity == sender_identity && &row.connection_id == sender_connection_id diff --git a/templates/chat-console-rs/spacetimedb/src/lib.rs b/templates/chat-console-rs/spacetimedb/src/lib.rs index de77f34bde2..d00949f5858 100644 --- a/templates/chat-console-rs/spacetimedb/src/lib.rs +++ b/templates/chat-console-rs/spacetimedb/src/lib.rs @@ -26,8 +26,8 @@ fn validate_name(name: String) -> Result { #[spacetimedb::reducer] pub fn set_name(ctx: &ReducerContext, name: String) -> Result<(), String> { let name = validate_name(name)?; - if let Some(user) = ctx.db.user().identity().find(ctx.sender) { - log::info!("User {} sets name to {name}", ctx.sender); + if let Some(user) = ctx.db.user().identity().find(ctx.sender()) { + log::info!("User {} sets name to {name}", ctx.sender()); ctx.db.user().identity().update(User { name: Some(name), ..user @@ -52,9 +52,9 @@ pub fn send_message(ctx: &ReducerContext, text: String) -> Result<(), String> { // - Rate-limit messages per-user. // - Reject messages from unnamed user. let text = validate_message(text)?; - log::info!("User {}: {text}", ctx.sender); + log::info!("User {}: {text}", ctx.sender()); ctx.db.message().insert(Message { - sender: ctx.sender, + sender: ctx.sender(), text, sent: ctx.timestamp, }); @@ -67,7 +67,7 @@ pub fn init(_ctx: &ReducerContext) {} #[spacetimedb::reducer(client_connected)] pub fn identity_connected(ctx: &ReducerContext) { - if let Some(user) = ctx.db.user().identity().find(ctx.sender) { + if let Some(user) = ctx.db.user().identity().find(ctx.sender()) { // If this is a returning user, i.e. we already have a `User` with this `Identity`, // set `online: true`, but leave `name` and `identity` unchanged. ctx.db.user().identity().update(User { online: true, ..user }); @@ -76,7 +76,7 @@ pub fn identity_connected(ctx: &ReducerContext) { // which is online, but hasn't set a name. ctx.db.user().insert(User { name: None, - identity: ctx.sender, + identity: ctx.sender(), online: true, }); } @@ -84,11 +84,11 @@ pub fn identity_connected(ctx: &ReducerContext) { #[spacetimedb::reducer(client_disconnected)] pub fn identity_disconnected(ctx: &ReducerContext) { - if let Some(user) = ctx.db.user().identity().find(ctx.sender) { + if let Some(user) = ctx.db.user().identity().find(ctx.sender()) { ctx.db.user().identity().update(User { online: false, ..user }); } else { // This branch should be unreachable, // as it doesn't make sense for a client to disconnect without connecting first. - log::warn!("Disconnect event for unknown user with identity {:?}", ctx.sender); + log::warn!("Disconnect event for unknown user with identity {:?}", ctx.sender()); } } From 645f88817cb83bae26b9c4b7e44ee125ba320eb8 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Mon, 26 Jan 2026 19:42:31 +0000 Subject: [PATCH 2/3] Update LLM benchmark results --- docs/llms/docs-benchmark-analysis.md | 799 ++++-------- docs/llms/docs-benchmark-comment.md | 18 +- docs/llms/docs-benchmark-details.json | 1725 +++++++++++-------------- docs/llms/docs-benchmark-summary.json | 70 +- 4 files changed, 1100 insertions(+), 1512 deletions(-) diff --git a/docs/llms/docs-benchmark-analysis.md b/docs/llms/docs-benchmark-analysis.md index 594bf72eafb..e55af14c396 100644 --- a/docs/llms/docs-benchmark-analysis.md +++ b/docs/llms/docs-benchmark-analysis.md @@ -4,548 +4,315 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do ## Summary -- **Total failures analyzed**: 34 +- **Total failures analyzed**: 33 --- # Analysis of SpacetimeDB Benchmark Test Failures -This analysis focuses on test failures within SpacetimeDB benchmarks, specifically categorized by language and mode, providing actionable insights for documentation improvements to reduce these errors. +This document analyzes the SpacetimeDB benchmark test failures organized by language and mode. Each section addresses specific failures, providing insights into their causes and recommending actionable solutions. + +## Rust / rustdoc_json Failures + +### Compile/Publish Errors + +#### t_002_scheduled_table & t_017_scheduled_columns + +1. **The generated code**: + ```rust + use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; + + #[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))] + pub struct TickTimer { + #[primary_key] + #[auto_inc] + scheduled_id: u64, + scheduled_at: ScheduleAt, + } + + #[reducer(init)] + pub fn init(ctx: &ReducerContext) { + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: ScheduleAt::repeat_micros(50_000), + }); + } + + #[reducer] + pub fn tick(_ctx: &ReducerContext, _row: TickTimer) { + } + ``` + +2. **The golden example**: + ```rust + use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; + use std::time::Duration; + + #[table(name = tick_timer, scheduled(tick))] + pub struct TickTimer { + #[primary_key] + #[auto_inc] + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, + } + + #[reducer] + pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) { + } + + #[reducer(init)] + pub fn init(ctx: &ReducerContext) { + let every_50ms: ScheduleAt = Duration::from_millis(50).into(); + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: every_50ms, + }); + } + ``` + +3. **The error**: + - `publish_error: spacetime publish failed (exit=1)` + +4. **Explain the difference**: + - The LLM generated code uses `ScheduleAt::repeat_micros(50_000)` while the expected code uses `ScheduleAt::Interval(Duration::from_millis(50).into())`. + - The `scheduled` attribute is incorrectly set. + +5. **Root cause**: + - The documentation lacks clear guidance on using the `ScheduleAt` type effectively and the format for specifying scheduled actions. + +6. **Recommendation**: + - Update documentation to clarify how to define timing for scheduled entries, using `ScheduleAt::Interval` instead of `ScheduleAt::repeat_*` methods. --- -## Rust / rustdoc_json Failures (7 total) - -### Compile/Publish Errors (3 failures) - -#### 1. t_002_scheduled_table -- **The generated code**: - ```rust - use spacetimedb::{ReducerContext, ScheduleAt, Table}; - - #[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - scheduled_id: u64, - scheduled_at: ScheduleAt, - } - - #[spacetimedb::reducer(init)] - pub fn init(ctx: &ReducerContext) { - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: ScheduleAt::repeat_micros(50_000), - }); - } - - #[spacetimedb::reducer] - pub fn tick(_ctx: &ReducerContext) { - log::info!("tick"); - } - ``` - -- **The golden example**: - ```rust - #[table(name = tick_timer, scheduled(tick))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - pub scheduled_id: u64, - pub scheduled_at: ScheduleAt, - } - - #[reducer] - pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) { - } - - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - let every_50ms: ScheduleAt = Duration::from_millis(50).into(); - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: every_50ms, - }); - } - ``` - -- **The error**: `publish_error: spacetime publish failed (exit=1)` - -- **Explain the difference**: - - The generated code incorrectly used `ScheduleAt::repeat_micros(50_000)` instead of `ScheduleAt::Interval(Duration::from_millis(50).into())`. - - The reducer function lacked parameters that were required. - -- **Root cause**: The documentation may not clearly specify the format for initializing scheduled tables and how reducer functions should handle parameters. - -- **Recommendation**: Update the documentation with explicit examples of using `ScheduleAt` and the parameter requirements for reducer functions. +#### t_003_struct_in_table + +1. **The generated code**: + ```rust + use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType}; + + #[derive(SpacetimeType)] + pub struct Position { + x: i32, + y: i32, + } + + #[table(name = entity)] + pub struct Entity { + #[primary_key] + id: i32, + pos: Position, + } + + #[reducer] + pub fn add_entity(ctx: &ReducerContext, id: i32, x: i32, y: i32) { + ctx.db.entity().insert(Entity { id, pos: Position { x, y } }); + } + ``` + +2. **The golden example**: + ```rust + use spacetimedb::{table, SpacetimeType}; + + #[derive(SpacetimeType, Clone, Debug)] + pub struct Position { + pub x: i32, + pub y: i32, + } + + #[table(name = entity)] + pub struct Entity { + #[primary_key] + pub id: i32, + pub pos: Position, + } + ``` + +3. **The error**: + - `schema_parity: reducers differ - expected [], got ["add_entity()"]` + +4. **Explain the difference**: + - The expected code does not include any reducers, while the generated code has an unnecessary `add_entity` reducer. + +5. **Root cause**: + - Misrepresentation of structural requirements regarding reducers in the schema. + +6. **Recommendation**: + - Update the documentation to clarify when reducers should or should not be defined for schema parity. --- -#### 2. t_007_crud -- **The generated code**: - ```rust - use spacetimedb::{reducer, table, ReducerContext}; - - #[table(name = user)] - pub struct User { - #[primary_key] - id: i32, - name: String, - age: i32, - active: bool, - } - - #[reducer] - pub fn crud(ctx: &ReducerContext) { - let users = ctx.db.user(); - - users.insert(User { - id: 1, - name: "Alice".to_string(), - age: 30, - active: true, - }); - - users.insert(User { - id: 2, - name: "Bob".to_string(), - age: 22, - active: false, - }); - - if let Some(mut u) = users.id().find(1) { - u.name = "Alice2".to_string(); - u.age = 31; - u.active = false; - users.id().update(u); - } - - users.id().delete(&2); - } - ``` - -- **The golden example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; - - #[table(name = user)] - pub struct User { - #[primary_key] - pub id: i32, - pub name: String, - pub age: i32, - pub active: bool, - } - - #[reducer] - pub fn crud(ctx: &ReducerContext) { - ctx.db.user().insert(User { id: 1, name: "Alice".into(), age: 30, active: true }); - ctx.db.user().insert(User { id: 2, name: "Bob".into(), age: 22, active: false }); - ctx.db.user().id().update(User { id: 1, name: "Alice2".into(), age: 31, active: false }); - ctx.db.user().id().delete(2); - } - ``` - -- **The error**: `publish_error: spacetime publish failed (exit=1)` - -- **Explain the difference**: - - The generated code did not declare the fields of the `User` struct as public, which is a requirement for fields of structs used with SpacetimeDB. - - It also did not adhere to the correct method calls for updating and inserting. - -- **Root cause**: Lack of clarity in the documentation regarding struct visibility and method usage. - -- **Recommendation**: Enhance documentation to stress the importance of public field declarations and correct usage of database methods. +#### t_018_constraints + +1. **The generated code**: + ```rust + use spacetimedb::{table, reducer, ReducerContext, Table}; + + #[table(name = account, index(name = by_name, btree(columns = [name])))] + pub struct Account { + #[primary_key] + id: i32, + #[unique] + email: String, + name: String, + } + + #[reducer] + pub fn seed(ctx: &ReducerContext) { + let t = ctx.db.account(); + t.insert(Account { id: 1, email: "a@example.com".to_string(), name: "Alice".to_string() }); + t.insert(Account { id: 2, email: "b@example.com".to_string(), name: "Bob".to_string() }); + } + ``` + +2. **The golden example**: + ```rust + use spacetimedb::{reducer, table, ReducerContext, Table}; + + #[table( + name = account, + index(name = by_name, btree(columns = [name])) + )] + pub struct Account { + #[primary_key] + pub id: i32, + #[unique] + pub email: String, + pub name: String, + } + + #[reducer] + pub fn seed(ctx: &ReducerContext) { + ctx.db.account().insert(Account { id: 1, email: "a@example.com".into(), name: "Alice".into() }); + ctx.db.account().insert(Account { id: 2, email: "b@example.com".into(), name: "Bob".into() }); + } + ``` + +3. **The error**: + - `constraints_row_parity_after_seed: spacetime sql failed: no such table: 'account'` + +4. **Explain the difference**: + - The generated code did not include the `pub` keyword in front of fields, which results in private access. + +5. **Root cause**: + - Lack of explicit guidelines on visibility modifiers for database fields and schema definition. + +6. **Recommendation**: + - Update the documentation to emphasize that struct fields in database models must be public. --- -#### 3. t_017_scheduled_columns -- **The generated code**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; - - #[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - scheduled_id: u64, - scheduled_at: ScheduleAt, - } - - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - if ctx.db.tick_timer().count() == 0 { - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: ScheduleAt::repeat(50_000), - }); - } - } - - #[reducer(scheduled)] - pub fn tick(_ctx: &ReducerContext, _row: TickTimer) { - } - ``` - -- **The golden example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; - use std::time::Duration; - - #[table(name = tick_timer, scheduled(tick))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - pub scheduled_id: u64, - pub scheduled_at: ScheduleAt, - } - - #[reducer] - pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) { - } - - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - let every_50ms: ScheduleAt = Duration::from_millis(50).into(); - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: every_50ms, - }); - } - ``` - -- **The error**: `publish_error: spacetime publish failed (exit=1)` - -- **Explain the difference**: - - The LLM incorrectly specified the scheduled column and did not update the initialization logic for accurate type conversion. - -- **Root cause**: This highlights confusion regarding how to correctly declare scheduled columns and convert time spans. - -- **Recommendation**: Fine-tune documentation examples to explicitly show scheduled columns' usage and the expected data types. +### Additional Recommendations ---- - -### Other Failures (4 failures) - -#### t_004_insert -- **The generated code**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; - -- **Golden Example**: - ```rust - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - let every_50ms: ScheduleAt = Duration::from_millis(50).into(); - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: every_50ms, - }); - } - ``` - -- **Error**: `publish_error: spacetime publish failed (exit=1)` - -- **Explanation**: The method for initializing `scheduled_at` is incorrect. Instead of using `repeat_micros()`, the code should convert a `Duration` to `ScheduleAt`. - -- **Root Cause**: Misunderstanding of the proper way to initialize scheduled columns could be reflected in lacking documentation details. - -- **Recommendation**: Clarify the documentation regarding initializing `ScheduleAt`, emphasizing conversion from `Duration`. - ---- - -### Other Failures (2 Failures) - -#### 4. **t_016_sum_type_columns** -- **Generated Code**: - ```rust - #[spacetimedb::table(name = drawing)] - pub struct Drawing { - #[primary_key] - id: i32, - a: Shape, - b: Shape, - } - ``` - -- **Golden Example**: - ```rust - #[table(name = drawing)] - pub struct Drawing { - #[primary_key] - pub id: i32, - pub a: Shape, - pub b: Shape, - } - ``` - -- **Error**: Errors regarding tables not found. +1. **Documentation Clarity**: + - Ensure clear examples defining the expected syntax for all relevant SpacetimeDB features (e.g., table structure, reducer signatures). + +2. **Example Consistency**: + - Modify the examples to guarantee consistency in field access levels (public/private) across all instances. -- **Explanation**: Missing the `pub` attribute on struct fields results in failure to compile. - -- **Root Cause**: Lack of clarity on the use of visibility attributes (`pub`) in struct definitions. - -- **Recommendation**: Revise documentation to instruct that fields must be public to work within SpacetimeDB. +3. **Error Handling**: + - Include a section on expected error messages and discrepancies that developers should look out for, which could help in debugging similar errors effectively. --- -#### 5. **t_020_ecs** -- **Generated Code**: - ```rust - #[spacetimedb::table(name = entity)] - pub struct Entity { - #[primary_key] - id: i32, - } - - #[spacetimedb::table(name = position)] - pub struct Position { - #[primary_key] - entity_id: i32, - x: i32, - y: i32, - } - ``` - -- **Golden Example**: - ```rust - #[table(name = entity)] - pub struct Entity { - #[primary_key] - pub id: i32, - } - - #[table(name = position)] - pub struct Position { - #[primary_key] - pub entity_id: i32, - pub x: i32, - pub y: i32, - } - ``` - -- **Error**: Errors regarding tables not found. - -- **Explanation**: Missing the `pub` attribute leads to the struct not being properly registered with SpacetimeDB. - -- **Root Cause**: Similar to previous errors, the need for public access to struct fields is unclear. - -- **Recommendation**: Ensure documentation explicitly states that public access is necessary for all fields in SpacetimeDB structs. - ---- +## Rust / docs Failures -## Rust / docs Failures (22 total) - -### Timeout Issues (8 Failures) - -- **Failures**: Various tasks timed out, indicating potential performance or configuration issues. - -- **Root Cause**: Specifics of timeout settings and performance optimization strategies should be more explicit in the documentation. - -- **Recommendation**: Include guidelines on optimizing performance for long-running tasks or emphasize best practices for structuring queries and data handling. - ---- - -### Other Failures (14 Failures) - -#### 6. **t_000_empty_reducers** -- **Generated Code**: - ```rust - #[spacetimedb::reducer] - pub fn empty_reducer_no_args(_ctx: &spacetimedb::ReducerContext) { - } - ``` - -- **Golden Example**: - ```rust - #[reducer] - pub fn empty_reducer_no_args(ctx: &ReducerContext) -> Result<(), String> { - Ok(()) - } - ``` - -- **Error**: Schema-related errors due to missing return type and proper handling. - -- **Explanation**: Missing return type (`Result<(), String>`) was not implemented. - -- **Root Cause**: The documentation may not explicitly mention that reducers should return results. - -- **Recommendation**: Adjust the documentation to specify that reducer functions must include appropriate return types. - ---- - -#### 7. **t_001_basic_tables** -- **Generated Code**: - ```rust - #[spacetimedb::table(name = user)] - pub struct User { - #[primary_key] - id: i32, - name: String, - age: i32, - active: bool, - } - ``` - -- **The golden example**: - ```rust - #[table(name = user)] - pub struct User { - #[primary_key] - pub id: i32, - pub name: String, - pub age: i32, - pub active: bool, - } - ``` - -- **The error**: `data_parity_insert_user: spacetime sql failed: no such table: user` - -- **Explain the difference**: - - The generated code didn’t mark struct fields as public, and failed to return a `Result` for the reducer function, which is required by the documentation. - -- **Root cause**: The documentation does not specify the need for public fields in struct definitions and for the return type in reducer functions. - -- **Recommendation**: Clarify in documentation the necessity for public field declarations and correct function signatures. - -#### t_011_helper_function -- **The generated code**: - ```rust - use spacetimedb::{table, reducer, ReducerContext, Table}; - - #[table(name = result)] - pub struct ResultRow { - #[primary_key] - id: i32, - sum: i32, - } - - fn add(a: i32, b: i32) -> i32 { - a + b - } - - #[reducer] - fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) { - let sum = add(a, b); - ctx.db.result().insert(ResultRow { id, sum }); - } - ``` - -- **The golden example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; - - #[table(name = result)] - pub struct ResultRow { - #[primary_key] - pub id: i32, - pub sum: i32, - } - - fn add(a: i32, b: i32) -> i32 { a + b } - - #[reducer] - pub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) { - ctx.db.result().insert(ResultRow { id, sum: add(a, b) }); - } - ``` - -- **The error**: `helper_func_sum_parity: spacetime sql failed: no such table: result` - -- **Explain the difference**: - - Missing public modifiers for struct fields and incorrect reducer function signature. - -- **Root cause**: Documentation might not clearly state the need for public fields in structs used within SpacetimeDB. - -- **Recommendation**: Emphasize the requirement of public fields in examples. - ---- +### Timeout Issues -### C# / docs Failures (5 total) - -#### 1. t_014_elementary_columns -- **The generated code**: - ```csharp - [SpacetimeDB.Table(Name = "Primitive", Public = true)] - public partial struct Primitive - { - [SpacetimeDB.Table(Name = "Primitive")] - public partial struct Primitive - { - [SpacetimeDB.PrimaryKey] - public int Id; - public int Count; - public long Total; - public float Price; - public double Ratio; - public bool Active; - public string Name; - } - - [SpacetimeDB.Reducer] - public static void Seed(ReducerContext ctx) - { - ctx.Db.Primitive.Insert(new Primitive - { - Id = 1, - Count = 2, - Total = 3000000000L, - Price = 1.5f, - Ratio = 2.25, - Active = true, - Name = "Alice" - }); - } - } - ``` - -- **The golden example**: - ```csharp - [Table(Name = "Primitive")] - public partial struct Primitive - { - [Table(Name = "Primitive")] - public partial struct Primitive - { - [PrimaryKey] public int Id; - public int Count; - public long Total; - public float Price; - public double Ratio; - public bool Active; - public string Name; - } - - [Reducer] - public static void Seed(ReducerContext ctx) - { - ctx.Db.Primitive.Insert(new Primitive - { - Id = 1, - Count = 2, - Total = 3000000000, - Price = 1.5f, - Ratio = 2.25, - Active = true, - Name = "Alice" - }); - } - } - ``` - -- **The error**: `no such table: primitive` - -- **Explain the difference**: Field visibility was not explicitly made public in the generated code, which is a requirement for SpacetimeDB. - -- **Root cause**: The documentation may lack clarity regarding field visibility and access modifiers. - -- **Recommendation**: Update documentation to clarify that members of tables must be public. +- **Failures**: t_013_spacetime_sum_type, t_015_product_type_columns, t_016_sum_type_columns, t_018_constraints, t_019_many_to_many, t_020_ecs +- **Recommendation**: + - Review the execution time of these benchmarks and possibly optimize the code for performance or provide a timeout setting for testing. --- -(Continue this format for the remaining C# failures...) +## C# / docs Failures + +### t_014_elementary_columns + +1. **The generated code**: + ```csharp + using SpacetimeDB; + + public static partial class Module + { + [SpacetimeDB.Table(Name = "Primitive", Public = true)] + public partial struct Primitive + { + [SpacetimeDB.PrimaryKey] + public int Id; + public int Count; + public long Total; + public float Price; + public double Ratio; + public bool Active; + public string Name; + } + + [SpacetimeDB.Reducer] + public static void Seed(ReducerContext ctx) + { + ctx.Db.Primitive.Insert(new Primitive + { + Id = 1, + Count = 2, + Total = 3000000000L, + Price = 1.5f, + Ratio = 2.25, + Active = true, + Name = "Alice" + }); + } + } + ``` + +2. **The golden example**: + ```csharp + using SpacetimeDB; + + public static partial class Module + { + [Table(Name = "Primitive")] + public partial struct Primitive + { + [PrimaryKey] public int Id; + public int Count; + public long Total; + public float Price; + public double Ratio; + public bool Active; + public string Name; + } + + [Reducer] + public static void Seed(ReducerContext ctx) + { + ctx.Db.Primitive.Insert(new Primitive { + Id = 1, + Count = 2, + Total = 3000000000, + Price = 1.5f, + Ratio = 2.25, + Active = true, + Name = "Alice" + }); + } + } + ``` + +3. **The error**: + - `no such table: 'primitive'` + +4. **Explain the difference**: + - The generated code has an extra public field visibility which is redundant in this context, leading to distraction. + +5. **Root cause**: + - Inconsistent handling of public annotations for struct attributes. + +6. **Recommendation**: + - Align the documentation to show proper usage of attributes and visibility appropriately. --- ### Conclusion -This comprehensive analysis of SpacetimeDB benchmark test failures highlights key areas where the documentation can improve self-guidance for developers. Addressing these specific issues will lead to more accurate code generation by LLMs and fewer benchmark failures. +This analysis serves to highlight the discrepancies noted in the benchmark test failures across Rust and C#, with actionable steps to amend recurring issues. Essential areas of improvement focus on explicit documentation, consistent field access levels, and clearer definitions of API requirements. By implementing these recommendations, we can streamline the development process and avoid common pitfalls. diff --git a/docs/llms/docs-benchmark-comment.md b/docs/llms/docs-benchmark-comment.md index de9b098fefa..907eb0e9b50 100644 --- a/docs/llms/docs-benchmark-comment.md +++ b/docs/llms/docs-benchmark-comment.md @@ -2,16 +2,16 @@ | Language | Mode | Category | Tests Passed | Task Pass % | |----------|------|----------|--------------|-------------| -| Rust | rustdoc_json | basics | 20/27 | 76.4% ⬆️ +2.1% | -| Rust | rustdoc_json | schema | 26/34 | 75.3% | -| Rust | rustdoc_json | **total** | 46/61 | **75.9%** ⬆️ +1.1% | +| Rust | rustdoc_json | basics | 25/27 | 83.3% ⬆️ +6.9% | +| Rust | rustdoc_json | schema | 24/34 | 68.7% ⬇️ -6.7% | +| Rust | rustdoc_json | **total** | 49/61 | **76.7%** ⬆️ +0.8% | | Rust | docs | basics | 5/27 | 11.1% | -| Rust | docs | schema | 8/34 | 20.5% ⬆️ +8.0% | -| Rust | docs | **total** | 13/61 | **15.4%** ⬆️ +3.6% | -| C# | docs | basics | 27/27 | 100.0% ⬆️ +8.3% | -| C# | docs | schema | 21/34 | 63.7% | -| C# | docs | **total** | 48/61 | **83.5%** ⬆️ +4.5% | +| Rust | docs | schema | 4/18 | 12.5% ⬇️ -8.0% | +| Rust | docs | **total** | 9/45 | **11.7%** ⬇️ -3.6% | +| C# | docs | basics | 27/27 | 100.0% | +| C# | docs | schema | 24/32 | 70.3% ⬆️ +6.7% | +| C# | docs | **total** | 51/59 | **86.5%** ⬆️ +3.0% | _Compared against master branch baseline_ -Generated at: 2026-01-26T15:42:12.228Z +Generated at: 2026-01-26T19:41:35.586Z diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json index d4d9e9065c2..37d0cc6adb8 100644 --- a/docs/llms/docs-benchmark-details.json +++ b/docs/llms/docs-benchmark-details.json @@ -19,7 +19,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(ctx: &ReducerContext, count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(ctx: &ReducerContext, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(ctx: &ReducerContext, count: i32, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(ctx: &ReducerContext, active: bool, ratio: f32, label: String) {}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -35,15 +35,15 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:44233", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975549166Z", - "finished_at": "2026-01-26T15:31:47.073747240Z" + "started_at": "2026-01-26T19:27:52.325853523Z", + "finished_at": "2026-01-26T19:28:19.691117111Z" }, "t_001_basic_tables": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -69,15 +69,15 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:44233", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.974869540Z", - "finished_at": "2026-01-26T15:31:51.422745193Z" + "started_at": "2026-01-26T19:27:55.244006331Z", + "finished_at": "2026-01-26T19:28:22.209162875Z" }, "t_002_scheduled_table": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -87,7 +87,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext) {\n log::info!(\"tick\");\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _scheduled_id: u64) {\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -99,24 +99,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling shlex v1.3.0\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling zerocopy v0.8.34\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling zmij v1.0.17\n Compiling arrayvec v0.7.6\n Compiling second-stack v0.3.5\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.4.2\n Compiling serde_json v1.0.149\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling bytemuck v1.24.0\n Compiling cc v1.2.54\n Compiling hex v0.4.3\n Compiling spacetimedb-lib v1.11.1\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling itoa v1.0.17\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:41\n |\n4 | #[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:14:32\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:12\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:16:35\n |\n16 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 4 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling serde v1.0.228\n Compiling zerocopy v0.8.34\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling bytes v1.11.0\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling keccak v0.1.5\n Compiling zmij v1.0.17\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling smallvec v1.15.1\n Compiling hex v0.4.3\n Compiling second-stack v0.3.5\n Compiling constant_time_eq v0.4.2\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling itertools v0.12.1\n Compiling cc v1.2.54\n Compiling itoa v1.0.17\n Compiling spacetimedb-lib v1.11.1\n Compiling serde_json v1.0.149\n Compiling rand_core v0.6.4\n Compiling bytemuck v1.24.0\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling blake3 v1.8.3\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975688871Z", - "finished_at": "2026-01-26T15:32:30.863567064Z" + "started_at": "2026-01-26T19:28:10.719260484Z", + "finished_at": "2026-01-26T19:29:24.230351153Z" }, "t_003_struct_in_table": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_003_struct_in_table", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 1, - "llm_output": "#[derive(spacetimedb::SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", + "passed_tests": 0, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}\n\n#[reducer]\npub fn add_entity(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n ctx.db.entity().insert(Entity { id, pos: Position { x, y } });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -125,22 +125,27 @@ "work_dir_llm": "target/llm-runs/basics/t_003_struct_in_table/rust/server/gpt-5/llm", "scorer_details": { "schema_parity": { - "pass": true, - "partial": 1.0, + "pass": false, + "partial": 0.0, "notes": { "golden_db": "basics-t-003-struct-in-table-golden", "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "reducers_diff": { + "only_golden": [], + "only_llm": [ + "add_entity()" + ] + }, + "reducers_equal": false, + "server": "http://127.0.0.1:44233", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975098187Z", - "finished_at": "2026-01-26T15:31:55.504039476Z" + "started_at": "2026-01-26T19:27:53.381607365Z", + "finished_at": "2026-01-26T19:28:20.526037052Z" }, "t_004_insert": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -149,8 +154,8 @@ "golden_published": true, "model_name": "GPT-5", "total_tests": 2, - "passed_tests": 1, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-004-insert-golden", @@ -174,7 +179,7 @@ "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", "query": "SELECT id, name, age, active FROM user WHERE id=1", "reducer": "insert_user", - "server": "http://127.0.0.1:41115" + "server": "http://127.0.0.1:44233" } }, "schema_parity": { @@ -185,23 +190,15 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:44233", "tables_diff": null, "tables_equal": true } - }, - "data_parity_insert_user": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002173ebf56f08f752bb2ca55d0547329e19fb14624413a7089817795d01ab/sql)\n", - "phase": "sql_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975407164Z", - "finished_at": "2026-01-26T15:31:42.241214465Z" + "started_at": "2026-01-26T19:28:03.162962728Z", + "finished_at": "2026-01-26T19:28:30.365311126Z" }, "t_005_update": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -211,7 +208,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let _ = ctx.db.user().id().update(User { id, name, age, active });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().id().update(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -219,23 +216,17 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { - "data_parity_update_user": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], "golden_db": "basics-t-005-update-golden", - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "llm_db": "basics-t-005-update-gpt-5-llm", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user", - "server": "http://127.0.0.1:34081" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:44233", + "tables_diff": null, + "tables_equal": true } }, "seed_users_row": { @@ -245,33 +236,39 @@ "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, - "schema_parity": { + "data_parity_update_user": { "pass": true, "partial": 1.0, "notes": { + "args": [ + 1, + "Alice2", + 31, + false + ], "golden_db": "basics-t-005-update-golden", + "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "llm_db": "basics-t-005-update-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true + "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "update_user", + "server": "http://127.0.0.1:44233" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:42.241280124Z", - "finished_at": "2026-01-26T15:32:22.818349314Z" + "started_at": "2026-01-26T19:27:50.273973455Z", + "finished_at": "2026-01-26T19:28:39.660285285Z" }, "t_006_delete": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_006_delete", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -279,6 +276,19 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-006-delete-golden", + "llm_db": "basics-t-006-delete-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:44233", + "tables_diff": null, + "tables_equal": true + } + }, "delete_user_count_zero": { "pass": true, "partial": 1.0, @@ -294,34 +304,21 @@ "notes": { "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-006-delete-golden", - "llm_db": "basics-t-006-delete-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:49.227905993Z", - "finished_at": "2026-01-26T15:32:24.290487857Z" + "started_at": "2026-01-26T19:27:19.862018853Z", + "finished_at": "2026-01-26T19:27:52.325711770Z" }, "t_007_crud": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_007_crud", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 4, - "passed_tests": 0, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n users.insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n users.insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n if let Some(mut u) = users.id().find(1) {\n u.name = \"Alice2\".to_string();\n u.age = 31;\n u.active = false;\n users.id().update(u);\n }\n\n users.id().delete(&2);\n}", + "passed_tests": 4, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n users.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n users.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n\n let updated = User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false };\n users.id().update(updated);\n\n users.id().delete(&2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -329,18 +326,55 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling zerocopy v0.8.34\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.8\n Compiling shlex v1.3.0\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling heck v0.4.1\n Compiling bytes v1.11.0\n Compiling zmij v1.0.17\n Compiling convert_case v0.4.0\n Compiling itoa v1.0.17\n Compiling serde_json v1.0.149\n Compiling smallvec v1.15.1\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling constant_time_eq v0.4.2\n Compiling generic-array v0.14.7\n Compiling spacetimedb-lib v1.11.1\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling cc v1.2.54\n Compiling itertools v0.12.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling digest v0.10.7\n Compiling blake3 v1.8.3\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm)\nerror[E0599]: no method named `insert` found for reference `&user__TableHandle` in the current scope\n --> src/lib.rs:17:11\n |\n17 | users.insert(User {\n | ------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n17 | users.try_insert(User {\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&user__TableHandle` in the current scope\n --> src/lib.rs:24:11\n |\n24 | users.insert(User {\n | ------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n24 | users.try_insert(User {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 2 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" + } + }, + "crud_total_count_one": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user" + } + }, + "crud_row_id1_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "basics-t-007-crud-golden", + "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "crud", + "server": "http://127.0.0.1:44233" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-007-crud-golden", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:44233", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975950122Z", - "finished_at": "2026-01-26T15:32:16.012153929Z" + "started_at": "2026-01-26T19:27:23.893528453Z", + "finished_at": "2026-01-26T19:28:07.775124578Z" }, "t_008_index_lookup": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -350,7 +384,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n let _ = ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow { id: user.id, name: user.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -358,11 +392,17 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", "scorer_details": { - "seed_user_row": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + "golden_db": "basics-t-008-index-lookup-golden", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:44233", + "tables_diff": null, + "tables_equal": true } }, "index_lookup_projection_parity": { @@ -378,26 +418,20 @@ "llm_out": "id | name ----+--------- 1 | \"Alice\"", "query": "SELECT id, name FROM result WHERE id=1", "reducer": "lookup_user_name", - "server": "http://127.0.0.1:34081" + "server": "http://127.0.0.1:44233" } }, - "schema_parity": { + "seed_user_row": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:47.511640110Z", - "finished_at": "2026-01-26T15:32:25.220589761Z" + "started_at": "2026-01-26T19:28:01.265667865Z", + "finished_at": "2026-01-26T19:28:38.153772708Z" }, "t_009_init": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -407,7 +441,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let user = ctx.db.user();\n user.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n user.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -415,13 +449,13 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { - "init_total_two": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } }, "schema_parity": { @@ -432,33 +466,33 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:44233", "tables_diff": null, "tables_equal": true } }, - "init_seed_alice": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM user" } }, - "init_seed_bob": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:47.073777338Z", - "finished_at": "2026-01-26T15:32:23.492928074Z" + "started_at": "2026-01-26T19:27:35.047568301Z", + "finished_at": "2026-01-26T19:28:10.719222494Z" }, "t_010_connect": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -468,7 +502,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"connected\".to_string() });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".to_string() });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"connected\".to_string() });\n}\n\n#[reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".to_string() });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -484,15 +518,15 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:44233", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975818475Z", - "finished_at": "2026-01-26T15:31:49.227873175Z" + "started_at": "2026-01-26T19:27:22.864510006Z", + "finished_at": "2026-01-26T19:28:01.265636007Z" }, "t_011_helper_function": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -501,7 +535,7 @@ "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 2, + "passed_tests": 3, "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\nfn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", "category": "basics", "route_api_model": "gpt-5", @@ -510,6 +544,24 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { + "helper_func_sum_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + 2, + 3 + ], + "golden_db": "basics-t-011-helper-function-golden", + "golden_out": "id | sum ----+----- 1 | 5", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "llm_out": "id | sum ----+----- 1 | 5", + "query": "SELECT id, sum FROM result WHERE id=1", + "reducer": "compute_sum", + "server": "http://127.0.0.1:44233" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -518,19 +570,11 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:44233", "tables_diff": null, "tables_equal": true } }, - "helper_func_sum_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c20018816d0c78b41d0a1ebdd62b788e682bdd9fd17555d6ba26e941ae120e74/sql)\n", - "phase": "sql_golden" - } - }, "helper_func_sum_abs": { "pass": true, "partial": 1.0, @@ -542,8 +586,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:16.975246967Z", - "finished_at": "2026-01-26T15:31:47.511599186Z" + "started_at": "2026-01-26T19:28:07.775155436Z", + "finished_at": "2026-01-26T19:28:36.996978868Z" }, "t_012_spacetime_product_type": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -553,7 +597,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Score {\n pub left: i32,\n pub right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow { id, value: Score { left, right } });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow { id, value: Score { left, right } });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -561,22 +605,13 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { - "product_type_row_parity": { + "product_type_row_count": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_db": "schema-t-012-spacetime-product-type-golden", - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score", - "server": "http://127.0.0.1:34081" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } }, "schema_parity": { @@ -587,24 +622,33 @@ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:44233", "tables_diff": null, "tables_equal": true } }, - "product_type_row_count": { + "product_type_row_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" + "args": [ + 1, + 2, + 3 + ], + "golden_db": "schema-t-012-spacetime-product-type-golden", + "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", + "query": "SELECT id, value FROM result WHERE id=1", + "reducer": "set_score", + "server": "http://127.0.0.1:44233" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:24.290533494Z", - "finished_at": "2026-01-26T15:32:53.440043766Z" + "started_at": "2026-01-26T19:27:19.861773654Z", + "finished_at": "2026-01-26T19:27:55.243933613Z" }, "t_013_spacetime_sum_type": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -613,8 +657,8 @@ "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db\n .result()\n .insert(ResultRow { id, value: Shape::Circle(radius) });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -622,6 +666,14 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { + "sum_type_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44233/v1/database/c200d2d3b6f126b42f6535fa1c89a272d1278f24e7645295553d055f0964bb68/sql)\n", + "phase": "sql_golden" + } + }, "sum_type_row_count": { "pass": true, "partial": 1.0, @@ -639,78 +691,53 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41115", + "server": "http://127.0.0.1:44233", "tables_diff": null, "tables_equal": true } + } + }, + "vendor": "openai", + "started_at": "2026-01-26T19:26:39.088386414Z", + "finished_at": "2026-01-26T19:27:19.723979899Z" + }, + "t_014_elementary_columns": { + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", + "task": "t_014_elementary_columns", + "lang": "rust", + "golden_published": true, + "model_name": "GPT-5", + "total_tests": 3, + "passed_tests": 3, + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3000000000,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "category": "schema", + "route_api_model": "gpt-5", + "golden_db": "schema-t-014-elementary-columns-golden", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", + "scorer_details": { + "elementary_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + } }, - "sum_type_row_parity": { + "elementary_columns_row_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - 10 - ], - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle", - "server": "http://127.0.0.1:34081" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-26T15:32:29.610801727Z", - "finished_at": "2026-01-26T15:33:11.012437922Z" - }, - "t_014_elementary_columns": { - "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", - "task": "t_014_elementary_columns", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000i64,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", - "scorer_details": { - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + "args": [], + "golden_db": "schema-t-014-elementary-columns-golden", + "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", + "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", + "reducer": "seed", + "server": "http://127.0.0.1:44233" } }, "schema_parity": { @@ -721,29 +748,15 @@ "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:44233", "tables_diff": null, "tables_equal": true } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_db": "schema-t-014-elementary-columns-golden", - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed", - "server": "http://127.0.0.1:34081" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:25.220640456Z", - "finished_at": "2026-01-26T15:32:53.137088661Z" + "started_at": "2026-01-26T19:27:08.469170041Z", + "finished_at": "2026-01-26T19:28:03.162920625Z" }, "t_015_product_type_columns": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -753,7 +766,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n for p in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(&p.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -761,6 +774,15 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", "scorer_details": { + "product_type_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -769,7 +791,7 @@ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:44233", "tables_diff": null, "tables_equal": true } @@ -785,22 +807,13 @@ "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", "query": "SELECT id, home, work, pos FROM profile WHERE id=1", "reducer": "seed", - "server": "http://127.0.0.1:34081" - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" + "server": "http://127.0.0.1:44233" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:30.863583165Z", - "finished_at": "2026-01-26T15:33:07.055207496Z" + "started_at": "2026-01-26T19:26:39.089258499Z", + "finished_at": "2026-01-26T19:27:08.469138831Z" }, "t_016_sum_type_columns": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -810,7 +823,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let drawing = ctx.db.drawing();\n if drawing.count() == 0 {\n drawing.insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let drawings = ctx.db.drawing();\n for row in drawings.iter() {\n drawings.id().delete(&row.id);\n }\n drawings.insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -818,20 +831,20 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_columns_row_parity": { + "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c20084ed1de024b608d5d036674102ffea5602feb56e936989c5737d1fa347f1/sql)\n", - "phase": "sql_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44233/v1/database/c20072beef7cc108a057376dd8b6be7e2a74798bdfaea38dde4e2e708af5e427/sql)\n", + "phase": "sql" } }, - "sum_type_columns_row_count": { + "sum_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c200f0f8cf7ce43e0b888fecfda5572f02aac26af6bea4ac6867cab46bd5e36f/sql)\n", - "phase": "sql" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44233/v1/database/c20076700591ec6cb5396663df5e1f56128de954277e753577b8dd7756ee998c/sql)\n", + "phase": "sql_golden" } }, "schema_parity": { @@ -842,15 +855,15 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:44233", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:47.540914487Z", - "finished_at": "2026-01-26T15:33:39.147225272Z" + "started_at": "2026-01-26T19:26:39.088020973Z", + "finished_at": "2026-01-26T19:27:22.864464632Z" }, "t_017_scheduled_columns": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -860,7 +873,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 0, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table};\n\n#[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat(50_000),\n });\n }\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -872,14 +885,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.34\n Compiling serde v1.0.228\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling zmij v1.0.17\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling humantime v2.3.0\n Compiling bytes v1.11.0\n Compiling arrayref v0.3.9\n Compiling smallvec v1.15.1\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling getrandom v0.2.17\n Compiling spacetimedb-lib v1.11.1\n Compiling serde_json v1.0.149\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling itoa v1.0.17\n Compiling constant_time_eq v0.4.2\n Compiling second-stack v0.3.5\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling cc v1.2.54\n Compiling generic-array v0.14.7\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling blake3 v1.8.3\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected `at`\n --> src/lib.rs:4:38\n |\n4 | #[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))]\n | ^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:22:11\n |\n22 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:42\n |\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat(50_000),\n | ^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling zerocopy v0.8.34\n Compiling shlex v1.3.0\n Compiling anyhow v1.0.100\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling zmij v1.0.17\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling humantime v2.3.0\n Compiling bytes v1.11.0\n Compiling itoa v1.0.17\n Compiling arrayref v0.3.9\n Compiling second-stack v0.3.5\n Compiling smallvec v1.15.1\n Compiling getrandom v0.2.17\n Compiling constant_time_eq v0.4.2\n Compiling cc v1.2.54\n Compiling itertools v0.12.1\n Compiling serde_json v1.0.149\n Compiling hex v0.4.3\n Compiling spacetimedb-lib v1.11.1\n Compiling bytemuck v1.24.0\n Compiling memchr v2.7.6\n Compiling generic-array v0.14.7\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling ethnum v1.5.2\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected `at`\n --> src/lib.rs:4:38\n |\n4 | #[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))]\n | ^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:14:32\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:21:42\n |\n21 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:12\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:16:35\n |\n16 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:21:8\n |\n 20 | #[reducer]\n | ---------- required by a bound introduced by this call\n 21 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:21:8\n |\n20 | #[reducer]\n | ---------- required by a bound introduced by this call\n21 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:23.493005867Z", - "finished_at": "2026-01-26T15:33:38.753749775Z" + "started_at": "2026-01-26T19:26:39.087192758Z", + "finished_at": "2026-01-26T19:27:53.381568310Z" }, "t_018_constraints": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -888,8 +901,8 @@ "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let account = ctx.db.account();\n let ids: Vec = account.iter().map(|r| r.id).collect();\n for id in ids {\n account.id().delete(&id);\n }\n account.insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n account.insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let t = ctx.db.account();\n t.insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n t.insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -898,17 +911,11 @@ "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, + "pass": false, + "partial": 0.0, "notes": { - "args": [], - "golden_db": "schema-t-018-constraints-golden", - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed", - "server": "http://127.0.0.1:34081" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `account`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44233/v1/database/c200ce532d202b333f35de1400d0f2b40c51bd06fe40198129dd9b2a5b0d196e/sql)\n", + "phase": "sql_golden" } }, "constraints_seed_two_rows": { @@ -928,15 +935,15 @@ "llm_db": "schema-t-018-constraints-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:44233", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:16.012174316Z", - "finished_at": "2026-01-26T15:32:54.091817951Z" + "started_at": "2026-01-26T19:26:39.086684099Z", + "finished_at": "2026-01-26T19:27:19.861734913Z" }, "t_019_many_to_many": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -946,7 +953,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n // Clear existing rows to ensure exact dataset\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(&m.id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(&u.user_id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(&g.group_id);\n }\n\n // Insert users\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n // Insert groups\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n // Insert memberships\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n {\n let tbl = ctx.db.membership();\n let ids: Vec = tbl.iter().map(|r| r.id).collect();\n for id in ids {\n tbl.id().delete(&id);\n }\n }\n {\n let tbl = ctx.db.user();\n let ids: Vec = tbl.iter().map(|r| r.user_id).collect();\n for id in ids {\n tbl.user_id().delete(&id);\n }\n }\n {\n let tbl = ctx.db.group();\n let ids: Vec = tbl.iter().map(|r| r.group_id).collect();\n for id in ids {\n tbl.group_id().delete(&id);\n }\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -954,19 +961,6 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34081", - "tables_diff": null, - "tables_equal": true - } - }, "m2m_has_1_10": { "pass": true, "partial": 1.0, @@ -976,13 +970,13 @@ "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" } }, - "m2m_has_2_20": { + "m2m_has_1_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" } }, "memberships_three_rows": { @@ -994,19 +988,32 @@ "sql": "SELECT COUNT(*) AS n FROM membership" } }, - "m2m_has_1_20": { + "m2m_has_2_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:44233", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:32:22.818452395Z", - "finished_at": "2026-01-26T15:33:12.862647549Z" + "started_at": "2026-01-26T19:26:39.088752482Z", + "finished_at": "2026-01-26T19:27:50.273815091Z" }, "t_020_ecs": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -1016,7 +1023,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entity 1\n if ctx.db.entity().id().find(1).is_none() {\n ctx.db.entity().insert(Entity { id: 1 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(1) {\n p.x = 0;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(1) {\n v.vx = 1;\n v.vy = 0;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n\n // Entity 2\n if ctx.db.entity().id().find(2).is_none() {\n ctx.db.entity().insert(Entity { id: 2 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(2) {\n p.x = 10;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(2) {\n v.vx = -2;\n v.vy = 3;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for p in ctx.db.position().iter() {\n if let Some(v) = ctx.db.velocity().entity_id().find(p.entity_id) {\n let nx = p.x + v.vx;\n let ny = p.y + v.vy;\n if let Some(mut np) = ctx.db.next_position().entity_id().find(p.entity_id) {\n np.x = nx;\n np.y = ny;\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db\n .next_position()\n .insert(NextPosition { entity_id: p.entity_id, x: nx, y: ny });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.entity().insert(Entity { id: 1 });\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n ctx.db.entity().insert(Entity { id: 2 });\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let next = NextPosition {\n entity_id: pos.entity_id,\n x: pos.x + vel.vx,\n y: pos.y + vel.vy,\n };\n\n if ctx.db.next_position().entity_id().find(pos.entity_id).is_some() {\n ctx.db.next_position().entity_id().update(next);\n } else {\n ctx.db.next_position().insert(next);\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -1028,15 +1035,15 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44233/v1/database/c200adf8fd49d0e7e167ec3aa9141c1b249dc335dfa97e666a41345612d7362a/sql)\n", "phase": "sql" } }, - "ecs_seed_positions_count": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44233/v1/database/c200adf8fd49d0e7e167ec3aa9141c1b249dc335dfa97e666a41345612d7362a/sql)\n", "phase": "sql" } }, @@ -1044,7 +1051,7 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44233/v1/database/c200adf8fd49d0e7e167ec3aa9141c1b249dc335dfa97e666a41345612d7362a/sql)\n", "phase": "sql" } }, @@ -1056,7 +1063,7 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:44233", "tables_diff": null, "tables_equal": true } @@ -1065,14 +1072,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34081/v1/database/c2002214a80a8c41eccac4e56c7e632cd82fed302ea0753203a8cb149cbfc10d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44233/v1/database/c200adf8fd49d0e7e167ec3aa9141c1b249dc335dfa97e666a41345612d7362a/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:51.422780227Z", - "finished_at": "2026-01-26T15:32:47.540737141Z" + "started_at": "2026-01-26T19:26:39.089004480Z", + "finished_at": "2026-01-26T19:27:35.047523671Z" }, "t_021_multi_column_index": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -1082,7 +1089,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.log();\n\n // Clear existing rows to ensure the table contains exactly the seeded rows\n let ids: Vec = tbl.iter().map(|r| r.id).collect();\n for id in ids {\n tbl.id().delete(&id);\n }\n\n tbl.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n tbl.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n tbl.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let ids: Vec = ctx.db.log().iter().map(|r| r.id).collect();\n for id in ids {\n ctx.db.log().id().delete(&id);\n }\n\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -1090,15 +1097,6 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -1107,7 +1105,7 @@ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34081", + "server": "http://127.0.0.1:44233", "tables_diff": null, "tables_equal": true } @@ -1121,6 +1119,15 @@ "sql": "SELECT COUNT(*) AS n FROM log" } }, + "mcindex_lookup_u7_d2": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" + } + }, "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, @@ -1132,8 +1139,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:31:55.504076208Z", - "finished_at": "2026-01-26T15:32:29.610771248Z" + "started_at": "2026-01-26T19:26:39.087595992Z", + "finished_at": "2026-01-26T19:27:23.893472645Z" } } } @@ -1141,21 +1148,21 @@ }, { "mode": "docs", - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_000_empty_reducers", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {\n}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, count: i32, name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, active: bool, ratio: f32, label: String) {\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -1173,11 +1180,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.378312906Z", - "finished_at": "2026-01-26T15:34:06.312973975Z" + "started_at": "2026-01-26T19:30:54.611008723Z", + "finished_at": "2026-01-26T19:31:59.326178772Z" }, "t_001_basic_tables": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_001_basic_tables", "lang": "rust", "golden_published": true, @@ -1202,18 +1209,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.366156833Z", - "finished_at": "2026-01-26T15:34:21.141281364Z" + "started_at": "2026-01-26T19:31:23.564110648Z", + "finished_at": "2026-01-26T19:32:09.696102989Z" }, "t_002_scheduled_table": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_002_scheduled_table", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n}", + "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n }\n Ok(())\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -1231,11 +1238,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.381646420Z", - "finished_at": "2026-01-26T15:35:08.176674329Z" + "started_at": "2026-01-26T19:32:03.232154280Z", + "finished_at": "2026-01-26T19:32:55.980604615Z" }, "t_003_struct_in_table": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_003_struct_in_table", "lang": "rust", "golden_published": true, @@ -1260,11 +1267,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.368608920Z", - "finished_at": "2026-01-26T15:34:21.997549378Z" + "started_at": "2026-01-26T19:31:11.581070012Z", + "finished_at": "2026-01-26T19:32:09.839951728Z" }, "t_004_insert": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_004_insert", "lang": "rust", "golden_published": true, @@ -1297,11 +1304,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.375009734Z", - "finished_at": "2026-01-26T15:35:05.975054748Z" + "started_at": "2026-01-26T19:31:53.617759187Z", + "finished_at": "2026-01-26T19:32:28.129265268Z" }, "t_005_update": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_005_update", "lang": "rust", "golden_published": true, @@ -1316,12 +1323,13 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { - "data_parity_update_user": { + "seed_users_row": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "call_reducer_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", + "phase": "sql_golden", + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, "schema_parity": { @@ -1339,23 +1347,14 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", "phase": "call_reducer_golden" } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:06.313026786Z", - "finished_at": "2026-01-26T15:34:38.434768448Z" + "started_at": "2026-01-26T19:30:54.604964500Z", + "finished_at": "2026-01-26T19:31:53.617639721Z" }, "t_006_delete": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_006_delete", "lang": "rust", "golden_published": true, @@ -1379,13 +1378,12 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" } }, - "seed_users_row": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", + "phase": "describe_golden" } }, "seed_users_row": { @@ -1399,18 +1397,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:38.434794482Z", - "finished_at": "2026-01-26T15:35:08.669675470Z" + "started_at": "2026-01-26T19:30:54.580009457Z", + "finished_at": "2026-01-26T19:32:03.232098066Z" }, "t_007_crud": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_007_crud", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".into(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".into(),\n age: 22,\n active: false,\n });\n if let Some(mut u) = ctx.db.user().id().find(1) {\n u.name = \"Alice2\".into();\n u.age = 31;\n u.active = false;\n ctx.db.user().id().update(u);\n }\n ctx.db.user().id().delete(2);\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n\n if ctx.db.user().id().find(1).is_some() {\n ctx.db.user().id().update(User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false });\n }\n\n ctx.db.user().id().delete(2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -1418,23 +1416,6 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "call_reducer_golden" - } - }, - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1452,28 +1433,37 @@ "sql": "SELECT COUNT(*) AS n FROM user" } }, - "schema_parity": { + "crud_row_id1_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", + "phase": "call_reducer_golden" + } + }, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.388241938Z", - "finished_at": "2026-01-26T15:35:08.142795838Z" + "started_at": "2026-01-26T19:30:54.591258099Z", + "finished_at": "2026-01-26T19:32:19.735936055Z" }, "t_008_index_lookup": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_008_index_lookup", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user, public)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result, public)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -1481,21 +1471,21 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "seed_user_row": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", - "phase": "describe_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", + "phase": "sql_golden", + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, - "seed_user_row": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", + "phase": "describe_golden" } }, "index_lookup_projection_parity": { @@ -1508,18 +1498,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:21.997572287Z", - "finished_at": "2026-01-26T15:34:53.278840948Z" + "started_at": "2026-01-26T19:31:52.996077836Z", + "finished_at": "2026-01-26T19:32:44.567947473Z" }, "t_009_init": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_009_init", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n Ok(())\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n Ok(())\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -1527,13 +1517,21 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { - "init_seed_bob": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" + } + }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-009-init-golden`.\n", + "phase": "describe_golden" } }, "init_total_two": { @@ -1545,30 +1543,22 @@ "sql": "SELECT COUNT(*) AS n FROM user" } }, - "init_seed_alice": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-009-init-golden`.\n", - "phase": "describe_golden" + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:21.141305493Z", - "finished_at": "2026-01-26T15:34:43.189086037Z" + "started_at": "2026-01-26T19:30:54.597845517Z", + "finished_at": "2026-01-26T19:32:17.483969682Z" }, "t_010_connect": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_010_connect", "lang": "rust", "golden_published": true, @@ -1593,18 +1583,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.384904874Z", - "finished_at": "2026-01-26T15:35:08.049424018Z" + "started_at": "2026-01-26T19:30:54.585220084Z", + "finished_at": "2026-01-26T19:31:52.995965049Z" }, "t_011_helper_function": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_011_helper_function", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let s = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum: s });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -1624,8 +1614,9 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n", - "phase": "call_reducer_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" } }, "schema_parity": { @@ -1638,18 +1629,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:33:39.371740140Z", - "finished_at": "2026-01-26T15:35:03.413658925Z" + "started_at": "2026-01-26T19:31:59.326285562Z", + "finished_at": "2026-01-26T19:32:34.857458085Z" }, "t_012_spacetime_product_type": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_012_spacetime_product_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db\n .result()\n .insert(ResultRow { id, value: Score { left, right } });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[derive(spacetimedb::SpacetimeType, Clone)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db\n .result()\n .insert(ResultRow { id, value: Score { left, right } });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -1684,11 +1675,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:08.176748927Z", - "finished_at": "2026-01-26T15:35:33.331707923Z" + "started_at": "2026-01-26T19:30:31.399995435Z", + "finished_at": "2026-01-26T19:31:11.581019739Z" }, "t_013_spacetime_sum_type": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_013_spacetime_sum_type", "lang": "rust", "golden_published": false, @@ -1707,41 +1698,24 @@ "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", - "phase": "describe_golden" - } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", - "phase": "call_reducer_golden" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:08.669696395Z", - "finished_at": "2026-01-26T15:35:36.590766489Z" + "started_at": "2026-01-26T19:32:55.980673255Z", + "finished_at": "2026-01-26T19:32:55.980673255Z" }, "t_014_elementary_columns": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_014_elementary_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5_f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive, public)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -1749,20 +1723,20 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "elementary_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", + "phase": "call_reducer_golden" } }, - "elementary_columns_row_parity": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", + "phase": "describe_golden" } }, "elementary_columns_row_count": { @@ -1776,18 +1750,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:08.180313713Z", - "finished_at": "2026-01-26T15:35:31.695987649Z" + "started_at": "2026-01-26T19:30:30.882003264Z", + "finished_at": "2026-01-26T19:31:23.563975616Z" }, "t_015_product_type_columns": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_015_product_type_columns", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(row.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", "golden_db": null, @@ -1795,45 +1769,28 @@ "work_dir_golden": null, "work_dir_llm": null, "scorer_details": { - "product_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - }, - "product_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", - "phase": "call_reducer_golden" - } - }, - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", - "phase": "describe_golden" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:17.784622487Z", - "finished_at": "2026-01-26T15:35:56.892022012Z" + "started_at": "2026-01-26T19:32:55.980689378Z", + "finished_at": "2026-01-26T19:32:55.980689378Z" }, "t_016_sum_type_columns": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_016_sum_type_columns", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.drawing().try_insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", "golden_db": null, @@ -1841,50 +1798,34 @@ "work_dir_golden": null, "work_dir_llm": null, "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "describe_golden" - } - }, - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c200e332ee39012f68748e8e0058b1df01bfd4bd42fa50c7430f6e03a52a13d7/sql)\n", - "phase": "sql" - } - }, - "sum_type_columns_row_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "call_reducer_golden" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:20.422565504Z", - "finished_at": "2026-01-26T15:35:53.522136009Z" + "started_at": "2026-01-26T19:32:55.980668485Z", + "finished_at": "2026-01-26T19:32:55.980668485Z" }, "t_017_scheduled_columns": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_017_scheduled_columns", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: spacetimedb::ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: spacetimedb::ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n Ok(())\n}", + "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n Ok(())\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-017-scheduled-columns-golden", + "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm", "scorer_details": { "schema_parity": { "pass": false, @@ -1905,18 +1846,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:08.142877410Z", - "finished_at": "2026-01-26T15:35:53.117891027Z" + "started_at": "2026-01-26T19:29:24.576470966Z", + "finished_at": "2026-01-26T19:30:31.399943290Z" }, "t_018_constraints": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_018_constraints", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().id().delete(&1);\n ctx.db.account().id().delete(&2);\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", "golden_db": null, @@ -1924,45 +1865,28 @@ "work_dir_golden": null, "work_dir_llm": null, "scorer_details": { - "constraints_seed_two_rows": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", - "phase": "describe_golden" - } - }, - "constraints_row_parity_after_seed": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", - "phase": "call_reducer_golden" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:03.413773673Z", - "finished_at": "2026-01-26T15:35:40.991202532Z" + "started_at": "2026-01-26T19:32:55.980661308Z", + "finished_at": "2026-01-26T19:32:55.980661308Z" }, "t_019_many_to_many": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_019_many_to_many", "lang": "rust", "golden_published": false, "model_name": "GPT-5", - "total_tests": 5, - "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(m.id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(u.user_id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(g.group_id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".into() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".into() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".into() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".into() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "total_tests": 1, + "passed_tests": 0, + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", "golden_db": null, @@ -1970,64 +1894,28 @@ "work_dir_golden": null, "work_dir_llm": null, "scorer_details": { - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - }, - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { "error": "LLM call timed out", "phase": "build_or_publish" } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" - } } }, "vendor": "openai", - "started_at": "2026-01-26T15:35:05.975134623Z", - "finished_at": "2026-01-26T15:35:55.889702970Z" + "started_at": "2026-01-26T19:32:55.980679692Z", + "finished_at": "2026-01-26T19:32:55.980679692Z" }, "t_020_ecs": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_020_ecs", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entity 1\n let id1 = 1;\n if ctx.db.entity().id().find(id1).is_none() {\n ctx.db.entity().insert(Entity { id: id1 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(id1) {\n p.x = 0;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: id1, x: 0, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(id1) {\n v.vx = 1;\n v.vy = 0;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: id1, vx: 1, vy: 0 });\n }\n\n // Entity 2\n let id2 = 2;\n if ctx.db.entity().id().find(id2).is_none() {\n ctx.db.entity().insert(Entity { id: id2 });\n }\n if let Some(mut p) = ctx.db.position().entity_id().find(id2) {\n p.x = 10;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: id2, x: 10, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(id2) {\n v.vx = -2;\n v.vy = 3;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: id2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(mut np) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n np.x = nx;\n np.y = ny;\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", "golden_db": null, @@ -2035,67 +1923,44 @@ "work_dir_golden": null, "work_dir_llm": null, "scorer_details": { - "ecs_seed_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", - "phase": "sql" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-020-ecs-golden`.\n", - "phase": "describe_golden" - } - }, - "ecs_next_pos_entity2": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", - "phase": "sql" - } - }, - "ecs_step_next_positions_count": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46385/v1/database/c20019f8f996e9e87f067a3b3fb3b441e459a573af67d60fda8d5cb14edf5876/sql)\n", - "phase": "sql" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:43.189115873Z", - "finished_at": "2026-01-26T15:35:20.422539334Z" + "started_at": "2026-01-26T19:32:55.980684633Z", + "finished_at": "2026-01-26T19:32:55.980684633Z" }, "t_021_multi_column_index": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "task": "t_021_multi_column_index", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let t = ctx.db.log();\n t.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n t.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n t.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(\n name = log,\n index(name = by_user_day, btree(columns = [user_id, day]))\n)]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.log().try_insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n let _ = ctx.db.log().try_insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n let _ = ctx.db.log().try_insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-021-multi-column-index-golden", + "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { + "mcindex_lookup_u7_d2": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" + } + }, "schema_parity": { "pass": false, "partial": 0.0, @@ -2104,15 +1969,6 @@ "phase": "describe_golden" } }, - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, @@ -2122,19 +1978,19 @@ "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" } }, - "mcindex_lookup_u7_d2": { + "mcindex_seed_count": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM log" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:34:53.278868473Z", - "finished_at": "2026-01-26T15:35:17.784590880Z" + "started_at": "2026-01-26T19:29:24.579594233Z", + "finished_at": "2026-01-26T19:30:30.881922637Z" } } } @@ -2325,14 +2181,14 @@ "modes": [ { "mode": "docs", - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_000_empty_reducers", "lang": "csharp", "golden_published": true, @@ -2355,25 +2211,25 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:44935", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:38:48.351519518Z", - "finished_at": "2026-01-26T15:39:15.916317553Z" + "started_at": "2026-01-26T19:38:54.385915075Z", + "finished_at": "2026-01-26T19:40:03.912043999Z" }, "t_001_basic_tables": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_001_basic_tables", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\")]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\")]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\", Public = true)]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\", Public = true)]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -2389,25 +2245,25 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:44935", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:37:49.611034643Z", - "finished_at": "2026-01-26T15:39:10.630797919Z" + "started_at": "2026-01-26T19:39:06.588178775Z", + "finished_at": "2026-01-26T19:40:26.686164444Z" }, "t_002_scheduled_table": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_002_scheduled_table", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer(SpacetimeDB.ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -2423,25 +2279,25 @@ "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:44935", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:38:55.763350829Z", - "finished_at": "2026-01-26T15:39:35.117748962Z" + "started_at": "2026-01-26T19:40:26.686215935Z", + "finished_at": "2026-01-26T19:41:35.359471807Z" }, "t_003_struct_in_table": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_003_struct_in_table", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -2457,18 +2313,18 @@ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:44935", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:37:49.614826701Z", - "finished_at": "2026-01-26T15:38:48.351485744Z" + "started_at": "2026-01-26T19:39:03.248071954Z", + "finished_at": "2026-01-26T19:40:22.292027026Z" }, "t_004_insert": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_004_insert", "lang": "csharp", "golden_published": true, @@ -2491,7 +2347,7 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:44935", "tables_diff": null, "tables_equal": true } @@ -2512,23 +2368,23 @@ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "InsertUser", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:44935" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:37:49.624534968Z", - "finished_at": "2026-01-26T15:38:55.763316545Z" + "started_at": "2026-01-26T19:40:05.942682308Z", + "finished_at": "2026-01-26T19:41:00.825583459Z" }, "t_005_update": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_005_update", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.User.Id.Update(user);\n }\n else\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active,\n });\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var user = ctx.Db.User.Id.Find(id) ?? throw new System.Exception(\"User not found\");\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.User.Id.Update(user);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -2536,17 +2392,11 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "seed_users_row": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-005-update-golden", - "llm_db": "basics-t-005-update-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true + "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" } }, "data_parity_update_user": { @@ -2565,30 +2415,36 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "UpdateUser", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:44935" } }, - "seed_users_row": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" + "golden_db": "basics-t-005-update-golden", + "llm_db": "basics-t-005-update-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:44935", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:15.916349271Z", - "finished_at": "2026-01-26T15:39:57.685388275Z" + "started_at": "2026-01-26T19:38:38.720018388Z", + "finished_at": "2026-01-26T19:40:05.937381701Z" }, "t_006_delete": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_006_delete", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -2596,22 +2452,6 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/csharp/server/gpt-5/llm", "scorer_details": { - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2620,7 +2460,7 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:44935", "tables_diff": null, "tables_equal": true } @@ -2631,21 +2471,30 @@ "notes": { "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" } + }, + "delete_user_count_zero": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:51.394765047Z", - "finished_at": "2026-01-26T15:40:20.535132716Z" + "started_at": "2026-01-26T19:37:37.167869929Z", + "finished_at": "2026-01-26T19:38:38.719937430Z" }, "t_007_crud": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_007_crud", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n // insert (Id=1, Name=\"Alice\", Age=30, Active=true)\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n\n // insert (Id=2, Name=\"Bob\", Age=22, Active=false)\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n // update (Id=1, Name=\"Alice2\", Age=31, Active=false)\n if (ctx.Db.User.Id.Find(1) is User user1)\n {\n user1.Name = \"Alice2\";\n user1.Age = 31;\n user1.Active = false;\n ctx.Db.User.Id.Update(user1);\n }\n\n // delete Id=2\n ctx.Db.User.Id.Delete(2);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User user1)\n {\n user1.Name = \"Alice2\";\n user1.Age = 31;\n user1.Active = false;\n ctx.Db.User.Id.Update(user1);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -2653,6 +2502,15 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm", "scorer_details": { + "crud_total_count_one": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM User" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2661,18 +2519,18 @@ "llm_db": "basics-t-007-crud-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:44935", "tables_diff": null, "tables_equal": true } }, - "crud_total_count_one": { + "crud_row_id2_deleted": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" } }, "crud_row_id1_parity": { @@ -2686,32 +2544,23 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "Crud", - "server": "http://127.0.0.1:41793" - } - }, - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" + "server": "http://127.0.0.1:44935" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:10.630849129Z", - "finished_at": "2026-01-26T15:39:51.394722435Z" + "started_at": "2026-01-26T19:37:59.027103288Z", + "finished_at": "2026-01-26T19:38:54.385590993Z" }, "t_008_index_lookup": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_008_index_lookup", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Id.Delete(id);\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -2727,7 +2576,7 @@ "llm_db": "basics-t-008-index-lookup-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:44935", "tables_diff": null, "tables_equal": true } @@ -2752,23 +2601,23 @@ "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", "query": "SELECT Id, Name FROM Result WHERE Id=1", "reducer": "LookupUserName", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:44935" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:35.117796353Z", - "finished_at": "2026-01-26T15:40:10.865109962Z" + "started_at": "2026-01-26T19:40:05.937447735Z", + "finished_at": "2026-01-26T19:41:10.697769188Z" }, "t_009_init": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_009_init", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -2776,7 +2625,7 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/csharp/server/gpt-5/llm", "scorer_details": { - "init_total_two": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { @@ -2784,56 +2633,52 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41045", + "server": "http://127.0.0.1:44935", "tables_diff": null, "tables_equal": true } }, - "init_seed_bob": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" } }, - "init_seed_alice": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" } }, - "schema_parity": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM User" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:33.038774271Z", - "finished_at": "2026-01-26T15:40:09.097960805Z" + "started_at": "2026-01-26T19:38:16.753424300Z", + "finished_at": "2026-01-26T19:39:06.588120002Z" }, "t_010_connect": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_010_connect", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\", Public = true)]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -2849,25 +2694,25 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:44935", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:07.129948444Z", - "finished_at": "2026-01-26T15:39:33.038733830Z" + "started_at": "2026-01-26T19:37:47.380111761Z", + "finished_at": "2026-01-26T19:39:03.248030401Z" }, "t_011_helper_function": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_011_helper_function", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n private static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b)\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -2892,7 +2737,7 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:44935", "tables_diff": null, "tables_equal": true } @@ -2912,32 +2757,23 @@ "llm_out": "Id | Sum ----+----- 1 | 5", "query": "SELECT Id, Sum FROM Result WHERE Id=1", "reducer": "ComputeSum", - "server": "http://127.0.0.1:41793" - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" + "server": "http://127.0.0.1:44935" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:37:49.619502537Z", - "finished_at": "2026-01-26T15:39:07.129901962Z" + "started_at": "2026-01-26T19:40:22.292081947Z", + "finished_at": "2026-01-26T19:41:08.915943296Z" }, "t_012_spacetime_product_type": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_012_spacetime_product_type", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right },\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score\n {\n Left = left,\n Right = right\n }\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -2945,19 +2781,6 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-012-spacetime-product-type-golden", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true - } - }, "product_type_row_parity": { "pass": true, "partial": 1.0, @@ -2973,7 +2796,7 @@ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", "query": "SELECT Id, Value FROM Result WHERE Id=1", "reducer": "SetScore", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:44935" } }, "product_type_row_count": { @@ -2984,21 +2807,34 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-012-spacetime-product-type-golden", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:44935", + "tables_diff": null, + "tables_equal": true + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:45.264170257Z", - "finished_at": "2026-01-26T15:41:10.281722543Z" + "started_at": "2026-01-26T19:37:30.236090594Z", + "finished_at": "2026-01-26T19:38:16.753346626Z" }, "t_013_spacetime_sum_type": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_013_spacetime_sum_type", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)>\n {\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var existing = ctx.Db.Result.Id.Find(id);\n if (existing != null)\n {\n ctx.Db.Result.Id.Delete(id);\n }\n\n var shape = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = shape\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var shape = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = shape\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -3006,17 +2842,21 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "sum_type_row_parity": { "pass": true, "partial": 1.0, "notes": { + "args": [ + 1, + 10 + ], "golden_db": "schema-t-013-spacetime-sum-type-golden", + "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true + "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", + "query": "SELECT Id, Value FROM Result WHERE Id=1", + "reducer": "SetCircle", + "server": "http://127.0.0.1:44935" } }, "sum_type_row_count": { @@ -3028,37 +2868,33 @@ "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } }, - "sum_type_row_parity": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - 10 - ], "golden_db": "schema-t-013-spacetime-sum-type-golden", - "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "SetCircle", - "server": "http://127.0.0.1:41793" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:44935", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-26T15:41:10.281769568Z", - "finished_at": "2026-01-26T15:42:04.928512771Z" + "started_at": "2026-01-26T19:35:59.196578278Z", + "finished_at": "2026-01-26T19:37:03.535790946Z" }, "t_014_elementary_columns": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_014_elementary_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Primitive\")]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Primitive\", Public = true)]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -3066,19 +2902,11 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20015ce3d6d6c97d4d92483e7212bbbdc2017cebf40569b0de2758a8c06d8c9/sql)\n", - "phase": "sql" - } - }, "elementary_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c2002dc67d9e1fd4e2f7be96328c9cadfe424ab52e14114ffe15bf0808847523/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44935/v1/database/c2000e37e733499f47b57d9f488477b94fccf2e4c5c6c32182171721cbaba64a/sql)\n", "phase": "sql_golden" } }, @@ -3090,25 +2918,33 @@ "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:44935", "tables_diff": null, "tables_equal": true } + }, + "elementary_columns_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44935/v1/database/c200e70d6c47b9f5b97a9584014efb7008b4c312687bfdbfe1b0ee1891420453/sql)\n", + "phase": "sql" + } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:59.748869772Z", - "finished_at": "2026-01-26T15:41:30.294907546Z" + "started_at": "2026-01-26T19:37:03.535862900Z", + "finished_at": "2026-01-26T19:37:59.026960569Z" }, "t_015_product_type_columns": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_015_product_type_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Profile.Iter())\n {\n ctx.Db.Profile.Id.Delete(row.Id);\n }\n\n var home = new Address { Street = \"1 Main\", Zip = 11111 };\n var work = new Address { Street = \"2 Broad\", Zip = 22222 };\n var pos = new Position { X = 7, Y = 9 };\n\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = home,\n Work = work,\n Pos = pos\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -3116,15 +2952,6 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm", "scorer_details": { - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3133,11 +2960,20 @@ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:44935", "tables_diff": null, "tables_equal": true } }, + "product_type_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" + } + }, "product_type_columns_row_parity": { "pass": true, "partial": 1.0, @@ -3149,73 +2985,52 @@ "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:44935" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:41:10.284990504Z", - "finished_at": "2026-01-26T15:41:51.914201320Z" + "started_at": "2026-01-26T19:36:38.953458698Z", + "finished_at": "2026-01-26T19:37:30.236009816Z" }, "t_016_sum_type_columns": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_016_sum_type_columns", "lang": "csharp", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var a = new Shape.Circle(new Circle { Radius = 10 });\n var b = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 });\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = a,\n B = b\n });\n }\n}", + "total_tests": 1, + "passed_tests": 0, + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", - "golden_db": "schema-t-016-sum-type-columns-golden", - "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-016-sum-type-columns-golden", - "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c2007921ce30371e511ce337c700f912b2dc1842029ec51095bcee487a1fb5a2/sql)\n", - "phase": "sql_golden" - } - }, - "sum_type_columns_row_count": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20008b060dbbbe2ff941855e14b04bb0c88b907fca414a2d82874c574b1cf60/sql)\n", - "phase": "sql" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:41:12.893136011Z", - "finished_at": "2026-01-26T15:42:12.178253367Z" + "started_at": "2026-01-26T19:41:35.359543125Z", + "finished_at": "2026-01-26T19:41:35.359543125Z" }, "t_017_scheduled_columns": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_017_scheduled_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -3223,14 +3038,6 @@ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/gpt-5/llm", "scorer_details": { - "scheduled_seeded_one_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20077e1b6738482c8dd0108623bedd0c70f80fb40fb09fda1f12bbe2c6972bf/sql)\n", - "phase": "sql" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3239,7 +3046,7 @@ "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:44935", "tables_diff": null, "tables_equal": true } @@ -3248,24 +3055,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c2000932373ef07be1f77db87f24b98a9819f6033c40da69c914e654c807ead2/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44935/v1/database/c2004563b2a3af718eb5a6d78c0e16523a6ad57949ac9dd820dbc2c2e11c8842/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:27.963020126Z", - "finished_at": "2026-01-26T15:41:12.893111985Z" + "started_at": "2026-01-26T19:35:08.495907944Z", + "finished_at": "2026-01-26T19:36:38.949408263Z" }, "t_018_constraints": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_018_constraints", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Account\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = new[] { \"Name\" })]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Account.Iter())\n {\n ctx.Db.Account.Id.Delete(row.Id);\n }\n\n ctx.Db.Account.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.Account.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Account\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = new[] { \"Name\" })]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Account.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.Account.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -3273,6 +3080,15 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/csharp/server/gpt-5/llm", "scorer_details": { + "constraints_seed_two_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3281,20 +3097,11 @@ "llm_db": "schema-t-018-constraints-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:44935", "tables_diff": null, "tables_equal": true } }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" - } - }, "constraints_row_parity_after_seed": { "pass": true, "partial": 1.0, @@ -3306,23 +3113,23 @@ "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:41793" + "server": "http://127.0.0.1:44935" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:10.865255262Z", - "finished_at": "2026-01-26T15:40:59.748828990Z" + "started_at": "2026-01-26T19:35:08.491947971Z", + "finished_at": "2026-01-26T19:35:59.196533076Z" }, "t_019_many_to_many": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_019_many_to_many", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Ensure exactly these rows by clearing existing data first\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n\n // Users\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n // Groups\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n // Memberships\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -3330,6 +3137,15 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm", "scorer_details": { + "memberships_three_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Membership" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3338,61 +3154,52 @@ "llm_db": "schema-t-019-many-to-many-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41793", + "server": "http://127.0.0.1:44935", "tables_diff": null, "tables_equal": true } }, - "m2m_has_1_20": { + "m2m_has_2_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" } }, - "m2m_has_1_10": { + "m2m_has_1_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" } }, - "m2m_has_2_20": { + "m2m_has_1_10": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:20.535170845Z", - "finished_at": "2026-01-26T15:41:10.109404307Z" + "started_at": "2026-01-26T19:36:32.689581659Z", + "finished_at": "2026-01-26T19:37:37.167783708Z" }, "t_020_ecs": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_020_ecs", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Insert entities\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n\n // Insert positions\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n\n // Insert velocities\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n if (ctx.Db.Velocity.EntityId.Find(pos.EntityId) is Velocity vel)\n {\n int nx = pos.X + vel.VX;\n int ny = pos.Y + vel.VY;\n\n if (ctx.Db.NextPosition.EntityId.Find(pos.EntityId) is NextPosition np)\n {\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n if (ctx.Db.Velocity.EntityId.Find(pos.EntityId) is Velocity vel)\n {\n int nx = pos.X + vel.VX;\n int ny = pos.Y + vel.VY;\n\n if (ctx.Db.NextPosition.EntityId.Find(pos.EntityId) is NextPosition np)\n {\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition { EntityId = pos.EntityId, X = nx, Y = ny });\n }\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -3400,19 +3207,24 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm", "scorer_details": { - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", - "phase": "sql" + "golden_db": "schema-t-020-ecs-golden", + "llm_db": "schema-t-020-ecs-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:44935", + "tables_diff": null, + "tables_equal": true } }, - "ecs_next_pos_entity2": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44935/v1/database/c2003766705eb1f0e9f55468e9bf8d13f967d7125178a2028f2dd6ca5f43756d/sql)\n", "phase": "sql" } }, @@ -3420,45 +3232,40 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44935/v1/database/c2003766705eb1f0e9f55468e9bf8d13f967d7125178a2028f2dd6ca5f43756d/sql)\n", "phase": "sql" } }, - "schema_parity": { - "pass": true, - "partial": 1.0, + "ecs_next_pos_entity2": { + "pass": false, + "partial": 0.0, "notes": { - "golden_db": "schema-t-020-ecs-golden", - "llm_db": "schema-t-020-ecs-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41793", - "tables_diff": null, - "tables_equal": true + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44935/v1/database/c2003766705eb1f0e9f55468e9bf8d13f967d7125178a2028f2dd6ca5f43756d/sql)\n", + "phase": "sql" } }, - "ecs_seed_positions_count": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41793/v1/database/c20044bdf120effd67afb7dfb8cbe79b0556f0430ae82eb89fa8b87a5ace8738/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44935/v1/database/c2003766705eb1f0e9f55468e9bf8d13f967d7125178a2028f2dd6ca5f43756d/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:39:57.685434816Z", - "finished_at": "2026-01-26T15:40:45.264108157Z" + "started_at": "2026-01-26T19:36:38.949546269Z", + "finished_at": "2026-01-26T19:37:47.379943704Z" }, "t_021_multi_column_index": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "task": "t_021_multi_column_index", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 4, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"Log\", Public = true)]\n [Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", + "passed_tests": 4, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -3466,12 +3273,26 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "mcindex_seed_count": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm/Lib.cs(7,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm/StdbModule.csproj]\n", - "phase": "build_or_publish" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Log" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-021-multi-column-index-golden", + "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:44935", + "tables_diff": null, + "tables_equal": true } }, "mcindex_lookup_u7_d1": { @@ -3483,19 +3304,19 @@ "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" } }, - "mcindex_seed_count": { + "mcindex_lookup_u7_d2": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" } } }, "vendor": "openai", - "started_at": "2026-01-26T15:40:09.098001666Z", - "finished_at": "2026-01-26T15:40:27.962996760Z" + "started_at": "2026-01-26T19:35:08.500770472Z", + "finished_at": "2026-01-26T19:36:32.689520614Z" } } } @@ -3682,4 +3503,4 @@ } } ] -} \ No newline at end of file +} diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json index 40e83fafc94..fa057f8e85f 100644 --- a/docs/llms/docs-benchmark-summary.json +++ b/docs/llms/docs-benchmark-summary.json @@ -1,11 +1,11 @@ { "version": 1, - "generated_at": "2026-01-26T15:42:12.228Z", + "generated_at": "2026-01-26T19:41:35.586Z", "by_language": { "csharp": { "modes": { "docs": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "2f6071a69fd778167c45e20453bc018eebc468cc87681c455f819adfa10148a0", "models": { "GPT-5": { "categories": { @@ -19,20 +19,20 @@ }, "schema": { "tasks": 10, - "total_tests": 34, - "passed_tests": 21, - "pass_pct": 61.764706, - "task_pass_equiv": 6.3666663, - "task_pass_pct": 63.666664 + "total_tests": 32, + "passed_tests": 24, + "pass_pct": 75.0, + "task_pass_equiv": 7.033333, + "task_pass_pct": 70.33333 } }, "totals": { "tasks": 22, - "total_tests": 61, - "passed_tests": 48, - "pass_pct": 78.68852, - "task_pass_equiv": 18.366667, - "task_pass_pct": 83.48485 + "total_tests": 59, + "passed_tests": 51, + "pass_pct": 86.44068, + "task_pass_equiv": 19.033333, + "task_pass_pct": 86.515144 } } } @@ -42,7 +42,7 @@ "rust": { "modes": { "docs": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "a9b7f59b2c00330aa9e00220d45b9d8f7d257f9b10dc9f37b56a1e954c2b5117", "models": { "GPT-5": { "categories": { @@ -56,20 +56,20 @@ }, "schema": { "tasks": 10, - "total_tests": 14, - "passed_tests": 0, - "pass_pct": 0.0, - "task_pass_equiv": 0.0, - "task_pass_pct": 0.0 + "total_tests": 18, + "passed_tests": 4, + "pass_pct": 22.222221, + "task_pass_equiv": 1.25, + "task_pass_pct": 12.5 } }, "totals": { "tasks": 22, - "total_tests": 41, - "passed_tests": 5, - "pass_pct": 12.195122, - "task_pass_equiv": 1.3333334, - "task_pass_pct": 6.060606 + "total_tests": 45, + "passed_tests": 9, + "pass_pct": 20.0, + "task_pass_equiv": 2.5833335, + "task_pass_pct": 11.742425 } } } @@ -82,27 +82,27 @@ "basics": { "tasks": 12, "total_tests": 27, - "passed_tests": 20, - "pass_pct": 74.07407, - "task_pass_equiv": 9.166667, - "task_pass_pct": 76.38889 + "passed_tests": 25, + "pass_pct": 92.59259, + "task_pass_equiv": 10.0, + "task_pass_pct": 83.33333 }, "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 26, - "pass_pct": 76.47059, - "task_pass_equiv": 7.5333333, - "task_pass_pct": 75.333336 + "passed_tests": 24, + "pass_pct": 70.588234, + "task_pass_equiv": 6.8666663, + "task_pass_pct": 68.666664 } }, "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 46, - "pass_pct": 75.409836, - "task_pass_equiv": 16.7, - "task_pass_pct": 75.909096 + "passed_tests": 49, + "pass_pct": 80.327866, + "task_pass_equiv": 16.866667, + "task_pass_pct": 76.666664 } } } From 5831494c64449a4f3aceea6e259d172d04d453b5 Mon Sep 17 00:00:00 2001 From: joshua-spacetime Date: Wed, 28 Jan 2026 09:53:35 -0800 Subject: [PATCH 3/3] [2.0 Breaking] `Sender` changes for C# modules (#4143) # Description of Changes Make `Sender` a method on [Reducer|View|Procedure|Tx]Context in C#. Equivalent changes to https://github.com/clockworklabs/SpacetimeDB/pull/4101. # API and ABI breaking changes API breaking # Expected complexity level and risk 1 # Testing Pure refactor, no additional testing. --- .../diag/snapshots/Module#FFI.verified.cs | 21 +++++++++--- .../server/snapshots/Module#FFI.verified.cs | 21 +++++++++--- crates/bindings-csharp/Codegen/Module.cs | 20 +++++++++--- .../Runtime/Internal/TxContext.cs | 7 ++-- .../Runtime/ProcedureContext.cs | 10 ++++-- demo/Blackholio/server-csharp/Lib.cs | 16 +++++----- .../00400-key-architecture.md | 2 +- .../00300-tutorials/00100-chat-app.md | 12 +++---- .../00300-unity-tutorial/00300-part-2.md | 4 +-- .../00300-unity-tutorial/00400-part-3.md | 10 +++--- .../00300-unity-tutorial/00500-part-4.md | 4 +-- .../00400-unreal-tutorial/00300-part-2.md | 4 +-- .../00400-unreal-tutorial/00400-part-3.md | 10 +++--- .../00400-unreal-tutorial/00500-part-4.md | 4 +-- .../00100-databases/00500-cheat-sheet.md | 4 +-- .../00300-reducers/00400-reducer-context.md | 4 +-- .../00300-reducers/00500-lifecycle.md | 6 ++-- .../00300-reducers/00600-error-handling.md | 2 +- .../00200-functions/00500-views.md | 6 ++-- docs/docs/00200-core-concepts/00300-tables.md | 2 +- .../00300-tables/00210-file-storage.md | 2 +- .../00300-tables/00400-access-permissions.md | 8 ++--- .../00100-how-to/00300-logging.md | 4 +-- docs/static/llms.md | 32 +++++++++---------- modules/module-test-cs/Lib.cs | 6 ++-- modules/sdk-test-connect-disconnect-cs/Lib.cs | 4 +-- modules/sdk-test-cs/Lib.cs | 8 ++--- .../examples~/regression-tests/server/Lib.cs | 24 +++++++------- templates/chat-console-cs/spacetimedb/Lib.cs | 12 +++---- 29 files changed, 155 insertions(+), 114 deletions(-) diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs index 90eb00d3eaf..0731b890923 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs @@ -1,4 +1,4 @@ -//HintName: FFI.cs +//HintName: FFI.cs // #nullable enable // The runtime already defines SpacetimeDB.Internal.LocalReadOnly in Runtime\Internal\Module.cs as an empty partial type. @@ -17,7 +17,6 @@ namespace SpacetimeDB { public sealed record ReducerContext : DbContext, Internal.IReducerContext { - public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; public readonly Timestamp Timestamp; @@ -29,6 +28,8 @@ public sealed record ReducerContext : DbContext, Internal.IReducerContext // We need this property to be non-static for parity with client SDK. public Identity Identity => Internal.IReducerContext.GetIdentity(); + private readonly Identity _sender; + internal ReducerContext( Identity identity, ConnectionId? connectionId, @@ -37,7 +38,7 @@ internal ReducerContext( AuthCtx? senderAuth = null ) { - Sender = identity; + _sender = identity; ConnectionId = connectionId; Rng = random; Timestamp = time; @@ -45,6 +46,11 @@ internal ReducerContext( CounterUuid = 0; } + /// + /// The identity of the client that invoked the reducer. + /// + public Identity Sender() => _sender; + /// /// Create a new random `v4` using the built-in RNG. /// @@ -209,13 +215,18 @@ public sealed class Local : global::SpacetimeDB.LocalBase public sealed record ViewContext : DbContext, Internal.IViewContext { - public Identity Sender { get; } + private readonly Identity _sender; internal ViewContext(Identity sender, Internal.LocalReadOnly db) : base(db) { - Sender = sender; + _sender = sender; } + + /// + /// The identity of the client that invoked the view. + /// + public Identity Sender() => _sender; } public sealed record AnonymousViewContext diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs index f4d86c2ddbc..86a559890c1 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs @@ -1,4 +1,4 @@ -//HintName: FFI.cs +//HintName: FFI.cs // #nullable enable // The runtime already defines SpacetimeDB.Internal.LocalReadOnly in Runtime\Internal\Module.cs as an empty partial type. @@ -17,7 +17,6 @@ namespace SpacetimeDB { public sealed record ReducerContext : DbContext, Internal.IReducerContext { - public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; public readonly Timestamp Timestamp; @@ -29,6 +28,8 @@ public sealed record ReducerContext : DbContext, Internal.IReducerContext // We need this property to be non-static for parity with client SDK. public Identity Identity => Internal.IReducerContext.GetIdentity(); + private readonly Identity _sender; + internal ReducerContext( Identity identity, ConnectionId? connectionId, @@ -37,7 +38,7 @@ internal ReducerContext( AuthCtx? senderAuth = null ) { - Sender = identity; + _sender = identity; ConnectionId = connectionId; Rng = random; Timestamp = time; @@ -45,6 +46,11 @@ internal ReducerContext( CounterUuid = 0; } + /// + /// The identity of the client that invoked the reducer. + /// + public Identity Sender() => _sender; + /// /// Create a new random `v4` using the built-in RNG. /// @@ -199,13 +205,18 @@ public sealed class Local : global::SpacetimeDB.LocalBase public sealed record ViewContext : DbContext, Internal.IViewContext { - public Identity Sender { get; } + private readonly Identity _sender; internal ViewContext(Identity sender, Internal.LocalReadOnly db) : base(db) { - Sender = sender; + _sender = sender; } + + /// + /// The identity of the client that invoked the view. + /// + public Identity Sender() => _sender; } public sealed record AnonymousViewContext diff --git a/crates/bindings-csharp/Codegen/Module.cs b/crates/bindings-csharp/Codegen/Module.cs index 63331bba5b6..e927f8832a4 100644 --- a/crates/bindings-csharp/Codegen/Module.cs +++ b/crates/bindings-csharp/Codegen/Module.cs @@ -1737,7 +1737,6 @@ public void Initialize(IncrementalGeneratorInitializationContext context) namespace SpacetimeDB { public sealed record ReducerContext : DbContext, Internal.IReducerContext { - public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; public readonly Timestamp Timestamp; @@ -1747,16 +1746,24 @@ public sealed record ReducerContext : DbContext, Internal.IReducerContext // We need this property to be non-static for parity with client SDK. public Identity Identity => Internal.IReducerContext.GetIdentity(); + private readonly Identity _sender; + internal ReducerContext(Identity identity, ConnectionId? connectionId, Random random, Timestamp time, AuthCtx? senderAuth = null) { - Sender = identity; + _sender = identity; ConnectionId = connectionId; Rng = random; Timestamp = time; SenderAuth = senderAuth ?? AuthCtx.BuildFromSystemTables(connectionId, identity); CounterUuid = 0; } + + /// + /// The identity of the client that invoked the reducer. + /// + public Identity Sender() => _sender; + /// /// Create a new random `v4` using the built-in RNG. /// @@ -1891,13 +1898,18 @@ public sealed class Local : global::SpacetimeDB.LocalBase { public sealed record ViewContext : DbContext, Internal.IViewContext { - public Identity Sender { get; } + private readonly Identity _sender; internal ViewContext(Identity sender, Internal.LocalReadOnly db) : base(db) { - Sender = sender; + _sender = sender; } + + /// + /// The identity of the client that invoked the view. + /// + public Identity Sender() => _sender; } public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext diff --git a/crates/bindings-csharp/Runtime/Internal/TxContext.cs b/crates/bindings-csharp/Runtime/Internal/TxContext.cs index d5bea2febd9..f8465e78d56 100644 --- a/crates/bindings-csharp/Runtime/Internal/TxContext.cs +++ b/crates/bindings-csharp/Runtime/Internal/TxContext.cs @@ -9,13 +9,16 @@ public sealed class TxContext( Random rng ) { + private readonly Identity _sender = sender; + public Local Db { get; } = db; - public Identity Sender { get; } = sender; public ConnectionId? ConnectionId { get; } = connectionId; public Timestamp Timestamp { get; } = timestamp; public AuthCtx SenderAuth { get; } = senderAuth; public Random Rng { get; } = rng; + public Identity Sender() => _sender; + public TxContext WithTimestamp(Timestamp ts) => - new(Db, Sender, ConnectionId, ts, SenderAuth, Rng); + new(Db, _sender, ConnectionId, ts, SenderAuth, Rng); } diff --git a/crates/bindings-csharp/Runtime/ProcedureContext.cs b/crates/bindings-csharp/Runtime/ProcedureContext.cs index 4eb8583b7d5..1b878b8e549 100644 --- a/crates/bindings-csharp/Runtime/ProcedureContext.cs +++ b/crates/bindings-csharp/Runtime/ProcedureContext.cs @@ -10,8 +10,11 @@ public abstract class ProcedureContextBase( Timestamp time ) : Internal.IInternalProcedureContext { + private readonly Identity _sender = sender; + + public Identity Sender() => _sender; + public static Identity Identity => Internal.IProcedureContext.GetIdentity(); - public Identity Sender { get; } = sender; public ConnectionId? ConnectionId { get; } = connectionId; public Random Rng { get; } = random; public Timestamp Timestamp { get; private set; } = time; @@ -47,7 +50,7 @@ public Internal.TxContext EnterTxContext(long timestampMicros) txContext?.WithTimestamp(timestamp) ?? new Internal.TxContext( CreateLocal(), - Sender, + _sender, ConnectionId, timestamp, SenderAuth, @@ -229,8 +232,9 @@ public abstract class ProcedureTxContextBase(Internal.TxContext inner) internal void Refresh(Internal.TxContext inner) => Inner = inner; + public Identity Sender() => Inner.Sender(); + public LocalBase Db => (LocalBase)Inner.Db; - public Identity Sender => Inner.Sender; public ConnectionId? ConnectionId => Inner.ConnectionId; public Timestamp Timestamp => Inner.Timestamp; public AuthCtx SenderAuth => Inner.SenderAuth; diff --git a/demo/Blackholio/server-csharp/Lib.cs b/demo/Blackholio/server-csharp/Lib.cs index 20e26cf99e6..542a7040037 100644 --- a/demo/Blackholio/server-csharp/Lib.cs +++ b/demo/Blackholio/server-csharp/Lib.cs @@ -138,7 +138,7 @@ public static void Init(ReducerContext ctx) [Reducer(ReducerKind.ClientConnected)] public static void Connect(ReducerContext ctx) { - var player = ctx.Db.logged_out_player.identity.Find(ctx.Sender); + var player = ctx.Db.logged_out_player.identity.Find(ctx.Sender()); if (player != null) { ctx.Db.player.Insert(player.Value); @@ -157,7 +157,7 @@ public static void Connect(ReducerContext ctx) { ctx.Db.player.Insert(new Player { - identity = ctx.Sender, + identity = ctx.Sender(), name = "", }); } @@ -166,7 +166,7 @@ public static void Connect(ReducerContext ctx) [Reducer(ReducerKind.ClientDisconnected)] public static void Disconnect(ReducerContext ctx) { - var player = ctx.Db.player.identity.Find(ctx.Sender) ?? throw new Exception("Player not found"); + var player = ctx.Db.player.identity.Find(ctx.Sender()) ?? throw new Exception("Player not found"); foreach (var circle in ctx.Db.circle.player_id.Filter(player.player_id)) { var entity = ctx.Db.entity.entity_id.Find(circle.entity_id) ?? throw new Exception("Could not find circle"); @@ -183,7 +183,7 @@ public static void Disconnect(ReducerContext ctx) public static void EnterGame(ReducerContext ctx, string name) { Log.Info($"Creating player with name {name}"); - var player = ctx.Db.player.identity.Find(ctx.Sender) ?? throw new Exception("Player not found"); + var player = ctx.Db.player.identity.Find(ctx.Sender()) ?? throw new Exception("Player not found"); player.name = name; ctx.Db.player.identity.Update(player); SpawnPlayerInitialCircle(ctx, player.player_id); @@ -192,7 +192,7 @@ public static void EnterGame(ReducerContext ctx, string name) [Reducer] public static void Respawn(ReducerContext ctx) { - var player = ctx.Db.player.identity.Find(ctx.Sender) ?? throw new Exception("No such player found"); + var player = ctx.Db.player.identity.Find(ctx.Sender()) ?? throw new Exception("No such player found"); SpawnPlayerInitialCircle(ctx, player.player_id); } @@ -200,7 +200,7 @@ public static void Respawn(ReducerContext ctx) [Reducer] public static void Suicide(ReducerContext ctx) { - var player = ctx.Db.player.identity.Find(ctx.Sender) ?? throw new Exception("No such player found"); + var player = ctx.Db.player.identity.Find(ctx.Sender()) ?? throw new Exception("No such player found"); foreach (var circle in ctx.Db.circle.player_id.Filter(player.player_id)) { @@ -246,7 +246,7 @@ public static Entity SpawnCircleAt(ReducerContext ctx, int player_id, int mass, [Reducer] public static void UpdatePlayerInput(ReducerContext ctx, DbVector2 direction) { - var player = ctx.Db.player.identity.Find(ctx.Sender) ?? throw new Exception("Player not found"); + var player = ctx.Db.player.identity.Find(ctx.Sender()) ?? throw new Exception("Player not found"); foreach (var c in ctx.Db.circle.player_id.Filter(player.player_id)) { var circle = c; @@ -449,7 +449,7 @@ public static void DestroyEntity(ReducerContext ctx, int entityId) [Reducer] public static void PlayerSplit(ReducerContext ctx) { - var player = ctx.Db.player.identity.Find(ctx.Sender) ?? throw new Exception("Sender has no player"); + var player = ctx.Db.player.identity.Find(ctx.Sender()) ?? throw new Exception("Sender has no player"); List circles = ctx.Db.circle.player_id.Filter(player.player_id).ToList(); var circle_count = circles.Count; if (circle_count >= MAX_CIRCLES_PER_PLAYER) diff --git a/docs/docs/00100-intro/00100-getting-started/00400-key-architecture.md b/docs/docs/00100-intro/00100-getting-started/00400-key-architecture.md index 63eff91f7b0..59ba40c581f 100644 --- a/docs/docs/00100-intro/00100-getting-started/00400-key-architecture.md +++ b/docs/docs/00100-intro/00100-getting-started/00400-key-architecture.md @@ -439,7 +439,7 @@ A view can be written in C# like so: [SpacetimeDB.View(Name = "MyPlayer", Public = true)] public static Player? MyPlayer(ViewContext ctx) { - return ctx.Db.Player.Identity.Find(ctx.Sender) as Player; + return ctx.Db.Player.Identity.Find(ctx.Sender()) as Player; } ``` diff --git a/docs/docs/00100-intro/00300-tutorials/00100-chat-app.md b/docs/docs/00100-intro/00300-tutorials/00100-chat-app.md index 06ef0c039ce..f8dc277f38c 100644 --- a/docs/docs/00100-intro/00300-tutorials/00100-chat-app.md +++ b/docs/docs/00100-intro/00300-tutorials/00100-chat-app.md @@ -322,7 +322,7 @@ public static void SetName(ReducerContext ctx, string name) { name = ValidateName(name); - if (ctx.Db.User.Identity.Find(ctx.Sender) is User user) + if (ctx.Db.User.Identity.Find(ctx.Sender()) is User user) { user.Name = name; ctx.Db.User.Identity.Update(user); @@ -411,7 +411,7 @@ public static void SendMessage(ReducerContext ctx, string text) ctx.Db.Message.Insert( new Message { - Sender = ctx.Sender, + Sender = ctx.Sender(), Text = text, Sent = ctx.Timestamp, } @@ -504,9 +504,9 @@ In `spacetimedb/Lib.cs`, add to the `Module` class: [Reducer(ReducerKind.ClientConnected)] public static void ClientConnected(ReducerContext ctx) { - Log.Info($"Connect {ctx.Sender}"); + Log.Info($"Connect {ctx.Sender()}"); - if (ctx.Db.User.Identity.Find(ctx.Sender) is User user) + if (ctx.Db.User.Identity.Find(ctx.Sender()) is User user) { user.Online = true; ctx.Db.User.Identity.Update(user); @@ -517,7 +517,7 @@ public static void ClientConnected(ReducerContext ctx) new User { Name = null, - Identity = ctx.Sender, + Identity = ctx.Sender(), Online = true, } ); @@ -527,7 +527,7 @@ public static void ClientConnected(ReducerContext ctx) [Reducer(ReducerKind.ClientDisconnected)] public static void ClientDisconnected(ReducerContext ctx) { - if (ctx.Db.User.Identity.Find(ctx.Sender) is User user) + if (ctx.Db.User.Identity.Find(ctx.Sender()) is User user) { user.Online = false; ctx.Db.User.Identity.Update(user); diff --git a/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00300-part-2.md b/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00300-part-2.md index 9f06001aa8a..d19945793c3 100644 --- a/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00300-part-2.md +++ b/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00300-part-2.md @@ -335,7 +335,7 @@ Add this function to the `Module` class in `Lib.cs`: [Reducer] public static void Debug(ReducerContext ctx) { - Log.Info($"This reducer was called by {ctx.Sender}"); + Log.Info($"This reducer was called by {ctx.Sender()}"); } ``` @@ -444,7 +444,7 @@ Next let's connect our client to our database. Let's start by modifying our `Deb [Reducer(ReducerKind.ClientConnected)] public static void Connect(ReducerContext ctx) { - Log.Info($"{ctx.Sender} just connected."); + Log.Info($"{ctx.Sender()} just connected."); } ``` diff --git a/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00400-part-3.md b/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00400-part-3.md index ab95e71294d..f1a0c99aa0d 100644 --- a/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00400-part-3.md +++ b/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00400-part-3.md @@ -359,7 +359,7 @@ Next, modify your `Connect` reducer and add a new `Disconnect` reducer below it: [Reducer(ReducerKind.ClientConnected)] public static void Connect(ReducerContext ctx) { - var player = ctx.Db.logged_out_player.identity.Find(ctx.Sender); + var player = ctx.Db.logged_out_player.identity.Find(ctx.Sender()); if (player != null) { ctx.Db.player.Insert(player.Value); @@ -369,7 +369,7 @@ public static void Connect(ReducerContext ctx) { ctx.Db.player.Insert(new Player { - identity = ctx.Sender, + identity = ctx.Sender(), name = "", }); } @@ -378,7 +378,7 @@ public static void Connect(ReducerContext ctx) [Reducer(ReducerKind.ClientDisconnected)] public static void Disconnect(ReducerContext ctx) { - var player = ctx.Db.player.identity.Find(ctx.Sender) ?? throw new Exception("Player not found"); + var player = ctx.Db.player.identity.Find(ctx.Sender()) ?? throw new Exception("Player not found"); ctx.Db.logged_out_player.Insert(player); ctx.Db.player.identity.Delete(player.identity); } @@ -457,7 +457,7 @@ const int START_PLAYER_MASS = 15; public static void EnterGame(ReducerContext ctx, string name) { Log.Info($"Creating player with name {name}"); - var player = ctx.Db.player.identity.Find(ctx.Sender) ?? throw new Exception("Player not found"); + var player = ctx.Db.player.identity.Find(ctx.Sender()) ?? throw new Exception("Player not found"); player.name = name; ctx.Db.player.identity.Update(player); SpawnPlayerInitialCircle(ctx, player.player_id); @@ -579,7 +579,7 @@ Let's also modify our `disconnect` reducer to remove the circles from the arena [Reducer(ReducerKind.ClientDisconnected)] public static void Disconnect(ReducerContext ctx) { - var player = ctx.Db.player.identity.Find(ctx.Sender) ?? throw new Exception("Player not found"); + var player = ctx.Db.player.identity.Find(ctx.Sender()) ?? throw new Exception("Player not found"); // Remove any circles from the arena foreach (var circle in ctx.Db.circle.player_id.Filter(player.player_id)) { diff --git a/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00500-part-4.md b/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00500-part-4.md index 45ad7ee8ea8..2f7ae934851 100644 --- a/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00500-part-4.md +++ b/docs/docs/00100-intro/00300-tutorials/00300-unity-tutorial/00500-part-4.md @@ -49,7 +49,7 @@ Next, add the following reducer to the `Module` class of your `Lib.cs` file. [Reducer] public static void UpdatePlayerInput(ReducerContext ctx, DbVector2 direction) { - var player = ctx.Db.player.identity.Find(ctx.Sender) ?? throw new Exception("Player not found"); + var player = ctx.Db.player.identity.Find(ctx.Sender()) ?? throw new Exception("Player not found"); foreach (var c in ctx.Db.circle.player_id.Filter(player.player_id)) { var circle = c; @@ -60,7 +60,7 @@ public static void UpdatePlayerInput(ReducerContext ctx, DbVector2 direction) } ``` -This is a simple reducer that takes the movement input from the client and applies them to all circles that that player controls. Note that it is not possible for a player to move another player's circles using this reducer, because the `ctx.Sender` value is not set by the client. Instead `ctx.Sender` is set by SpacetimeDB after it has authenticated that sender. You can rest assured that the caller has been authenticated as that player by the time this reducer is called. +This is a simple reducer that takes the movement input from the client and applies them to all circles that that player controls. Note that it is not possible for a player to move another player's circles using this reducer, because the `ctx.Sender()` value is not set by the client. Instead `ctx.Sender()` is set by SpacetimeDB after it has authenticated that sender. You can rest assured that the caller has been authenticated as that player by the time this reducer is called. diff --git a/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00300-part-2.md b/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00300-part-2.md index bde4659ba26..a97a65abec8 100644 --- a/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00300-part-2.md +++ b/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00300-part-2.md @@ -333,7 +333,7 @@ Add this function to the `Module` class in `Lib.cs`: [Reducer] public static void Debug(ReducerContext ctx) { - Log.Info($"This reducer was called by {ctx.Sender}"); + Log.Info($"This reducer was called by {ctx.Sender()}"); } ``` @@ -440,7 +440,7 @@ Next let's connect our client to our database. Let's start by modifying our `Deb [Reducer(ReducerKind.ClientConnected)] public static void Connect(ReducerContext ctx) { - Log.Info($"{ctx.Sender} just connected."); + Log.Info($"{ctx.Sender()} just connected."); } ``` diff --git a/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00400-part-3.md b/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00400-part-3.md index 192c7d3ebc8..d3543050874 100644 --- a/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00400-part-3.md +++ b/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00400-part-3.md @@ -352,7 +352,7 @@ Next, modify your `Connect` reducer and add a new `Disconnect` reducer below it: [Reducer(ReducerKind.ClientConnected)] public static void Connect(ReducerContext ctx) { - var player = ctx.Db.logged_out_player.identity.Find(ctx.Sender); + var player = ctx.Db.logged_out_player.identity.Find(ctx.Sender()); if (player != null) { ctx.Db.player.Insert(player.Value); @@ -362,7 +362,7 @@ public static void Connect(ReducerContext ctx) { ctx.Db.player.Insert(new Player { - identity = ctx.Sender, + identity = ctx.Sender(), name = "", }); } @@ -371,7 +371,7 @@ public static void Connect(ReducerContext ctx) [Reducer(ReducerKind.ClientDisconnected)] public static void Disconnect(ReducerContext ctx) { - var player = ctx.Db.player.identity.Find(ctx.Sender) ?? throw new Exception("Player not found"); + var player = ctx.Db.player.identity.Find(ctx.Sender()) ?? throw new Exception("Player not found"); ctx.Db.logged_out_player.Insert(player); ctx.Db.player.identity.Delete(player.identity); } @@ -446,7 +446,7 @@ const int START_PLAYER_MASS = 15; public static void EnterGame(ReducerContext ctx, string name) { Log.Info($"Creating player with name {name}"); - var player = ctx.Db.player.identity.Find(ctx.Sender) ?? throw new Exception("Player not found"); + var player = ctx.Db.player.identity.Find(ctx.Sender()) ?? throw new Exception("Player not found"); player.name = name; ctx.Db.player.identity.Update(player); SpawnPlayerInitialCircle(ctx, player.player_id); @@ -566,7 +566,7 @@ Let's also modify our `disconnect` reducer to remove the circles from the arena [Reducer(ReducerKind.ClientDisconnected)] public static void Disconnect(ReducerContext ctx) { - var player = ctx.Db.player.identity.Find(ctx.Sender) ?? throw new Exception("Player not found"); + var player = ctx.Db.player.identity.Find(ctx.Sender()) ?? throw new Exception("Player not found"); // Remove any circles from the arena foreach (var circle in ctx.Db.circle.player_id.Filter(player.player_id)) { diff --git a/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00500-part-4.md b/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00500-part-4.md index ae8eb620633..2acda18aac8 100644 --- a/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00500-part-4.md +++ b/docs/docs/00100-intro/00300-tutorials/00400-unreal-tutorial/00500-part-4.md @@ -50,7 +50,7 @@ Next, add the following reducer to the `Module` class of your `Lib.cs` file. [Reducer] public static void UpdatePlayerInput(ReducerContext ctx, DbVector2 direction) { - var player = ctx.Db.player.identity.Find(ctx.Sender) ?? throw new Exception("Player not found"); + var player = ctx.Db.player.identity.Find(ctx.Sender()) ?? throw new Exception("Player not found"); foreach (var c in ctx.Db.circle.player_id.Filter(player.player_id)) { var circle = c; @@ -61,7 +61,7 @@ public static void UpdatePlayerInput(ReducerContext ctx, DbVector2 direction) } ``` -This is a simple reducer that takes the movement input from the client and applies them to all circles that that player controls. Note that it is not possible for a player to move another player's circles using this reducer, because the `ctx.Sender` value is not set by the client. Instead `ctx.Sender` is set by SpacetimeDB after it has authenticated that sender. You can rest assured that the caller has been authenticated as that player by the time this reducer is called. +This is a simple reducer that takes the movement input from the client and applies them to all circles that that player controls. Note that it is not possible for a player to move another player's circles using this reducer, because the `ctx.Sender()` value is not set by the client. Instead `ctx.Sender()` is set by SpacetimeDB after it has authenticated that sender. You can rest assured that the caller has been authenticated as that player by the time this reducer is called. Let's start by building out a simple math library to help us do collision calculations. Create a new `math.rs` file in the `blackholio/spacetimedb/src` directory and add the following contents. Let's also move the `DbVector2` type from `lib.rs` into this file. diff --git a/docs/docs/00200-core-concepts/00100-databases/00500-cheat-sheet.md b/docs/docs/00200-core-concepts/00100-databases/00500-cheat-sheet.md index b49afcd4c0b..da6b28f2e02 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00500-cheat-sheet.md +++ b/docs/docs/00200-core-concepts/00100-databases/00500-cheat-sheet.md @@ -440,7 +440,7 @@ using SpacetimeDB; [SpacetimeDB.View(Public = true)] public static Player? MyPlayer(ViewContext ctx) { - return ctx.Db.Player.Identity.Find(ctx.Sender); + return ctx.Db.Player.Identity.Find(ctx.Sender()); } // Return multiple rows @@ -493,7 +493,7 @@ ctx.identity // Module's identity ```csharp ctx.Db // Database access -ctx.Sender // Identity of caller +ctx.Sender() // Identity of caller ctx.ConnectionId // ConnectionId? ctx.Timestamp // Timestamp ctx.Identity // Module's identity diff --git a/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00400-reducer-context.md b/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00400-reducer-context.md index 38f447125d4..d789845ab29 100644 --- a/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00400-reducer-context.md +++ b/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00400-reducer-context.md @@ -145,7 +145,7 @@ public static partial class Module public static void UpdateScore(ReducerContext ctx, uint newScore) { // Get the caller's identity - Identity caller = ctx.Sender; + Identity caller = ctx.Sender(); // Find and update their player record if (ctx.Db.Player.Identity.Find(caller) is Player player) @@ -262,7 +262,7 @@ public static partial class Module public static void SendReminder(ReducerContext ctx, ScheduledTask task) { // Only allow the scheduler (module identity) to call this - if (ctx.Sender != ctx.Identity) + if (ctx.Sender() != ctx.Identity) { throw new Exception("This reducer can only be called by the scheduler"); } diff --git a/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00500-lifecycle.md b/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00500-lifecycle.md index 20f43bc8394..605ad09dd09 100644 --- a/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00500-lifecycle.md +++ b/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00500-lifecycle.md @@ -110,7 +110,7 @@ spacetimedb.clientConnected((ctx) => { [SpacetimeDB.Reducer(ReducerKind.ClientConnected)] public static void OnConnect(ReducerContext ctx) { - Log.Info($"Client connected: {ctx.Sender}"); + Log.Info($"Client connected: {ctx.Sender()}"); // ctx.ConnectionId is guaranteed to be non-null var connId = ctx.ConnectionId!.Value; @@ -119,7 +119,7 @@ public static void OnConnect(ReducerContext ctx) ctx.Db.Session.Insert(new Session { ConnectionId = connId, - Identity = ctx.Sender, + Identity = ctx.Sender(), ConnectedAt = ctx.Timestamp }); } @@ -182,7 +182,7 @@ spacetimedb.clientDisconnected((ctx) => { [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)] public static void OnDisconnect(ReducerContext ctx) { - Log.Info($"Client disconnected: {ctx.Sender}"); + Log.Info($"Client disconnected: {ctx.Sender()}"); // ctx.ConnectionId is guaranteed to be non-null var connId = ctx.ConnectionId!.Value; diff --git a/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00600-error-handling.md b/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00600-error-handling.md index 714f68d1110..661e5870853 100644 --- a/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00600-error-handling.md +++ b/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00600-error-handling.md @@ -61,7 +61,7 @@ Throw an exception: [SpacetimeDB.Reducer] public static void TransferCredits(ReducerContext ctx, ulong toUser, uint amount) { - var fromUser = ctx.Db.User.Id.Find(ctx.Sender); + var fromUser = ctx.Db.User.Id.Find(ctx.Sender()); if (fromUser == null) { throw new InvalidOperationException("User not found"); diff --git a/docs/docs/00200-core-concepts/00200-functions/00500-views.md b/docs/docs/00200-core-concepts/00200-functions/00500-views.md index 0603fa97dad..7b8c43ea1b7 100644 --- a/docs/docs/00200-core-concepts/00200-functions/00500-views.md +++ b/docs/docs/00200-core-concepts/00200-functions/00500-views.md @@ -130,7 +130,7 @@ public static partial class Module [SpacetimeDB.View(Name = "MyPlayer", Public = true)] public static Player? MyPlayer(ViewContext ctx) { - return ctx.Db.Player.Identity.Find(ctx.Sender) as Player; + return ctx.Db.Player.Identity.Find(ctx.Sender()) as Player; } // Multiple rows: return a list @@ -283,7 +283,7 @@ spacetimedb.view( [SpacetimeDB.View(Name = "MyPlayer", Public = true)] public static Player? MyPlayer(ViewContext ctx) { - return ctx.Db.Player.Identity.Find(ctx.Sender); + return ctx.Db.Player.Identity.Find(ctx.Sender()); } ``` @@ -485,7 +485,7 @@ public partial class Module [SpacetimeDB.View(Name = "EntitiesInMyChunk", Public = true)] public static List EntitiesInMyChunk(ViewContext ctx) { - if (ctx.Db.Player.Identity.Find(ctx.Sender) is not Player player) + if (ctx.Db.Player.Identity.Find(ctx.Sender()) is not Player player) { return new List(); } diff --git a/docs/docs/00200-core-concepts/00300-tables.md b/docs/docs/00200-core-concepts/00300-tables.md index cb241bc1ae1..e6ca5262ff7 100644 --- a/docs/docs/00200-core-concepts/00300-tables.md +++ b/docs/docs/00200-core-concepts/00300-tables.md @@ -344,7 +344,7 @@ ctx.Db.Player.Insert(new Player { /* ... */ }); ctx.Db.LoggedOutPlayer.Insert(new Player { /* ... */ }); // Move a row between tables -var player = ctx.Db.LoggedOutPlayer.Identity.Find(ctx.Sender); +var player = ctx.Db.LoggedOutPlayer.Identity.Find(ctx.Sender()); if (player != null) { ctx.Db.Player.Insert(player.Value); diff --git a/docs/docs/00200-core-concepts/00300-tables/00210-file-storage.md b/docs/docs/00200-core-concepts/00300-tables/00210-file-storage.md index 0a102b8377f..80b0eb5a19a 100644 --- a/docs/docs/00200-core-concepts/00300-tables/00210-file-storage.md +++ b/docs/docs/00200-core-concepts/00300-tables/00210-file-storage.md @@ -221,7 +221,7 @@ public static partial class Module ctx.Db.Document.Insert(new Document { Id = 0, // auto-increment - OwnerId = ctx.Sender, + OwnerId = ctx.Sender(), Filename = filename, MimeType = mimeType, SizeBytes = sizeBytes, diff --git a/docs/docs/00200-core-concepts/00300-tables/00400-access-permissions.md b/docs/docs/00200-core-concepts/00300-tables/00400-access-permissions.md index 27ae27f34a6..3293e90e05f 100644 --- a/docs/docs/00200-core-concepts/00300-tables/00400-access-permissions.md +++ b/docs/docs/00200-core-concepts/00300-tables/00400-access-permissions.md @@ -372,8 +372,8 @@ public partial class Module public static List MyMessages(ViewContext ctx) { // Look up messages by index where caller is sender or recipient - var sent = ctx.Db.Message.Sender.Filter(ctx.Sender).ToList(); - var received = ctx.Db.Message.Recipient.Filter(ctx.Sender).ToList(); + var sent = ctx.Db.Message.Sender.Filter(ctx.Sender()).ToList(); + var received = ctx.Db.Message.Recipient.Filter(ctx.Sender()).ToList(); sent.AddRange(received); return sent; } @@ -505,7 +505,7 @@ public partial class Module public static PublicUserProfile? MyProfile(ViewContext ctx) { // Look up the caller's account by their identity (unique index) - if (ctx.Db.UserAccount.Identity.Find(ctx.Sender) is not UserAccount user) + if (ctx.Db.UserAccount.Identity.Find(ctx.Sender()) is not UserAccount user) { return null; } @@ -662,7 +662,7 @@ public partial class Module public static List MyTeam(ViewContext ctx) { // Find the caller's employee record by identity (unique index) - if (ctx.Db.Employee.Identity.Find(ctx.Sender) is not Employee me) + if (ctx.Db.Employee.Identity.Find(ctx.Sender()) is not Employee me) { return new List(); } diff --git a/docs/docs/00300-resources/00100-how-to/00300-logging.md b/docs/docs/00300-resources/00100-how-to/00300-logging.md index 1958b01634e..1b77fd7bcc7 100644 --- a/docs/docs/00300-resources/00100-how-to/00300-logging.md +++ b/docs/docs/00300-resources/00100-how-to/00300-logging.md @@ -69,7 +69,7 @@ public static partial class Module throw new ArgumentException("Value cannot be zero"); } - Log.Debug($"Debug information: ctx.Sender = {ctx.Sender}"); + Log.Debug($"Debug information: ctx.Sender() = {ctx.Sender()}"); } } ``` @@ -202,7 +202,7 @@ Include relevant context in your log messages: [SpacetimeDB.Reducer] public static void TransferCredits(ReducerContext ctx, ulong toUser, uint amount) { - Log.Info($"Credit transfer: from={ctx.Sender}, to={toUser}, amount={amount}"); + Log.Info($"Credit transfer: from={ctx.Sender()}, to={toUser}, amount={amount}"); // ... transfer logic } diff --git a/docs/static/llms.md b/docs/static/llms.md index 876a91b31ad..f35907bf85d 100644 --- a/docs/static/llms.md +++ b/docs/static/llms.md @@ -1449,7 +1449,7 @@ Reducers are the functions within your server module responsible for atomically - **Return Type:** Reducers should typically return `void`. Errors are signaled by throwing exceptions. - **Reducer Context:** The `ReducerContext` (`ctx`) provides access to: - `ctx.Db`: Handles for interacting with database tables. - - `ctx.Sender`: The `Identity` of the caller. + - `ctx.Sender()`: The `Identity` of the caller. - `ctx.Identity`: The `Identity` of the module itself. - `ctx.Timestamp`: The `Timestamp` of the invocation. - `ctx.ConnectionId`: The nullable `ConnectionId` of the caller. @@ -1478,7 +1478,7 @@ public static partial class Module [Reducer] public static void UpdatePlayerData(ReducerContext ctx, string? newName) { - var playerId = ctx.Sender; + var playerId = ctx.Sender(); // Find player by primary key var player = ctx.Db.player_state.PlayerId.Find(playerId); @@ -1521,10 +1521,10 @@ public static partial class Module if (string.IsNullOrWhiteSpace(name)) { throw new ArgumentException("Name cannot be empty."); } - Log.Info($"Attempting to register player: {name} ({ctx.Sender})"); + Log.Info($"Attempting to register player: {name} ({ctx.Sender()})"); // Check if player identity or name already exists - if (ctx.Db.player_state.PlayerId.Find(ctx.Sender) != null || ctx.Db.player_state.Name.Find(name) != null) + if (ctx.Db.player_state.PlayerId.Find(ctx.Sender()) != null || ctx.Db.player_state.Name.Find(name) != null) { throw new Exception("Player already registered or name taken."); } @@ -1532,7 +1532,7 @@ public static partial class Module // Create new player instance var newPlayer = new PlayerState { - PlayerId = ctx.Sender, + PlayerId = ctx.Sender(), Name = name, Health = 100, Level = 1, @@ -1541,14 +1541,14 @@ public static partial class Module // Insert the new player. This will throw on constraint violation. ctx.Db.player_state.Insert(newPlayer); - Log.Info($"Player registered successfully: {ctx.Sender}"); + Log.Info($"Player registered successfully: {ctx.Sender()}"); } // Example: Basic reducer showing deletion [Reducer] public static void DeleteMyItems(ReducerContext ctx) { - var ownerId = ctx.Sender; + var ownerId = ctx.Sender(); int deletedCount = 0; // Find items by owner (Requires an index on OwnerId for efficiency) @@ -1632,13 +1632,13 @@ These reducers cannot take arguments beyond `&ReducerContext`. // Example init reducer is shown in Scheduled Reducers section [Reducer(ReducerKind.ClientConnected)] public static void HandleConnect(ReducerContext ctx) { - Log.Info($"Client connected: {ctx.Sender}"); + Log.Info($"Client connected: {ctx.Sender()}"); // ... setup initial state for ctx.sender ... } [Reducer(ReducerKind.ClientDisconnected)] public static void HandleDisconnect(ReducerContext ctx) { - Log.Info($"Client disconnected: {ctx.Sender}"); + Log.Info($"Client disconnected: {ctx.Sender()}"); // ... cleanup state for ctx.sender ... } ``` @@ -1692,7 +1692,7 @@ public static partial class Module public static void SendMessage(ReducerContext ctx, SendMessageSchedule scheduleArgs) { // Security check is important! - if (!ctx.Sender.Equals(ctx.Identity)) + if (!ctx.Sender().Equals(ctx.Identity)) { throw new Exception("Reducer SendMessage may not be invoked by clients, only via scheduling."); } @@ -1740,12 +1740,12 @@ public static partial class Module - **Best-Effort Scheduling:** Scheduled reducers are called on a best-effort basis and may be slightly delayed in their execution when a database is under heavy load. -- **Restricting Access (Security):** Scheduled reducers are normal reducers and _can_ still be called directly by clients. If a scheduled reducer should _only_ be called by the scheduler, it is crucial to begin the reducer with a check comparing the caller's identity (`ctx.Sender`) to the module's own identity (`ctx.Identity`). +- **Restricting Access (Security):** Scheduled reducers are normal reducers and _can_ still be called directly by clients. If a scheduled reducer should _only_ be called by the scheduler, it is crucial to begin the reducer with a check comparing the caller's identity (`ctx.Sender()`) to the module's own identity (`ctx.Identity`). ```csharp [Reducer] // Assuming linked via [Table(Scheduled=...)] public static void MyScheduledTask(ReducerContext ctx, MyScheduleArgs args) { - if (!ctx.Sender.Equals(ctx.Identity)) + if (!ctx.Sender().Equals(ctx.Identity)) { throw new Exception("Reducer MyScheduledTask may not be invoked by clients, only via scheduling."); } @@ -1757,7 +1757,7 @@ public static partial class Module ``` :::info Scheduled Reducers and Connections -Scheduled reducer calls originate from the SpacetimeDB scheduler itself, not from an external client connection. Therefore, within a scheduled reducer, `ctx.Sender` will be the module's own identity, and `ctx.ConnectionId` will be `null`. +Scheduled reducer calls originate from the SpacetimeDB scheduler itself, not from an external client connection. Therefore, within a scheduled reducer, `ctx.Sender()` will be the module's own identity, and `ctx.ConnectionId` will be `null`. ::: ##### Error Handling: Exceptions @@ -1827,7 +1827,7 @@ public static partial class Module [SpacetimeDB.View(Name = "MyPlayer", Public = true)] public static Player? MyPlayer(ViewContext ctx) { - return ctx.Db.Player.Identity.Find(ctx.Sender); + return ctx.Db.Player.Identity.Find(ctx.Sender()); } // View that returns all players at a specific level (same for all callers) @@ -1858,7 +1858,7 @@ public static partial class Module Views use one of two context types: -- **`ViewContext`**: Provides access to the caller's `Identity` through `ctx.Sender`. Use this when the view depends on who is querying it (e.g., "get my player"). +- **`ViewContext`**: Provides access to the caller's `Identity` through `ctx.Sender()`. Use this when the view depends on who is querying it (e.g., "get my player"). - **`AnonymousViewContext`**: Does not provide caller information. Use this when the view produces the same results regardless of who queries it (e.g., "get top 10 players"). Both contexts provide read-only access to tables and indexes through `ctx.Db`. @@ -1887,7 +1887,7 @@ The query builder provides a fluent API for constructing type-safe SQL queries: public static Query MyMessages(ViewContext ctx) { return ctx.Db.Message - .Filter(msg => msg.Sender == ctx.Sender) + .Filter(msg => msg.Sender == ctx.Sender()) .Build(); } diff --git a/modules/module-test-cs/Lib.cs b/modules/module-test-cs/Lib.cs index a8d7d7a05ba..72b69649960 100644 --- a/modules/module-test-cs/Lib.cs +++ b/modules/module-test-cs/Lib.cs @@ -219,7 +219,7 @@ static partial class Module [View(Name = "my_player", Public = true)] public static Player? my_player(ViewContext ctx) { - return (Player?)ctx.Db.player.identity.Find(ctx.Sender); + return (Player?)ctx.Db.player.identity.Find(ctx.Sender()); } // This reducer is run at module initialization. @@ -278,7 +278,7 @@ public static void log_module_identity(ReducerContext ctx) public static void test(ReducerContext ctx, TestAlias arg, TestB arg2, TestC arg3, TestF arg4) { Log.Info("BEGIN"); - Log.Info($"sender: {ctx.Sender}"); + Log.Info($"sender: {ctx.Sender()}"); Log.Info($"timestamp: {ctx.Timestamp}"); Log.Info($"bar: {arg2.foo}"); @@ -462,7 +462,7 @@ public static void test_btree_index_args(ReducerContext ctx) [Reducer] public static void assert_caller_identity_is_module_identity(ReducerContext ctx) { - var caller = ctx.Sender; + var caller = ctx.Sender(); var owner = ctx.Identity; if (!caller.Equals(owner)) { diff --git a/modules/sdk-test-connect-disconnect-cs/Lib.cs b/modules/sdk-test-connect-disconnect-cs/Lib.cs index 79af81d8cc4..200457fa616 100644 --- a/modules/sdk-test-connect-disconnect-cs/Lib.cs +++ b/modules/sdk-test-connect-disconnect-cs/Lib.cs @@ -19,12 +19,12 @@ static partial class Module [SpacetimeDB.Reducer(ReducerKind.ClientConnected)] public static void identity_connected(ReducerContext ctx) { - ctx.Db.connected.Insert(new Connected { identity = ctx.Sender}); + ctx.Db.connected.Insert(new Connected { identity = ctx.Sender()}); } [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)] public static void identity_disconnected(ReducerContext ctx) { - ctx.Db.disconnected.Insert(new Disconnected { identity = ctx.Sender}); + ctx.Db.disconnected.Insert(new Disconnected { identity = ctx.Sender()}); } } diff --git a/modules/sdk-test-cs/Lib.cs b/modules/sdk-test-cs/Lib.cs index 06ddacbb631..0387d77abae 100644 --- a/modules/sdk-test-cs/Lib.cs +++ b/modules/sdk-test-cs/Lib.cs @@ -1887,25 +1887,25 @@ public static void update_pk_simple_enum(ReducerContext ctx, SimpleEnum a, int d [SpacetimeDB.Reducer] public static void insert_caller_one_identity(ReducerContext ctx) { - ctx.Db.one_identity.Insert(new OneIdentity { i = ctx.Sender }); + ctx.Db.one_identity.Insert(new OneIdentity { i = ctx.Sender() }); } [SpacetimeDB.Reducer] public static void insert_caller_vec_identity(ReducerContext ctx) { - ctx.Db.vec_identity.Insert(new VecIdentity { i = new List { ctx.Sender } }); + ctx.Db.vec_identity.Insert(new VecIdentity { i = new List { ctx.Sender() } }); } [SpacetimeDB.Reducer] public static void insert_caller_unique_identity(ReducerContext ctx, int data) { - ctx.Db.unique_identity.Insert(new UniqueIdentity { i = ctx.Sender, data = data }); + ctx.Db.unique_identity.Insert(new UniqueIdentity { i = ctx.Sender(), data = data }); } [SpacetimeDB.Reducer] public static void insert_caller_pk_identity(ReducerContext ctx, int data) { - ctx.Db.pk_identity.Insert(new PkIdentity { i = ctx.Sender, data = data }); + ctx.Db.pk_identity.Insert(new PkIdentity { i = ctx.Sender(), data = data }); } [SpacetimeDB.Reducer] diff --git a/sdks/csharp/examples~/regression-tests/server/Lib.cs b/sdks/csharp/examples~/regression-tests/server/Lib.cs index 5adc5d46a30..cbdab48eae6 100644 --- a/sdks/csharp/examples~/regression-tests/server/Lib.cs +++ b/sdks/csharp/examples~/regression-tests/server/Lib.cs @@ -150,13 +150,13 @@ public partial struct NullStringNullable [SpacetimeDB.View(Name = "my_player", Public = true)] public static Player? MyPlayer(ViewContext ctx) { - return ctx.Db.player.Identity.Find(ctx.Sender); + return ctx.Db.player.Identity.Find(ctx.Sender()); } [SpacetimeDB.View(Name = "my_account", Public = true)] public static Account? MyAccount(ViewContext ctx) { - return ctx.Db.account.Identity.Find(ctx.Sender) as Account; + return ctx.Db.account.Identity.Find(ctx.Sender()) as Account; } [SpacetimeDB.View(Name = "my_account_missing", Public = true)] @@ -280,23 +280,23 @@ public static void InsertNullStringIntoNullable(ReducerContext ctx) [Reducer(ReducerKind.ClientConnected)] public static void ClientConnected(ReducerContext ctx) { - Log.Info($"Connect {ctx.Sender}"); + Log.Info($"Connect {ctx.Sender()}"); - if (ctx.Db.player.Identity.Find(ctx.Sender) is Player player) + if (ctx.Db.player.Identity.Find(ctx.Sender()) is Player player) { // We are not logging player login status, so do nothing } else { // Lets setup a new player with a level of 1 - ctx.Db.player.Insert(new Player { Identity = ctx.Sender, Name = "NewPlayer" }); - var playerId = (ctx.Db.player.Identity.Find(ctx.Sender)!).Value.Id; + ctx.Db.player.Insert(new Player { Identity = ctx.Sender(), Name = "NewPlayer" }); + var playerId = (ctx.Db.player.Identity.Find(ctx.Sender())!).Value.Id; ctx.Db.player_level.Insert(new PlayerLevel { PlayerId = playerId, Level = 1 }); } - if (ctx.Db.account.Identity.Find(ctx.Sender) is null) + if (ctx.Db.account.Identity.Find(ctx.Sender()) is null) { - ctx.Db.account.Insert(new Account { Identity = ctx.Sender, Name = "Account" }); + ctx.Db.account.Insert(new Account { Identity = ctx.Sender(), Name = "Account" }); } if (ctx.Db.nullable_vec.Id.Find(1) is null) @@ -614,10 +614,10 @@ public static ReturnStruct TxContextCapabilities(ProcedureContext ctx) } // Test 3: Verify transaction context properties are accessible - var txSender = tx.Sender; + var txSender = tx.Sender(); var txTimestamp = tx.Timestamp; - if (txSender.Equals(ctx.Sender) == false) + if (txSender.Equals(ctx.Sender()) == false) { throw new InvalidOperationException("Transaction sender should match procedure sender"); } @@ -654,14 +654,14 @@ public static ReturnStruct AuthenticationCapabilities(ProcedureContext ctx) { // Test 1: Verify authentication context is accessible from procedure context var procAuth = ctx.SenderAuth; - var procSender = ctx.Sender; + var procSender = ctx.Sender(); var procConnectionId = ctx.ConnectionId; var result = ctx.WithTx(tx => { // Test 2: Verify authentication context is accessible from transaction context var txAuth = tx.SenderAuth; - var txSender = tx.Sender; + var txSender = tx.Sender(); var txConnectionId = tx.ConnectionId; // Test 3: Authentication contexts should be consistent diff --git a/templates/chat-console-cs/spacetimedb/Lib.cs b/templates/chat-console-cs/spacetimedb/Lib.cs index 3d82175859a..020bf6c072b 100644 --- a/templates/chat-console-cs/spacetimedb/Lib.cs +++ b/templates/chat-console-cs/spacetimedb/Lib.cs @@ -24,7 +24,7 @@ public static void SetName(ReducerContext ctx, string name) { name = ValidateName(name); - if (ctx.Db.User.Identity.Find(ctx.Sender) is User user) + if (ctx.Db.User.Identity.Find(ctx.Sender()) is User user) { user.Name = name; ctx.Db.User.Identity.Update(user); @@ -49,7 +49,7 @@ public static void SendMessage(ReducerContext ctx, string text) ctx.Db.Message.Insert( new Message { - Sender = ctx.Sender, + Sender = ctx.Sender(), Text = text, Sent = ctx.Timestamp, } @@ -69,9 +69,9 @@ private static string ValidateMessage(string text) [Reducer(ReducerKind.ClientConnected)] public static void ClientConnected(ReducerContext ctx) { - Log.Info($"Connect {ctx.Sender}"); + Log.Info($"Connect {ctx.Sender()}"); - if (ctx.Db.User.Identity.Find(ctx.Sender) is User user) + if (ctx.Db.User.Identity.Find(ctx.Sender()) is User user) { // If this is a returning user, i.e., we already have a `User` with this `Identity`, // set `Online: true`, but leave `Name` and `Identity` unchanged. @@ -86,7 +86,7 @@ public static void ClientConnected(ReducerContext ctx) new User { Name = null, - Identity = ctx.Sender, + Identity = ctx.Sender(), Online = true, } ); @@ -96,7 +96,7 @@ public static void ClientConnected(ReducerContext ctx) [Reducer(ReducerKind.ClientDisconnected)] public static void ClientDisconnected(ReducerContext ctx) { - if (ctx.Db.User.Identity.Find(ctx.Sender) is User user) + if (ctx.Db.User.Identity.Find(ctx.Sender()) is User user) { // This user should exist, so set `Online: false`. user.Online = false;