From e6f0d2089d232d5a5f0fc1f17a380cdac153d930 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 27 Jan 2026 16:49:12 -0500 Subject: [PATCH 1/2] Update 00250-zen-of-spacetimedb.md Signed-off-by: Tyler Cloutier --- .../00100-getting-started/00250-zen-of-spacetimedb.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/docs/00100-intro/00100-getting-started/00250-zen-of-spacetimedb.md b/docs/docs/00100-intro/00100-getting-started/00250-zen-of-spacetimedb.md index bcf7da63793..eb62cbebc58 100644 --- a/docs/docs/00100-intro/00100-getting-started/00250-zen-of-spacetimedb.md +++ b/docs/docs/00100-intro/00100-getting-started/00250-zen-of-spacetimedb.md @@ -26,14 +26,18 @@ Traditional stack: SpacetimeDB: ## Everything is Persistent -You will ask, does everything need to be persistent? Won't that be a lot of data? Won't that be slow. The answers are yes, no, and no. SpacetimeDB persists everything by default, even the full history of everything that has every changed. It should be **your choice** to delete data, not the database's. SpacetimeDB gives you that choice. +SpacetimeDB persists everything by default, including the full history of any rows that have ever changed. + +You will ask, does everything need to be persistent? Won't that be a lot of data? Well, you would be surprised! For example, updating 1 million player transforms 10 times per second for a year uses roughly 10 petabytes of data, uncompressed. SpacetimeDB can compress that sort of data by about 5-10x, meaning that keeping every position for every player for a game with a million concurrent players uses only about 1-2 petabytes per year. Storing that much data in Amazon S3 would only cost you between $2,300 and $5,600 per month. A fraction of the cost of a single engineer or data scientist! + +You can of course choose to delete the historical data, but it should be **your choice** to delete data, not the database's. SpacetimeDB gives you that choice. + +Won't it be slow to persist everything? No. SpacetimeDB is designed so that persistence guarantees only ever increase latency and never decrease throughput! Modern SSDs can write upwards of 15 GB/s of data to disk. DRAM can only do about 4x more. Let's actually use that Samsung-given bandwidth. SpacetimeDB holds all your data in memory for blazing-fast access, but automatically persists everything to disk. You get the speed of in-memory computing with the durability of a traditional database. You will be tempted to ask for "ephemeral state". This is a mistake. Persistent everything allows your app to recover to the *exact* state it was in. In principle, you could even debug your production app in the state it was in in the past with a time-traveling debugger. -SpacetimeDB is designed so that persistence guarantees only ever increase latency and never decrease throughput! Modern SSDs can write upwards of 15 GB/s of data to disk. DRAM can only do about 4x more. Let's actually use that Samsung-given bandwidth. - Write your code as if memory were infinite and permanent. Insert rows freely. Query without fear. SpacetimeDB handles the persistence, you handle the logic. ## Everything is Real-Time From 555dfecfd29744fc873b29d4f16c8e7a582f3300 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Tue, 27 Jan 2026 22:08:01 +0000 Subject: [PATCH 2/2] Update LLM benchmark results --- docs/llms/docs-benchmark-analysis.md | 515 ++++----- docs/llms/docs-benchmark-comment.md | 16 +- docs/llms/docs-benchmark-details.json | 1545 +++++++++++++------------ docs/llms/docs-benchmark-summary.json | 42 +- 4 files changed, 1062 insertions(+), 1056 deletions(-) diff --git a/docs/llms/docs-benchmark-analysis.md b/docs/llms/docs-benchmark-analysis.md index e9fac33c1d4..f3701018925 100644 --- a/docs/llms/docs-benchmark-analysis.md +++ b/docs/llms/docs-benchmark-analysis.md @@ -8,307 +8,246 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do --- -# Analysis of SpacetimeDB Benchmark Failures - -## Rust / rustdoc_json Failures (9 total) - -### Compile/Publish Errors - -#### t_002_scheduled_table & t_017_scheduled_columns - -1. **The generated code**: - ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; - - #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - scheduled_id: u64, - scheduled_at: ScheduleAt, - } - - #[reducer(scheduled)] - pub fn tick(_ctx: &ReducerContext, _row: TickTimer) { - } - - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - if ctx.db.tick_timer().count() == 0 { - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: ScheduleAt::repeat_micros(50_000), - }); - } - } - ``` - -2. **The golden example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; - use std::time::Duration; - - #[table(name = tick_timer, scheduled(tick))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - pub scheduled_id: u64, - pub scheduled_at: ScheduleAt, - } - - #[reducer] - pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) { - } - - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - let every_50ms: ScheduleAt = Duration::from_millis(50).into(); - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: every_50ms, - }); - } - ``` - -3. **The error**: - ```plaintext - publish_error: spacetime publish failed (exit=1) - ``` - -4. **Explain the difference**: - - The LLM used incorrect syntax for reducer attributes and data type for `scheduled_at`. The expected code uses `scheduled` as an attribute and specifies data converts. - -5. **Root cause**: - - The documentation lacks clarity on the correct use of the `scheduled` attribute and how to handle the `ScheduleAt` type properly. - -6. **Recommendation**: - - Update documentation to clearly explain the usage and requirement of reducer attributes and how to define the `ScheduleAt` type. Show examples covering both attribute definitions and data types. +# Analysis of SpacetimeDB Benchmark Test Failures + +## Rust / Rustdoc_JSON Failures + +### Compile/Publish Errors (4 failures) + +#### Failure Group: Scheduled Functions + +**1. Code Examples** + +**LLM Output**: +```rust +#[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] +pub struct TickTimer { + #[primary_key] + #[auto_inc] + scheduled_id: u64, + scheduled_at: ScheduleAt, +} + +#[spacetimedb::reducer(init)] +pub fn init(ctx: &ReducerContext) { + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()), + }); +} + +#[spacetimedb::reducer] +pub fn tick(_ctx: &ReducerContext, _row: TickTimer) { +} +``` + +**Expected (golden)**: +```rust +#[table(name = tick_timer, scheduled(tick))] +pub struct TickTimer { + #[primary_key] + #[auto_inc] + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, +} + +#[reducer(init)] +pub fn init(ctx: &ReducerContext) -> Result<(), String> { + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: ScheduleAt::Interval(Duration::from_millis(50).into()), + }); + Ok(()) +} + +#[reducer] +pub fn tick(_ctx: &ReducerContext, _row: TickTimer) -> Result<(), String> { + Ok(()) +} +``` + +**2. Error**: `publish_error: spacetime publish failed (exit=1)` + +**3. Differences**: +- The expected code uses the correct syntax for initializing the scheduled function and includes return types (`Result<(), String>`). +- The `scheduled_at` value is initialized using `Duration::from_millis(50)` instead of `Duration::from_micros(50_000)`. + +**4. Root Cause**: +The generated functions lacked return types, which are critical for reducers in Rust, and the duration initialization was incorrect. + +**5. Recommendation**: +- Update the documentation to specify that all reducer functions should have return types. +- Clarify the correct usage of time duration initialization for `ScheduleAt`. + +#### Failure Group: Structs in Tables + +**1. Code Examples** + +**LLM Output**: +```rust +#[derive(SpacetimeType)] +pub struct Position { + x: i32, + y: i32, +} + +#[table(name = entity)] +pub struct Entity { + #[primary_key] + id: i32, + pos: Position, +} +``` + +**Expected (golden)**: +```rust +#[derive(SpacetimeType, Clone, Debug)] +pub struct Position { + pub x: i32, + pub y: i32, +} + +#[table(name = entity)] +pub struct Entity { + #[primary_key] + pub id: i32, + pub pos: Position, +} +``` + +**2. Error**: `schema_parity: reducers differ - expected [], got ["add_entity()"]` + +**3. Differences**: +- The original code lacked public access modifiers for struct fields, which is required for the SpacetimeDB framework. + +**4. Root Cause**: +LLM output does not follow the requirement for public fields in data structures intended for use with SpacetimeDB. + +**5. Recommendation**: +- Update the docstring examples to include access modifiers for struct fields when using `SpacetimeType`. --- -#### t_019_many_to_many - -1. **The generated code**: - ```rust - use spacetimedb::{table, reducer, ReducerContext, Table}; - - #[table(name = user)] - pub struct User { - #[primary_key] - user_id: i32, - name: String, - } - - #[table(name = group)] - pub struct Group { - #[primary_key] - group_id: i32, - title: String, - } - - #[table( - name = membership, - index(name = by_user, btree(columns = [user_id])), - index(name = by_group, btree(columns = [group_id])) - )] - pub struct Membership { - #[primary_key] - id: i32, - user_id: i32, - group_id: i32, - } - ``` - -2. **The golden example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; - - #[table(name = user)] - pub struct User { - #[primary_key] - pub user_id: i32, - pub name: String, - } - - #[table(name = group)] - pub struct Group { - #[primary_key] - pub group_id: i32, - pub title: String, - } - - #[table( - name = membership, - index(name = by_user, btree(columns = [user_id])), - index(name = by_group, btree(columns = [group_id])) - )] - pub struct Membership { - #[primary_key] - pub id: i32, - pub user_id: i32, - pub group_id: i32, - } - ``` - -3. **The error**: - ```plaintext - publish_error: spacetime publish failed (exit=1) - ``` - -4. **Explain the difference**: - - The generated code does not mark primary key fields as public, whereas the expected code does, which can cause access issues. - -5. **Root cause**: - - Lack of clarity in the documentation regarding the access level of struct fields in SpacetimeDB. - -6. **Recommendation**: - - Add notes in the documentation stating that public access is required for fields in database schemas. +### Other Failures (4 failures) ---- - -### Other Failures - -#### t_003_struct_in_table - -1. **The generated code**: - ```rust - use spacetimedb::ReducerContext; - use spacetimedb::SpacetimeType; - - #[derive(SpacetimeType)] - pub struct Position { - pub x: i32, - pub y: i32, - } +#### Failure Group: Incomplete Functions and Missing Return Types - #[spacetimedb::table(name = entity)] - pub struct Entity { - #[primary_key] - pub id: i32, - pub pos: Position, - } +**1. Code Examples** - #[spacetimedb::reducer(init)] - pub fn init(_ctx: &ReducerContext) {} - ``` +**LLM Output**: +```rust +#[spacetimedb::reducer] +pub fn empty_reducer_no_args(_ctx: &ReducerContext) { } -2. **The golden example**: - ```rust - use spacetimedb::{table, SpacetimeType}; +#[spacetimedb::reducer] +pub fn empty_reducer_with_int(_ctx: &ReducerContext, count: i32) { } +``` - #[derive(SpacetimeType, Clone, Debug)] - pub struct Position { - pub x: i32, - pub y: i32, - } +**Expected (golden)**: +```rust +#[reducer] +pub fn empty_reducer_no_args(ctx: &ReducerContext) -> Result<(), String> { + Ok(()) +} - #[table(name = entity)] - pub struct Entity { - #[primary_key] - pub id: i32, - pub pos: Position, - } - ``` +#[reducer] +pub fn empty_reducer_with_int(ctx: &ReducerContext, count: i32) -> Result<(), String> { + Ok(()) +} +``` -3. **The error**: - ```plaintext - schema_parity: reducers differ - expected [], got ["init()"] - ``` +**2. Error**: `schema_parity: describe failed` -4. **Explain the difference**: - - The generated function `init` is not necessary, and it's leading to additional unwanted complexity that results in a failure. +**3. Differences**: +- The reducers in the generated code did not have the required return types (`Result<(), String>`) and failed to match the expected function signatures. -5. **Root cause**: - - The documentation does not specify when reducers are required, which should be clarified. +**4. Root Cause**: +Inconsistent documentation specifying both function signatures and expected return types for reducers. -6. **Recommendation**: - - Clarify in the docs that reducers are only needed when initializing data or other specific purposes. +**5. Recommendation**: +- Ensure that all documentation explicitly mentions that all reducer functions need to return `Result<(), String>`. --- -#### t_013_spacetime_sum_type - -1. **The generated code**: - ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType}; - - #[derive(SpacetimeType)] - pub struct Rect { - width: i32, - height: i32, - } - - #[derive(SpacetimeType)] - pub enum Shape { - Circle(i32), - Rectangle(Rect), - } - - #[table(name = result)] - pub struct ResultRow { - #[primary_key] - id: i32, - value: Shape, - } - - #[reducer] - fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) { - ctx.db.result().insert(ResultRow { - id, - value: Shape::Circle(radius), - }); - } - ``` - -2. **The golden example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table}; - - #[derive(SpacetimeType, Clone, Debug)] - pub struct Rect { - pub width: i32, - pub height: i32, - } - - #[derive(SpacetimeType, Clone, Debug)] - pub enum Shape { - Circle(i32), - Rectangle(Rect), - } - - #[table(name = result)] - pub struct ResultRow { - #[primary_key] - pub id: i32, - pub value: Shape, - } - - #[reducer] - pub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) { - ctx.db.result().insert(ResultRow { id, value: Shape::Circle(radius) }); - } - ``` - -3. **The error**: - ```plaintext - sum_type_row_parity: spacetime sql failed: no such table: `result`. - ``` - -4. **Explain the difference**: - - The LLM did not make the necessary fields public, which can lead to visibility issues with database inserts. - -5. **Root cause**: - - The documentation may not be clear about the visibility of struct fields. - -6. **Recommendation**: - - Update the documentation to stress the importance of public access for all fields that interact with the database. - ---- - -### Summary Recommendations - -- **Visibility**: The documentation should consistently emphasize that public access is necessary for struct fields used in SpacetimeDB. -- **Reducer Necessity**: Clarify when reducers should be defined in accordance with specific use cases (e.g., initial data setup). -- **Attribute Usage**: Provide specific examples of attribute usage, including trait bounds for types like `ScheduleAt` and when/how to annotate functions within data models. - -By addressing the above gaps in the documentation, future errors can be mitigated, making it easier for developers to utilize SpacetimeDB effectively. +## C# / Docs Failures + +### Other Failures (4 failures) + +#### Failure Group: Table Not Found Errors + +**1. Code Examples** + +**LLM Output**: +```csharp +[SpacetimeDB.Table(Name = "Primitive", Public = true)] +public partial struct Primitive +{ + [SpacetimeDB.PrimaryKey] + public int Id; + public int Count; + public long Total; + public float Price; + public double Ratio; + public bool Active; + public string Name; +} + +[SpacetimeDB.Reducer] +public static void Seed(ReducerContext ctx) +{ + ctx.Db.Primitive.Insert(new Primitive + { + Id = 1, + Count = 2, + Total = 3000000000L, + Price = 1.5f, + Ratio = 2.25, + Active = true, + Name = "Alice" + }); +} +``` + +**Expected (golden)**: +```csharp +[Table(Name = "Primitive")] +public partial struct Primitive +{ + [PrimaryKey] public int Id; + public int Count; + public long Total; + public float Price; + public double Ratio; + public bool Active; + public string Name; +} + +[Reducer] +public static void Seed(ReducerContext ctx) +{ + ctx.Db.Primitive.Insert(new Primitive { + Id = 1, + Count = 2, + Total = 3000000000, + Price = 1.5f, + Ratio = 2.25, + Active = true, + Name = "Alice" + }); +} +``` + +**2. Error**: `no such table: 'primitive'` + +**3. Differences**: +- The generated code defined the table with the `Public` modifier unnecessarily and lacked proper annotations in the expected format. + +**4. Root Cause**: +Inadequate understanding of how to define database tables properly in the SpacetimeDB context from the documentation. + +**5. Recommendation**: +- Revise the documentation to emphasize how tables should be correctly defined with respect to visibility modifiers. + +--- + +This analysis highlights common patterns in the failures, including issues with function return types, public access modifiers, and correct syntax for API functions in both Rust and C#. The recommendations aim to enhance clarity and ensure adherence to SpacetimeDB’s requirements. diff --git a/docs/llms/docs-benchmark-comment.md b/docs/llms/docs-benchmark-comment.md index 02fff94c87c..4e0c404474c 100644 --- a/docs/llms/docs-benchmark-comment.md +++ b/docs/llms/docs-benchmark-comment.md @@ -2,16 +2,16 @@ | Language | Mode | Category | Tests Passed | Task Pass % | |----------|------|----------|--------------|-------------| -| Rust | rustdoc_json | basics | 25/27 | 83.3% ⬆️ +6.9% | -| Rust | rustdoc_json | schema | 18/34 | 55.3% ⬇️ -20.0% | -| Rust | rustdoc_json | **total** | 43/61 | **70.6%** ⬇️ -5.3% | +| Rust | rustdoc_json | basics | 25/27 | 83.3% | +| Rust | rustdoc_json | schema | 22/34 | 62.0% ⬆️ +6.7% | +| Rust | rustdoc_json | **total** | 47/61 | **73.6%** ⬆️ +3.0% | | Rust | docs | basics | 5/27 | 11.1% | -| Rust | docs | schema | 4/30 | 12.5% ⬇️ -8.0% | -| Rust | docs | **total** | 9/57 | **11.7%** ⬇️ -3.6% | +| Rust | docs | schema | 8/34 | 20.5% ⬆️ +8.0% | +| Rust | docs | **total** | 13/61 | **15.4%** ⬆️ +3.6% | | C# | docs | basics | 27/27 | 100.0% | -| C# | docs | schema | 25/34 | 73.7% ⬆️ +10.0% | -| C# | docs | **total** | 52/61 | **88.0%** ⬆️ +4.5% | +| C# | docs | schema | 25/34 | 73.7% | +| C# | docs | **total** | 52/61 | **88.0%** | _Compared against master branch baseline_ -Generated at: 2026-01-27T16:47:52.904Z +Generated at: 2026-01-27T22:07:18.787Z diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json index 1f525cb5996..ab048c26b9f 100644 --- a/docs/llms/docs-benchmark-details.json +++ b/docs/llms/docs-benchmark-details.json @@ -19,7 +19,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, count: i32, name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, active: bool, ratio: f32, label: String) {\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -35,15 +35,15 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35791", + "server": "http://127.0.0.1:45511", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:32:33.607423222Z", - "finished_at": "2026-01-27T16:34:08.828085880Z" + "started_at": "2026-01-27T21:56:30.430313727Z", + "finished_at": "2026-01-27T21:57:10.467939149Z" }, "t_001_basic_tables": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -69,15 +69,15 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35791", + "server": "http://127.0.0.1:45511", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:34:07.172274692Z", - "finished_at": "2026-01-27T16:35:04.564067732Z" + "started_at": "2026-01-27T21:56:53.005501208Z", + "finished_at": "2026-01-27T21:57:21.645370449Z" }, "t_002_scheduled_table": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -87,7 +87,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table, ScheduleAt};\n\n#[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db\n .tick_timer()\n .insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext) {\n // Implement your periodic logic here\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -99,14 +99,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling zerocopy v0.8.34\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling find-msvc-tools v0.1.8\n Compiling shlex v1.3.0\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling humantime v2.3.0\n Compiling bytes v1.11.0\n Compiling zmij v1.0.17\n Compiling second-stack v0.3.5\n Compiling spacetimedb-lib v1.11.1\n Compiling itoa v1.0.17\n Compiling serde_json v1.0.149\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling arrayref v0.3.9\n Compiling rand_core v0.6.4\n Compiling constant_time_eq v0.4.2\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling itertools v0.12.1\n Compiling generic-array v0.14.7\n Compiling cc v1.2.54\n Compiling syn v2.0.114\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling blake3 v1.8.3\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:12:11\n |\n12 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:13:42\n |\n13 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:19:36\n |\n19 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:18:15\n |\n18 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:19:16\n |\n19 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:21:39\n |\n21 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling zerocopy v0.8.34\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling zmij v1.0.17\n Compiling bytes v1.11.0\n Compiling humantime v2.3.0\n Compiling serde_json v1.0.149\n Compiling bytemuck v1.24.0\n Compiling second-stack v0.3.5\n Compiling itoa v1.0.17\n Compiling getrandom v0.2.17\n Compiling arrayref v0.3.9\n Compiling smallvec v1.15.1\n Compiling hex v0.4.3\n Compiling constant_time_eq v0.4.2\n Compiling spacetimedb-lib v1.11.1\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling generic-array v0.14.7\n Compiling cc v1.2.54\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling blake3 v1.8.3\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:41\n |\n4 | #[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:17:21\n |\n17 | .insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:16:14\n |\n15 | / ctx.db\n16 | | .tick_timer()\n | | -^^^^^^^^^^ method not found in `Local`\n | |_____________|\n |\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:19:43\n |\n19 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:34:08.828209290Z", - "finished_at": "2026-01-27T16:35:26.918284244Z" + "started_at": "2026-01-27T21:57:21.645397645Z", + "finished_at": "2026-01-27T21:58:33.883597995Z" }, "t_003_struct_in_table": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -116,7 +116,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::ReducerContext;\nuse spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(_ctx: &ReducerContext) {}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}\n\n#[reducer]\npub fn add_entity(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n ctx.db.entity().insert(Entity { id, pos: Position { x, y } });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -133,19 +133,19 @@ "reducers_diff": { "only_golden": [], "only_llm": [ - "init()" + "add_entity()" ] }, "reducers_equal": false, - "server": "http://127.0.0.1:35791", + "server": "http://127.0.0.1:45511", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:34:07.169845549Z", - "finished_at": "2026-01-27T16:35:07.116018289Z" + "started_at": "2026-01-27T21:57:21.911599247Z", + "finished_at": "2026-01-27T21:57:57.958186477Z" }, "t_004_insert": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -155,7 +155,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 2, - "llm_output": "use spacetimedb::Table;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn insert_user(ctx: &spacetimedb::ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-004-insert-golden", @@ -179,7 +179,7 @@ "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", "query": "SELECT id, name, age, active FROM user WHERE id=1", "reducer": "insert_user", - "server": "http://127.0.0.1:35791" + "server": "http://127.0.0.1:45511" } }, "schema_parity": { @@ -190,15 +190,15 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35791", + "server": "http://127.0.0.1:45511", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:34:08.208097234Z", - "finished_at": "2026-01-27T16:35:13.309948498Z" + "started_at": "2026-01-27T21:56:44.401349330Z", + "finished_at": "2026-01-27T21:57:14.670545355Z" }, "t_005_update": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -208,7 +208,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table, UniqueColumn};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let user = User { id, name, age, active };\n ctx.db.user().id().update(user);\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let table = ctx.db.user();\n if table.id().find(id).is_some() {\n let _ = table.id().update(User { id, name, age, active });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -216,23 +216,17 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { - "data_parity_update_user": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], "golden_db": "basics-t-005-update-golden", - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "llm_db": "basics-t-005-update-gpt-5-llm", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user", - "server": "http://127.0.0.1:35791" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:45511", + "tables_diff": null, + "tables_equal": true } }, "seed_users_row": { @@ -242,23 +236,29 @@ "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, - "schema_parity": { + "data_parity_update_user": { "pass": true, "partial": 1.0, "notes": { + "args": [ + 1, + "Alice2", + 31, + false + ], "golden_db": "basics-t-005-update-golden", + "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "llm_db": "basics-t-005-update-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35791", - "tables_diff": null, - "tables_equal": true + "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "update_user", + "server": "http://127.0.0.1:45511" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:32:30.244848815Z", - "finished_at": "2026-01-27T16:34:12.499371947Z" + "started_at": "2026-01-27T21:56:26.482433771Z", + "finished_at": "2026-01-27T21:57:16.618669083Z" }, "t_006_delete": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -268,7 +268,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -293,7 +293,7 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35791", + "server": "http://127.0.0.1:45511", "tables_diff": null, "tables_equal": true } @@ -307,8 +307,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T16:32:20.195987848Z", - "finished_at": "2026-01-27T16:34:08.193093663Z" + "started_at": "2026-01-27T21:56:52.270086833Z", + "finished_at": "2026-01-27T21:58:00.013810045Z" }, "t_007_crud": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -318,7 +318,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n let _ = users.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n let _ = users.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n\n let updated = User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false };\n let _ = users.id().update(updated);\n\n users.id().delete(&2);\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n let _ = ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n let _ = ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n let updated = User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false };\n let _ = ctx.db.user().id().update(updated);\n ctx.db.user().id().delete(&2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -326,13 +326,18 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "crud_row_id2_deleted": { + "crud_row_id1_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" + "args": [], + "golden_db": "basics-t-007-crud-golden", + "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "crud", + "server": "http://127.0.0.1:45511" } }, "crud_total_count_one": { @@ -344,37 +349,32 @@ "sql": "SELECT COUNT(*) AS n FROM user" } }, - "schema_parity": { + "crud_row_id2_deleted": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-007-crud-golden", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35791", - "tables_diff": null, - "tables_equal": true + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" } }, - "crud_row_id1_parity": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [], "golden_db": "basics-t-007-crud-golden", - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "llm_db": "basics-t-007-crud-gpt-5-llm", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud", - "server": "http://127.0.0.1:35791" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:45511", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:32:23.303951905Z", - "finished_at": "2026-01-27T16:34:10.621743269Z" + "started_at": "2026-01-27T21:57:16.618713310Z", + "finished_at": "2026-01-27T21:57:59.328188811Z" }, "t_008_index_lookup": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -384,7 +384,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n let _ = ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -399,6 +399,19 @@ "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-008-index-lookup-golden", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:45511", + "tables_diff": null, + "tables_equal": true + } + }, "index_lookup_projection_parity": { "pass": true, "partial": 1.0, @@ -412,26 +425,13 @@ "llm_out": "id | name ----+--------- 1 | \"Alice\"", "query": "SELECT id, name FROM result WHERE id=1", "reducer": "lookup_user_name", - "server": "http://127.0.0.1:35791" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35791", - "tables_diff": null, - "tables_equal": true + "server": "http://127.0.0.1:45511" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:34:08.199928363Z", - "finished_at": "2026-01-27T16:35:12.331943993Z" + "started_at": "2026-01-27T21:57:14.670616741Z", + "finished_at": "2026-01-27T21:58:00.233395415Z" }, "t_009_init": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -441,7 +441,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let _ = ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n let _ = ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let user = ctx.db.user();\n user.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n user.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -449,22 +449,22 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { - "init_seed_alice": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM user" } }, - "init_seed_bob": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" } }, "schema_parity": { @@ -475,24 +475,24 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35791", + "server": "http://127.0.0.1:45511", "tables_diff": null, "tables_equal": true } }, - "init_total_two": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:32:30.244175665Z", - "finished_at": "2026-01-27T16:34:05.774125311Z" + "started_at": "2026-01-27T21:57:10.467974395Z", + "finished_at": "2026-01-27T21:57:46.281944467Z" }, "t_010_connect": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -518,15 +518,15 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35791", + "server": "http://127.0.0.1:45511", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:32:22.434743282Z", - "finished_at": "2026-01-27T16:34:04.716053382Z" + "started_at": "2026-01-27T21:57:12.175282252Z", + "finished_at": "2026-01-27T21:57:47.463202429Z" }, "t_011_helper_function": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -559,16 +559,7 @@ "llm_out": "id | sum ----+----- 1 | 5", "query": "SELECT id, sum FROM result WHERE id=1", "reducer": "compute_sum", - "server": "http://127.0.0.1:35791" - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" + "server": "http://127.0.0.1:45511" } }, "schema_parity": { @@ -579,15 +570,24 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35791", + "server": "http://127.0.0.1:45511", "tables_diff": null, "tables_equal": true } + }, + "helper_func_sum_abs": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" + } } }, "vendor": "openai", - "started_at": "2026-01-27T16:34:08.208709446Z", - "finished_at": "2026-01-27T16:35:04.173989578Z" + "started_at": "2026-01-27T21:56:53.341032329Z", + "finished_at": "2026-01-27T21:57:21.911571496Z" }, "t_012_spacetime_product_type": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -596,8 +596,8 @@ "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow { id, value: Score { left, right } });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -605,24 +605,6 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_db": "schema-t-012-spacetime-product-type-golden", - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score", - "server": "http://127.0.0.1:35791" - } - }, "product_type_row_count": { "pass": true, "partial": 1.0, @@ -632,6 +614,14 @@ "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } }, + "product_type_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45511/v1/database/c20066de900672af903bd323213825ab123d970ed990ba3708988e33f63407f2/sql)\n", + "phase": "sql_golden" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -640,15 +630,15 @@ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35791", + "server": "http://127.0.0.1:45511", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:32:19.074381549Z", - "finished_at": "2026-01-27T16:34:04.285926069Z" + "started_at": "2026-01-27T21:55:50.881200151Z", + "finished_at": "2026-01-27T21:56:26.171932993Z" }, "t_013_spacetime_sum_type": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -658,7 +648,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 2, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\nfn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow { id, value: Shape::Circle(radius) });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -674,19 +664,11 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35791", + "server": "http://127.0.0.1:45511", "tables_diff": null, "tables_equal": true } }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20002a07d9e6caa3aff0cce8d70357fc0dab19fc7bec72594d9469cfd9e0d34/sql)\n", - "phase": "sql_golden" - } - }, "sum_type_row_count": { "pass": true, "partial": 1.0, @@ -695,11 +677,19 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } + }, + "sum_type_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45511/v1/database/c200e5fbecfa12b09aedee1725f81fee4cc7816888145963c4128b275957d1f9/sql)\n", + "phase": "sql_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-27T16:30:59.526974942Z", - "finished_at": "2026-01-27T16:32:33.607356316Z" + "started_at": "2026-01-27T21:55:50.881434595Z", + "finished_at": "2026-01-27T21:56:44.401279988Z" }, "t_014_elementary_columns": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -709,7 +699,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -717,6 +707,15 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { + "elementary_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -725,7 +724,7 @@ "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35791", + "server": "http://127.0.0.1:45511", "tables_diff": null, "tables_equal": true } @@ -741,22 +740,13 @@ "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", "reducer": "seed", - "server": "http://127.0.0.1:35791" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + "server": "http://127.0.0.1:45511" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:32:12.570231067Z", - "finished_at": "2026-01-27T16:34:07.169774438Z" + "started_at": "2026-01-27T21:56:16.721976906Z", + "finished_at": "2026-01-27T21:56:53.005468623Z" }, "t_015_product_type_columns": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -766,7 +756,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 2, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n if ctx.db.profile().id().find(1i32).is_none() {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let home = Address { street: \"1 Main\".to_string(), zip: 11111 };\n let work = Address { street: \"2 Broad\".to_string(), zip: 22222 };\n let pos = Position { x: 7, y: 9 };\n ctx.db.profile().insert(Profile { id: 1, home, work, pos });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -787,7 +777,7 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profile`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20022cf10da13cb5686a3d982e8ce97257e0a609cd8cdbf4b75448c0605a05c/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profile`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45511/v1/database/c200ccf4f06f41761313705e411551f93d64b4aff2922f89a4c80c47c5e7316c/sql)\n", "phase": "sql_golden" } }, @@ -799,15 +789,15 @@ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35791", + "server": "http://127.0.0.1:45511", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:30:59.529279381Z", - "finished_at": "2026-01-27T16:32:19.074245877Z" + "started_at": "2026-01-27T21:55:50.882135186Z", + "finished_at": "2026-01-27T21:56:26.482395588Z" }, "t_016_sum_type_columns": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -817,7 +807,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let drawings = ctx.db.drawing();\n if drawings.count() == 0 {\n drawings.insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -829,7 +819,7 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c200d48d9295216edd01ffee50fc27f7162d3c6ce2ff1442fb683d8707911992/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45511/v1/database/c2005cb32f9e8c92d208ab76edf6fb54138728597e085ad576ab57cdd18ad47e/sql)\n", "phase": "sql_golden" } }, @@ -841,7 +831,7 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35791", + "server": "http://127.0.0.1:45511", "tables_diff": null, "tables_equal": true } @@ -850,14 +840,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c200ed636d1809b5fcb164f6ee86a539d4bdcbe0e1bcc7e6343341fa806bb61a/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45511/v1/database/c20037932e455399a735ccb0f4441a8f89c3046647a5b00c855c2802bbee0b65/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:30:59.526533974Z", - "finished_at": "2026-01-27T16:32:27.393830604Z" + "started_at": "2026-01-27T21:55:50.881577887Z", + "finished_at": "2026-01-27T21:56:53.340999329Z" }, "t_017_scheduled_columns": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -867,7 +857,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::RepeatEveryMicros(50_000),\n });\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -879,14 +869,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.34\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling zmij v1.0.17\n Compiling bytes v1.11.0\n Compiling keccak v0.1.5\n Compiling humantime v2.3.0\n Compiling constant_time_eq v0.4.2\n Compiling second-stack v0.3.5\n Compiling serde_json v1.0.149\n Compiling itoa v1.0.17\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling arrayref v0.3.9\n Compiling spacetimedb-lib v1.11.1\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling rand_core v0.6.4\n Compiling memchr v2.7.6\n Compiling cc v1.2.54\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling itertools v0.12.1\n Compiling num-traits v0.2.19\n Compiling generic-array v0.14.7\n Compiling syn v2.0.114\n Compiling http v1.4.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling blake3 v1.8.3\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling ethnum v1.5.2\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:12:11\n |\n12 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:13:42\n |\n13 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:19:36\n |\n19 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:18:15\n |\n18 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:19:16\n |\n19 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:21:39\n |\n21 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.34\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling bytes v1.11.0\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling convert_case v0.4.0\n Compiling arrayvec v0.7.6\n Compiling zmij v1.0.17\n Compiling humantime v2.3.0\n Compiling smallvec v1.15.1\n Compiling itoa v1.0.17\n Compiling spacetimedb-lib v1.11.1\n Compiling second-stack v0.3.5\n Compiling getrandom v0.2.17\n Compiling arrayref v0.3.9\n Compiling bytemuck v1.24.0\n Compiling cc v1.2.54\n Compiling itertools v0.12.1\n Compiling constant_time_eq v0.4.2\n Compiling hex v0.4.3\n Compiling serde_json v1.0.149\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:14:32\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:21:42\n |\n21 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:12\n |\n14 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `RepeatEveryMicros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:16:35\n |\n16 | scheduled_at: ScheduleAt::RepeatEveryMicros(50_000),\n | ^^^^^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:21:8\n |\n 20 | #[reducer]\n | ---------- required by a bound introduced by this call\n 21 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:21:8\n |\n20 | #[reducer]\n | ---------- required by a bound introduced by this call\n21 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:30:59.525424836Z", - "finished_at": "2026-01-27T16:32:23.303916011Z" + "started_at": "2026-01-27T21:55:50.882262207Z", + "finished_at": "2026-01-27T21:57:12.175265404Z" }, "t_018_constraints": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -896,7 +886,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 2, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let account = ctx.db.account();\n account.insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n account.insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db\n .account()\n .insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db\n .account()\n .insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -908,10 +898,19 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `account`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c2002a3ca814e8fc4e610af01b0cba5740eac41619685f4959b780669796967a/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `account`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45511/v1/database/c200ff6e2bd0570e541deaad776ea03f5b0bf8dc103f206aa00417919d001d2b/sql)\n", "phase": "sql_golden" } }, + "constraints_seed_two_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -920,34 +919,25 @@ "llm_db": "schema-t-018-constraints-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35791", + "server": "http://127.0.0.1:45511", "tables_diff": null, "tables_equal": true } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } } }, "vendor": "openai", - "started_at": "2026-01-27T16:30:59.520529105Z", - "finished_at": "2026-01-27T16:32:22.433999235Z" + "started_at": "2026-01-27T21:55:50.881725580Z", + "finished_at": "2026-01-27T21:56:16.721950170Z" }, "t_019_many_to_many": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", "task": "t_019_many_to_many", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 5, - "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n for m in ctx.db.membership().iter() {\n ctx.db.membership().delete(&m);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().delete(&u);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().delete(&g);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "passed_tests": 5, + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let membership = ctx.db.membership();\n for row in membership.iter() {\n membership.id().delete(&row.id);\n }\n\n let group = ctx.db.group();\n for row in group.iter() {\n group.group_id().delete(&row.group_id);\n }\n\n let user = ctx.db.user();\n for row in user.iter() {\n user.user_id().delete(&row.user_id);\n }\n\n user.insert(User { user_id: 1, name: \"Alice\".to_string() });\n user.insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n group.insert(Group { group_id: 10, title: \"Admin\".to_string() });\n group.insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n membership.insert(Membership { id: 1, user_id: 1, group_id: 10 });\n membership.insert(Membership { id: 2, user_id: 1, group_id: 20 });\n membership.insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -955,18 +945,59 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "m2m_has_1_10": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.34\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling zmij v1.0.17\n Compiling bytes v1.11.0\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling smallvec v1.15.1\n Compiling constant_time_eq v0.4.2\n Compiling hex v0.4.3\n Compiling second-stack v0.3.5\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling arrayref v0.3.9\n Compiling serde_json v1.0.149\n Compiling itoa v1.0.17\n Compiling spacetimedb-lib v1.11.1\n Compiling memchr v2.7.6\n Compiling cc v1.2.54\n Compiling rand_core v0.6.4\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling itertools v0.12.1\n Compiling num-traits v0.2.19\n Compiling blake3 v1.8.3\n Compiling syn v2.0.114\n Compiling http v1.4.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling ppv-lite86 v0.2.21\n Compiling sha3 v0.10.8\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm)\nerror[E0308]: mismatched types\n --> src/lib.rs:33:36\n |\n 33 | ctx.db.membership().delete(&m);\n | ------ ^^ expected `Membership`, found `&Membership`\n | |\n | arguments to this method are incorrect\n |\nnote: method defined here\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/table.rs:104:8\n |\n104 | fn delete(&self, row: Self::Row) -> bool {\n | ^^^^^^\nhelp: consider removing the borrow\n |\n 33 - ctx.db.membership().delete(&m);\n 33 + ctx.db.membership().delete(m);\n |\n\nerror[E0308]: mismatched types\n --> src/lib.rs:36:30\n |\n 36 | ctx.db.user().delete(&u);\n | ------ ^^ expected `User`, found `&User`\n | |\n | arguments to this method are incorrect\n |\nnote: method defined here\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/table.rs:104:8\n |\n104 | fn delete(&self, row: Self::Row) -> bool {\n | ^^^^^^\nhelp: consider removing the borrow\n |\n 36 - ctx.db.user().delete(&u);\n 36 + ctx.db.user().delete(u);\n |\n\nerror[E0308]: mismatched types\n --> src/lib.rs:39:31\n |\n 39 | ctx.db.group().delete(&g);\n | ------ ^^ expected `Group`, found `&Group`\n | |\n | arguments to this method are incorrect\n |\nnote: method defined here\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/table.rs:104:8\n |\n104 | fn delete(&self, row: Self::Row) -> bool {\n | ^^^^^^\nhelp: consider removing the borrow\n |\n 39 - ctx.db.group().delete(&g);\n 39 + ctx.db.group().delete(g);\n |\n\nFor more information about this error, try `rustc --explain E0308`.\nerror: could not compile `spacetime-module` (lib) due to 3 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:45511", + "tables_diff": null, + "tables_equal": true + } + }, + "m2m_has_1_20": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" + } + }, + "memberships_three_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM membership" + } + }, + "m2m_has_2_20": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:30:59.527880905Z", - "finished_at": "2026-01-27T16:32:20.195931017Z" + "started_at": "2026-01-27T21:56:26.171964152Z", + "finished_at": "2026-01-27T21:57:30.262139056Z" }, "t_020_ecs": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -976,7 +1007,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table, UniqueColumn};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entities\n let entity_tbl = ctx.db.entity();\n entity_tbl.id().delete(&1);\n entity_tbl.id().delete(&2);\n entity_tbl.insert(Entity { id: 1 });\n entity_tbl.insert(Entity { id: 2 });\n\n // Positions\n let pos_tbl = ctx.db.position();\n pos_tbl.entity_id().delete(&1);\n pos_tbl.entity_id().delete(&2);\n pos_tbl.insert(Position { entity_id: 1, x: 0, y: 0 });\n pos_tbl.insert(Position { entity_id: 2, x: 10, y: 0 });\n\n // Velocities\n let vel_tbl = ctx.db.velocity();\n vel_tbl.entity_id().delete(&1);\n vel_tbl.entity_id().delete(&2);\n vel_tbl.insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n vel_tbl.insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n let pos_tbl = ctx.db.position();\n let vel_idx = ctx.db.velocity().entity_id();\n let next_tbl = ctx.db.next_position();\n\n for p in pos_tbl.iter() {\n if let Some(v) = vel_idx.find(p.entity_id) {\n let new_x = p.x + v.vx;\n let new_y = p.y + v.vy;\n\n if let Some(mut existing) = next_tbl.entity_id().find(p.entity_id) {\n existing.x = new_x;\n existing.y = new_y;\n let _updated = next_tbl.entity_id().update(existing);\n } else {\n next_tbl.insert(NextPosition {\n entity_id: p.entity_id,\n x: new_x,\n y: new_y,\n });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.entity().try_insert(Entity { id: 1 });\n let _ = ctx.db.entity().try_insert(Entity { id: 2 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 1, x: 0, y: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 2, x: 10, y: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n let positions = ctx.db.position();\n let velocities = ctx.db.velocity();\n let next = ctx.db.next_position();\n\n for p in positions.iter() {\n if let Some(v) = velocities.entity_id().find(p.entity_id) {\n let nx = p.x + v.vx;\n let ny = p.y + v.vy;\n let row = NextPosition { entity_id: p.entity_id, x: nx, y: ny };\n if next.entity_id().find(p.entity_id).is_some() {\n let _ = next.entity_id().update(row);\n } else {\n let _ = next.insert(row);\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -984,11 +1015,19 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { + "ecs_seed_positions_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45511/v1/database/c200f539c39322a26f09c5d34ffb0854ca4bb2f56b340ad9ddabef31e297f8de/sql)\n", + "phase": "sql" + } + }, "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20072c315c8f431236d9e6b480d290295aa297deff4a50cf8d26d63cc0359c8/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45511/v1/database/c200f539c39322a26f09c5d34ffb0854ca4bb2f56b340ad9ddabef31e297f8de/sql)\n", "phase": "sql" } }, @@ -996,7 +1035,15 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20072c315c8f431236d9e6b480d290295aa297deff4a50cf8d26d63cc0359c8/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45511/v1/database/c200f539c39322a26f09c5d34ffb0854ca4bb2f56b340ad9ddabef31e297f8de/sql)\n", + "phase": "sql" + } + }, + "ecs_next_pos_entity1": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45511/v1/database/c200f539c39322a26f09c5d34ffb0854ca4bb2f56b340ad9ddabef31e297f8de/sql)\n", "phase": "sql" } }, @@ -1008,31 +1055,15 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35791", + "server": "http://127.0.0.1:45511", "tables_diff": null, "tables_equal": true } - }, - "ecs_seed_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20072c315c8f431236d9e6b480d290295aa297deff4a50cf8d26d63cc0359c8/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35791/v1/database/c20072c315c8f431236d9e6b480d290295aa297deff4a50cf8d26d63cc0359c8/sql)\n", - "phase": "sql" - } } }, "vendor": "openai", - "started_at": "2026-01-27T16:30:59.528443334Z", - "finished_at": "2026-01-27T16:32:30.244044278Z" + "started_at": "2026-01-27T21:55:50.881999154Z", + "finished_at": "2026-01-27T21:56:52.270043237Z" }, "t_021_multi_column_index": { "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", @@ -1042,7 +1073,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let table = ctx.db.log();\n let ids: Vec = table.iter().map(|row| row.id).collect();\n for id in ids {\n table.id().delete(&id);\n }\n table.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n table.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n table.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -1050,6 +1081,15 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { + "mcindex_seed_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM log" + } + }, "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, @@ -1067,20 +1107,11 @@ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35791", + "server": "http://127.0.0.1:45511", "tables_diff": null, "tables_equal": true } }, - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, "mcindex_lookup_u7_d2": { "pass": true, "partial": 1.0, @@ -1092,8 +1123,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T16:30:59.525967187Z", - "finished_at": "2026-01-27T16:32:12.570150592Z" + "started_at": "2026-01-27T21:55:50.881865222Z", + "finished_at": "2026-01-27T21:56:30.430284110Z" } } } @@ -1101,21 +1132,21 @@ }, { "mode": "docs", - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_000_empty_reducers", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) { }\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, count: i32) { }\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, name: String) { }\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, count: i32, name: String) { }\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, active: bool, ratio: f32, label: String) { }", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -1133,18 +1164,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T16:37:07.698626526Z", - "finished_at": "2026-01-27T16:38:22.803963062Z" + "started_at": "2026-01-27T21:59:28.738404956Z", + "finished_at": "2026-01-27T22:00:05.222156504Z" }, "t_001_basic_tables": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_001_basic_tables", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", + "llm_output": "use spacetimedb::Table;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -1162,18 +1193,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T16:37:11.008540155Z", - "finished_at": "2026-01-27T16:38:33.131965123Z" + "started_at": "2026-01-27T21:59:40.437506321Z", + "finished_at": "2026-01-27T22:00:09.706859982Z" }, "t_002_scheduled_table": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_002_scheduled_table", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) { }\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n Ok(())\n}", + "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: spacetimedb::ScheduleAt,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: spacetimedb::ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n }\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -1191,11 +1222,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T16:37:40.463756751Z", - "finished_at": "2026-01-27T16:38:46.674647986Z" + "started_at": "2026-01-27T22:00:09.776019508Z", + "finished_at": "2026-01-27T22:01:03.100111948Z" }, "t_003_struct_in_table": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_003_struct_in_table", "lang": "rust", "golden_published": true, @@ -1220,11 +1251,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T16:37:10.913641809Z", - "finished_at": "2026-01-27T16:38:28.540002548Z" + "started_at": "2026-01-27T22:00:09.779396910Z", + "finished_at": "2026-01-27T22:00:39.022195202Z" }, "t_004_insert": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_004_insert", "lang": "rust", "golden_published": true, @@ -1239,29 +1270,29 @@ "work_dir_golden": "target/llm-runs/basics/t_004_insert/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_004_insert/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "data_parity_insert_user": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", + "phase": "call_reducer_golden" } }, - "data_parity_insert_user": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:37:12.398406350Z", - "finished_at": "2026-01-27T16:38:29.957852183Z" + "started_at": "2026-01-27T21:59:33.320363016Z", + "finished_at": "2026-01-27T22:00:09.225655385Z" }, "t_005_update": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_005_update", "lang": "rust", "golden_published": true, @@ -1303,11 +1334,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T16:37:05.840143086Z", - "finished_at": "2026-01-27T16:38:27.673339086Z" + "started_at": "2026-01-27T21:59:28.647622669Z", + "finished_at": "2026-01-27T22:00:17.523842310Z" }, "t_006_delete": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_006_delete", "lang": "rust", "golden_published": true, @@ -1350,18 +1381,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T16:36:39.493868942Z", - "finished_at": "2026-01-27T16:37:10.913606401Z" + "started_at": "2026-01-27T21:59:34.492198269Z", + "finished_at": "2026-01-27T22:00:08.795354268Z" }, "t_007_crud": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_007_crud", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n if let Some(mut u) = ctx.db.user().id().find(1) {\n u.name = \"Alice2\".into();\n u.age = 31;\n u.active = false;\n ctx.db.user().id().update(u);\n }\n ctx.db.user().id().delete(2);\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n\n if let Some(mut u) = ctx.db.user().id().find(1) {\n u.name = \"Alice2\".to_string();\n u.age = 31;\n u.active = false;\n ctx.db.user().id().update(u);\n }\n\n ctx.db.user().id().delete(2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -1378,12 +1409,13 @@ "sql": "SELECT COUNT(*) AS n FROM user" } }, - "schema_parity": { - "pass": false, - "partial": 0.0, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "describe_golden" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" } }, "crud_row_id1_parity": { @@ -1394,29 +1426,28 @@ "phase": "call_reducer_golden" } }, - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, + "schema_parity": { + "pass": false, + "partial": 0.0, "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:36:46.682301461Z", - "finished_at": "2026-01-27T16:37:30.046001557Z" + "started_at": "2026-01-27T22:00:09.225690701Z", + "finished_at": "2026-01-27T22:00:53.098859615Z" }, "t_008_index_lookup": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_008_index_lookup", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n let row = ResultRow { id: u.id, name: u.name.clone() };\n if ctx.db.result().id().find(u.id).is_some() {\n ctx.db.result().id().update(row);\n } else {\n ctx.db.result().insert(row);\n }\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n if ctx.db.result().id().find(id).is_some() {\n ctx.db.result().id().update(ResultRow { id, name: user.name });\n } else {\n ctx.db.result().insert(ResultRow { id, name: user.name });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -1451,18 +1482,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T16:37:11.012380843Z", - "finished_at": "2026-01-27T16:38:41.409499176Z" + "started_at": "2026-01-27T22:00:08.795543885Z", + "finished_at": "2026-01-27T22:00:42.342192764Z" }, "t_009_init": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_009_init", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n Ok(())\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -1470,15 +1501,6 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - }, "init_seed_alice": { "pass": true, "partial": 1.0, @@ -1497,6 +1519,15 @@ "sql": "SELECT COUNT(*) AS n FROM user" } }, + "init_seed_bob": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + } + }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1507,18 +1538,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T16:36:57.771441165Z", - "finished_at": "2026-01-27T16:37:40.463630543Z" + "started_at": "2026-01-27T21:59:50.287538471Z", + "finished_at": "2026-01-27T22:00:17.757690411Z" }, "t_010_connect": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_010_connect", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"connected\".to_string() });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".to_string() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.event().insert(Event { id: 0, kind: \"connected\".into() });\n Ok(())\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".into() });\n Ok(())\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -1536,18 +1567,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T16:36:39.717206337Z", - "finished_at": "2026-01-27T16:37:11.008504607Z" + "started_at": "2026-01-27T22:00:05.222185543Z", + "finished_at": "2026-01-27T22:00:38.284150630Z" }, "t_011_helper_function": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_011_helper_function", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.result().insert(ResultRow { id, sum: add(a, b) });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -1582,18 +1613,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T16:37:30.046112807Z", - "finished_at": "2026-01-27T16:38:42.478505901Z" + "started_at": "2026-01-27T21:59:47.134179939Z", + "finished_at": "2026-01-27T22:00:09.775866314Z" }, "t_012_spacetime_product_type": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_012_spacetime_product_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[derive(spacetimedb::SpacetimeType, Clone, Debug)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -1601,13 +1632,12 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { - "product_type_row_count": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", + "phase": "describe_golden" } }, "product_type_row_parity": { @@ -1618,28 +1648,29 @@ "phase": "call_reducer_golden" } }, - "schema_parity": { + "product_type_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", - "phase": "describe_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:36:36.282196616Z", - "finished_at": "2026-01-27T16:37:12.398375835Z" + "started_at": "2026-01-27T21:58:34.113911342Z", + "finished_at": "2026-01-27T21:59:34.492169941Z" }, "t_013_spacetime_sum_type": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_013_spacetime_sum_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db\n .result()\n .insert(ResultRow { id, value: Shape::Circle(radius) });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().id().delete(id);\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -1647,6 +1678,14 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", + "phase": "describe_golden" + } + }, "sum_type_row_parity": { "pass": false, "partial": 0.0, @@ -1663,29 +1702,21 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", - "phase": "describe_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-27T16:35:27.761679764Z", - "finished_at": "2026-01-27T16:36:39.717100673Z" + "started_at": "2026-01-27T21:58:34.116466048Z", + "finished_at": "2026-01-27T21:59:07.957810697Z" }, "t_014_elementary_columns": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_014_elementary_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000_i64,\n price: 1.5_f32,\n ratio: 2.25_f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3000000000i64,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -1693,21 +1724,21 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_count": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", + "phase": "describe_golden" } }, - "schema_parity": { + "elementary_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", - "phase": "describe_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" } }, "elementary_columns_row_parity": { @@ -1720,18 +1751,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T16:36:31.329332201Z", - "finished_at": "2026-01-27T16:37:07.698588033Z" + "started_at": "2026-01-27T21:59:07.957842631Z", + "finished_at": "2026-01-27T21:59:40.437477451Z" }, "t_015_product_type_columns": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_015_product_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(row.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".into(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".into(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n if ctx.db.profile().id().find(1).is_none() {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -1739,15 +1770,6 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "product_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1763,21 +1785,30 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", "phase": "call_reducer_golden" } + }, + "product_type_columns_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" + } } }, "vendor": "openai", - "started_at": "2026-01-27T16:35:27.778895055Z", - "finished_at": "2026-01-27T16:36:36.282016589Z" + "started_at": "2026-01-27T21:58:34.132272186Z", + "finished_at": "2026-01-27T21:59:33.320275639Z" }, "t_016_sum_type_columns": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_016_sum_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[derive(spacetimedb::SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(spacetimedb::SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -1785,44 +1816,44 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "sum_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", + "phase": "call_reducer_golden" } }, - "sum_type_columns_row_parity": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", + "phase": "describe_golden" } }, "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45829/v1/database/c2008da059ad558df05ae36956ea06f6277a53e08f40f6950324728e3d63cfb8/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45981/v1/database/c200620d156430770305d35225398f5bdccd8e1edab109eb44e4284657fdb0d3/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:35:27.755209342Z", - "finished_at": "2026-01-27T16:37:05.840061596Z" + "started_at": "2026-01-27T21:58:34.119405994Z", + "finished_at": "2026-01-27T21:59:28.738324004Z" }, "t_017_scheduled_columns": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_017_scheduled_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let interval = ScheduleAt::Interval(Duration::from_micros(50_000).into());\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: interval,\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n Ok(())\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -1830,6 +1861,14 @@ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n", + "phase": "describe_golden" + } + }, "scheduled_seeded_one_row": { "pass": true, "partial": 1.0, @@ -1838,29 +1877,21 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n", - "phase": "describe_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-27T16:35:27.744787479Z", - "finished_at": "2026-01-27T16:37:10.715377293Z" + "started_at": "2026-01-27T21:58:34.135515898Z", + "finished_at": "2026-01-27T21:59:47.134125164Z" }, "t_018_constraints": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_018_constraints", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".into(),\n name: \"Alice\".into(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".into(),\n name: \"Bob\".into(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.account().iter() {\n ctx.db.account().id().delete(row.id);\n }\n\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -1868,21 +1899,21 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { - "constraints_seed_two_rows": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", + "phase": "describe_golden" } }, - "schema_parity": { + "constraints_seed_two_rows": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", - "phase": "describe_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" } }, "constraints_row_parity_after_seed": { @@ -1895,47 +1926,83 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T16:35:27.741095372Z", - "finished_at": "2026-01-27T16:36:39.493832562Z" + "started_at": "2026-01-27T21:58:34.122551959Z", + "finished_at": "2026-01-27T21:59:28.647541973Z" }, "t_019_many_to_many": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_019_many_to_many", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 5, + "passed_tests": 4, + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user, public)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group, public)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n public,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Clear existing rows to ensure we end up with exactly the specified data set\n for m in ctx.db.membership().iter().collect::>() {\n ctx.db.membership().id().delete(&m.id);\n }\n for g in ctx.db.r#group().iter().collect::>() {\n ctx.db.r#group().group_id().delete(&g.group_id);\n }\n for u in ctx.db.user().iter().collect::>() {\n ctx.db.user().user_id().delete(&u.user_id);\n }\n\n // Insert users\n ctx.db.user().insert(User {\n user_id: 1,\n name: \"Alice\".into(),\n });\n ctx.db.user().insert(User {\n user_id: 2,\n name: \"Bob\".into(),\n });\n\n // Insert groups\n ctx.db.r#group().insert(Group {\n group_id: 10,\n title: \"Admin\".into(),\n });\n ctx.db.r#group().insert(Group {\n group_id: 20,\n title: \"Dev\".into(),\n });\n\n // Insert memberships\n ctx.db.membership().insert(Membership {\n id: 1,\n user_id: 1,\n group_id: 10,\n });\n ctx.db.membership().insert(Membership {\n id: 2,\n user_id: 1,\n group_id: 20,\n });\n ctx.db.membership().insert(Membership {\n id: 3,\n user_id: 2,\n group_id: 20,\n });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "m2m_has_1_20": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" + } + }, + "m2m_has_1_10": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" + } + }, + "m2m_has_2_20": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" + } + }, + "memberships_three_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM membership" + } + }, + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "LLM call timed out", - "phase": "build_or_publish" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-019-many-to-many-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:38:46.674706560Z", - "finished_at": "2026-01-27T16:38:46.674706560Z" + "started_at": "2026-01-27T21:59:22.401446796Z", + "finished_at": "2026-01-27T22:00:29.002067239Z" }, "t_020_ecs": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_020_ecs", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.entity().insert(Entity { id: 1 });\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n ctx.db.entity().insert(Entity { id: 2 });\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(mut existing) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n existing.x = nx;\n existing.y = ny;\n ctx.db.next_position().entity_id().update(existing);\n } else {\n ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // entity 1\n let _ = ctx.db.entity().try_insert(Entity { id: 1 });\n if let Some(mut p) = ctx.db.position().entity_id().find(1) {\n p.x = 0;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(1) {\n v.vx = 1;\n v.vy = 0;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n\n // entity 2\n let _ = ctx.db.entity().try_insert(Entity { id: 2 });\n if let Some(mut p) = ctx.db.position().entity_id().find(2) {\n p.x = 10;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n if let Some(mut v) = ctx.db.velocity().entity_id().find(2) {\n v.vx = -2;\n v.vy = 3;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(mut np) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n np.x = nx;\n np.y = ny;\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -1943,60 +2010,60 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "ecs_next_pos_entity1": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45829/v1/database/c200c119a59fe6bcdb676eb3fd09b54c4485b76f3252e779561170a7fdfdbca2/sql)\n", - "phase": "sql" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-020-ecs-golden`.\n", + "phase": "describe_golden" } }, - "ecs_next_pos_entity2": { + "ecs_step_next_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45829/v1/database/c200c119a59fe6bcdb676eb3fd09b54c4485b76f3252e779561170a7fdfdbca2/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45981/v1/database/c200af561dfe31790d5f75e7e6eb859acf922a8ce783c7752ecf0686e5aa8a05/sql)\n", "phase": "sql" } }, - "schema_parity": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-020-ecs-golden`.\n", - "phase": "describe_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45981/v1/database/c200af561dfe31790d5f75e7e6eb859acf922a8ce783c7752ecf0686e5aa8a05/sql)\n", + "phase": "sql" } }, - "ecs_seed_positions_count": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45829/v1/database/c200c119a59fe6bcdb676eb3fd09b54c4485b76f3252e779561170a7fdfdbca2/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45981/v1/database/c200af561dfe31790d5f75e7e6eb859acf922a8ce783c7752ecf0686e5aa8a05/sql)\n", "phase": "sql" } }, - "ecs_step_next_positions_count": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45829/v1/database/c200c119a59fe6bcdb676eb3fd09b54c4485b76f3252e779561170a7fdfdbca2/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:45981/v1/database/c200af561dfe31790d5f75e7e6eb859acf922a8ce783c7752ecf0686e5aa8a05/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:35:27.774072550Z", - "finished_at": "2026-01-27T16:36:31.328546056Z" + "started_at": "2026-01-27T21:58:34.128964652Z", + "finished_at": "2026-01-27T21:59:50.287487076Z" }, "t_021_multi_column_index": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "task": "t_021_multi_column_index", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let ids: Vec = ctx.db.log().iter().map(|r| r.id).collect();\n for id in ids {\n ctx.db.log().id().delete(&id);\n }\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -2004,6 +2071,23 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", + "phase": "describe_golden" + } + }, + "mcindex_seed_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM log" + } + }, "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, @@ -2021,28 +2105,11 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" } - }, - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", - "phase": "describe_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-27T16:35:27.749626445Z", - "finished_at": "2026-01-27T16:36:46.682079929Z" + "started_at": "2026-01-27T21:58:34.125789398Z", + "finished_at": "2026-01-27T21:59:22.401369156Z" } } } @@ -2233,14 +2300,14 @@ "modes": [ { "mode": "docs", - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_000_empty_reducers", "lang": "csharp", "golden_published": true, @@ -2263,25 +2330,25 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:32995", + "server": "http://127.0.0.1:42031", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:45:33.624143092Z", - "finished_at": "2026-01-27T16:46:27.480022955Z" + "started_at": "2026-01-27T22:05:00.112638697Z", + "finished_at": "2026-01-27T22:05:46.121861290Z" }, "t_001_basic_tables": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_001_basic_tables", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\")]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\")]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\", Public = true)]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\", Public = true)]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -2297,25 +2364,25 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:32995", + "server": "http://127.0.0.1:42031", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:45:58.037479404Z", - "finished_at": "2026-01-27T16:46:46.072177909Z" + "started_at": "2026-01-27T22:05:46.121885196Z", + "finished_at": "2026-01-27T22:06:22.443409459Z" }, "t_002_scheduled_table": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_002_scheduled_table", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n if (ctx.Db.TickTimer.Count == 0)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -2331,25 +2398,25 @@ "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:32995", + "server": "http://127.0.0.1:42031", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:46:46.072324097Z", - "finished_at": "2026-01-27T16:47:52.804047987Z" + "started_at": "2026-01-27T22:06:22.566930178Z", + "finished_at": "2026-01-27T22:07:08.516744730Z" }, "t_003_struct_in_table": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_003_struct_in_table", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -2365,18 +2432,18 @@ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:32995", + "server": "http://127.0.0.1:42031", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:45:44.449789708Z", - "finished_at": "2026-01-27T16:46:27.856049534Z" + "started_at": "2026-01-27T22:06:56.852214977Z", + "finished_at": "2026-01-27T22:07:18.733479497Z" }, "t_004_insert": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_004_insert", "lang": "csharp", "golden_published": true, @@ -2391,6 +2458,19 @@ "work_dir_golden": "target/llm-runs/basics/t_004_insert/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_004_insert/csharp/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-004-insert-golden", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:42031", + "tables_diff": null, + "tables_equal": true + } + }, "data_parity_insert_user": { "pass": true, "partial": 1.0, @@ -2407,36 +2487,23 @@ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "InsertUser", - "server": "http://127.0.0.1:32995" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-004-insert-golden", - "llm_db": "basics-t-004-insert-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:32995", - "tables_diff": null, - "tables_equal": true + "server": "http://127.0.0.1:42031" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:46:29.820748855Z", - "finished_at": "2026-01-27T16:47:18.121969583Z" + "started_at": "2026-01-27T22:05:21.015736490Z", + "finished_at": "2026-01-27T22:05:47.405778904Z" }, "t_005_update": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_005_update", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Id.Update(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var row = ctx.Db.User.Id.Find(id);\n if (row is User)\n {\n ctx.Db.User.Id.Update(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -2444,19 +2511,6 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-005-update-golden", - "llm_db": "basics-t-005-update-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:32995", - "tables_diff": null, - "tables_equal": true - } - }, "data_parity_update_user": { "pass": true, "partial": 1.0, @@ -2473,7 +2527,7 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "UpdateUser", - "server": "http://127.0.0.1:32995" + "server": "http://127.0.0.1:42031" } }, "seed_users_row": { @@ -2482,21 +2536,34 @@ "notes": { "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-005-update-golden", + "llm_db": "basics-t-005-update-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:42031", + "tables_diff": null, + "tables_equal": true + } } }, "vendor": "openai", - "started_at": "2026-01-27T16:45:30.312651937Z", - "finished_at": "2026-01-27T16:46:29.809622619Z" + "started_at": "2026-01-27T22:04:50.875700846Z", + "finished_at": "2026-01-27T22:05:50.454024469Z" }, "t_006_delete": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_006_delete", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -2504,6 +2571,19 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/csharp/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-006-delete-golden", + "llm_db": "basics-t-006-delete-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:42031", + "tables_diff": null, + "tables_equal": true + } + }, "delete_user_count_zero": { "pass": true, "partial": 1.0, @@ -2519,34 +2599,21 @@ "notes": { "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-006-delete-golden", - "llm_db": "basics-t-006-delete-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:32995", - "tables_diff": null, - "tables_equal": true - } } }, "vendor": "openai", - "started_at": "2026-01-27T16:44:29.592100157Z", - "finished_at": "2026-01-27T16:45:30.312571408Z" + "started_at": "2026-01-27T22:05:42.457690222Z", + "finished_at": "2026-01-27T22:06:15.967919960Z" }, "t_007_crud": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_007_crud", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User u1)\n {\n u1.Name = \"Alice2\";\n u1.Age = 31;\n u1.Active = false;\n ctx.Db.User.Id.Update(u1);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User u)\n {\n u.Name = \"Alice2\";\n u.Age = 31;\n u.Active = false;\n ctx.Db.User.Id.Update(u);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -2554,22 +2621,17 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm", "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" - } - }, - "crud_total_count_one": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User" + "golden_db": "basics-t-007-crud-golden", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:42031", + "tables_diff": null, + "tables_equal": true } }, "crud_row_id1_parity": { @@ -2583,36 +2645,41 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "Crud", - "server": "http://127.0.0.1:32995" + "server": "http://127.0.0.1:42031" } }, - "schema_parity": { + "crud_total_count_one": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-007-crud-golden", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:32995", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM User" + } + }, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:44:36.788013061Z", - "finished_at": "2026-01-27T16:45:44.449720987Z" + "started_at": "2026-01-27T22:06:22.563944298Z", + "finished_at": "2026-01-27T22:07:08.425768578Z" }, "t_008_index_lookup": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_008_index_lookup", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Insert(new Result { Id = user.Id, Name = user.Name });\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -2620,6 +2687,19 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-008-index-lookup-golden", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:42031", + "tables_diff": null, + "tables_equal": true + } + }, "seed_user_row": { "pass": true, "partial": 1.0, @@ -2640,29 +2720,16 @@ "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", "query": "SELECT Id, Name FROM Result WHERE Id=1", "reducer": "LookupUserName", - "server": "http://127.0.0.1:32995" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:32995", - "tables_diff": null, - "tables_equal": true + "server": "http://127.0.0.1:42031" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:46:29.809976029Z", - "finished_at": "2026-01-27T16:47:52.369916015Z" + "started_at": "2026-01-27T22:06:18.642783143Z", + "finished_at": "2026-01-27T22:07:03.442207582Z" }, "t_009_init": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_009_init", "lang": "csharp", "golden_published": true, @@ -2677,60 +2744,60 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:32995", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" } }, - "init_total_two": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" } }, - "init_seed_alice": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM User" } }, - "init_seed_bob": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" + "golden_db": "basics-t-009-init-golden", + "llm_db": "basics-t-009-init-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:42031", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:45:00.485893120Z", - "finished_at": "2026-01-27T16:45:58.037338911Z" + "started_at": "2026-01-27T22:05:50.454149883Z", + "finished_at": "2026-01-27T22:06:22.563910581Z" }, "t_010_connect": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_010_connect", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -2746,25 +2813,25 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:32995", + "server": "http://127.0.0.1:42031", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:44:29.599798551Z", - "finished_at": "2026-01-27T16:45:33.624088924Z" + "started_at": "2026-01-27T22:06:15.967964050Z", + "finished_at": "2026-01-27T22:06:56.852187767Z" }, "t_011_helper_function": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_011_helper_function", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b)\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n private static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -2780,7 +2847,7 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:32995", + "server": "http://127.0.0.1:42031", "tables_diff": null, "tables_equal": true } @@ -2800,7 +2867,7 @@ "llm_out": "Id | Sum ----+----- 1 | 5", "query": "SELECT Id, Sum FROM Result WHERE Id=1", "reducer": "ComputeSum", - "server": "http://127.0.0.1:32995" + "server": "http://127.0.0.1:42031" } }, "helper_func_sum_abs": { @@ -2814,18 +2881,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T16:46:29.832685293Z", - "finished_at": "2026-01-27T16:47:21.941754653Z" + "started_at": "2026-01-27T22:05:47.405807074Z", + "finished_at": "2026-01-27T22:06:18.642743657Z" }, "t_012_spacetime_product_type": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_012_spacetime_product_type", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score\n {\n Left = left,\n Right = right\n }\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right }\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -2833,15 +2900,6 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/gpt-5/llm", "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - }, "product_type_row_parity": { "pass": true, "partial": 1.0, @@ -2857,7 +2915,16 @@ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", "query": "SELECT Id, Value FROM Result WHERE Id=1", "reducer": "SetScore", - "server": "http://127.0.0.1:32995" + "server": "http://127.0.0.1:42031" + } + }, + "product_type_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } }, "schema_parity": { @@ -2868,25 +2935,25 @@ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:32995", + "server": "http://127.0.0.1:42031", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:44:13.602642076Z", - "finished_at": "2026-01-27T16:45:00.485815025Z" + "started_at": "2026-01-27T22:02:31.393119815Z", + "finished_at": "2026-01-27T22:03:52.200736367Z" }, "t_013_spacetime_sum_type": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_013_spacetime_sum_type", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(\n Circle Circle,\n Rectangle Rectangle\n )> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Circle { Radius = radius })\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var circle = new Circle { Radius = radius };\n var shape = new Shape.Circle(circle);\n ctx.Db.Result.Insert(new Result { Id = id, Value = shape });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -2894,19 +2961,6 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:32995", - "tables_diff": null, - "tables_equal": true - } - }, "sum_type_row_parity": { "pass": true, "partial": 1.0, @@ -2921,7 +2975,20 @@ "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", "query": "SELECT Id, Value FROM Result WHERE Id=1", "reducer": "SetCircle", - "server": "http://127.0.0.1:32995" + "server": "http://127.0.0.1:42031" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-013-spacetime-sum-type-golden", + "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:42031", + "tables_diff": null, + "tables_equal": true } }, "sum_type_row_count": { @@ -2935,11 +3002,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-27T16:42:33.200019323Z", - "finished_at": "2026-01-27T16:43:38.115938270Z" + "started_at": "2026-01-27T22:02:31.395674019Z", + "finished_at": "2026-01-27T22:04:09.953769907Z" }, "t_014_elementary_columns": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_014_elementary_columns", "lang": "csharp", "golden_published": true, @@ -2954,49 +3021,49 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, + "elementary_columns_row_parity": { + "pass": false, + "partial": 0.0, "notes": { - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:32995", - "tables_diff": null, - "tables_equal": true + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42031/v1/database/c20043b845b8085a07c0b477548855329e4796100f0f03364d36a179a90383cc/sql)\n", + "phase": "sql_golden" } }, "elementary_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c2004f34121dd1f4ae79dcdbda66d549feb82a5d63960ba14565d5cc8b739c97/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42031/v1/database/c2007013a11da860089b5483efb500927c3a045571f20e8c11d044d87f4af1db/sql)\n", "phase": "sql" } }, - "elementary_columns_row_parity": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c200543789d2e744179bd83fb1415d87abe1ef9050fbc5d65555128998b27490/sql)\n", - "phase": "sql_golden" + "golden_db": "schema-t-014-elementary-columns-golden", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:42031", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:43:38.116010007Z", - "finished_at": "2026-01-27T16:44:36.787949888Z" + "started_at": "2026-01-27T22:04:26.310123052Z", + "finished_at": "2026-01-27T22:05:00.112608530Z" }, "t_015_product_type_columns": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_015_product_type_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Profile.Iter())\n {\n ctx.Db.Profile.Id.Delete(row.Id);\n }\n\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 },\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -3004,15 +3071,6 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm", "scorer_details": { - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" - } - }, "product_type_columns_row_parity": { "pass": true, "partial": 1.0, @@ -3024,7 +3082,7 @@ "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:32995" + "server": "http://127.0.0.1:42031" } }, "schema_parity": { @@ -3035,25 +3093,34 @@ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:32995", + "server": "http://127.0.0.1:42031", "tables_diff": null, "tables_equal": true } + }, + "product_type_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" + } } }, "vendor": "openai", - "started_at": "2026-01-27T16:43:18.813807927Z", - "finished_at": "2026-01-27T16:44:27.109398198Z" + "started_at": "2026-01-27T22:03:56.262251047Z", + "finished_at": "2026-01-27T22:04:31.236049938Z" }, "t_016_sum_type_columns": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_016_sum_type_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\", Public = true)]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var a = new Shape.Circle(new Circle { Radius = 10 });\n var b = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 });\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = a,\n B = b\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -3061,11 +3128,19 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm", "scorer_details": { + "sum_type_columns_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42031/v1/database/c20070cd03a56bffcc38458dcb1029b1991f9498b6a12c9a46a7878bffe10d49/sql)\n", + "phase": "sql" + } + }, "sum_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c200a6ce7b1295896bce91792742020e2d963fc68bfca6d8ff9804106ed485f8/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42031/v1/database/c200eaa8f05c885ac60455478b9b68d6b18bb1b92f299c08c43cdd29ef1b9a97/sql)\n", "phase": "sql_golden" } }, @@ -3077,33 +3152,25 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:32995", + "server": "http://127.0.0.1:42031", "tables_diff": null, "tables_equal": true } - }, - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c200bbb059f1df73468f61a994e99b02f150ebdc9dce5396fc2cc064f5ebb917/sql)\n", - "phase": "sql" - } } }, "vendor": "openai", - "started_at": "2026-01-27T16:41:37.626585140Z", - "finished_at": "2026-01-27T16:43:18.813740398Z" + "started_at": "2026-01-27T22:02:31.398828215Z", + "finished_at": "2026-01-27T22:03:56.262197561Z" }, "t_017_scheduled_columns": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_017_scheduled_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n // Scheduled tick; no-op\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer(SpacetimeDB.ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer TickTimer)\n {\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -3119,7 +3186,7 @@ "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:32995", + "server": "http://127.0.0.1:42031", "tables_diff": null, "tables_equal": true } @@ -3128,17 +3195,17 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c200283fa6a6ce49e85887cd3c05f6cd55c39e71c5cdd5dc78a67d45df2c41c4/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42031/v1/database/c200e44961ff20a44bd250567c5dcaf9c11e2455c3afcfdfd773be3c4042003a/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:41:37.619157633Z", - "finished_at": "2026-01-27T16:42:40.765400962Z" + "started_at": "2026-01-27T22:04:09.953815313Z", + "finished_at": "2026-01-27T22:04:50.875665709Z" }, "t_018_constraints": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_018_constraints", "lang": "csharp", "golden_published": true, @@ -3153,6 +3220,15 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/csharp/server/gpt-5/llm", "scorer_details": { + "constraints_seed_two_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" + } + }, "constraints_row_parity_after_seed": { "pass": true, "partial": 1.0, @@ -3164,7 +3240,7 @@ "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:32995" + "server": "http://127.0.0.1:42031" } }, "schema_parity": { @@ -3175,34 +3251,25 @@ "llm_db": "schema-t-018-constraints-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:32995", + "server": "http://127.0.0.1:42031", "tables_diff": null, "tables_equal": true } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" - } } }, "vendor": "openai", - "started_at": "2026-01-27T16:41:37.615926753Z", - "finished_at": "2026-01-27T16:42:33.199931492Z" + "started_at": "2026-01-27T22:02:31.402276922Z", + "finished_at": "2026-01-27T22:03:52.031753547Z" }, "t_019_many_to_many": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_019_many_to_many", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { nameof(UserId) })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { nameof(GroupId) })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -3210,19 +3277,6 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:32995", - "tables_diff": null, - "tables_equal": true - } - }, "m2m_has_1_20": { "pass": true, "partial": 1.0, @@ -3241,6 +3295,15 @@ "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" } }, + "memberships_three_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Membership" + } + }, "m2m_has_2_20": { "pass": true, "partial": 1.0, @@ -3250,29 +3313,33 @@ "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" } }, - "memberships_three_rows": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:42031", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:42:40.765467116Z", - "finished_at": "2026-01-27T16:44:13.602547935Z" + "started_at": "2026-01-27T22:04:31.236092692Z", + "finished_at": "2026-01-27T22:05:42.457644003Z" }, "t_020_ecs": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_020_ecs", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var p in ctx.Db.Position.Iter())\n {\n if (ctx.Db.Velocity.EntityId.Find(p.EntityId) is Velocity v)\n {\n int nx = p.X + v.VX;\n int ny = p.Y + v.VY;\n\n if (ctx.Db.NextPosition.EntityId.Find(p.EntityId) is NextPosition np)\n {\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = p.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n UpsertEntity(ctx, 1);\n UpsertPosition(ctx, 1, 0, 0);\n UpsertVelocity(ctx, 1, 1, 0);\n\n UpsertEntity(ctx, 2);\n UpsertPosition(ctx, 2, 10, 0);\n UpsertVelocity(ctx, 2, -2, 3);\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n var velOpt = ctx.Db.Velocity.EntityId.Find(pos.EntityId);\n if (velOpt == null)\n {\n continue;\n }\n\n var vel = velOpt.Value;\n var nx = pos.X + vel.VX;\n var ny = pos.Y + vel.VY;\n\n var npOpt = ctx.Db.NextPosition.EntityId.Find(pos.EntityId);\n if (npOpt == null)\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n else\n {\n var np = npOpt.Value;\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n }\n }\n\n static void UpsertEntity(ReducerContext ctx, int id)\n {\n var existing = ctx.Db.Entity.Id.Find(id);\n if (existing == null)\n {\n ctx.Db.Entity.Insert(new Entity { Id = id });\n }\n }\n\n static void UpsertPosition(ReducerContext ctx, int entityId, int x, int y)\n {\n var existing = ctx.Db.Position.EntityId.Find(entityId);\n if (existing == null)\n {\n ctx.Db.Position.Insert(new Position { EntityId = entityId, X = x, Y = y });\n }\n else\n {\n var p = existing.Value;\n p.X = x;\n p.Y = y;\n ctx.Db.Position.EntityId.Update(p);\n }\n }\n\n static void UpsertVelocity(ReducerContext ctx, int entityId, int vx, int vy)\n {\n var existing = ctx.Db.Velocity.EntityId.Find(entityId);\n if (existing == null)\n {\n ctx.Db.Velocity.Insert(new Velocity { EntityId = entityId, VX = vx, VY = vy });\n }\n else\n {\n var v = existing.Value;\n v.VX = vx;\n v.VY = vy;\n ctx.Db.Velocity.EntityId.Update(v);\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -3280,11 +3347,11 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm", "scorer_details": { - "ecs_step_next_positions_count": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c20010df49a5a8796254ab11e03e6f0cf6c22f260028d174841bda846335004f/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42031/v1/database/c200dcde9ccab700de2ea96db18b1fe51e357b2dca9c454c0c9e2f4d6cd478fa/sql)\n", "phase": "sql" } }, @@ -3292,7 +3359,7 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c20010df49a5a8796254ab11e03e6f0cf6c22f260028d174841bda846335004f/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42031/v1/database/c200dcde9ccab700de2ea96db18b1fe51e357b2dca9c454c0c9e2f4d6cd478fa/sql)\n", "phase": "sql" } }, @@ -3304,41 +3371,41 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:32995", + "server": "http://127.0.0.1:42031", "tables_diff": null, "tables_equal": true } }, - "ecs_seed_positions_count": { + "ecs_step_next_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c20010df49a5a8796254ab11e03e6f0cf6c22f260028d174841bda846335004f/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42031/v1/database/c200dcde9ccab700de2ea96db18b1fe51e357b2dca9c454c0c9e2f4d6cd478fa/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity2": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:32995/v1/database/c20010df49a5a8796254ab11e03e6f0cf6c22f260028d174841bda846335004f/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:42031/v1/database/c200dcde9ccab700de2ea96db18b1fe51e357b2dca9c454c0c9e2f4d6cd478fa/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-27T16:43:06.400065322Z", - "finished_at": "2026-01-27T16:44:29.591963094Z" + "started_at": "2026-01-27T22:03:52.204029385Z", + "finished_at": "2026-01-27T22:05:21.015673517Z" }, "t_021_multi_column_index": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "task": "t_021_multi_column_index", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Log.Iter())\n {\n ctx.Db.Log.Id.Delete(row.Id);\n }\n\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -3346,6 +3413,15 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm", "scorer_details": { + "mcindex_lookup_u7_d1": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" + } + }, "mcindex_seed_count": { "pass": true, "partial": 1.0, @@ -3364,15 +3440,6 @@ "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" } }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3381,15 +3448,15 @@ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:32995", + "server": "http://127.0.0.1:42031", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-27T16:41:37.622718575Z", - "finished_at": "2026-01-27T16:43:06.399960112Z" + "started_at": "2026-01-27T22:03:52.200781109Z", + "finished_at": "2026-01-27T22:04:26.310079475Z" } } } diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json index 56cb6e89647..34a640fb40e 100644 --- a/docs/llms/docs-benchmark-summary.json +++ b/docs/llms/docs-benchmark-summary.json @@ -1,11 +1,11 @@ { "version": 1, - "generated_at": "2026-01-27T16:47:52.904Z", + "generated_at": "2026-01-27T22:07:18.787Z", "by_language": { "csharp": { "modes": { "docs": { - "hash": "b22d989c00281f7f3e8912a08e8322746fa6cba271164f4cad2be9954b7f6ec9", + "hash": "bd31092a0af7d25ac3ccd5ede135c9840ed8cad3c2d6006ee1ad0f5f69e3a3e1", "models": { "GPT-5": { "categories": { @@ -42,7 +42,7 @@ "rust": { "modes": { "docs": { - "hash": "f09c4aa335b00d6e1e55573dfbc47f13accb0a1c631339ddb4dc8a6237c520cd", + "hash": "d86518d2ac099e60eb25705cd317e30df95f4d6a4787a2fe5aae12753b8ab4f7", "models": { "GPT-5": { "categories": { @@ -56,20 +56,20 @@ }, "schema": { "tasks": 10, - "total_tests": 30, - "passed_tests": 4, - "pass_pct": 13.333333, - "task_pass_equiv": 1.25, - "task_pass_pct": 12.5 + "total_tests": 34, + "passed_tests": 8, + "pass_pct": 23.529411, + "task_pass_equiv": 2.05, + "task_pass_pct": 20.5 } }, "totals": { "tasks": 22, - "total_tests": 57, - "passed_tests": 9, - "pass_pct": 15.789474, - "task_pass_equiv": 2.5833335, - "task_pass_pct": 11.742425 + "total_tests": 61, + "passed_tests": 13, + "pass_pct": 21.311476, + "task_pass_equiv": 3.3833334, + "task_pass_pct": 15.378788 } } } @@ -90,19 +90,19 @@ "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 18, - "pass_pct": 52.941177, - "task_pass_equiv": 5.5333333, - "task_pass_pct": 55.333336 + "passed_tests": 22, + "pass_pct": 64.70588, + "task_pass_equiv": 6.2, + "task_pass_pct": 62.0 } }, "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 43, - "pass_pct": 70.491806, - "task_pass_equiv": 15.533334, - "task_pass_pct": 70.606064 + "passed_tests": 47, + "pass_pct": 77.04918, + "task_pass_equiv": 16.2, + "task_pass_pct": 73.63637 } } }