From 7fee358cf98c80b3f82e32f7e88f6d64f701b24a Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 09:19:23 +0300 Subject: [PATCH 01/21] feat(gen): migrate to gen-contract v4.0.1 / gen-sdk v2.0.0, adopt Sdk.Sigs Pins gen/Deps/Contract.dhall (new) and bumps gen/Deps/Sdk.dhall to gen-sdk v2.0.0, removing the gen/Deps/package.dhall barrel. Rewires Structures/CustomKind.dhall and all 11 Interpreters/*.dhall plus all 10 Templates/*.dhall to import Lude/Prelude/Contract/Sdk directly instead of through the barrel, replacing the local gen/Algebras/{Interpreter,Template} sig-constructors with gen-sdk's Sdk.Sigs.interpreter / Sdk.Sigs.template (now deleted). Matches the shape java.gen's current release already uses. Query.dhall, Result.dhall, and ResultColumns.dhall keep their pre-existing bare-export shape (like Member.dhall/ParamsMember.dhall) instead of adopting Sdk.Sigs.interpreter: their `run` functions take 3/4 curried arguments (config, lookup, [rowClassName], input), which doesn't fit Sigs.interpreter's fixed 2-argument Config -> Input -> Result shape. This extends the Member/ParamsMember exception already called out in the migration plan to these three files, confirmed against their actual call sites in Project.dhall/Query.dhall. gen/compile.dhall and gen/Gen.dhall still reference the removed barrel and are intentionally left broken until the next task rewrites the root entry point (Config.dhall/compile.dhall -> Interpret.dhall, Gen.dhall -> package.dhall). --- gen/Algebras/Interpreter.dhall | 21 ------------- gen/Algebras/Template.dhall | 5 --- gen/Algebras/package.dhall | 1 - gen/Deps/Contract.dhall | 2 ++ gen/Deps/Sdk.dhall | 4 +-- gen/Deps/package.dhall | 4 --- gen/Interpreters/CustomType.dhall | 25 +++++++++------ gen/Interpreters/Member.dhall | 27 ++++++++++------ gen/Interpreters/ParamsMember.dhall | 45 ++++++++++++++++----------- gen/Interpreters/Primitive.dhall | 23 +++++++++----- gen/Interpreters/Project.dhall | 25 +++++++++------ gen/Interpreters/Query.dhall | 23 +++++++++----- gen/Interpreters/QueryFragments.dhall | 23 ++++++++------ gen/Interpreters/Result.dhall | 26 ++++++++++------ gen/Interpreters/ResultColumns.dhall | 24 +++++++++----- gen/Interpreters/Scalar.dhall | 21 ++++++++----- gen/Interpreters/Value.dhall | 23 +++++++++----- gen/Structures/CustomKind.dhall | 4 +-- gen/Templates/CompositeModule.dhall | 12 +++---- gen/Templates/CoreModule.dhall | 4 +-- gen/Templates/EnumModule.dhall | 20 ++++++------ gen/Templates/FacadeModule.dhall | 8 ++--- gen/Templates/InitModule.dhall | 4 +-- gen/Templates/RegisterModule.dhall | 8 ++--- gen/Templates/RowsModule.dhall | 12 +++---- gen/Templates/RuntimeModule.dhall | 4 +-- gen/Templates/StatementModule.dhall | 12 +++---- gen/Templates/TypesInit.dhall | 8 ++--- 28 files changed, 230 insertions(+), 188 deletions(-) delete mode 100644 gen/Algebras/Interpreter.dhall delete mode 100644 gen/Algebras/Template.dhall delete mode 100644 gen/Algebras/package.dhall create mode 100644 gen/Deps/Contract.dhall delete mode 100644 gen/Deps/package.dhall diff --git a/gen/Algebras/Interpreter.dhall b/gen/Algebras/Interpreter.dhall deleted file mode 100644 index 7532153..0000000 --- a/gen/Algebras/Interpreter.dhall +++ /dev/null @@ -1,21 +0,0 @@ -let Deps = ../Deps/package.dhall - -let OnUnsupported = ../Structures/OnUnsupported.dhall - -let Config = - { packageName : Text - , importName : Text - , emitSync : Bool - , onUnsupported : OnUnsupported.Mode - } - -let module = - \(Input : Type) -> - \(Output : Type) -> - let Result = Deps.Lude.Compiled.Type Output - - let Run = Config -> Input -> Result - - in \(run : Run) -> { Input, Output, Result, Run, run } - -in { Config, module } diff --git a/gen/Algebras/Template.dhall b/gen/Algebras/Template.dhall deleted file mode 100644 index 15f2624..0000000 --- a/gen/Algebras/Template.dhall +++ /dev/null @@ -1,5 +0,0 @@ -let module = - \(Params : Type) -> - let Run = Params -> Text in \(run : Run) -> { Params, Run, run } - -in { module } diff --git a/gen/Algebras/package.dhall b/gen/Algebras/package.dhall deleted file mode 100644 index f0f64bd..0000000 --- a/gen/Algebras/package.dhall +++ /dev/null @@ -1 +0,0 @@ -{ Interpreter = ./Interpreter.dhall, Template = ./Template.dhall } diff --git a/gen/Deps/Contract.dhall b/gen/Deps/Contract.dhall new file mode 100644 index 0000000..ea9e805 --- /dev/null +++ b/gen/Deps/Contract.dhall @@ -0,0 +1,2 @@ +https://raw.githubusercontent.com/pgenie-io/gen-contract/v4.0.1/src/package.dhall + sha256:4a130ba7fbaa152a776babbb1bf2994a4833931ca76bde9bf6930d354225651e diff --git a/gen/Deps/Sdk.dhall b/gen/Deps/Sdk.dhall index 6cf6014..5602c43 100644 --- a/gen/Deps/Sdk.dhall +++ b/gen/Deps/Sdk.dhall @@ -1,3 +1,3 @@ -https://raw.githubusercontent.com/pgenie-io/gen-sdk/v0.11.0/dhall/package.dhall - sha256:8d43544ecb0e612406af3133bdbca51138c704a77a5a29ef62fe034d0e77a3a6 +https://raw.githubusercontent.com/pgenie-io/gen-sdk/v2.0.0/src/package.dhall + sha256:b9def6ab1179bc4aaae7fc6e91977f094f75934cd5755175c294a9e97ca71b15 as Source diff --git a/gen/Deps/package.dhall b/gen/Deps/package.dhall deleted file mode 100644 index b395cd9..0000000 --- a/gen/Deps/package.dhall +++ /dev/null @@ -1,4 +0,0 @@ -{ Sdk = ./Sdk.dhall -, Lude = ./Lude.dhall -, Prelude = ./Prelude.dhall -} diff --git a/gen/Interpreters/CustomType.dhall b/gen/Interpreters/CustomType.dhall index 7927f0a..676f9ab 100644 --- a/gen/Interpreters/CustomType.dhall +++ b/gen/Interpreters/CustomType.dhall @@ -1,16 +1,16 @@ -let Deps = ../Deps/package.dhall +let Lude = ../Deps/Lude.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Prelude = ../Deps/Prelude.dhall -let CustomKind = ../Structures/CustomKind.dhall +let Model = ../Deps/Contract.dhall -let Algebra = ../Algebras/Interpreter.dhall +let Sdk = ../Deps/Sdk.dhall -let Lude = Deps.Lude +let ImportSet = ../Structures/ImportSet.dhall -let Prelude = Deps.Prelude +let CustomKind = ../Structures/CustomKind.dhall -let Model = Deps.Sdk.Project +let OnUnsupported = ../Structures/OnUnsupported.dhall let MemberGen = ./Member.dhall @@ -18,6 +18,13 @@ let EnumModule = ../Templates/EnumModule.dhall let CompositeModule = ../Templates/CompositeModule.dhall +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } + let Input = Model.CustomType let TypeKind = < Enum | Composite > @@ -75,7 +82,7 @@ let renderExtraImports = ) let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(input : Input) -> let typeName = input.name.inPascalCase @@ -174,4 +181,4 @@ let run = } input.definition -in { Input, Output, TypeKind, run } +in Sdk.Sigs.interpreter Config Input Output run diff --git a/gen/Interpreters/Member.dhall b/gen/Interpreters/Member.dhall index 672dd18..8d2d27c 100644 --- a/gen/Interpreters/Member.dhall +++ b/gen/Interpreters/Member.dhall @@ -1,4 +1,8 @@ -let Deps = ../Deps/package.dhall +let Lude = ../Deps/Lude.dhall + +let Prelude = ../Deps/Prelude.dhall + +let Model = ../Deps/Contract.dhall let ImportSet = ../Structures/ImportSet.dhall @@ -6,14 +10,17 @@ let CustomKind = ../Structures/CustomKind.dhall let PyIdent = ../Structures/PyIdent.dhall -let Algebra = ../Algebras/Interpreter.dhall - -let Lude = Deps.Lude - -let Model = Deps.Sdk.Project +let OnUnsupported = ../Structures/OnUnsupported.dhall let Value = ./Value.dhall +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } + let Input = Model.Member -- decodeExpr is a Dhall function: given the source expression (e.g. row["x"] or @@ -29,7 +36,7 @@ let Output = } let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(lookup : CustomKind.Lookup) -> \(input : Input) -> -- Result-column / composite-field name becomes a dataclass field and decode @@ -95,7 +102,7 @@ let run = \(fields : List CustomKind.CompositeField) -> \(src : Text) -> let fieldTypes = - Deps.Prelude.Text.concatMapSep + Prelude.Text.concatMapSep ", " CustomKind.CompositeField (\(f : CustomKind.CompositeField) -> f.pyType) @@ -120,7 +127,7 @@ let run = , decodeExpr = passthroughDecode } , Custom = - Deps.Prelude.Optional.fold + Prelude.Optional.fold Model.Name value.scalar.customRef (Lude.Compiled.Type Output) @@ -235,6 +242,6 @@ let run = in Lude.Compiled.flatMap Value.Output Output buildOutput compiledValue -let Run = Algebra.Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output +let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output in { Input, Output, Run, run } diff --git a/gen/Interpreters/ParamsMember.dhall b/gen/Interpreters/ParamsMember.dhall index 47bef3f..2a4b469 100644 --- a/gen/Interpreters/ParamsMember.dhall +++ b/gen/Interpreters/ParamsMember.dhall @@ -1,17 +1,24 @@ -let Deps = ../Deps/package.dhall +let Lude = ../Deps/Lude.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Prelude = ../Deps/Prelude.dhall -let CustomKind = ../Structures/CustomKind.dhall +let Model = ../Deps/Contract.dhall -let Algebra = ../Algebras/Interpreter.dhall +let ImportSet = ../Structures/ImportSet.dhall -let Lude = Deps.Lude +let CustomKind = ../Structures/CustomKind.dhall -let Model = Deps.Sdk.Project +let OnUnsupported = ../Structures/OnUnsupported.dhall let Value = ./Value.dhall +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } + let Input = Model.Member let PyIdent = ../Structures/PyIdent.dhall @@ -192,7 +199,7 @@ let scalarIsJsonb = let valueIsArray = \(value : Model.Value) -> - Deps.Prelude.Optional.fold + Prelude.Optional.fold Model.ArraySettings value.arraySettings Bool @@ -204,15 +211,15 @@ let valueIsArray = -- the two must not be collapsed. let isJsonbScalar = \(value : Model.Value) -> - Deps.Prelude.Bool.and - [ scalarIsJsonb value, Deps.Prelude.Bool.not (valueIsArray value) ] + Prelude.Bool.and + [ scalarIsJsonb value, Prelude.Bool.not (valueIsArray value) ] let isJsonScalar = \(value : Model.Value) -> - Deps.Prelude.Bool.and + Prelude.Bool.and [ scalarIsJson value - , Deps.Prelude.Bool.not (scalarIsJsonb value) - , Deps.Prelude.Bool.not (valueIsArray value) + , Prelude.Bool.not (scalarIsJsonb value) + , Prelude.Bool.not (valueIsArray value) ] -- A json/jsonb ARRAY param has no faithful psycopg bind (Jsonb wraps a scalar, @@ -221,10 +228,10 @@ let isJsonScalar = -- in SQL is the supported route (the param then types as text[], not json[]). let isJsonArray = \(value : Model.Value) -> - Deps.Prelude.Bool.and [ scalarIsJson value, valueIsArray value ] + Prelude.Bool.and [ scalarIsJson value, valueIsArray value ] let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(lookup : CustomKind.Lookup) -> \(input : Input) -> let fieldName = pySafeName input.name.inSnakeCase @@ -251,7 +258,7 @@ let run = let compositeBind = \(fields : List CustomKind.CompositeField) -> let joinedFields = - Deps.Prelude.Text.concatMapSep + Prelude.Text.concatMapSep ", " CustomKind.CompositeField ( \(f : CustomKind.CompositeField) -> @@ -265,8 +272,8 @@ let run = -- as a 1-tuple, so force it for exactly one field; concatMapSep -- already inserts the internal comma for two or more. let trailingComma = - if Deps.Prelude.Natural.equal - ( Deps.Prelude.List.length + if Prelude.Natural.equal + ( Prelude.List.length CustomKind.CompositeField fields ) @@ -301,7 +308,7 @@ let run = , needsJsonbImport } - in Deps.Prelude.Optional.fold + in Prelude.Optional.fold Model.Name value.scalar.customRef (Lude.Compiled.Type Output) @@ -377,6 +384,6 @@ let run = in Lude.Compiled.flatMap Value.Output Output buildOutput compiledValue -let Run = Algebra.Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output +let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output in { Input, Output, Run, run } diff --git a/gen/Interpreters/Primitive.dhall b/gen/Interpreters/Primitive.dhall index 0e1cb70..052ef1b 100644 --- a/gen/Interpreters/Primitive.dhall +++ b/gen/Interpreters/Primitive.dhall @@ -1,10 +1,19 @@ -let Deps = ../Deps/package.dhall +let Lude = ../Deps/Lude.dhall + +let Model = ../Deps/Contract.dhall + +let Sdk = ../Deps/Sdk.dhall let ImportSet = ../Structures/ImportSet.dhall -let Algebra = ../Algebras/Interpreter.dhall +let OnUnsupported = ../Structures/OnUnsupported.dhall -let Model = Deps.Sdk.Project +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } let Input = Model.Primitive @@ -13,16 +22,16 @@ let Output = { pyType : Text, imports : ImportSet.Type } let supported = \(pyType : Text) -> \(imports : ImportSet.Type) -> - Deps.Lude.Compiled.ok Output { pyType, imports } + Lude.Compiled.ok Output { pyType, imports } let unsupported = \(pgType : Text) -> - Deps.Lude.Compiled.report Output [ pgType ] "Unsupported type" + Lude.Compiled.report Output [ pgType ] "Unsupported type" let plain = \(pyType : Text) -> supported pyType ImportSet.empty let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(input : Input) -> merge { Bit = unsupported "bit" @@ -89,4 +98,4 @@ let run = } input -in Algebra.module Input Output run +in Sdk.Sigs.interpreter Config Input Output run diff --git a/gen/Interpreters/Project.dhall b/gen/Interpreters/Project.dhall index d51438f..d4fa6d6 100644 --- a/gen/Interpreters/Project.dhall +++ b/gen/Interpreters/Project.dhall @@ -1,12 +1,10 @@ -let Deps = ../Deps/package.dhall +let Lude = ../Deps/Lude.dhall -let Algebra = ../Algebras/Interpreter.dhall +let Prelude = ../Deps/Prelude.dhall -let Lude = Deps.Lude +let Model = ../Deps/Contract.dhall -let Prelude = Deps.Prelude - -let Model = Deps.Sdk.Project +let Sdk = ../Deps/Sdk.dhall let CustomKind = ../Structures/CustomKind.dhall @@ -40,6 +38,13 @@ let OnUnsupported = ../Structures/OnUnsupported.dhall let Report = { path : List Text, message : Text } +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } + let Input = Model.Project let Output = Lude.Files.Type @@ -67,7 +72,7 @@ let withHeader = -- Internal interpreter config (importName etc.) is only needed to satisfy -- Value.run's signature; rendering a composite field's pyType does not read it. let lookupConfig - : Algebra.Config + : Config = { packageName = "" , importName = "" , emitSync = False @@ -175,7 +180,7 @@ let buildLookup = (\(_ : Model.Name) -> CustomKind.TypeKind.Absent) let combineOutputs = - \(config : Algebra.Config) -> + \(config : Config) -> \(input : Input) -> \(queries : List QueryGen.Output) -> -- Already the post-Skip-filter surviving set (see `run`); equal to @@ -430,7 +435,7 @@ let combineOutputs = let QueryCheck = { query : Model.Query, keep : Bool, warning : Optional Report } let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(input : Input) -> let skip = merge { Fail = False, Skip = True } config.onUnsupported @@ -542,4 +547,4 @@ let run = in Lude.Compiled.appendWarnings Output skipWarnings combined -in Algebra.module Input Output run +in Sdk.Sigs.interpreter Config Input Output run diff --git a/gen/Interpreters/Query.dhall b/gen/Interpreters/Query.dhall index b0e688d..daa1d47 100644 --- a/gen/Interpreters/Query.dhall +++ b/gen/Interpreters/Query.dhall @@ -1,6 +1,6 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Interpreter.dhall +let Lude = ../Deps/Lude.dhall let ImportSet = ../Structures/ImportSet.dhall @@ -10,6 +10,8 @@ let PyIdent = ../Structures/PyIdent.dhall let Surface = ../Structures/Surface.dhall +let OnUnsupported = ../Structures/OnUnsupported.dhall + let RowsModule = ../Templates/RowsModule.dhall let ResultModule = ./Result.dhall @@ -20,13 +22,16 @@ let ParamsMember = ./ParamsMember.dhall let StatementModule = ../Templates/StatementModule.dhall -let Prelude = Deps.Prelude - -let Lude = Deps.Lude +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } let Compiled = Lude.Compiled -let Model = Deps.Sdk.Project +let Model = ../Deps/Contract.dhall let Input = Model.Query @@ -46,7 +51,7 @@ let Output = } let render = - \(config : Algebra.Config) -> + \(config : Config) -> \(input : Input) -> \(result : ResultModule.Output) -> \(fragments : QueryFragmentsModule.Output) -> @@ -131,7 +136,7 @@ let render = } let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(lookup : CustomKind.Lookup) -> \(input : Input) -> let rowClassName = input.name.inPascalCase ++ "Row" @@ -172,4 +177,6 @@ let run = ) ) +let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output + in { Input, Output, run } diff --git a/gen/Interpreters/QueryFragments.dhall b/gen/Interpreters/QueryFragments.dhall index 066bc75..7813900 100644 --- a/gen/Interpreters/QueryFragments.dhall +++ b/gen/Interpreters/QueryFragments.dhall @@ -1,16 +1,21 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Interpreter.dhall +let Lude = ../Deps/Lude.dhall -let Prelude = Deps.Prelude +let Sdk = ../Deps/Sdk.dhall -let Sdk = Deps.Sdk +let Compiled = Lude.Compiled -let Lude = Deps.Lude +let Model = ../Deps/Contract.dhall -let Compiled = Lude.Compiled +let OnUnsupported = ../Structures/OnUnsupported.dhall -let Model = Deps.Sdk.Project +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } let Input = Model.QueryFragments @@ -47,8 +52,8 @@ let renderSql Prelude.Text.concatMap Model.QueryFragment renderFragment fragments let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(input : Input) -> Compiled.ok Output { sqlLiteral = renderSql input } -in Algebra.module Input Output run +in Sdk.Sigs.interpreter Config Input Output run diff --git a/gen/Interpreters/Result.dhall b/gen/Interpreters/Result.dhall index a6638e3..83f3ef1 100644 --- a/gen/Interpreters/Result.dhall +++ b/gen/Interpreters/Result.dhall @@ -1,20 +1,25 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Interpreter.dhall +let Lude = ../Deps/Lude.dhall + +let Model = ../Deps/Contract.dhall let ImportSet = ../Structures/ImportSet.dhall let CustomKind = ../Structures/CustomKind.dhall -let ResultColumns = ./ResultColumns.dhall - -let Prelude = Deps.Prelude +let OnUnsupported = ../Structures/OnUnsupported.dhall -let Lude = Deps.Lude +let ResultColumns = ./ResultColumns.dhall let Compiled = Lude.Compiled -let Model = Deps.Sdk.Project +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } let Input = Model.Result @@ -57,7 +62,7 @@ let cardinalityShape cardinality let rowsOutput = - \(config : Algebra.Config) -> + \(config : Config) -> \(lookup : CustomKind.Lookup) -> \(rowClassName : Text) -> \(rows : Model.ResultRows) -> @@ -84,7 +89,7 @@ let rowsOutput = (ResultColumns.run config lookup rowClassName columns) let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(lookup : CustomKind.Lookup) -> \(rowClassName : Text) -> \(input : Input) -> @@ -96,4 +101,7 @@ let run = } input +let Run = + Config -> CustomKind.Lookup -> Text -> Input -> Lude.Compiled.Type Output + in { Input, Output, RowClass, run } diff --git a/gen/Interpreters/ResultColumns.dhall b/gen/Interpreters/ResultColumns.dhall index ef550ec..b2fc737 100644 --- a/gen/Interpreters/ResultColumns.dhall +++ b/gen/Interpreters/ResultColumns.dhall @@ -1,20 +1,25 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Interpreter.dhall +let Lude = ../Deps/Lude.dhall + +let Model = ../Deps/Contract.dhall let ImportSet = ../Structures/ImportSet.dhall let CustomKind = ../Structures/CustomKind.dhall -let Member = ./Member.dhall - -let Prelude = Deps.Prelude +let OnUnsupported = ../Structures/OnUnsupported.dhall -let Lude = Deps.Lude +let Member = ./Member.dhall let Compiled = Lude.Compiled -let Model = Deps.Sdk.Project +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } let Input = List Model.Member @@ -53,7 +58,7 @@ let assemble } let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(lookup : CustomKind.Lookup) -> \(rowClassName : Text) -> \(input : Input) -> @@ -73,4 +78,7 @@ let run = input ) +let Run = + Config -> CustomKind.Lookup -> Text -> Input -> Lude.Compiled.Type Output + in { Input, Output, run } diff --git a/gen/Interpreters/Scalar.dhall b/gen/Interpreters/Scalar.dhall index b0eff70..63ec788 100644 --- a/gen/Interpreters/Scalar.dhall +++ b/gen/Interpreters/Scalar.dhall @@ -1,15 +1,22 @@ -let Deps = ../Deps/package.dhall +let Lude = ../Deps/Lude.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Model = ../Deps/Contract.dhall -let Algebra = ../Algebras/Interpreter.dhall +let Sdk = ../Deps/Sdk.dhall -let Lude = Deps.Lude +let ImportSet = ../Structures/ImportSet.dhall -let Model = Deps.Sdk.Project +let OnUnsupported = ../Structures/OnUnsupported.dhall let Primitive = ./Primitive.dhall +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } + let Input = Model.Scalar -- Passthrough for primitives; Custom is opaque here. The enum-vs-composite @@ -24,7 +31,7 @@ let Output = } let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(input : Input) -> merge { Primitive = @@ -52,4 +59,4 @@ let run = } input -in Algebra.module Input Output run /\ { ScalarDecode } +in Sdk.Sigs.interpreter Config Input Output run /\ { ScalarDecode } diff --git a/gen/Interpreters/Value.dhall b/gen/Interpreters/Value.dhall index de69fd0..e1c8d6a 100644 --- a/gen/Interpreters/Value.dhall +++ b/gen/Interpreters/Value.dhall @@ -1,17 +1,24 @@ -let Deps = ../Deps/package.dhall +let Lude = ../Deps/Lude.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Interpreter.dhall +let Model = ../Deps/Contract.dhall -let Lude = Deps.Lude +let Sdk = ../Deps/Sdk.dhall -let Prelude = Deps.Prelude +let ImportSet = ../Structures/ImportSet.dhall -let Model = Deps.Sdk.Project +let OnUnsupported = ../Structures/OnUnsupported.dhall let Scalar = ./Scalar.dhall +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } + let Input = Model.Value let Output = @@ -23,7 +30,7 @@ let Output = } let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(input : Input) -> Lude.Compiled.map Scalar.Output @@ -62,4 +69,4 @@ let run = ) (Scalar.run config input.scalar) -in Algebra.module Input Output run +in Sdk.Sigs.interpreter Config Input Output run diff --git a/gen/Structures/CustomKind.dhall b/gen/Structures/CustomKind.dhall index 22d3031..f2c4a1b 100644 --- a/gen/Structures/CustomKind.dhall +++ b/gen/Structures/CustomKind.dhall @@ -1,6 +1,4 @@ -let Deps = ../Deps/package.dhall - -let Model = Deps.Sdk.Project +let Model = ../Deps/Contract.dhall -- A composite field as the decode/encode sites need it: the Python attribute -- name and the rendered Python type (already nullability-applied). Threaded so diff --git a/gen/Templates/CompositeModule.dhall b/gen/Templates/CompositeModule.dhall index 198c77d..b1ab3e0 100644 --- a/gen/Templates/CompositeModule.dhall +++ b/gen/Templates/CompositeModule.dhall @@ -1,6 +1,6 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Template.dhall +let Sdk = ../Deps/Sdk.dhall let Field = { fieldName : Text, fieldType : Text } @@ -12,7 +12,7 @@ let Params = { typeName : Text, extraImports : List Text, fields : List Field } let run = \(params : Params) -> let fieldLines = - Deps.Prelude.Text.concatMapSep + Prelude.Text.concatMapSep "\n" Field ( \(field : Field) -> @@ -21,12 +21,12 @@ let run = params.fields let imports = - if Deps.Prelude.List.null Text params.extraImports + if Prelude.List.null Text params.extraImports then "from dataclasses import dataclass" else '' from dataclasses import dataclass - ${Deps.Prelude.Text.concatSep "\n" params.extraImports}'' + ${Prelude.Text.concatSep "\n" params.extraImports}'' in '' ${imports} @@ -43,4 +43,4 @@ let run = ${fieldLines} '' -in Algebra.module Params run /\ { Field } +in Sdk.Sigs.template Params run /\ { Field } diff --git a/gen/Templates/CoreModule.dhall b/gen/Templates/CoreModule.dhall index d3938c2..34a4943 100644 --- a/gen/Templates/CoreModule.dhall +++ b/gen/Templates/CoreModule.dhall @@ -1,4 +1,4 @@ -let Algebra = ../Algebras/Template.dhall +let Sdk = ../Deps/Sdk.dhall -- The surface-agnostic core of a generated package, emitted once at -- _generated/_core.py. It owns the names shared by every module and by both the @@ -43,4 +43,4 @@ let content = ) '' -in Algebra.module {} (\(_ : {}) -> content) +in Sdk.Sigs.template {} (\(_ : {}) -> content) diff --git a/gen/Templates/EnumModule.dhall b/gen/Templates/EnumModule.dhall index 7f82ebf..1ebf486 100644 --- a/gen/Templates/EnumModule.dhall +++ b/gen/Templates/EnumModule.dhall @@ -1,6 +1,6 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Template.dhall +let Sdk = ../Deps/Sdk.dhall let Variant = { memberName : Text, pgValue : Text } @@ -16,18 +16,18 @@ let run = let escapeLabel : Text -> Text = \(raw : Text) -> - Deps.Prelude.Function.composeList + Prelude.Function.composeList Text - [ Deps.Prelude.Text.replace "\\" "\\\\" - , Deps.Prelude.Text.replace "\r" "\\r" - , Deps.Prelude.Text.replace "\n" "\\n" - , Deps.Prelude.Text.replace "\t" "\\t" - , Deps.Prelude.Text.replace "\"" "\\\"" + [ Prelude.Text.replace "\\" "\\\\" + , Prelude.Text.replace "\r" "\\r" + , Prelude.Text.replace "\n" "\\n" + , Prelude.Text.replace "\t" "\\t" + , Prelude.Text.replace "\"" "\\\"" ] raw let memberLines = - Deps.Prelude.Text.concatMapSep + Prelude.Text.concatMapSep "\n" Variant ( \(variant : Variant) -> @@ -43,4 +43,4 @@ let run = ${memberLines} '' -in Algebra.module Params run /\ { Variant } +in Sdk.Sigs.template Params run /\ { Variant } diff --git a/gen/Templates/FacadeModule.dhall b/gen/Templates/FacadeModule.dhall index 2beb74c..18d6f2e 100644 --- a/gen/Templates/FacadeModule.dhall +++ b/gen/Templates/FacadeModule.dhall @@ -1,8 +1,6 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Template.dhall - -let Prelude = Deps.Prelude +let Sdk = ../Deps/Sdk.dhall -- A statement's public surface: the function and, when the query returns rows, -- its frozen Row dataclass. functionName doubles as the leaf module name. @@ -126,4 +124,4 @@ let run = ${allEntries}] '' -in Algebra.module Params run /\ { StatementExport, TypeExport } +in Sdk.Sigs.template Params run /\ { StatementExport, TypeExport } diff --git a/gen/Templates/InitModule.dhall b/gen/Templates/InitModule.dhall index f609ed6..cf5fc9f 100644 --- a/gen/Templates/InitModule.dhall +++ b/gen/Templates/InitModule.dhall @@ -1,4 +1,4 @@ -let Algebra = ../Algebras/Template.dhall +let Sdk = ../Deps/Sdk.dhall -- A minimal package __init__.py: a module docstring plus an empty __all__. -- Used for the _generated subpackage root and the statements subpackage. The @@ -15,4 +15,4 @@ let render = __all__: list[str] = [] '' -in Algebra.module Params render +in Sdk.Sigs.template Params render diff --git a/gen/Templates/RegisterModule.dhall b/gen/Templates/RegisterModule.dhall index ec6886b..cc94bfb 100644 --- a/gen/Templates/RegisterModule.dhall +++ b/gen/Templates/RegisterModule.dhall @@ -1,11 +1,9 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Template.dhall +let Sdk = ../Deps/Sdk.dhall let Surface = ../Structures/Surface.dhall -let Prelude = Deps.Prelude - -- Per-connection type registration, emitted once per surface. psycopg decodes an -- unregistered composite as a text string; registering its CompositeInfo makes -- it decode to a namedtuple, which the generated decode then splats into the @@ -96,4 +94,4 @@ let run = in Prelude.Text.concatSep "\n" allLines ++ "\n" -in Algebra.module Params run +in Sdk.Sigs.template Params run diff --git a/gen/Templates/RowsModule.dhall b/gen/Templates/RowsModule.dhall index 0107fa2..18da31b 100644 --- a/gen/Templates/RowsModule.dhall +++ b/gen/Templates/RowsModule.dhall @@ -1,10 +1,10 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Template.dhall +let Lude = ../Deps/Lude.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Sdk = ../Deps/Sdk.dhall -let Prelude = Deps.Prelude +let ImportSet = ../Structures/ImportSet.dhall -- The shared row module: every query's frozen Row dataclass and its decode -- function live here once, so the async and sync statement modules import the @@ -25,7 +25,7 @@ let indentAll \(text : Text) -> let pad = Prelude.Text.replicate n " " - in pad ++ Deps.Lude.Text.indentNonEmpty n text + in pad ++ Lude.Text.indentNonEmpty n text let importLineIf : Bool -> Text -> List Text @@ -118,4 +118,4 @@ let run = ++ Prelude.Text.concatMapSep "\n\n\n" RowDef renderRow params.rows ++ "\n" -in Algebra.module Params run /\ { RowDef } +in Sdk.Sigs.template Params run /\ { RowDef } diff --git a/gen/Templates/RuntimeModule.dhall b/gen/Templates/RuntimeModule.dhall index ddaf53a..baa56bc 100644 --- a/gen/Templates/RuntimeModule.dhall +++ b/gen/Templates/RuntimeModule.dhall @@ -1,4 +1,4 @@ -let Algebra = ../Algebras/Template.dhall +let Sdk = ../Deps/Sdk.dhall -- The fixed _runtime.py body, emitted once per generated package. No per-query -- customization. Mirrors DESIGN section 3 with two strict-clean adjustments the @@ -160,5 +160,5 @@ let syncContent = _ = cur.execute(sql, params) '' -in Algebra.module {} (\(_ : {}) -> content) +in Sdk.Sigs.template {} (\(_ : {}) -> content) /\ { runSync = \(_ : {}) -> syncContent } diff --git a/gen/Templates/StatementModule.dhall b/gen/Templates/StatementModule.dhall index f717aa7..f2a6f61 100644 --- a/gen/Templates/StatementModule.dhall +++ b/gen/Templates/StatementModule.dhall @@ -1,13 +1,13 @@ -let Algebra = ../Algebras/Template.dhall +let Prelude = ../Deps/Prelude.dhall -let Deps = ../Deps/package.dhall +let Lude = ../Deps/Lude.dhall + +let Sdk = ../Deps/Sdk.dhall let ImportSet = ../Structures/ImportSet.dhall let Surface = ../Structures/Surface.dhall -let Prelude = Deps.Prelude - -- Prefix every line (including the first) with `n` spaces, leaving blank lines -- untouched so trailing whitespace never appears. let indentAll @@ -16,7 +16,7 @@ let indentAll \(text : Text) -> let pad = Prelude.Text.replicate n " " - in pad ++ Deps.Lude.Text.indentNonEmpty n text + in pad ++ Lude.Text.indentNonEmpty n text -- A statement module is the thin per-surface I/O wrapper: it imports its Row -- dataclass and decode function from the shared `_rows` module and renders one @@ -187,7 +187,7 @@ let renderCall then "return ${await}${params.helperName}(conn, _SQL, params, ${params.decodeName})" else "return ${await}${params.helperName}(conn, _SQL, params)" -in Algebra.module +in Sdk.Sigs.template Params ( \(params : Params) -> renderImports params diff --git a/gen/Templates/TypesInit.dhall b/gen/Templates/TypesInit.dhall index 7583526..3182085 100644 --- a/gen/Templates/TypesInit.dhall +++ b/gen/Templates/TypesInit.dhall @@ -1,6 +1,6 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Template.dhall +let Sdk = ../Deps/Sdk.dhall let Export = { moduleName : Text, typeName : Text } @@ -9,7 +9,7 @@ let Params = { exports : List Export } let run = \(params : Params) -> let exportLines = - Deps.Prelude.Text.concatMapSep + Prelude.Text.concatMapSep "\n" Export ( \(export : Export) -> @@ -21,4 +21,4 @@ let run = ${exportLines} '' -in Algebra.module Params run /\ { Export } +in Sdk.Sigs.template Params run /\ { Export } From a5b4530f477c839541374ac485fa7892e76f52ae Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 09:31:19 +0300 Subject: [PATCH 02/21] feat(gen): rewrite compile.dhall/Gen.dhall as Interpret.dhall/package.dhall Task 4 of the Dhall dependency/layout migration: consumes Interpreters/Project.dhall's Sdk.Sigs.interpreter-shaped run and produces the Sdk.Sigs.generator-built entry point, now that the Deps barrel is gone (Tasks 1-3). - gen/compile.dhall -> gen/Interpret.dhall: drops the outer Optional Config unwrap (Sdk.Sigs.generator substitutes defaultConfig for the omitted-block case), keeps the per-field Prelude.Optional.fold defaults and importName derivation; imports Contract/Prelude directly instead of through the deleted barrel. - gen/Gen.dhall -> gen/package.dhall: builds Sdk.Sigs.generator Config Config/default interpret, with Config/default supplying an all-None Config. --- gen/Gen.dhall | 3 -- gen/Interpret.dhall | 54 ++++++++++++++++++++++++++++++++++++ gen/compile.dhall | 67 --------------------------------------------- gen/package.dhall | 16 +++++++++++ 4 files changed, 70 insertions(+), 70 deletions(-) delete mode 100644 gen/Gen.dhall create mode 100644 gen/Interpret.dhall delete mode 100644 gen/compile.dhall create mode 100644 gen/package.dhall diff --git a/gen/Gen.dhall b/gen/Gen.dhall deleted file mode 100644 index 9cb87fb..0000000 --- a/gen/Gen.dhall +++ /dev/null @@ -1,3 +0,0 @@ -let Sdk = ./Deps/Sdk.dhall - -in Sdk.module ./Config.dhall ./compile.dhall diff --git a/gen/Interpret.dhall b/gen/Interpret.dhall new file mode 100644 index 0000000..de4642f --- /dev/null +++ b/gen/Interpret.dhall @@ -0,0 +1,54 @@ +let Contract = ./Deps/Contract.dhall + +let Prelude = ./Deps/Prelude.dhall + +let Config = ./Config.dhall + +let OnUnsupported = ./Structures/OnUnsupported.dhall + +let ProjectInterpreter = ./Interpreters/Project.dhall + +-- Entry point handed to gen-sdk's Sdk.Sigs.generator as `interpret`. Each +-- field of Config is independently Optional, so a project may omit the +-- whole config block (Sdk.Sigs.generator substitutes an all-None +-- defaultConfig, see package.dhall) or any subset of its keys; `defaults` +-- collects every fallback in one place (packageName from the project name in +-- kebab case, emitSync off, onUnsupported Fail). The async surface is always +-- emitted; emitSync adds the sync mirror. +in \(config : Config) -> + \(project : Contract.Project) -> + let defaults = + { packageName = project.name.inKebabCase + , emitSync = False + , onUnsupported = OnUnsupported.Mode.Fail + } + + let packageName = + Prelude.Optional.fold + Text + config.packageName + Text + (\(t : Text) -> t) + defaults.packageName + + let emitSync = + Prelude.Optional.fold + Bool + config.emitSync + Bool + (\(b : Bool) -> b) + defaults.emitSync + + let onUnsupported = + Prelude.Optional.fold + OnUnsupported.Mode + config.onUnsupported + OnUnsupported.Mode + (\(m : OnUnsupported.Mode) -> m) + defaults.onUnsupported + + let importName = Prelude.Text.replace "-" "_" packageName + + let interpreterConfig = { packageName, importName, emitSync, onUnsupported } + + in ProjectInterpreter.run interpreterConfig project diff --git a/gen/compile.dhall b/gen/compile.dhall deleted file mode 100644 index cd12642..0000000 --- a/gen/compile.dhall +++ /dev/null @@ -1,67 +0,0 @@ -let Deps = ./Deps/package.dhall - -let Model = Deps.Sdk.Project - -let Prelude = Deps.Prelude - -let Config = ./Config.dhall - -let OnUnsupported = ./Structures/OnUnsupported.dhall - -let ProjectInterpreter = ./Interpreters/Project.dhall - --- Entry point handed to gen-sdk's module. `config` and each of its fields are --- Optional, so a project may omit the whole config block or any subset of its --- keys; `defaults` collects every fallback in one place (packageName from the --- project name in kebab case, emitSync off, onUnsupported Fail), so a future --- knob's default is added here alongside the others. The async surface is --- always emitted; emitSync adds the sync mirror. -in \(config : Optional Config) -> - \(project : Model.Project) -> - let defaults = - { packageName = project.name.inKebabCase - , emitSync = False - , onUnsupported = OnUnsupported.Mode.Fail - } - - let packageName = - Prelude.Optional.fold - Config - config - Text - (\(c : Config) -> - Prelude.Optional.fold Text c.packageName Text (\(t : Text) -> t) defaults.packageName - ) - defaults.packageName - - let emitSync = - Prelude.Optional.fold - Config - config - Bool - (\(c : Config) -> - Prelude.Optional.fold Bool c.emitSync Bool (\(b : Bool) -> b) defaults.emitSync - ) - defaults.emitSync - - let onUnsupported = - Prelude.Optional.fold - Config - config - OnUnsupported.Mode - ( \(c : Config) -> - Prelude.Optional.fold - OnUnsupported.Mode - c.onUnsupported - OnUnsupported.Mode - (\(m : OnUnsupported.Mode) -> m) - defaults.onUnsupported - ) - defaults.onUnsupported - - let importName = Prelude.Text.replace "-" "_" packageName - - let interpreterConfig = - { packageName, importName, emitSync, onUnsupported } - - in ProjectInterpreter.run interpreterConfig project diff --git a/gen/package.dhall b/gen/package.dhall new file mode 100644 index 0000000..3d07cf7 --- /dev/null +++ b/gen/package.dhall @@ -0,0 +1,16 @@ +let Sdk = ./Deps/Sdk.dhall + +let OnUnsupported = ./Structures/OnUnsupported.dhall + +let Config = ./Config.dhall + +let Config/default + : Config + = { packageName = None Text + , emitSync = None Bool + , onUnsupported = None OnUnsupported.Mode + } + +let interpret = ./Interpret.dhall + +in Sdk.Sigs.generator Config Config/default interpret From b13e42764765280e075d9390e9cea31dd5e3db8c Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 09:38:42 +0300 Subject: [PATCH 03/21] refactor(gen): rename gen/ to src/, move tests/Exhaustive.dhall to demos/ Pure directory move for gen/ -> src/ (relative imports inside are unaffected). tests/Exhaustive.dhall becomes demos/Exhaustive.dhall and is rewritten to use Sdk.Output.toFileMap, since Sdk.Sigs.generator-built modules no longer expose compileToFileMap. --- {tests => demos}/Exhaustive.dhall | 16 ++++++++-------- {gen => src}/Config.dhall | 0 {gen => src}/Deps/Contract.dhall | 0 {gen => src}/Deps/Lude.dhall | 0 {gen => src}/Deps/Prelude.dhall | 0 {gen => src}/Deps/Sdk.dhall | 0 {gen => src}/Interpret.dhall | 0 {gen => src}/Interpreters/CustomType.dhall | 0 {gen => src}/Interpreters/Member.dhall | 0 {gen => src}/Interpreters/ParamsMember.dhall | 0 {gen => src}/Interpreters/Primitive.dhall | 0 {gen => src}/Interpreters/Project.dhall | 0 {gen => src}/Interpreters/Query.dhall | 0 {gen => src}/Interpreters/QueryFragments.dhall | 0 {gen => src}/Interpreters/Result.dhall | 0 {gen => src}/Interpreters/ResultColumns.dhall | 0 {gen => src}/Interpreters/Scalar.dhall | 0 {gen => src}/Interpreters/Value.dhall | 0 {gen => src}/Structures/CustomKind.dhall | 0 {gen => src}/Structures/ImportSet.dhall | 0 {gen => src}/Structures/OnUnsupported.dhall | 0 {gen => src}/Structures/PyIdent.dhall | 0 {gen => src}/Structures/Surface.dhall | 0 {gen => src}/Templates/CompositeModule.dhall | 0 {gen => src}/Templates/CoreModule.dhall | 0 {gen => src}/Templates/EnumModule.dhall | 0 {gen => src}/Templates/FacadeModule.dhall | 0 {gen => src}/Templates/InitModule.dhall | 0 {gen => src}/Templates/RegisterModule.dhall | 0 {gen => src}/Templates/RowsModule.dhall | 0 {gen => src}/Templates/RuntimeModule.dhall | 0 {gen => src}/Templates/StatementModule.dhall | 0 {gen => src}/Templates/TypesInit.dhall | 0 {gen => src}/package.dhall | 0 34 files changed, 8 insertions(+), 8 deletions(-) rename {tests => demos}/Exhaustive.dhall (63%) rename {gen => src}/Config.dhall (100%) rename {gen => src}/Deps/Contract.dhall (100%) rename {gen => src}/Deps/Lude.dhall (100%) rename {gen => src}/Deps/Prelude.dhall (100%) rename {gen => src}/Deps/Sdk.dhall (100%) rename {gen => src}/Interpret.dhall (100%) rename {gen => src}/Interpreters/CustomType.dhall (100%) rename {gen => src}/Interpreters/Member.dhall (100%) rename {gen => src}/Interpreters/ParamsMember.dhall (100%) rename {gen => src}/Interpreters/Primitive.dhall (100%) rename {gen => src}/Interpreters/Project.dhall (100%) rename {gen => src}/Interpreters/Query.dhall (100%) rename {gen => src}/Interpreters/QueryFragments.dhall (100%) rename {gen => src}/Interpreters/Result.dhall (100%) rename {gen => src}/Interpreters/ResultColumns.dhall (100%) rename {gen => src}/Interpreters/Scalar.dhall (100%) rename {gen => src}/Interpreters/Value.dhall (100%) rename {gen => src}/Structures/CustomKind.dhall (100%) rename {gen => src}/Structures/ImportSet.dhall (100%) rename {gen => src}/Structures/OnUnsupported.dhall (100%) rename {gen => src}/Structures/PyIdent.dhall (100%) rename {gen => src}/Structures/Surface.dhall (100%) rename {gen => src}/Templates/CompositeModule.dhall (100%) rename {gen => src}/Templates/CoreModule.dhall (100%) rename {gen => src}/Templates/EnumModule.dhall (100%) rename {gen => src}/Templates/FacadeModule.dhall (100%) rename {gen => src}/Templates/InitModule.dhall (100%) rename {gen => src}/Templates/RegisterModule.dhall (100%) rename {gen => src}/Templates/RowsModule.dhall (100%) rename {gen => src}/Templates/RuntimeModule.dhall (100%) rename {gen => src}/Templates/StatementModule.dhall (100%) rename {gen => src}/Templates/TypesInit.dhall (100%) rename {gen => src}/package.dhall (100%) diff --git a/tests/Exhaustive.dhall b/demos/Exhaustive.dhall similarity index 63% rename from tests/Exhaustive.dhall rename to demos/Exhaustive.dhall index bfe8b84..35e8f28 100644 --- a/tests/Exhaustive.dhall +++ b/demos/Exhaustive.dhall @@ -1,9 +1,9 @@ -- Applies this generator to gen-sdk's shared cross-backend fixture project --- (the same "music_catalogue" project java.gen's own tests/Exhaustive.dhall +-- (the same "music_catalogue" project java.gen's own demos/Exhaustive.dhall -- exercises), so a Python client compiles from it and passes basedpyright -- strict. Pinned directly at gen-sdk's package.dhall, separately from --- gen/Deps/Sdk.dhall: that file only imports gen-sdk's `module.dhall` (the --- generator-construction function), which has no `Fixtures` field. +-- src/Deps/Sdk.dhall: that file only imports gen-sdk's `package.dhall` `as +-- Source` for RAM, and this fixture load doesn't need that mode. -- -- The fixture project deliberately covers PG types this generator does not -- support (box, inet, money, ranges, ...), so onUnsupported is set to Skip: @@ -13,13 +13,13 @@ -- Intended to be executed with: -- -- ```bash --- dhall to-directory-tree --file tests/Exhaustive.dhall --output --allow-path-separators +-- dhall to-directory-tree --file demos/Exhaustive.dhall --output --allow-path-separators -- ``` -let Sdk = ../gen/Deps/Sdk.dhall +let Sdk = ../src/Deps/Sdk.dhall -let Gen = ../gen/Gen.dhall +let Gen = ../src/package.dhall -let OnUnsupported = ../gen/Structures/OnUnsupported.dhall +let OnUnsupported = ../src/Structures/OnUnsupported.dhall let project = Sdk.Fixtures.Exhaustive @@ -30,4 +30,4 @@ let config = , onUnsupported = Some OnUnsupported.Mode.Skip } -in Gen.compileToFileMap config project +in Sdk.Output.toFileMap (Gen.compile config project) diff --git a/gen/Config.dhall b/src/Config.dhall similarity index 100% rename from gen/Config.dhall rename to src/Config.dhall diff --git a/gen/Deps/Contract.dhall b/src/Deps/Contract.dhall similarity index 100% rename from gen/Deps/Contract.dhall rename to src/Deps/Contract.dhall diff --git a/gen/Deps/Lude.dhall b/src/Deps/Lude.dhall similarity index 100% rename from gen/Deps/Lude.dhall rename to src/Deps/Lude.dhall diff --git a/gen/Deps/Prelude.dhall b/src/Deps/Prelude.dhall similarity index 100% rename from gen/Deps/Prelude.dhall rename to src/Deps/Prelude.dhall diff --git a/gen/Deps/Sdk.dhall b/src/Deps/Sdk.dhall similarity index 100% rename from gen/Deps/Sdk.dhall rename to src/Deps/Sdk.dhall diff --git a/gen/Interpret.dhall b/src/Interpret.dhall similarity index 100% rename from gen/Interpret.dhall rename to src/Interpret.dhall diff --git a/gen/Interpreters/CustomType.dhall b/src/Interpreters/CustomType.dhall similarity index 100% rename from gen/Interpreters/CustomType.dhall rename to src/Interpreters/CustomType.dhall diff --git a/gen/Interpreters/Member.dhall b/src/Interpreters/Member.dhall similarity index 100% rename from gen/Interpreters/Member.dhall rename to src/Interpreters/Member.dhall diff --git a/gen/Interpreters/ParamsMember.dhall b/src/Interpreters/ParamsMember.dhall similarity index 100% rename from gen/Interpreters/ParamsMember.dhall rename to src/Interpreters/ParamsMember.dhall diff --git a/gen/Interpreters/Primitive.dhall b/src/Interpreters/Primitive.dhall similarity index 100% rename from gen/Interpreters/Primitive.dhall rename to src/Interpreters/Primitive.dhall diff --git a/gen/Interpreters/Project.dhall b/src/Interpreters/Project.dhall similarity index 100% rename from gen/Interpreters/Project.dhall rename to src/Interpreters/Project.dhall diff --git a/gen/Interpreters/Query.dhall b/src/Interpreters/Query.dhall similarity index 100% rename from gen/Interpreters/Query.dhall rename to src/Interpreters/Query.dhall diff --git a/gen/Interpreters/QueryFragments.dhall b/src/Interpreters/QueryFragments.dhall similarity index 100% rename from gen/Interpreters/QueryFragments.dhall rename to src/Interpreters/QueryFragments.dhall diff --git a/gen/Interpreters/Result.dhall b/src/Interpreters/Result.dhall similarity index 100% rename from gen/Interpreters/Result.dhall rename to src/Interpreters/Result.dhall diff --git a/gen/Interpreters/ResultColumns.dhall b/src/Interpreters/ResultColumns.dhall similarity index 100% rename from gen/Interpreters/ResultColumns.dhall rename to src/Interpreters/ResultColumns.dhall diff --git a/gen/Interpreters/Scalar.dhall b/src/Interpreters/Scalar.dhall similarity index 100% rename from gen/Interpreters/Scalar.dhall rename to src/Interpreters/Scalar.dhall diff --git a/gen/Interpreters/Value.dhall b/src/Interpreters/Value.dhall similarity index 100% rename from gen/Interpreters/Value.dhall rename to src/Interpreters/Value.dhall diff --git a/gen/Structures/CustomKind.dhall b/src/Structures/CustomKind.dhall similarity index 100% rename from gen/Structures/CustomKind.dhall rename to src/Structures/CustomKind.dhall diff --git a/gen/Structures/ImportSet.dhall b/src/Structures/ImportSet.dhall similarity index 100% rename from gen/Structures/ImportSet.dhall rename to src/Structures/ImportSet.dhall diff --git a/gen/Structures/OnUnsupported.dhall b/src/Structures/OnUnsupported.dhall similarity index 100% rename from gen/Structures/OnUnsupported.dhall rename to src/Structures/OnUnsupported.dhall diff --git a/gen/Structures/PyIdent.dhall b/src/Structures/PyIdent.dhall similarity index 100% rename from gen/Structures/PyIdent.dhall rename to src/Structures/PyIdent.dhall diff --git a/gen/Structures/Surface.dhall b/src/Structures/Surface.dhall similarity index 100% rename from gen/Structures/Surface.dhall rename to src/Structures/Surface.dhall diff --git a/gen/Templates/CompositeModule.dhall b/src/Templates/CompositeModule.dhall similarity index 100% rename from gen/Templates/CompositeModule.dhall rename to src/Templates/CompositeModule.dhall diff --git a/gen/Templates/CoreModule.dhall b/src/Templates/CoreModule.dhall similarity index 100% rename from gen/Templates/CoreModule.dhall rename to src/Templates/CoreModule.dhall diff --git a/gen/Templates/EnumModule.dhall b/src/Templates/EnumModule.dhall similarity index 100% rename from gen/Templates/EnumModule.dhall rename to src/Templates/EnumModule.dhall diff --git a/gen/Templates/FacadeModule.dhall b/src/Templates/FacadeModule.dhall similarity index 100% rename from gen/Templates/FacadeModule.dhall rename to src/Templates/FacadeModule.dhall diff --git a/gen/Templates/InitModule.dhall b/src/Templates/InitModule.dhall similarity index 100% rename from gen/Templates/InitModule.dhall rename to src/Templates/InitModule.dhall diff --git a/gen/Templates/RegisterModule.dhall b/src/Templates/RegisterModule.dhall similarity index 100% rename from gen/Templates/RegisterModule.dhall rename to src/Templates/RegisterModule.dhall diff --git a/gen/Templates/RowsModule.dhall b/src/Templates/RowsModule.dhall similarity index 100% rename from gen/Templates/RowsModule.dhall rename to src/Templates/RowsModule.dhall diff --git a/gen/Templates/RuntimeModule.dhall b/src/Templates/RuntimeModule.dhall similarity index 100% rename from gen/Templates/RuntimeModule.dhall rename to src/Templates/RuntimeModule.dhall diff --git a/gen/Templates/StatementModule.dhall b/src/Templates/StatementModule.dhall similarity index 100% rename from gen/Templates/StatementModule.dhall rename to src/Templates/StatementModule.dhall diff --git a/gen/Templates/TypesInit.dhall b/src/Templates/TypesInit.dhall similarity index 100% rename from gen/Templates/TypesInit.dhall rename to src/Templates/TypesInit.dhall diff --git a/gen/package.dhall b/src/package.dhall similarity index 100% rename from gen/package.dhall rename to src/package.dhall From 2c9073fc3716a54536e77f2cebfe2878c2b21d0a Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 10:01:44 +0300 Subject: [PATCH 04/21] refactor(gen): update path references from gen/ to src/, tests/Exhaustive.dhall to demos/ Tasks 1-5 renamed gen/ to src/, gen/Gen.dhall to src/package.dhall, and tests/Exhaustive.dhall to demos/Exhaustive.dhall. This updates every external reference to the old paths across CI workflows, dev scripts, docs, and the bench harness. - .github/workflows/ci.yml, release.yml, build-contract-shell.sh: path refs. - README.md, AGENTS.md, DESIGN.md: doc path refs; DESIGN.md's section 10 tree diagram and entry-point description rewritten to match the actual post-move src/ layout (Interpret.dhall/package.dhall, no more Algebras/, Sdk.Sigs.generator instead of Sdk.module). - build.bash, mise.toml: dev-script path refs. - bench/generate.sh, bench/as-source.sh: gen/ -> src/ copies; the as-Source/plain-import sha256 substitution pair for gen-sdk is verified via `dhall hash` to be a no-op for v2.0.0 (both modes hash to the same value, unlike the old v0.11.0 pair), so only a comment remains; the lude pair is untouched since lude's pin is unchanged by this migration. - tests/fixture-project/project1.pgn.yaml: the fixture project's own `gen:` key still pointed at the now-deleted gen/Gen.dhall, which would break the harness and the `golden` mise task; updated to src/package.dhall. --- .github/scripts/build-contract-shell.sh | 2 +- .github/workflows/ci.yml | 8 ++--- .github/workflows/release.yml | 2 +- AGENTS.md | 2 +- DESIGN.md | 46 ++++++++++++++----------- README.md | 10 +++--- bench/as-source.sh | 24 ++++++++++--- bench/generate.sh | 26 ++++++++++---- mise.toml | 8 ++--- tests/fixture-project/project1.pgn.yaml | 14 ++++---- 10 files changed, 87 insertions(+), 55 deletions(-) diff --git a/.github/scripts/build-contract-shell.sh b/.github/scripts/build-contract-shell.sh index 3845a1d..9fbc79c 100755 --- a/.github/scripts/build-contract-shell.sh +++ b/.github/scripts/build-contract-shell.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Wraps the Dhall-generated package (from tests/Exhaustive.dhall) in a minimal +# Wraps the Dhall-generated package (from demos/Exhaustive.dhall) in a minimal # consumer shell, mirroring the hand-written tests/golden/ shell (pyproject.toml # + py.typed) so basedpyright strict runs against the same layout a real # consumer would import, per the full_package pattern in tests/conftest.py. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7664a04..b5d526f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,7 +121,7 @@ jobs: with: ref: ${{ inputs.ref || github.sha }} - # The `as Source` import mode on gen/Deps only changes how pgn loads the + # The `as Source` import mode on src/Deps only changes how pgn loads the # remote packages (unnormalized, to save RAM), not what they evaluate to. # The pinned action below bundles a dhall fork that predates the mode, so # strip it here; the evaluation is semantically identical either way. The @@ -132,16 +132,16 @@ jobs: shell: bash run: | set -euo pipefail - sed -i -e 's/^[[:space:]]*as Source$//' -e 's/^[[:space:]]*sha256:[0-9a-f]\{64\}$//' gen/Deps/*.dhall + sed -i -e 's/^[[:space:]]*as Source$//' -e 's/^[[:space:]]*sha256:[0-9a-f]\{64\}$//' src/Deps/*.dhall - # A plain, standard-Dhall evaluator cannot run this: gen/Interpreters/Project.dhall's + # A plain, standard-Dhall evaluator cannot run this: src/Interpreters/Project.dhall's # buildLookup and gen-sdk's own Fixtures.Exhaustive both use Text/equal, a builtin # from pgn's forked Dhall, absent from the upstream dhall-lang Prelude. This action # bundles that same fork. - name: Generate output from Dhall uses: nikita-volkov/dhall-directory-tree.github-action@60a18dc647d6daea805263ea0fed7bb8011f3bcd # v2 with: - dhall_file: tests/Exhaustive.dhall + dhall_file: demos/Exhaustive.dhall output_dir: contract-output - name: Assert the compile did not fail diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 25e92c7..91764bc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -86,7 +86,7 @@ jobs: id: resolve uses: nikita-volkov/dhall-resolve.github-action@7caaf1fdb40ac864bc02e37575f577ee084713a8 # v3 with: - file: gen/Gen.dhall + file: src/package.dhall minify: true - name: Prepare changelog for release diff --git a/AGENTS.md b/AGENTS.md index f6d2533..7da3b7b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,6 +33,6 @@ already says. ## Dhall -`gen/` pins its remote imports by sha256 (`gen/Deps/*.dhall`). Bump those +`src/` pins its remote imports by sha256 (`src/Deps/*.dhall`). Bump those deliberately, one at a time, and re-run the harness before committing a pin change. diff --git a/DESIGN.md b/DESIGN.md index bff5a2c..79e8a7c 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,7 +1,7 @@ # python.gen DESIGN Status: current. This describes the generator as shipped, not a plan. The code -is the source of truth: the Dhall generator under `gen/`, the fixture project +is the source of truth: the Dhall generator under `src/`, the fixture project and golden output under `tests/`. When this doc and the tree disagree, the tree wins. @@ -378,34 +378,37 @@ overwritten on every run. Do not hand-edit it. ## 10. Generator decomposition -`gen/Gen.dhall` is the entry point handed to gen-sdk: +`src/package.dhall` is the entry point handed to gen-sdk: ```dhall let Sdk = ./Deps/Sdk.dhall -in Sdk ./Config.dhall ./compile.dhall +let Config = ./Config.dhall + +let interpret = ./Interpret.dhall + +in Sdk.Sigs.generator Config Config/default interpret ``` -The Sdk `module` function has signature `\(Config : Type) -> \(compile) -> -{ contractVersion, Config, compile, compileToFileMap }`, and -`compile : Optional Config -> Project -> Lude.Compiled.Type Lude.Files.Type`, -where `Files.Type = List { path : Text, content : Text }`. `compile.dhall` -folds the optional user config into the internal interpreter config and -calls `Interpreters/Project.dhall`, which traverses queries and custom types -and assembles the file list. +`Sdk.Sigs.generator` has signature `\(Config : Type) -> \(defaultConfig : Config) -> +\(interpret : Config -> Contract.Project -> Contract.Output) -> ...`; it curries +`interpret` against `defaultConfig` whenever the user config is absent and hands +the result to gen-contract's `Contract.module`. `Interpret.dhall` folds the +optional user config into the internal interpreter config and calls +`Interpreters/Project.dhall`, which traverses queries and custom types and +assembles the file list (`Contract.Output`). -`gen/` mirrors a typical pgn gen-sdk generator, Python-flavored. The -algebra/interpreter/template split keeps assembly separate from rendering. +`src/` mirrors a typical pgn gen-sdk generator, Python-flavored: `Interpreters/` +assembles data, `Templates/` renders it to Python text. The interpreter/template +algebra signatures themselves live in gen-sdk's `Sdk.Sigs` (`interpreter.dhall`/ +`template.dhall`), not a local `Algebras/` dir. ```text -gen/ - Gen.dhall # Sdk Config compile (entry handed to gen-sdk) +src/ + package.dhall # Sdk.Sigs.generator Config Config/default interpret (entry handed to gen-sdk) Config.dhall # user config TYPE: { packageName, emitSync, onUnsupported } - compile.dhall # derive interpreter Config from user Config, call Project.run - Deps/ # pinned remote imports (gen-sdk module + Project, lude, Prelude) - Algebras/ - Interpreter.dhall # Config + `module Input Output run` ; Run = Config -> Input -> Compiled Output - Template.dhall # `module Params run` ; Run = Params -> Text + Interpret.dhall # derive interpreter Config from user Config, call Project.run + Deps/ # pinned remote imports: gen-sdk, gen-contract, lude, dhall Prelude Structures/ Surface.dhall # async/sync token table (section 4) CustomKind.dhall # Lookup : Name -> < Enum | Composite | Absent > + composite fields @@ -426,6 +429,7 @@ gen/ Project.dhall # traverse queries+customTypes, assemble all files + facade + header, # apply the Skip filter (section 11) Templates/ + CoreModule.dhall # shared _core.py: JsonValue, NoRowError/DecodeError, require_array RuntimeModule.dhall # async + sync _runtime.py bodies RowsModule.dhall # shared _rows.py (Row dataclasses + decode fns) StatementModule.dhall # one per-surface statement wrapper @@ -629,11 +633,11 @@ SQL rendering are largely driver-agnostic. CI runs two independent jobs (`.github/workflows/ci.yml`): `harness` (the pytest suite against a live Postgres) and `contract` (compiles gen-sdk's -`Fixtures.Exhaustive` via `tests/Exhaustive.dhall` and runs basedpyright +`Fixtures.Exhaustive` via `demos/Exhaustive.dhall` and runs basedpyright strict on the result). The `contract` job needs `nikita-volkov/dhall-directory-tree.github-action`, a Docker action bundling a forked Dhall evaluator; the local `dhall` CLI most people have installed is the standard dhall-lang build and does not -understand `Text/equal`, so it cannot run `tests/Exhaustive.dhall` directly. +understand `Text/equal`, so it cannot run `demos/Exhaustive.dhall` directly. Reproduce the `contract` job locally with [`act`](https://github.com/nektos/act) (not installed in this environment; `act -j contract` pulls the same pinned Docker action and runs the job as GitHub would). diff --git a/README.md b/README.md index db3226f..9e9a17f 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ name: ```yaml artifacts: python: - gen: https://raw.githubusercontent.com/slavashvets/python.gen/master/gen/Gen.dhall + gen: https://raw.githubusercontent.com/slavashvets/python.gen/master/src/package.dhall config: packageName: my-db-client emitSync: true @@ -61,10 +61,10 @@ database. | form | example | works? | | --------------- | --------------------------------------- | ------------------------------------------ | -| plain http(s) | `https://.../python.gen/gen/Gen.dhall` | yes; pgn fetches it and every relative import over HTTP | -| relative path | `../path/to/python.gen/gen/Gen.dhall` | yes, if you keep a local checkout next to your project | -| absolute path | `/abs/path/to/python.gen/gen/Gen.dhall` | yes, but the resulting freeze key is machine-specific | -| `file://` URL | `file:///abs/.../Gen.dhall` | rejected; pgn's project schema does not accept `file://` | +| plain http(s) | `https://.../python.gen/src/package.dhall` | yes; pgn fetches it and every relative import over HTTP | +| relative path | `../path/to/python.gen/src/package.dhall` | yes, if you keep a local checkout next to your project | +| absolute path | `/abs/path/to/python.gen/src/package.dhall` | yes, but the resulting freeze key is machine-specific | +| `file://` URL | `file:///abs/.../package.dhall` | rejected; pgn's project schema does not accept `file://` | Whichever form you use, the freeze file that caches the resolved generator (section "Freeze lifecycle" below) keys on the literal `gen:` value and diff --git a/bench/as-source.sh b/bench/as-source.sh index 203b973..c16945b 100755 --- a/bench/as-source.sh +++ b/bench/as-source.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Cold-cache benchmark of `pgn generate` on the fixture project, comparing the -# current gen (Deps imported `as Source`) against the same tree with the +# current src (Deps imported `as Source`) against the same tree with the # pre-as-Source Deps (plain pinned imports). Both variants run the same pgn # binary by default, so the measurement isolates the import mode itself. # @@ -24,7 +24,7 @@ trap 'rm -rf "$work"' EXIT prepare() { # $1 = variant root; mirrors the repo layout the fixture expects rm -rf "$1" && mkdir -p "$1/tests" - cp -R "$root/gen" "$1/gen" + cp -R "$root/src" "$1/src" cp -R "$root/tests/fixture-project" "$1/tests/fixture-project" rm -f "$1/tests/fixture-project/freeze1.pgn.yaml" rm -rf "$1/tests/fixture-project/artifacts" @@ -34,9 +34,23 @@ prepare() { # $1 = variant root; mirrors the repo layout the fixture expects # normalized-expression pins (an `as Source` pin hashes the import's source, # so the two modes need different sha256 values for the same version). strip_as_source() { # $1 = variant root - perl -i -ne 'print unless /^\s*as Source$/' "$1"/gen/Deps/*.dhall - perl -i -pe 's/8d43544ecb0e612406af3133bdbca51138c704a77a5a29ef62fe034d0e77a3a6/b9f7bb842345f3864c71e877fda4200306ba5c044a43e6f7713a23bc4769b91a/' "$1/gen/Deps/Sdk.dhall" - perl -i -pe 's/46b527b071eba96a17e76b4bc5774645714dd5b4355974d221e705aa7c126e77/14c43eec97972ae27afe3386ff937d04db66f84273d5551476361db12d2c4b50/' "$1/gen/Deps/Lude.dhall" + perl -i -ne 'print unless /^\s*as Source$/' "$1"/src/Deps/*.dhall + + # gen-sdk v2.0.0's src/package.dhall: `mise x -- dhall hash` against the + # live GitHub-hosted package (both `... as Source` and the plain import) + # returns the SAME sha256 (b9def6ab1179bc4aaae7fc6e91977f094f75934cd5755175c294a9e97ca71b15), + # matching the value already committed in src/Deps/Sdk.dhall -- so, unlike + # the old v0.11.0 pin this pair used to target (where the two genuinely + # differed: 8d43544e...->b9f7bb84...), no character swap is needed here + # after the strip above; the committed pin already equals the plain-import + # target. See docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md + # Task 6 for how this was verified (dhall's local import cache made the + # lookup instant; a cold, uncached fetch of a *different* URL hung in this + # sandbox, so treat network reachability here as best-effort, not given). + + # lude v5.1.0 is unchanged by this migration (see src/Deps/Lude.dhall), so + # its existing as-Source -> plain-import hash swap below is still correct. + perl -i -pe 's/46b527b071eba96a17e76b4bc5774645714dd5b4355974d221e705aa7c126e77/14c43eec97972ae27afe3386ff937d04db66f84273d5551476361db12d2c4b50/' "$1/src/Deps/Lude.dhall" } measure() { # $1 = label, $2 = variant root, $3 = pgn binary diff --git a/bench/generate.sh b/bench/generate.sh index a2acd50..2769621 100755 --- a/bench/generate.sh +++ b/bench/generate.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Build the fixture client from the working-tree gen, in one of two variants: -# with - gen/Deps as committed (imports `as Source`) +# Build the fixture client from the working-tree src, in one of two variants: +# with - src/Deps as committed (imports `as Source`) # without - the pre-as-Source Deps (mode stripped, normalized-expression # pins restored; byte-identical to commit fb78869's Deps) # Output lands in ./demo-with-as-source or ./demo-without-as-source. @@ -15,15 +15,29 @@ url="${PGN_TEST_DATABASE_URL:-postgresql://postgres:postgres@localhost:5432/post out="$root/demo-$variant-as-source" rm -rf "$out" && mkdir -p "$out/tests" -cp -R "$root/gen" "$out/gen" +cp -R "$root/src" "$out/src" cp -R "$root/tests/fixture-project" "$out/tests/fixture-project" rm -f "$out/tests/fixture-project/freeze1.pgn.yaml" rm -rf "$out/tests/fixture-project/artifacts" if [ "$variant" = without ]; then - perl -i -ne 'print unless /^\s*as Source$/' "$out"/gen/Deps/*.dhall - perl -i -pe 's/8d43544ecb0e612406af3133bdbca51138c704a77a5a29ef62fe034d0e77a3a6/b9f7bb842345f3864c71e877fda4200306ba5c044a43e6f7713a23bc4769b91a/' "$out/gen/Deps/Sdk.dhall" - perl -i -pe 's/46b527b071eba96a17e76b4bc5774645714dd5b4355974d221e705aa7c126e77/14c43eec97972ae27afe3386ff937d04db66f84273d5551476361db12d2c4b50/' "$out/gen/Deps/Lude.dhall" + perl -i -ne 'print unless /^\s*as Source$/' "$out"/src/Deps/*.dhall + + # gen-sdk v2.0.0's src/package.dhall: `mise x -- dhall hash` against the + # live GitHub-hosted package (both `... as Source` and the plain import) + # returns the SAME sha256 (b9def6ab1179bc4aaae7fc6e91977f094f75934cd5755175c294a9e97ca71b15), + # matching the value already committed in src/Deps/Sdk.dhall -- so, unlike + # the old v0.11.0 pin this pair used to target (where the two genuinely + # differed: 8d43544e...->b9f7bb84...), no character swap is needed here + # after the strip above; the committed pin already equals the plain-import + # target. See docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md + # Task 6 for how this was verified (dhall's local import cache made the + # lookup instant; a cold, uncached fetch of a *different* URL hung in this + # sandbox, so treat network reachability here as best-effort, not given). + + # lude v5.1.0 is unchanged by this migration (see src/Deps/Lude.dhall), so + # its existing as-Source -> plain-import hash swap below is still correct. + perl -i -pe 's/46b527b071eba96a17e76b4bc5774645714dd5b4355974d221e705aa7c126e77/14c43eec97972ae27afe3386ff937d04db66f84273d5551476361db12d2c4b50/' "$out/src/Deps/Lude.dhall" fi cd "$out/tests/fixture-project" diff --git a/mise.toml b/mise.toml index 040920b..68903bf 100644 --- a/mise.toml +++ b/mise.toml @@ -35,10 +35,10 @@ run = "bench/generate.sh without" # Regenerates only the "python" artifact: a full 7-artifact generate peaks at # ~31 GB RSS (memory goes to normalizing the generator closure per artifact, # see ci.yml), a single-artifact one at ~10 GB. The temp copy guarantees a -# fresh resolve of the working-tree gen/ (no stale freeze) and keeps pgn's +# fresh resolve of the working-tree src/ (no stale freeze) and keeps pgn's # scratch files out of the repo. [tasks.golden] -description = "Refresh tests/golden from the working-tree gen/ (single-artifact run)" +description = "Refresh tests/golden from the working-tree src/ (single-artifact run)" run = ''' #!/usr/bin/env bash set -euo pipefail @@ -48,14 +48,14 @@ trap 'rm -rf "$tmp"' EXIT cp -R "$root/tests/fixture-project/." "$tmp/fixture" rm -f "$tmp/fixture/freeze1.pgn.yaml" rm -rf "$tmp/fixture/artifacts" -python3 - "$tmp/fixture/project1.pgn.yaml" "$root/gen/Gen.dhall" <<'EOF' +python3 - "$tmp/fixture/project1.pgn.yaml" "$root/src/package.dhall" <<'EOF' import sys from pathlib import Path p = Path(sys.argv[1]) head, sep, _ = p.read_text().partition(" # The variants below") assert sep, "variants marker not found in project1.pgn.yaml" -_ = p.write_text(head.rstrip().replace("../../gen/Gen.dhall", sys.argv[2]) + "\n") +_ = p.write_text(head.rstrip().replace("../../src/package.dhall", sys.argv[2]) + "\n") EOF cd "$tmp/fixture" pgn --database-url "${PGN_TEST_DATABASE_URL:-postgresql://postgres:postgres@localhost:5432/postgres?sslmode=disable}" generate diff --git a/tests/fixture-project/project1.pgn.yaml b/tests/fixture-project/project1.pgn.yaml index f6bec82..ff75999 100644 --- a/tests/fixture-project/project1.pgn.yaml +++ b/tests/fixture-project/project1.pgn.yaml @@ -4,7 +4,7 @@ version: 0.0.0 postgres: 18 artifacts: python: - gen: ../../gen/Gen.dhall + gen: ../../src/package.dhall config: packageName: specimen-client emitSync: true @@ -12,25 +12,25 @@ artifacts: # Optional Config knobs (undocumented by pgn itself); see # tests/test_config_variants.py for the assertions. python-name-only: - gen: ../../gen/Gen.dhall + gen: ../../src/package.dhall config: packageName: name-only-client python-sync-only: - gen: ../../gen/Gen.dhall + gen: ../../src/package.dhall config: emitSync: true python-empty: - gen: ../../gen/Gen.dhall + gen: ../../src/package.dhall config: {} python-bare: - gen: ../../gen/Gen.dhall + gen: ../../src/package.dhall python-unknown-key: - gen: ../../gen/Gen.dhall + gen: ../../src/package.dhall config: packageName: unknown-key-client bogusField: 1 python-null: - gen: ../../gen/Gen.dhall + gen: ../../src/package.dhall config: packageName: null-client emitSync: null From a30ac3209dfc7556fe82bacb2b7ca7ac52d5f359 Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 10:05:34 +0300 Subject: [PATCH 05/21] fix(gen): update gen/ path references in the pytest harness to src/ tests/_harness.py's GEN_DIR (and its two consumers) still pointed the harness at the repo-root gen/ directory, which Task 5 deleted when it renamed gen/ to src/. Every test using the session-scoped generated_tree fixture, or building its own copytree in test_unsupported_types.py, would have hit FileNotFoundError on the very first pgn run. Renamed the constant to SRC_DIR (was misleading otherwise), pointed it at src/, matched the copied-tree destination directory name to what project1.pgn.yaml's `gen: ../../src/package.dhall` (fixed in the prior commit) actually resolves against, and fixed the two hardcoded yaml literals in test_unsupported_types.py plus conftest.py's docstring. Confirmed via `python3 -m py_compile` and a repo-wide grep for any remaining gen/-as-directory or GEN_DIR references in .py files (none). --- tests/_harness.py | 2 +- tests/conftest.py | 10 +++++----- tests/test_unsupported_types.py | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/_harness.py b/tests/_harness.py index 037ac41..9e5ffbd 100644 --- a/tests/_harness.py +++ b/tests/_harness.py @@ -23,7 +23,7 @@ DEFAULT_MAX_RSS_GB = 40.0 HERE = Path(__file__).resolve().parent -GEN_DIR = HERE.parent / "gen" +SRC_DIR = HERE.parent / "src" FIXTURE_PROJECT = HERE / "fixture-project" GOLDEN_DIR = HERE / "golden" diff --git a/tests/conftest.py b/tests/conftest.py index d0ccc23..5bac571 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,9 +15,9 @@ from tests._harness import ( FIXTURE_PROJECT, - GEN_DIR, GOLDEN_DIR, HERE, + SRC_DIR, admin_database_url, effective_database_name, run_pgn, @@ -67,15 +67,15 @@ def fixture_copy(tmp_path: Path) -> Path: def generated_tree(pgn_bin: str, pgn_admin_url: str, tmp_path_factory: pytest.TempPathFactory) -> Path: """Generate the fixture client once and return its artifacts/python dir. - The generator path in project1.pgn.yaml is `../../gen/Gen.dhall`, + The generator path in project1.pgn.yaml is `../../src/package.dhall`, relative to the fixture project. To keep it resolvable the copy mirrors the - real layout: `/gen` and `/tests/fixture-project`. The freeze + real layout: `/src` and `/tests/fixture-project`. The freeze file is dropped so pgn re-resolves the working-tree generator instead of a - cached hash (a stale freeze makes pgn ignore gen/ edits and silently + cached hash (a stale freeze makes pgn ignore src/ edits and silently emit the old output). """ root = tmp_path_factory.mktemp("pygen") - _ = shutil.copytree(GEN_DIR, root / "gen") + _ = shutil.copytree(SRC_DIR, root / "src") project = shutil.copytree(FIXTURE_PROJECT, root / "tests" / "fixture-project") (project / "freeze1.pgn.yaml").unlink(missing_ok=True) shutil.rmtree(project / "artifacts", ignore_errors=True) diff --git a/tests/test_unsupported_types.py b/tests/test_unsupported_types.py index 9d99eab..642db87 100644 --- a/tests/test_unsupported_types.py +++ b/tests/test_unsupported_types.py @@ -24,14 +24,14 @@ import pytest -from tests._harness import FIXTURE_PROJECT, GEN_DIR, HERE, run_pgn +from tests._harness import FIXTURE_PROJECT, HERE, SRC_DIR, run_pgn HARNESS_ROOT = HERE.parent def test_unsupported_pg_type_fails_loudly(pgn_bin: str, pgn_admin_url: str, tmp_path: Path) -> None: root = tmp_path / "pygen" - _ = shutil.copytree(GEN_DIR, root / "gen") + _ = shutil.copytree(SRC_DIR, root / "src") project = shutil.copytree(FIXTURE_PROJECT, root / "tests" / "fixture-project") (project / "freeze1.pgn.yaml").unlink(missing_ok=True) shutil.rmtree(project / "artifacts", ignore_errors=True) @@ -51,7 +51,7 @@ def test_unsupported_pg_type_fails_loudly(pgn_bin: str, pgn_admin_url: str, tmp_ def test_json_array_param_fails_loudly(pgn_bin: str, pgn_admin_url: str, tmp_path: Path) -> None: root = tmp_path / "pygen" - _ = shutil.copytree(GEN_DIR, root / "gen") + _ = shutil.copytree(SRC_DIR, root / "src") project = shutil.copytree(FIXTURE_PROJECT, root / "tests" / "fixture-project") (project / "freeze1.pgn.yaml").unlink(missing_ok=True) shutil.rmtree(project / "artifacts", ignore_errors=True) @@ -66,7 +66,7 @@ def test_json_array_param_fails_loudly(pgn_bin: str, pgn_admin_url: str, tmp_pat "postgres: 18\n" "artifacts:\n" " python:\n" - " gen: ../../gen/Gen.dhall\n" + " gen: ../../src/package.dhall\n" " config:\n" " onUnsupported: Fail\n" ) @@ -104,7 +104,7 @@ def test_skip_unsupported_drops_offending_units_and_cascades( golden package is held to. """ root = tmp_path / "pygen" - _ = shutil.copytree(GEN_DIR, root / "gen") + _ = shutil.copytree(SRC_DIR, root / "src") project = shutil.copytree(FIXTURE_PROJECT, root / "tests" / "fixture-project") (project / "freeze1.pgn.yaml").unlink(missing_ok=True) shutil.rmtree(project / "artifacts", ignore_errors=True) @@ -129,7 +129,7 @@ def test_skip_unsupported_drops_offending_units_and_cascades( "postgres: 18\n" "artifacts:\n" " python:\n" - " gen: ../../gen/Gen.dhall\n" + " gen: ../../src/package.dhall\n" " config:\n" " onUnsupported: Skip\n" ) From ffea69fe9ccccfafaef73b343fc9514cd6b21840 Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 14:19:59 +0300 Subject: [PATCH 06/21] docs(gen): add Task 7 CHANGELOG entry, fix stale Config.dhall comment Documents the gen-contract v4.0.1 / gen-sdk v2.0.0 migration and gen/->src/ layout restructure under # Upcoming, and updates Config.dhall's doc comment to reference Interpret.dhall instead of the old compile.dhall name. --- CHANGELOG.md | 12 ++++++++++++ src/Config.dhall | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cc7b2c..f98fe04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Upcoming +- Migrated the generator's internal dependencies to `gen-contract` v4.0.1 + and `gen-sdk` v2.0.0, adopting `Sdk.Sigs` in place of the local + `Algebras/` module, and restructured the repository layout to match the + pGenie generator architecture: implementation moved from `gen/` to + `src/`, the public entry point renamed from `gen/Gen.dhall` to + `src/package.dhall`, and the fixture driver moved from + `tests/Exhaustive.dhall` to `demos/Exhaustive.dhall`. No change to + generated output or the public Dhall interface (`artifacts..gen` + URLs pointing at a previously-released `resolved.dhall` are unaffected; + only the next release's URL path changes, from `.../gen/Gen.dhall` — the + unresolved source path some projects may reference directly instead of a + frozen release — to `.../src/package.dhall`). - The test harness now runs every pgn subprocess in its own process group under an RSS watchdog: a thread polls `ps -o rss=` every 2 s and, on breach of `PGN_MAX_RSS_GB` (default 40 GB), kills the whole group and fails the test with diff --git a/src/Config.dhall b/src/Config.dhall index 7ec1fe6..1377127 100644 --- a/src/Config.dhall +++ b/src/Config.dhall @@ -5,7 +5,7 @@ -- unsupported statement/type and its dependents, with a warning) when a query -- or custom type hits a PG shape the generator cannot render; see -- Structures/OnUnsupported.dhall. All fields are Optional so a project may omit --- the whole config block or any subset of its keys; compile.dhall supplies the +-- the whole config block or any subset of its keys; Interpret.dhall supplies the -- defaults. let OnUnsupported = ./Structures/OnUnsupported.dhall From 9c33a23dbbec070c530430900ea7f770054a4fcd Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 14:29:51 +0300 Subject: [PATCH 07/21] refactor(gen): drop dead Run type-alias bindings in Query/Result/ResultColumns These were declared but never exported (unlike Member.dhall/ParamsMember.dhall, which do export Run), left over from documenting each file's actual multi-arg call signature during the gen-sdk v2 migration. Flagged by final review as harmless but pointless; removing rather than exporting keeps these three files' tails exactly as they were pre-migration, matching the migration's "keep bare exports verbatim" instruction for this exception group. --- src/Interpreters/Query.dhall | 2 -- src/Interpreters/Result.dhall | 3 --- src/Interpreters/ResultColumns.dhall | 3 --- 3 files changed, 8 deletions(-) diff --git a/src/Interpreters/Query.dhall b/src/Interpreters/Query.dhall index daa1d47..cce3fcd 100644 --- a/src/Interpreters/Query.dhall +++ b/src/Interpreters/Query.dhall @@ -177,6 +177,4 @@ let run = ) ) -let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output - in { Input, Output, run } diff --git a/src/Interpreters/Result.dhall b/src/Interpreters/Result.dhall index 83f3ef1..5d652a0 100644 --- a/src/Interpreters/Result.dhall +++ b/src/Interpreters/Result.dhall @@ -101,7 +101,4 @@ let run = } input -let Run = - Config -> CustomKind.Lookup -> Text -> Input -> Lude.Compiled.Type Output - in { Input, Output, RowClass, run } diff --git a/src/Interpreters/ResultColumns.dhall b/src/Interpreters/ResultColumns.dhall index b2fc737..e0bbfaa 100644 --- a/src/Interpreters/ResultColumns.dhall +++ b/src/Interpreters/ResultColumns.dhall @@ -78,7 +78,4 @@ let run = input ) -let Run = - Config -> CustomKind.Lookup -> Text -> Input -> Lude.Compiled.Type Output - in { Input, Output, run } From 15c28361de3192aa2d0e896fa32076c5f841ece3 Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 16:30:39 +0300 Subject: [PATCH 08/21] The plan --- .../plans/2026-07-11-gen-sdk-v2-migration.md | 666 ++++++++++++++++++ 1 file changed, 666 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md diff --git a/docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md b/docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md new file mode 100644 index 0000000..5e0499c --- /dev/null +++ b/docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md @@ -0,0 +1,666 @@ +# python.gen: migrate to gen-contract v4.0.1 / gen-sdk v2.0.0, adopt architecture layout + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Bring `python.gen` up to the same structural shape as `java.gen`'s +last release (`v1.1.0`): pin `gen-contract`/`gen-sdk` directly (no bundled +SDK), adopt `Sdk.Sigs` in place of the local `Algebras/` folder, and move to +the `src/`-rooted repo layout the normative architecture doc describes — with +**zero change to generated output**. + +**Architecture:** Reference is +`gen-sdk/docs/generator-architecture.md` (already read in full) and +`java.gen`'s current `master` (tag `v1.1.0`, commit `658508c`). `python.gen` +is currently on pre-split `gen-sdk v0.11.0` (no `Sigs`, bundles `Project` +itself) with a hand-rolled `gen/Algebras/{Interpreter,Template}.dhall`, a +`gen/Deps/package.dhall` barrel, and the old `gen/` + `tests/Exhaustive.dhall` +layout — i.e. it has never done *any* of the steps `java.gen` went through +(`gen-migration-plan.md` at the pgenie repo root covers `rust.gen`/`haskell.gen`, +which already had `Sigs`; it does not cover `python.gen` at all). This plan +folds all of `java.gen`'s historical steps into one destination state, since +there is no reason to recreate `java.gen`'s intermediate commits. + +**Tech Stack:** Dhall (fork `pgn`'s dhall, package name `dhll`, needed for the +`Text/equal` builtin and `as Source` import mode — see Global Constraints), +Python 3.12 / `uv` / `pytest` for the harness, `mise` for tool pinning. + +## Global Constraints + +- **No behavior change.** This is a dependency/layout migration, not a + feature change. If `demos/Exhaustive.dhall` (see Task 5) produces different + file paths or content than `tests/Exhaustive.dhall` did before the move, + that is a bug in the migration, not an expected diff. +- **Do not touch interpreter algorithms.** `Interpreters/Project.dhall`'s + `Skip`/`Fail` logic (`typeSucceeds`/`queryChecks`/`effectiveQueries`) is + deliberately hand-rolled instead of using `Typeclasses.Classes.Alternative` + the way `java.gen` does, per an explicit in-file comment: multiple + `QueryGen.run`/`CustomTypeGen.run` call sites for the same query measurably + multiplied Dhall normalization time (seconds → minutes), confirmed by wall-time + bisection. This is orthogonal to the Deps/Sigs migration — leave it as is. + Do **not** add `src/Deps/Typeclasses.dhall` since nothing will use it. +- **`Interpreters/Member.dhall` and `Interpreters/ParamsMember.dhall` keep + their 3-argument `Run` type** (`Config -> CustomKind.Lookup -> Input -> + Compiled Output`) instead of conforming to `Sdk.Sigs.interpreter`'s fixed + 2-argument shape (`Config -> Input -> Result`). `java.gen`'s own + `Member.dhall` doesn't need a lookup table; `python.gen`'s does (custom-type + name resolution — see the `buildLookup`/`IndexedCustomType` comments in + `Interpreters/Project.dhall`). Bundling `Lookup` into `Config` or `Input` + to force-fit the Sig is a bigger, separate refactor — out of scope here. + These two files get the Deps-import and `Algebra.Config` → local `Config` + changes (Task 2) but keep their existing bare `{ Input, Output, Run, run }` + export, not `Sdk.Sigs.interpreter Config Input Output run`. +- **Config narrowing is deferred.** The architecture doc's ideal is each + interpreter declaring only the `Config` fields it (and its children) needs. + `java.gen` does this trivially (`{ useOptional : Bool }` everywhere, since + that's its whole config). `python.gen`'s config has 4 fields + (`packageName`, `importName`, `emitSync`, `onUnsupported`); auditing exactly + which fields each of the 11 interpreters actually reads and narrowing each + is real, separate work with real risk of missing a field some deeply nested + path needs. This plan keeps the **same 4-field `Config` record, declared + locally and verbatim in each interpreter module** (no more shared + `Algebra.Config` alias) — this satisfies "`Config` is a parameter each + module declares itself," just not narrowed. Note it as a follow-up, don't + do it now. +- **Local verification gap.** This sandbox has a global `dhall`/`pgn` (cabal + build, `dhll-1.42.3`) that is **not** the `mise`-pinned `pgn v0.9.1` this + repo's CI/tests actually use — a plain `dhall type --file tests/Exhaustive.dhall` + here fails on the *pre-migration* tree already, with an `as Source` + hash-integrity mismatch on `gen/Deps/Sdk.dhall` (confirmed during planning: + expected `8d43544e...`, actual `573b4655...`). This is a toolchain mismatch, + not evidence of a real problem. **Whoever executes this plan must run + verification through `mise x -- dhall ...` / `mise x -- pgn ...` / `mise x + -- uv run pytest`**, matching `mise.toml`'s pin, not a bare global `dhall`. + If `mise` isn't available in the execution environment either, at minimum + run `dhall format --transitive` (syntax-only, tool-version-agnostic) and + flag that deeper verification (`dhall type`, fixture diff, pytest, + basedpyright) still needs to happen on a properly provisioned machine/CI + before this is considered done. + +--- + +## File Structure + +``` +src/ (was gen/) + package.dhall (was Gen.dhall — now built via Sdk.Sigs.generator) + Config.dhall (unchanged content, moved) + Interpret.dhall (was compile.dhall — Config no longer Optional at top) + Deps/ + Contract.dhall (NEW — gen-contract v4.0.1 pin) + Sdk.dhall (bumped gen-sdk v0.11.0 → v2.0.0) + Lude.dhall (unchanged content, moved) + Prelude.dhall (unchanged content, moved) + (package.dhall barrel REMOVED) + Interpreters/ (11 files: Deps.Sdk.Project → Deps.Contract, Algebra → Sdk.Sigs) + Templates/ (10 files: Algebra → Sdk.Sigs; 7 also de-barrel Deps) + Structures/ (CustomKind.dhall: Deps.Sdk.Project → Deps.Contract; others untouched) + (Algebras/ REMOVED) +demos/ + Exhaustive.dhall (was tests/Exhaustive.dhall — rewritten for Sdk.Output.toFileMap) +tests/ (Python pytest harness — unchanged except any gen/-path references) +.github/workflows/{ci,release}.yml, .github/scripts/build-contract-shell.sh, +README.md, AGENTS.md, DESIGN.md, build.bash, bench/*.sh, mise.toml + (path references updated: gen/ → src/, tests/Exhaustive.dhall → demos/Exhaustive.dhall) +``` + +--- + +### Task 1: Add the new Deps pins and remove the `Deps/package.dhall` barrel + +**Files:** +- Create: `gen/Deps/Contract.dhall` +- Modify: `gen/Deps/Sdk.dhall` +- Delete: `gen/Deps/package.dhall` +- Modify (de-barrel): every file that currently has `let Deps = ../Deps/package.dhall` (see the full list in Tasks 2–3 — do this as part of those tasks, not twice) + +This task only stages the new pins; Task 2 is where the fallout (broken +`Sdk.Project`/`Sdk.Fixtures` references, `Algebras/` removal) gets fixed. Do +not try to get `dhall type` green after this task alone — it won't be, and +that's expected (same as `gen-migration-plan.md`'s phase-0 commit 2 for +`java.gen`). Fold Task 1 and Task 2 into one commit if you'd rather not carry +a known-broken intermediate state. + +- [ ] **Step 1: Create `gen/Deps/Contract.dhall`** — the exact pin `java.gen` + and `gen-sdk` itself use: + +```dhall +https://raw.githubusercontent.com/pgenie-io/gen-contract/v4.0.1/src/package.dhall + sha256:4a130ba7fbaa152a776babbb1bf2994a4833931ca76bde9bf6930d354225651e +``` + +- [ ] **Step 2: Bump `gen/Deps/Sdk.dhall`** to `gen-sdk v2.0.0` (same pin + `java.gen`'s `src/Deps/Sdk.dhall` uses). Preserve the existing `as Source` + import mode (see `AGENTS.md`/CI comments on why `python.gen` uses it — + `java.gen` doesn't, but that's an intentional `python.gen`-specific RAM + optimization, not something this migration should undo): + +```dhall +https://raw.githubusercontent.com/pgenie-io/gen-sdk/v2.0.0/src/package.dhall + sha256:b9def6ab1179bc4aaae7fc6e91977f094f75934cd5755175c294a9e97ca71b15 + as Source +``` + + If keeping `as Source` here, its hash is a *source*-text hash, not the + semantic hash above copied from `java.gen` (which imports plainly). Verify + with `dhall hash` against the raw URL using the repo's pinned toolchain + (`mise x -- dhall hash <<< 'https://raw.githubusercontent.com/pgenie-io/gen-sdk/v2.0.0/src/package.dhall as Source'` + or equivalent) before trusting the semantic hash verbatim in `as Source` + mode — don't guess. + +- [ ] **Step 3: Delete `gen/Deps/package.dhall`.** + +- [ ] **Step 4: Commit** (or fold into Task 2's commit). + +--- + +### Task 2: Rewire `Structures/CustomKind.dhall` and all 11 `Interpreters/*.dhall` + +**Files:** +- Modify: `gen/Structures/CustomKind.dhall` +- Modify: `gen/Interpreters/{CustomType,Member,ParamsMember,Primitive,Project,Query,QueryFragments,Result,ResultColumns,Scalar,Value}.dhall` +- Delete: `gen/Algebras/` (all three files) + +**The mechanical recipe, applied to every file above:** + +1. Replace `let Deps = ../Deps/package.dhall` with direct imports of exactly + what the file uses. Every one of these files uses `Deps.Sdk.Project` + (→ becomes a direct `Deps/Contract.dhall` import) and `Deps.Lude`/`Deps.Prelude` + (→ direct imports). Concretely, replace: + ```dhall + let Deps = ../Deps/package.dhall + ``` + with (only the lines this particular file actually needs — check with + `grep -n 'Deps\.' ` first): + ```dhall + let Lude = ../Deps/Lude.dhall + + let Prelude = ../Deps/Prelude.dhall + + let Model = ../Deps/Contract.dhall + ``` + and change every remaining `Deps.Lude` → `Lude`, `Deps.Prelude` → `Prelude` + in the body. `QueryFragments.dhall` additionally has an unused + `let Sdk = Deps.Sdk` line (line 7) — drop it, nothing in the file + references the `Sdk` binding. + +2. Replace the line `let Model = Deps.Sdk.Project` (now redundant with step + 1's `Model` binding) — don't duplicate it, step 1 already introduces + `Model` pointed at `Deps/Contract.dhall`. + +3. Delete `let Algebra = ../Algebras/Interpreter.dhall`. + +4. Add a local `Config` type declaration (same 4 fields everywhere per the + Global Constraints note on deferred narrowing): + ```dhall + let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } + ``` + This needs `OnUnsupported = ../Structures/OnUnsupported.dhall` imported in + any file that doesn't already import it (check first — `Project.dhall` + already does). + +5. Change every `\(config : Algebra.Config) ->` to `\(config : Config) ->`. + +6. Change the tail: + - **10 of the 11 files** (`CustomType`, `Primitive`, `Project`, `Query`, + `QueryFragments`, `Result`, `ResultColumns`, `Scalar`, `Value` — 9 files, + not 10; `Member`/`ParamsMember` are the exception below) end with + `Algebra.module Input Output run` (or, for `Scalar.dhall`, + `Algebra.module Input Output run /\ { ScalarDecode }`). Change to: + ```dhall + Sdk.Sigs.interpreter Config Input Output run + ``` + (`Scalar.dhall`: `Sdk.Sigs.interpreter Config Input Output run /\ { ScalarDecode }`), + which needs `let Sdk = ../Deps/Sdk.dhall` imported (it isn't currently, + since `Deps.Sdk.Project` used to come through the barrel — add it). + - **`Member.dhall` and `ParamsMember.dhall`** keep their existing tail + verbatim: `in { Input, Output, Run, run }`, just with `Algebra.Config` → + `Config` in the `Run` type alias line + (`let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output`). + No `Sdk.Sigs.interpreter` here — see Global Constraints. + +7. `Interpreters/Project.dhall` specifically: its two `\(config : Algebra.Config) ->` + occurrences (the `combineOutputs` and `run` functions) both become + `\(config : Config) ->`; its `lookupConfig : Algebra.Config` type + annotation (used to type-check `Value.run` calls for composite-field + rendering) becomes `lookupConfig : Config`. No other logic in this file + changes — the `Skip`/`Fail` machinery is untouched per Global Constraints. + +- [ ] **Step 1: Apply the recipe to all 11 `Interpreters/*.dhall` files and `Structures/CustomKind.dhall`.** +- [ ] **Step 2: Delete `gen/Algebras/`.** +- [ ] **Step 3: Verify** (with the `mise`-pinned toolchain, not the bare local `dhall` — see Global Constraints): + ```bash + mise x -- dhall type --file gen/Interpreters/Project.dhall + ``` + Expected: prints the interpreter's type (a record with `Input`, `Output`, + `Result`, `Run`, `run` fields) with no error. This alone pulls in every + other `Interpreters/*.dhall` transitively, so it's a full check of this task. +- [ ] **Step 4: Commit.** + +--- + +### Task 3: Rewire the 10 `Templates/*.dhall` files + +**Files:** +- Modify: `gen/Templates/{CompositeModule,CoreModule,EnumModule,FacadeModule,InitModule,RegisterModule,RowsModule,RuntimeModule,StatementModule,TypesInit}.dhall` + +**Recipe:** + +1. `CoreModule.dhall`, `InitModule.dhall`, `RuntimeModule.dhall` don't import + `Deps` at all (no `Prelude`/`Lude` need) — only change: drop + `let Algebra = ../Algebras/Template.dhall`, add `let Sdk = ../Deps/Sdk.dhall`, + and change the tail. `CoreModule`/`RuntimeModule` end with + `Algebra.module {} (\(_ : {}) -> content)` → `Sdk.Sigs.template {} (\(_ : {}) -> content)`. + `InitModule` ends with `Algebra.module Params render` → + `Sdk.Sigs.template Params render`. + +2. The other 7 (`CompositeModule`, `EnumModule`, `FacadeModule`, + `RegisterModule`, `RowsModule`, `StatementModule`, `TypesInit`) currently + have `let Deps = ../Deps/package.dhall`, and only ever use + `Deps.Prelude.*` (all seven) and, additionally, `Deps.Lude.Text.indentNonEmpty` + (`RowsModule`, `StatementModule` only — confirmed by + `grep -n 'Deps\.' gen/Templates/*.dhall` during planning). Replace the + barrel import with: + ```dhall + let Prelude = ../Deps/Prelude.dhall + ``` + adding `let Lude = ../Deps/Lude.dhall` only in `RowsModule.dhall` and + `StatementModule.dhall`. Then replace `Deps.Prelude.` → `Prelude.` and + `Deps.Lude.` → `Lude.` throughout each file's body. Drop + `let Algebra = ../Algebras/Template.dhall`, add `let Sdk = ../Deps/Sdk.dhall`. + +3. Tails for these 7: `Algebra.module Params run` → + `Sdk.Sigs.template Params run` (`FacadeModule`, `RegisterModule`, + `StatementModule`); with a combined record for the other 4: + `Algebra.module Params run /\ { Field }` (`CompositeModule`) → + `Sdk.Sigs.template Params run /\ { Field }`; `/\ { Variant }` (`EnumModule`); + `/\ { StatementExport, TypeExport }` (`FacadeModule` — check which of + `FacadeModule`/others actually has this combinator vs a plain + `Algebra.module Params run`, per the earlier grep output, before editing — + don't assume, re-`grep -n 'Algebra.module' gen/Templates/*.dhall` and + match each file's exact current tail); `/\ { RowDef }` (`RowsModule`); + `/\ { Export }` (`TypesInit`). + +- [ ] **Step 1: Apply the recipe to all 10 files.** +- [ ] **Step 2: Verify:** + ```bash + mise x -- dhall type --file gen/Interpreters/Project.dhall + ``` + (Templates are only reachable transitively through `Interpreters/Project.dhall` + and its children, same as Task 2 — this single check covers both tasks once + both are done. If running Task 2 and 3 as separate commits, this step will + fail after Task 2 alone if any interpreter references a not-yet-updated + template's old shape; if so, do Tasks 2 and 3 as one commit instead.) +- [ ] **Step 3: Commit.** + +--- + +### Task 4: Rewrite the root entry point (`Config.dhall`, `compile.dhall` → `Interpret.dhall`, `Gen.dhall` → `package.dhall`) + +**Files:** +- Modify (move, content unchanged): `gen/Config.dhall` +- Modify (move + rewrite): `gen/compile.dhall` → `gen/Interpret.dhall` +- Modify (move + rewrite): `gen/Gen.dhall` → `gen/package.dhall` + +(Paths shown as `gen/...` here since Task 5 does the `gen/` → `src/` directory +move; do this task first, in place, then Task 5 is a pure `git mv` sweep with +no further content changes.) + +**Interfaces:** +- Consumes: `Interpreters/Project.dhall`'s `run` (produced by Task 2, + now `Sdk.Sigs.interpreter`-shaped: `.run : Config -> Contract.Project -> Compiled Output`). +- Produces: `package.dhall`'s `Sdk.Sigs.generator`-built value + (`{ contractVersion, Config, compile }`), consumed by Task 5's + `demos/Exhaustive.dhall` and by any pGenie project's `artifacts..gen` URL. + +`Sdk.Sigs.generator`'s shape (from the architecture doc): +```dhall +\(Config : Type) -> +\(defaultConfig : Config) -> +\(interpret : Config -> Contract.Project -> Contract.Output) -> + let compile = \(config : Optional Config) -> + merge { None = interpret defaultConfig, Some = interpret } config + in Contract.module Config compile +``` +Note `interpret` takes a **bare** `Config`, not `Optional Config` — the outer +"config block omitted entirely" case is handled once, by substituting +`defaultConfig`, not by `interpret` itself. `python.gen`'s current +`compile.dhall` handles *two* levels of optionality (the whole block, and +each field within it) with a doubled `Prelude.Optional.fold`. Only the outer +level goes away; each field inside `Config` stays individually `Optional` (so +a project can supply `emitSync: true` alone and still get default +`packageName`/`onUnsupported`) — that per-field defaulting is +`python.gen`-specific richness `java.gen` doesn't have (its `Config` has one +non-Optional `Bool` field), and this migration must not lose it. + +- [ ] **Step 1: `gen/Config.dhall`** — content unchanged, just confirm it + still reads (no edits needed here; listed for completeness since Task 5 + moves the file). + +- [ ] **Step 2: Rewrite `gen/compile.dhall` as `gen/Interpret.dhall`** — + drop the outer `Optional Config` unwrap (the two outermost + `Prelude.Optional.fold Config config Text (\(c : Config) -> ...)` / + `... Bool ...` / `... OnUnsupported.Mode ...` wrappers), keep everything + else (the per-field defaults, `importName` derivation) as is: + +```dhall +let Deps = ./Deps/package.dhall + +-- NOTE: Task 5 changes this to ./Deps/Contract.dhall / ./Deps/Prelude.dhall +-- directly once the Deps barrel is gone (Task 1) — write it that way now, +-- don't reintroduce the barrel: +let Contract = ./Deps/Contract.dhall + +let Prelude = ./Deps/Prelude.dhall + +let Config = ./Config.dhall + +let OnUnsupported = ./Structures/OnUnsupported.dhall + +let ProjectInterpreter = ./Interpreters/Project.dhall + +-- Entry point handed to gen-sdk's Sdk.Sigs.generator as `interpret`. Each +-- field of Config is independently Optional, so a project may omit the +-- whole config block (Sdk.Sigs.generator substitutes an all-None +-- defaultConfig, see package.dhall) or any subset of its keys; `defaults` +-- collects every fallback in one place (packageName from the project name in +-- kebab case, emitSync off, onUnsupported Fail). The async surface is always +-- emitted; emitSync adds the sync mirror. +in \(config : Config) -> + \(project : Contract.Project) -> + let defaults = + { packageName = project.name.inKebabCase + , emitSync = False + , onUnsupported = OnUnsupported.Mode.Fail + } + + let packageName = + Prelude.Optional.fold + Text + config.packageName + Text + (\(t : Text) -> t) + defaults.packageName + + let emitSync = + Prelude.Optional.fold + Bool + config.emitSync + Bool + (\(b : Bool) -> b) + defaults.emitSync + + let onUnsupported = + Prelude.Optional.fold + OnUnsupported.Mode + config.onUnsupported + OnUnsupported.Mode + (\(m : OnUnsupported.Mode) -> m) + defaults.onUnsupported + + let importName = Prelude.Text.replace "-" "_" packageName + + let interpreterConfig = { packageName, importName, emitSync, onUnsupported } + + in ProjectInterpreter.run interpreterConfig project +``` + +- [ ] **Step 3: Rewrite `gen/Gen.dhall` as `gen/package.dhall`:** + +```dhall +let Sdk = ./Deps/Sdk.dhall + +let OnUnsupported = ./Structures/OnUnsupported.dhall + +let Config = ./Config.dhall + +let Config/default + : Config + = { packageName = None Text + , emitSync = None Bool + , onUnsupported = None OnUnsupported.Mode + } + +let interpret = ./Interpret.dhall + +in Sdk.Sigs.generator Config Config/default interpret +``` + +- [ ] **Step 4: Verify:** + ```bash + mise x -- dhall type --file gen/package.dhall + ``` + Expected type: a record with `contractVersion`, `Config`, `compile` fields + (`compile : Optional Config -> Contract.Project -> Contract.Output`). +- [ ] **Step 5: Commit.** + +--- + +### Task 5: Move `gen/` → `src/`, `tests/Exhaustive.dhall` → `demos/Exhaustive.dhall` + +**Files:** +- Move: `gen/` → `src/` (whole tree, `git mv`) +- Move + rewrite: `tests/Exhaustive.dhall` → `demos/Exhaustive.dhall` + +- [ ] **Step 1:** + ```bash + git mv gen src + mkdir -p demos + git mv tests/Exhaustive.dhall demos/Exhaustive.dhall + ``` + All the `../Deps/...`, `./Interpreters/...`, `../Templates/...` style + relative imports inside `src/` are untouched by this move (they're relative + to their own file, not to the repo root), so no content changes are needed + inside `src/` itself from the move alone. + +- [ ] **Step 2: Rewrite `demos/Exhaustive.dhall`.** Its old body called + `Gen.compileToFileMap config project` — `Sdk.Sigs.generator`-built modules + don't have a `compileToFileMap` field (per the architecture doc: "there is + no `compileToFileMap` on the module — turning an `Output` into files is the + caller's job, via `Sdk.Output.toFileMap`"). New content: + +```dhall +-- Applies this generator to gen-sdk's shared cross-backend fixture project +-- (the same "music_catalogue" project java.gen's own demos/Exhaustive.dhall +-- exercises), so a Python client compiles from it and passes basedpyright +-- strict. Pinned directly at gen-sdk's package.dhall, separately from +-- src/Deps/Sdk.dhall: that file only imports gen-sdk's `package.dhall` `as +-- Source` for RAM, and this fixture load doesn't need that mode. +-- +-- The fixture project deliberately covers PG types this generator does not +-- support (box, inet, money, ranges, ...), so onUnsupported is set to Skip: +-- those statements/types are dropped with a warning instead of aborting the +-- whole compile. +-- +-- Intended to be executed with: +-- +-- ```bash +-- dhall to-directory-tree --file demos/Exhaustive.dhall --output --allow-path-separators +-- ``` +let Sdk = ../src/Deps/Sdk.dhall + +let Gen = ../src/package.dhall + +let OnUnsupported = ../src/Structures/OnUnsupported.dhall + +let project = Sdk.Fixtures.Exhaustive + +let config = + Some + { packageName = None Text + , emitSync = Some True + , onUnsupported = Some OnUnsupported.Mode.Skip + } + +in Sdk.Output.toFileMap (Gen.compile config project) +``` + +- [ ] **Step 3: Verify:** + ```bash + mise x -- dhall type --file demos/Exhaustive.dhall + ``` + Expected: `List { mapKey : Text, mapValue : Text }` (or however this + fork/version of Dhall renders `Prelude.Map.Type Text Text`), no error. +- [ ] **Step 4: Commit.** + +--- + +### Task 6: Update every external reference to the old paths + +**Files:** +- Modify: `.github/workflows/ci.yml` +- Modify: `.github/workflows/release.yml` +- Modify: `.github/scripts/build-contract-shell.sh` +- Modify: `README.md` +- Modify: `AGENTS.md` +- Modify: `DESIGN.md` +- Modify: `build.bash` +- Modify: `bench/generate.sh`, `bench/as-source.sh` +- Modify: `mise.toml` (the `golden` task) + +**Path substitutions to apply everywhere they occur** (verify each hit with +`grep -rn` first — don't blind-sed across the whole repo, `tests/golden/` +contains generated Python that must NOT be touched): + +| Old | New | +|---|---| +| `gen/Gen.dhall` | `src/package.dhall` | +| `gen/Deps/*.dhall` | `src/Deps/*.dhall` | +| `tests/Exhaustive.dhall` | `demos/Exhaustive.dhall` | +| `gen/` (prose/dir references) | `src/` | + +Specific known hits (from `grep -rn "gen/Gen\.dhall\|gen/Deps\|tests/Exhaustive"` +run during planning): + +- `.github/workflows/ci.yml`: the `contract` job's "Strip `as Source`..." + step does `sed -i ... gen/Deps/*.dhall` → `src/Deps/*.dhall`; the + "Generate output from Dhall" step's `dhall_file: tests/Exhaustive.dhall` → + `demos/Exhaustive.dhall`. +- `.github/workflows/release.yml`: the "Resolve Dhall" step's + `file: gen/Gen.dhall` → `file: src/package.dhall`. +- `.github/scripts/build-contract-shell.sh`: comment references + `tests/Exhaustive.dhall` → `demos/Exhaustive.dhall` (comment only, verify + no functional path argument needs changing — it's invoked with + `contract-output` as a positional arg per `ci.yml`, not a hardcoded path). +- `README.md`: line ~38 `gen: https://raw.githubusercontent.com/slavashvets/python.gen/master/gen/Gen.dhall` + → `.../src/package.dhall`; lines ~64-66, the three example URLs + (`.../gen/Gen.dhall`) → `.../src/package.dhall`. +- `AGENTS.md`: line ~36 "`gen/` pins its remote imports by sha256 + (`gen/Deps/*.dhall`)" → "`src/` pins its remote imports by sha256 + (`src/Deps/*.dhall`)". +- `DESIGN.md`: line 4 (`gen/`), line 381 (`gen/Gen.dhall` "is the entry point + handed to gen-sdk"), line 397 (`gen/` mirrors...), line 401 (the `gen/` + tree diagram — replace with the new `src/` tree, matching Task 5's actual + post-move layout), lines 632/636 (`tests/Exhaustive.dhall` → `demos/Exhaustive.dhall`). +- `build.bash`: this is a scratch/dev script (mostly commented-out lines) — + update the live lines: `target=tests/Exhaustive.dhall` → + `target=demos/Exhaustive.dhall`; the commented `# target=gen/Gen.dhall` and + `# dhall freeze gen/Deps/*.dhall` lines → `src/` equivalents (keep them + commented, just fix the paths so they're not stale if uncommented later). +- `bench/generate.sh`, `bench/as-source.sh`: both `cp -R "$root/gen" "$out/gen"` / + `"$1/gen"` → `"$root/src" "$out/src"` (and update the `gen/Deps/*.dhall` + perl substitutions to `src/Deps/*.dhall`). **The `as Source` → plain-import + sha256 substitutions in both scripts' `strip_as_source` are pinned to the + *old* `gen-sdk v0.11.0`/`lude v5.1.0` source-vs-normalized hash pairs** + (`8d43544e...`→`b9f7bb84...` for Sdk, `46b527b0...`→`14c43eec...` for Lude). + Since `Task 1` bumps `gen-sdk` to `v2.0.0`, these substitution pairs are now + wrong and must be recomputed for the new pin using the repo's actual pinned + toolchain (`mise x -- dhall hash` on the plain, non-`as-Source` import) — + don't guess these; if the recompute can't happen in this pass, leave a + `# TODO` in the script rather than shipping a silently-wrong benchmark. +- `mise.toml`'s `golden` task: the `python3 - ... "$root/gen/Gen.dhall"` arg + and the fixture-project string replace target `"../../gen/Gen.dhall"` → + `"../../src/package.dhall"`. + +- [ ] **Step 1: Apply all substitutions above.** +- [ ] **Step 2: Confirm no stragglers:** + ```bash + grep -rn "gen/Gen\.dhall\|gen/Deps\|gen/Interpreters\|gen/Templates\|gen/Structures\|gen/Config\.dhall\|gen/compile\.dhall\|tests/Exhaustive" \ + --include="*.md" --include="*.yml" --include="*.yaml" --include="*.toml" --include="*.sh" --include="*.bash" . + ``` + Expected: no output (everything left under `tests/golden/` or `tests/fixture-project/` + that isn't a generator-path reference is fine and out of scope — check any + hit manually rather than assuming). +- [ ] **Step 3: Commit.** + +--- + +### Task 7: Format, verify end-to-end, update CHANGELOG + +- [ ] **Step 1: Format everything:** + ```bash + mise x -- dhall format --transitive src/package.dhall + mise x -- dhall format --transitive demos/Exhaustive.dhall + ``` + +- [ ] **Step 2: Full type-check:** + ```bash + mise x -- dhall type --file src/package.dhall + mise x -- dhall type --file demos/Exhaustive.dhall + ``` + +- [ ] **Step 3: Regenerate the Exhaustive fixture and confirm no diff in + output** (this is the load-bearing check — everything above only proves + the Dhall type-checks, not that it still produces the same files): + ```bash + mise x -- dhall to-directory-tree --allow-path-separators --file demos/Exhaustive.dhall --output /tmp/pygen-after + ``` + Compare against a snapshot taken from the pre-migration tree the same way + (`git stash`, regenerate to `/tmp/pygen-before`, `git stash pop`, `diff -rq + /tmp/pygen-before /tmp/pygen-after`). Expected: **no diff**. Any diff here + is a migration bug, not an intentional update — per Global Constraints, go + fix it rather than accepting the new output. + +- [ ] **Step 4: Run the Python harness:** + ```bash + mise x -- uv sync + mise x -- uv run pytest tests -v + ``` + (Needs a reachable Postgres — `PGN_TEST_DATABASE_URL`, see `ci.yml` for the + Docker Compose equivalent — and the `mise`-pinned `pgn 0.9.1`, since the + harness shells out to it.) Expected: all green, no new failures relative to + a pre-migration run. + +- [ ] **Step 5: Add a CHANGELOG.md entry** under `# Upcoming` (the file + already starts with that heading), non-breaking, modeled on `java.gen`'s + own `v1.1.0` entry: + ```markdown + - Migrated the generator's internal dependencies to `gen-contract` v4.0.1 + and `gen-sdk` v2.0.0, adopting `Sdk.Sigs` in place of the local + `Algebras/` module, and restructured the repository layout to match the + pGenie generator architecture: implementation moved from `gen/` to + `src/`, the public entry point renamed from `gen/Gen.dhall` to + `src/package.dhall`, and the fixture driver moved from + `tests/Exhaustive.dhall` to `demos/Exhaustive.dhall`. No change to + generated output or the public Dhall interface (`artifacts..gen` + URLs pointing at a previously-released `resolved.dhall` are unaffected; + only the next release's URL path changes, from `.../gen/Gen.dhall` — the + unresolved source path some projects may reference directly instead of a + frozen release — to `.../src/package.dhall`). + ``` + Adjust the last parenthetical if no project in practice points at the + unresolved source path (check `README.md`'s own recommended usage — if it + only ever recommends the frozen `resolved.dhall` release asset, simplify + this to "no change to generated output or the public Dhall interface"). + +- [ ] **Step 6: Commit.** + +## Deferred / explicitly out of scope (record as follow-ups, don't do now) + +- Narrowing each interpreter's `Config` to only the fields it needs (see + Global Constraints). +- Giving `Member.dhall`/`ParamsMember.dhall` a real `Sdk.Sigs.interpreter` + shape by folding `CustomKind.Lookup` into `Config` or `Input` (see Global + Constraints). +- Adding a `Name` interpreter (`java.gen` has `Interpreters/Name.dhall` + centralizing identifier casing/escaping; `python.gen` calls + `Structures/PyIdent.dhall` ad hoc from several interpreters instead). This + is a real architecture-doc deviation but not something `java.gen`'s last + release changed — separate task if wanted. +- Recomputing the `as Source` bench-script hash pairs for `gen-sdk v2.0.0` / + `lude v5.1.0` (flagged inline in Task 6 — needs the real toolchain, not + guessable). From 81ea82c81dee169431839b06bfa530175aebf92f Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 17:31:09 +0300 Subject: [PATCH 09/21] Revert "The plan" This reverts commit 15c28361de3192aa2d0e896fa32076c5f841ece3. --- .../plans/2026-07-11-gen-sdk-v2-migration.md | 666 ------------------ 1 file changed, 666 deletions(-) delete mode 100644 docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md diff --git a/docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md b/docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md deleted file mode 100644 index 5e0499c..0000000 --- a/docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md +++ /dev/null @@ -1,666 +0,0 @@ -# python.gen: migrate to gen-contract v4.0.1 / gen-sdk v2.0.0, adopt architecture layout - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Bring `python.gen` up to the same structural shape as `java.gen`'s -last release (`v1.1.0`): pin `gen-contract`/`gen-sdk` directly (no bundled -SDK), adopt `Sdk.Sigs` in place of the local `Algebras/` folder, and move to -the `src/`-rooted repo layout the normative architecture doc describes — with -**zero change to generated output**. - -**Architecture:** Reference is -`gen-sdk/docs/generator-architecture.md` (already read in full) and -`java.gen`'s current `master` (tag `v1.1.0`, commit `658508c`). `python.gen` -is currently on pre-split `gen-sdk v0.11.0` (no `Sigs`, bundles `Project` -itself) with a hand-rolled `gen/Algebras/{Interpreter,Template}.dhall`, a -`gen/Deps/package.dhall` barrel, and the old `gen/` + `tests/Exhaustive.dhall` -layout — i.e. it has never done *any* of the steps `java.gen` went through -(`gen-migration-plan.md` at the pgenie repo root covers `rust.gen`/`haskell.gen`, -which already had `Sigs`; it does not cover `python.gen` at all). This plan -folds all of `java.gen`'s historical steps into one destination state, since -there is no reason to recreate `java.gen`'s intermediate commits. - -**Tech Stack:** Dhall (fork `pgn`'s dhall, package name `dhll`, needed for the -`Text/equal` builtin and `as Source` import mode — see Global Constraints), -Python 3.12 / `uv` / `pytest` for the harness, `mise` for tool pinning. - -## Global Constraints - -- **No behavior change.** This is a dependency/layout migration, not a - feature change. If `demos/Exhaustive.dhall` (see Task 5) produces different - file paths or content than `tests/Exhaustive.dhall` did before the move, - that is a bug in the migration, not an expected diff. -- **Do not touch interpreter algorithms.** `Interpreters/Project.dhall`'s - `Skip`/`Fail` logic (`typeSucceeds`/`queryChecks`/`effectiveQueries`) is - deliberately hand-rolled instead of using `Typeclasses.Classes.Alternative` - the way `java.gen` does, per an explicit in-file comment: multiple - `QueryGen.run`/`CustomTypeGen.run` call sites for the same query measurably - multiplied Dhall normalization time (seconds → minutes), confirmed by wall-time - bisection. This is orthogonal to the Deps/Sigs migration — leave it as is. - Do **not** add `src/Deps/Typeclasses.dhall` since nothing will use it. -- **`Interpreters/Member.dhall` and `Interpreters/ParamsMember.dhall` keep - their 3-argument `Run` type** (`Config -> CustomKind.Lookup -> Input -> - Compiled Output`) instead of conforming to `Sdk.Sigs.interpreter`'s fixed - 2-argument shape (`Config -> Input -> Result`). `java.gen`'s own - `Member.dhall` doesn't need a lookup table; `python.gen`'s does (custom-type - name resolution — see the `buildLookup`/`IndexedCustomType` comments in - `Interpreters/Project.dhall`). Bundling `Lookup` into `Config` or `Input` - to force-fit the Sig is a bigger, separate refactor — out of scope here. - These two files get the Deps-import and `Algebra.Config` → local `Config` - changes (Task 2) but keep their existing bare `{ Input, Output, Run, run }` - export, not `Sdk.Sigs.interpreter Config Input Output run`. -- **Config narrowing is deferred.** The architecture doc's ideal is each - interpreter declaring only the `Config` fields it (and its children) needs. - `java.gen` does this trivially (`{ useOptional : Bool }` everywhere, since - that's its whole config). `python.gen`'s config has 4 fields - (`packageName`, `importName`, `emitSync`, `onUnsupported`); auditing exactly - which fields each of the 11 interpreters actually reads and narrowing each - is real, separate work with real risk of missing a field some deeply nested - path needs. This plan keeps the **same 4-field `Config` record, declared - locally and verbatim in each interpreter module** (no more shared - `Algebra.Config` alias) — this satisfies "`Config` is a parameter each - module declares itself," just not narrowed. Note it as a follow-up, don't - do it now. -- **Local verification gap.** This sandbox has a global `dhall`/`pgn` (cabal - build, `dhll-1.42.3`) that is **not** the `mise`-pinned `pgn v0.9.1` this - repo's CI/tests actually use — a plain `dhall type --file tests/Exhaustive.dhall` - here fails on the *pre-migration* tree already, with an `as Source` - hash-integrity mismatch on `gen/Deps/Sdk.dhall` (confirmed during planning: - expected `8d43544e...`, actual `573b4655...`). This is a toolchain mismatch, - not evidence of a real problem. **Whoever executes this plan must run - verification through `mise x -- dhall ...` / `mise x -- pgn ...` / `mise x - -- uv run pytest`**, matching `mise.toml`'s pin, not a bare global `dhall`. - If `mise` isn't available in the execution environment either, at minimum - run `dhall format --transitive` (syntax-only, tool-version-agnostic) and - flag that deeper verification (`dhall type`, fixture diff, pytest, - basedpyright) still needs to happen on a properly provisioned machine/CI - before this is considered done. - ---- - -## File Structure - -``` -src/ (was gen/) - package.dhall (was Gen.dhall — now built via Sdk.Sigs.generator) - Config.dhall (unchanged content, moved) - Interpret.dhall (was compile.dhall — Config no longer Optional at top) - Deps/ - Contract.dhall (NEW — gen-contract v4.0.1 pin) - Sdk.dhall (bumped gen-sdk v0.11.0 → v2.0.0) - Lude.dhall (unchanged content, moved) - Prelude.dhall (unchanged content, moved) - (package.dhall barrel REMOVED) - Interpreters/ (11 files: Deps.Sdk.Project → Deps.Contract, Algebra → Sdk.Sigs) - Templates/ (10 files: Algebra → Sdk.Sigs; 7 also de-barrel Deps) - Structures/ (CustomKind.dhall: Deps.Sdk.Project → Deps.Contract; others untouched) - (Algebras/ REMOVED) -demos/ - Exhaustive.dhall (was tests/Exhaustive.dhall — rewritten for Sdk.Output.toFileMap) -tests/ (Python pytest harness — unchanged except any gen/-path references) -.github/workflows/{ci,release}.yml, .github/scripts/build-contract-shell.sh, -README.md, AGENTS.md, DESIGN.md, build.bash, bench/*.sh, mise.toml - (path references updated: gen/ → src/, tests/Exhaustive.dhall → demos/Exhaustive.dhall) -``` - ---- - -### Task 1: Add the new Deps pins and remove the `Deps/package.dhall` barrel - -**Files:** -- Create: `gen/Deps/Contract.dhall` -- Modify: `gen/Deps/Sdk.dhall` -- Delete: `gen/Deps/package.dhall` -- Modify (de-barrel): every file that currently has `let Deps = ../Deps/package.dhall` (see the full list in Tasks 2–3 — do this as part of those tasks, not twice) - -This task only stages the new pins; Task 2 is where the fallout (broken -`Sdk.Project`/`Sdk.Fixtures` references, `Algebras/` removal) gets fixed. Do -not try to get `dhall type` green after this task alone — it won't be, and -that's expected (same as `gen-migration-plan.md`'s phase-0 commit 2 for -`java.gen`). Fold Task 1 and Task 2 into one commit if you'd rather not carry -a known-broken intermediate state. - -- [ ] **Step 1: Create `gen/Deps/Contract.dhall`** — the exact pin `java.gen` - and `gen-sdk` itself use: - -```dhall -https://raw.githubusercontent.com/pgenie-io/gen-contract/v4.0.1/src/package.dhall - sha256:4a130ba7fbaa152a776babbb1bf2994a4833931ca76bde9bf6930d354225651e -``` - -- [ ] **Step 2: Bump `gen/Deps/Sdk.dhall`** to `gen-sdk v2.0.0` (same pin - `java.gen`'s `src/Deps/Sdk.dhall` uses). Preserve the existing `as Source` - import mode (see `AGENTS.md`/CI comments on why `python.gen` uses it — - `java.gen` doesn't, but that's an intentional `python.gen`-specific RAM - optimization, not something this migration should undo): - -```dhall -https://raw.githubusercontent.com/pgenie-io/gen-sdk/v2.0.0/src/package.dhall - sha256:b9def6ab1179bc4aaae7fc6e91977f094f75934cd5755175c294a9e97ca71b15 - as Source -``` - - If keeping `as Source` here, its hash is a *source*-text hash, not the - semantic hash above copied from `java.gen` (which imports plainly). Verify - with `dhall hash` against the raw URL using the repo's pinned toolchain - (`mise x -- dhall hash <<< 'https://raw.githubusercontent.com/pgenie-io/gen-sdk/v2.0.0/src/package.dhall as Source'` - or equivalent) before trusting the semantic hash verbatim in `as Source` - mode — don't guess. - -- [ ] **Step 3: Delete `gen/Deps/package.dhall`.** - -- [ ] **Step 4: Commit** (or fold into Task 2's commit). - ---- - -### Task 2: Rewire `Structures/CustomKind.dhall` and all 11 `Interpreters/*.dhall` - -**Files:** -- Modify: `gen/Structures/CustomKind.dhall` -- Modify: `gen/Interpreters/{CustomType,Member,ParamsMember,Primitive,Project,Query,QueryFragments,Result,ResultColumns,Scalar,Value}.dhall` -- Delete: `gen/Algebras/` (all three files) - -**The mechanical recipe, applied to every file above:** - -1. Replace `let Deps = ../Deps/package.dhall` with direct imports of exactly - what the file uses. Every one of these files uses `Deps.Sdk.Project` - (→ becomes a direct `Deps/Contract.dhall` import) and `Deps.Lude`/`Deps.Prelude` - (→ direct imports). Concretely, replace: - ```dhall - let Deps = ../Deps/package.dhall - ``` - with (only the lines this particular file actually needs — check with - `grep -n 'Deps\.' ` first): - ```dhall - let Lude = ../Deps/Lude.dhall - - let Prelude = ../Deps/Prelude.dhall - - let Model = ../Deps/Contract.dhall - ``` - and change every remaining `Deps.Lude` → `Lude`, `Deps.Prelude` → `Prelude` - in the body. `QueryFragments.dhall` additionally has an unused - `let Sdk = Deps.Sdk` line (line 7) — drop it, nothing in the file - references the `Sdk` binding. - -2. Replace the line `let Model = Deps.Sdk.Project` (now redundant with step - 1's `Model` binding) — don't duplicate it, step 1 already introduces - `Model` pointed at `Deps/Contract.dhall`. - -3. Delete `let Algebra = ../Algebras/Interpreter.dhall`. - -4. Add a local `Config` type declaration (same 4 fields everywhere per the - Global Constraints note on deferred narrowing): - ```dhall - let Config = - { packageName : Text - , importName : Text - , emitSync : Bool - , onUnsupported : OnUnsupported.Mode - } - ``` - This needs `OnUnsupported = ../Structures/OnUnsupported.dhall` imported in - any file that doesn't already import it (check first — `Project.dhall` - already does). - -5. Change every `\(config : Algebra.Config) ->` to `\(config : Config) ->`. - -6. Change the tail: - - **10 of the 11 files** (`CustomType`, `Primitive`, `Project`, `Query`, - `QueryFragments`, `Result`, `ResultColumns`, `Scalar`, `Value` — 9 files, - not 10; `Member`/`ParamsMember` are the exception below) end with - `Algebra.module Input Output run` (or, for `Scalar.dhall`, - `Algebra.module Input Output run /\ { ScalarDecode }`). Change to: - ```dhall - Sdk.Sigs.interpreter Config Input Output run - ``` - (`Scalar.dhall`: `Sdk.Sigs.interpreter Config Input Output run /\ { ScalarDecode }`), - which needs `let Sdk = ../Deps/Sdk.dhall` imported (it isn't currently, - since `Deps.Sdk.Project` used to come through the barrel — add it). - - **`Member.dhall` and `ParamsMember.dhall`** keep their existing tail - verbatim: `in { Input, Output, Run, run }`, just with `Algebra.Config` → - `Config` in the `Run` type alias line - (`let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output`). - No `Sdk.Sigs.interpreter` here — see Global Constraints. - -7. `Interpreters/Project.dhall` specifically: its two `\(config : Algebra.Config) ->` - occurrences (the `combineOutputs` and `run` functions) both become - `\(config : Config) ->`; its `lookupConfig : Algebra.Config` type - annotation (used to type-check `Value.run` calls for composite-field - rendering) becomes `lookupConfig : Config`. No other logic in this file - changes — the `Skip`/`Fail` machinery is untouched per Global Constraints. - -- [ ] **Step 1: Apply the recipe to all 11 `Interpreters/*.dhall` files and `Structures/CustomKind.dhall`.** -- [ ] **Step 2: Delete `gen/Algebras/`.** -- [ ] **Step 3: Verify** (with the `mise`-pinned toolchain, not the bare local `dhall` — see Global Constraints): - ```bash - mise x -- dhall type --file gen/Interpreters/Project.dhall - ``` - Expected: prints the interpreter's type (a record with `Input`, `Output`, - `Result`, `Run`, `run` fields) with no error. This alone pulls in every - other `Interpreters/*.dhall` transitively, so it's a full check of this task. -- [ ] **Step 4: Commit.** - ---- - -### Task 3: Rewire the 10 `Templates/*.dhall` files - -**Files:** -- Modify: `gen/Templates/{CompositeModule,CoreModule,EnumModule,FacadeModule,InitModule,RegisterModule,RowsModule,RuntimeModule,StatementModule,TypesInit}.dhall` - -**Recipe:** - -1. `CoreModule.dhall`, `InitModule.dhall`, `RuntimeModule.dhall` don't import - `Deps` at all (no `Prelude`/`Lude` need) — only change: drop - `let Algebra = ../Algebras/Template.dhall`, add `let Sdk = ../Deps/Sdk.dhall`, - and change the tail. `CoreModule`/`RuntimeModule` end with - `Algebra.module {} (\(_ : {}) -> content)` → `Sdk.Sigs.template {} (\(_ : {}) -> content)`. - `InitModule` ends with `Algebra.module Params render` → - `Sdk.Sigs.template Params render`. - -2. The other 7 (`CompositeModule`, `EnumModule`, `FacadeModule`, - `RegisterModule`, `RowsModule`, `StatementModule`, `TypesInit`) currently - have `let Deps = ../Deps/package.dhall`, and only ever use - `Deps.Prelude.*` (all seven) and, additionally, `Deps.Lude.Text.indentNonEmpty` - (`RowsModule`, `StatementModule` only — confirmed by - `grep -n 'Deps\.' gen/Templates/*.dhall` during planning). Replace the - barrel import with: - ```dhall - let Prelude = ../Deps/Prelude.dhall - ``` - adding `let Lude = ../Deps/Lude.dhall` only in `RowsModule.dhall` and - `StatementModule.dhall`. Then replace `Deps.Prelude.` → `Prelude.` and - `Deps.Lude.` → `Lude.` throughout each file's body. Drop - `let Algebra = ../Algebras/Template.dhall`, add `let Sdk = ../Deps/Sdk.dhall`. - -3. Tails for these 7: `Algebra.module Params run` → - `Sdk.Sigs.template Params run` (`FacadeModule`, `RegisterModule`, - `StatementModule`); with a combined record for the other 4: - `Algebra.module Params run /\ { Field }` (`CompositeModule`) → - `Sdk.Sigs.template Params run /\ { Field }`; `/\ { Variant }` (`EnumModule`); - `/\ { StatementExport, TypeExport }` (`FacadeModule` — check which of - `FacadeModule`/others actually has this combinator vs a plain - `Algebra.module Params run`, per the earlier grep output, before editing — - don't assume, re-`grep -n 'Algebra.module' gen/Templates/*.dhall` and - match each file's exact current tail); `/\ { RowDef }` (`RowsModule`); - `/\ { Export }` (`TypesInit`). - -- [ ] **Step 1: Apply the recipe to all 10 files.** -- [ ] **Step 2: Verify:** - ```bash - mise x -- dhall type --file gen/Interpreters/Project.dhall - ``` - (Templates are only reachable transitively through `Interpreters/Project.dhall` - and its children, same as Task 2 — this single check covers both tasks once - both are done. If running Task 2 and 3 as separate commits, this step will - fail after Task 2 alone if any interpreter references a not-yet-updated - template's old shape; if so, do Tasks 2 and 3 as one commit instead.) -- [ ] **Step 3: Commit.** - ---- - -### Task 4: Rewrite the root entry point (`Config.dhall`, `compile.dhall` → `Interpret.dhall`, `Gen.dhall` → `package.dhall`) - -**Files:** -- Modify (move, content unchanged): `gen/Config.dhall` -- Modify (move + rewrite): `gen/compile.dhall` → `gen/Interpret.dhall` -- Modify (move + rewrite): `gen/Gen.dhall` → `gen/package.dhall` - -(Paths shown as `gen/...` here since Task 5 does the `gen/` → `src/` directory -move; do this task first, in place, then Task 5 is a pure `git mv` sweep with -no further content changes.) - -**Interfaces:** -- Consumes: `Interpreters/Project.dhall`'s `run` (produced by Task 2, - now `Sdk.Sigs.interpreter`-shaped: `.run : Config -> Contract.Project -> Compiled Output`). -- Produces: `package.dhall`'s `Sdk.Sigs.generator`-built value - (`{ contractVersion, Config, compile }`), consumed by Task 5's - `demos/Exhaustive.dhall` and by any pGenie project's `artifacts..gen` URL. - -`Sdk.Sigs.generator`'s shape (from the architecture doc): -```dhall -\(Config : Type) -> -\(defaultConfig : Config) -> -\(interpret : Config -> Contract.Project -> Contract.Output) -> - let compile = \(config : Optional Config) -> - merge { None = interpret defaultConfig, Some = interpret } config - in Contract.module Config compile -``` -Note `interpret` takes a **bare** `Config`, not `Optional Config` — the outer -"config block omitted entirely" case is handled once, by substituting -`defaultConfig`, not by `interpret` itself. `python.gen`'s current -`compile.dhall` handles *two* levels of optionality (the whole block, and -each field within it) with a doubled `Prelude.Optional.fold`. Only the outer -level goes away; each field inside `Config` stays individually `Optional` (so -a project can supply `emitSync: true` alone and still get default -`packageName`/`onUnsupported`) — that per-field defaulting is -`python.gen`-specific richness `java.gen` doesn't have (its `Config` has one -non-Optional `Bool` field), and this migration must not lose it. - -- [ ] **Step 1: `gen/Config.dhall`** — content unchanged, just confirm it - still reads (no edits needed here; listed for completeness since Task 5 - moves the file). - -- [ ] **Step 2: Rewrite `gen/compile.dhall` as `gen/Interpret.dhall`** — - drop the outer `Optional Config` unwrap (the two outermost - `Prelude.Optional.fold Config config Text (\(c : Config) -> ...)` / - `... Bool ...` / `... OnUnsupported.Mode ...` wrappers), keep everything - else (the per-field defaults, `importName` derivation) as is: - -```dhall -let Deps = ./Deps/package.dhall - --- NOTE: Task 5 changes this to ./Deps/Contract.dhall / ./Deps/Prelude.dhall --- directly once the Deps barrel is gone (Task 1) — write it that way now, --- don't reintroduce the barrel: -let Contract = ./Deps/Contract.dhall - -let Prelude = ./Deps/Prelude.dhall - -let Config = ./Config.dhall - -let OnUnsupported = ./Structures/OnUnsupported.dhall - -let ProjectInterpreter = ./Interpreters/Project.dhall - --- Entry point handed to gen-sdk's Sdk.Sigs.generator as `interpret`. Each --- field of Config is independently Optional, so a project may omit the --- whole config block (Sdk.Sigs.generator substitutes an all-None --- defaultConfig, see package.dhall) or any subset of its keys; `defaults` --- collects every fallback in one place (packageName from the project name in --- kebab case, emitSync off, onUnsupported Fail). The async surface is always --- emitted; emitSync adds the sync mirror. -in \(config : Config) -> - \(project : Contract.Project) -> - let defaults = - { packageName = project.name.inKebabCase - , emitSync = False - , onUnsupported = OnUnsupported.Mode.Fail - } - - let packageName = - Prelude.Optional.fold - Text - config.packageName - Text - (\(t : Text) -> t) - defaults.packageName - - let emitSync = - Prelude.Optional.fold - Bool - config.emitSync - Bool - (\(b : Bool) -> b) - defaults.emitSync - - let onUnsupported = - Prelude.Optional.fold - OnUnsupported.Mode - config.onUnsupported - OnUnsupported.Mode - (\(m : OnUnsupported.Mode) -> m) - defaults.onUnsupported - - let importName = Prelude.Text.replace "-" "_" packageName - - let interpreterConfig = { packageName, importName, emitSync, onUnsupported } - - in ProjectInterpreter.run interpreterConfig project -``` - -- [ ] **Step 3: Rewrite `gen/Gen.dhall` as `gen/package.dhall`:** - -```dhall -let Sdk = ./Deps/Sdk.dhall - -let OnUnsupported = ./Structures/OnUnsupported.dhall - -let Config = ./Config.dhall - -let Config/default - : Config - = { packageName = None Text - , emitSync = None Bool - , onUnsupported = None OnUnsupported.Mode - } - -let interpret = ./Interpret.dhall - -in Sdk.Sigs.generator Config Config/default interpret -``` - -- [ ] **Step 4: Verify:** - ```bash - mise x -- dhall type --file gen/package.dhall - ``` - Expected type: a record with `contractVersion`, `Config`, `compile` fields - (`compile : Optional Config -> Contract.Project -> Contract.Output`). -- [ ] **Step 5: Commit.** - ---- - -### Task 5: Move `gen/` → `src/`, `tests/Exhaustive.dhall` → `demos/Exhaustive.dhall` - -**Files:** -- Move: `gen/` → `src/` (whole tree, `git mv`) -- Move + rewrite: `tests/Exhaustive.dhall` → `demos/Exhaustive.dhall` - -- [ ] **Step 1:** - ```bash - git mv gen src - mkdir -p demos - git mv tests/Exhaustive.dhall demos/Exhaustive.dhall - ``` - All the `../Deps/...`, `./Interpreters/...`, `../Templates/...` style - relative imports inside `src/` are untouched by this move (they're relative - to their own file, not to the repo root), so no content changes are needed - inside `src/` itself from the move alone. - -- [ ] **Step 2: Rewrite `demos/Exhaustive.dhall`.** Its old body called - `Gen.compileToFileMap config project` — `Sdk.Sigs.generator`-built modules - don't have a `compileToFileMap` field (per the architecture doc: "there is - no `compileToFileMap` on the module — turning an `Output` into files is the - caller's job, via `Sdk.Output.toFileMap`"). New content: - -```dhall --- Applies this generator to gen-sdk's shared cross-backend fixture project --- (the same "music_catalogue" project java.gen's own demos/Exhaustive.dhall --- exercises), so a Python client compiles from it and passes basedpyright --- strict. Pinned directly at gen-sdk's package.dhall, separately from --- src/Deps/Sdk.dhall: that file only imports gen-sdk's `package.dhall` `as --- Source` for RAM, and this fixture load doesn't need that mode. --- --- The fixture project deliberately covers PG types this generator does not --- support (box, inet, money, ranges, ...), so onUnsupported is set to Skip: --- those statements/types are dropped with a warning instead of aborting the --- whole compile. --- --- Intended to be executed with: --- --- ```bash --- dhall to-directory-tree --file demos/Exhaustive.dhall --output --allow-path-separators --- ``` -let Sdk = ../src/Deps/Sdk.dhall - -let Gen = ../src/package.dhall - -let OnUnsupported = ../src/Structures/OnUnsupported.dhall - -let project = Sdk.Fixtures.Exhaustive - -let config = - Some - { packageName = None Text - , emitSync = Some True - , onUnsupported = Some OnUnsupported.Mode.Skip - } - -in Sdk.Output.toFileMap (Gen.compile config project) -``` - -- [ ] **Step 3: Verify:** - ```bash - mise x -- dhall type --file demos/Exhaustive.dhall - ``` - Expected: `List { mapKey : Text, mapValue : Text }` (or however this - fork/version of Dhall renders `Prelude.Map.Type Text Text`), no error. -- [ ] **Step 4: Commit.** - ---- - -### Task 6: Update every external reference to the old paths - -**Files:** -- Modify: `.github/workflows/ci.yml` -- Modify: `.github/workflows/release.yml` -- Modify: `.github/scripts/build-contract-shell.sh` -- Modify: `README.md` -- Modify: `AGENTS.md` -- Modify: `DESIGN.md` -- Modify: `build.bash` -- Modify: `bench/generate.sh`, `bench/as-source.sh` -- Modify: `mise.toml` (the `golden` task) - -**Path substitutions to apply everywhere they occur** (verify each hit with -`grep -rn` first — don't blind-sed across the whole repo, `tests/golden/` -contains generated Python that must NOT be touched): - -| Old | New | -|---|---| -| `gen/Gen.dhall` | `src/package.dhall` | -| `gen/Deps/*.dhall` | `src/Deps/*.dhall` | -| `tests/Exhaustive.dhall` | `demos/Exhaustive.dhall` | -| `gen/` (prose/dir references) | `src/` | - -Specific known hits (from `grep -rn "gen/Gen\.dhall\|gen/Deps\|tests/Exhaustive"` -run during planning): - -- `.github/workflows/ci.yml`: the `contract` job's "Strip `as Source`..." - step does `sed -i ... gen/Deps/*.dhall` → `src/Deps/*.dhall`; the - "Generate output from Dhall" step's `dhall_file: tests/Exhaustive.dhall` → - `demos/Exhaustive.dhall`. -- `.github/workflows/release.yml`: the "Resolve Dhall" step's - `file: gen/Gen.dhall` → `file: src/package.dhall`. -- `.github/scripts/build-contract-shell.sh`: comment references - `tests/Exhaustive.dhall` → `demos/Exhaustive.dhall` (comment only, verify - no functional path argument needs changing — it's invoked with - `contract-output` as a positional arg per `ci.yml`, not a hardcoded path). -- `README.md`: line ~38 `gen: https://raw.githubusercontent.com/slavashvets/python.gen/master/gen/Gen.dhall` - → `.../src/package.dhall`; lines ~64-66, the three example URLs - (`.../gen/Gen.dhall`) → `.../src/package.dhall`. -- `AGENTS.md`: line ~36 "`gen/` pins its remote imports by sha256 - (`gen/Deps/*.dhall`)" → "`src/` pins its remote imports by sha256 - (`src/Deps/*.dhall`)". -- `DESIGN.md`: line 4 (`gen/`), line 381 (`gen/Gen.dhall` "is the entry point - handed to gen-sdk"), line 397 (`gen/` mirrors...), line 401 (the `gen/` - tree diagram — replace with the new `src/` tree, matching Task 5's actual - post-move layout), lines 632/636 (`tests/Exhaustive.dhall` → `demos/Exhaustive.dhall`). -- `build.bash`: this is a scratch/dev script (mostly commented-out lines) — - update the live lines: `target=tests/Exhaustive.dhall` → - `target=demos/Exhaustive.dhall`; the commented `# target=gen/Gen.dhall` and - `# dhall freeze gen/Deps/*.dhall` lines → `src/` equivalents (keep them - commented, just fix the paths so they're not stale if uncommented later). -- `bench/generate.sh`, `bench/as-source.sh`: both `cp -R "$root/gen" "$out/gen"` / - `"$1/gen"` → `"$root/src" "$out/src"` (and update the `gen/Deps/*.dhall` - perl substitutions to `src/Deps/*.dhall`). **The `as Source` → plain-import - sha256 substitutions in both scripts' `strip_as_source` are pinned to the - *old* `gen-sdk v0.11.0`/`lude v5.1.0` source-vs-normalized hash pairs** - (`8d43544e...`→`b9f7bb84...` for Sdk, `46b527b0...`→`14c43eec...` for Lude). - Since `Task 1` bumps `gen-sdk` to `v2.0.0`, these substitution pairs are now - wrong and must be recomputed for the new pin using the repo's actual pinned - toolchain (`mise x -- dhall hash` on the plain, non-`as-Source` import) — - don't guess these; if the recompute can't happen in this pass, leave a - `# TODO` in the script rather than shipping a silently-wrong benchmark. -- `mise.toml`'s `golden` task: the `python3 - ... "$root/gen/Gen.dhall"` arg - and the fixture-project string replace target `"../../gen/Gen.dhall"` → - `"../../src/package.dhall"`. - -- [ ] **Step 1: Apply all substitutions above.** -- [ ] **Step 2: Confirm no stragglers:** - ```bash - grep -rn "gen/Gen\.dhall\|gen/Deps\|gen/Interpreters\|gen/Templates\|gen/Structures\|gen/Config\.dhall\|gen/compile\.dhall\|tests/Exhaustive" \ - --include="*.md" --include="*.yml" --include="*.yaml" --include="*.toml" --include="*.sh" --include="*.bash" . - ``` - Expected: no output (everything left under `tests/golden/` or `tests/fixture-project/` - that isn't a generator-path reference is fine and out of scope — check any - hit manually rather than assuming). -- [ ] **Step 3: Commit.** - ---- - -### Task 7: Format, verify end-to-end, update CHANGELOG - -- [ ] **Step 1: Format everything:** - ```bash - mise x -- dhall format --transitive src/package.dhall - mise x -- dhall format --transitive demos/Exhaustive.dhall - ``` - -- [ ] **Step 2: Full type-check:** - ```bash - mise x -- dhall type --file src/package.dhall - mise x -- dhall type --file demos/Exhaustive.dhall - ``` - -- [ ] **Step 3: Regenerate the Exhaustive fixture and confirm no diff in - output** (this is the load-bearing check — everything above only proves - the Dhall type-checks, not that it still produces the same files): - ```bash - mise x -- dhall to-directory-tree --allow-path-separators --file demos/Exhaustive.dhall --output /tmp/pygen-after - ``` - Compare against a snapshot taken from the pre-migration tree the same way - (`git stash`, regenerate to `/tmp/pygen-before`, `git stash pop`, `diff -rq - /tmp/pygen-before /tmp/pygen-after`). Expected: **no diff**. Any diff here - is a migration bug, not an intentional update — per Global Constraints, go - fix it rather than accepting the new output. - -- [ ] **Step 4: Run the Python harness:** - ```bash - mise x -- uv sync - mise x -- uv run pytest tests -v - ``` - (Needs a reachable Postgres — `PGN_TEST_DATABASE_URL`, see `ci.yml` for the - Docker Compose equivalent — and the `mise`-pinned `pgn 0.9.1`, since the - harness shells out to it.) Expected: all green, no new failures relative to - a pre-migration run. - -- [ ] **Step 5: Add a CHANGELOG.md entry** under `# Upcoming` (the file - already starts with that heading), non-breaking, modeled on `java.gen`'s - own `v1.1.0` entry: - ```markdown - - Migrated the generator's internal dependencies to `gen-contract` v4.0.1 - and `gen-sdk` v2.0.0, adopting `Sdk.Sigs` in place of the local - `Algebras/` module, and restructured the repository layout to match the - pGenie generator architecture: implementation moved from `gen/` to - `src/`, the public entry point renamed from `gen/Gen.dhall` to - `src/package.dhall`, and the fixture driver moved from - `tests/Exhaustive.dhall` to `demos/Exhaustive.dhall`. No change to - generated output or the public Dhall interface (`artifacts..gen` - URLs pointing at a previously-released `resolved.dhall` are unaffected; - only the next release's URL path changes, from `.../gen/Gen.dhall` — the - unresolved source path some projects may reference directly instead of a - frozen release — to `.../src/package.dhall`). - ``` - Adjust the last parenthetical if no project in practice points at the - unresolved source path (check `README.md`'s own recommended usage — if it - only ever recommends the frozen `resolved.dhall` release asset, simplify - this to "no change to generated output or the public Dhall interface"). - -- [ ] **Step 6: Commit.** - -## Deferred / explicitly out of scope (record as follow-ups, don't do now) - -- Narrowing each interpreter's `Config` to only the fields it needs (see - Global Constraints). -- Giving `Member.dhall`/`ParamsMember.dhall` a real `Sdk.Sigs.interpreter` - shape by folding `CustomKind.Lookup` into `Config` or `Input` (see Global - Constraints). -- Adding a `Name` interpreter (`java.gen` has `Interpreters/Name.dhall` - centralizing identifier casing/escaping; `python.gen` calls - `Structures/PyIdent.dhall` ad hoc from several interpreters instead). This - is a real architecture-doc deviation but not something `java.gen`'s last - release changed — separate task if wanted. -- Recomputing the `as Source` bench-script hash pairs for `gen-sdk v2.0.0` / - `lude v5.1.0` (flagged inline in Task 6 — needs the real toolchain, not - guessable). From 7b3ef57f68105b5c5dbfee2a1e41117fea135e96 Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sun, 12 Jul 2026 06:16:12 +0300 Subject: [PATCH 10/21] Fix --- CHANGELOG.md | 40 ++ DESIGN.md | 294 +++++--- ...26-07-11-encounter-order-custom-imports.md | 327 +++++++++ .../2026-07-11-reusable-custom-type-codecs.md | 633 ++++++++++++++++++ docs/upstream-asks.md | 12 + src/Config.dhall | 15 - src/Deps/Lude.dhall | 5 +- src/Deps/Prelude.dhall | 1 - src/Deps/Sdk.dhall | 1 - src/Interpret.dhall | 54 -- src/Interpreters/CustomType.dhall | 14 +- src/Interpreters/Member.dhall | 180 ++--- src/Interpreters/ParamsMember.dhall | 121 +--- src/Interpreters/Project.dhall | 180 ++--- src/Interpreters/Query.dhall | 11 +- src/Interpreters/Result.dhall | 27 +- src/Interpreters/ResultColumns.dhall | 10 +- src/Structures/CustomKind.dhall | 28 - src/Structures/ImportSet.dhall | 115 +--- src/Templates/CompositeModule.dhall | 42 +- src/Templates/EnumModule.dhall | 11 + src/Templates/RowsModule.dhall | 2 +- src/Templates/StatementModule.dhall | 2 +- src/package.dhall | 21 +- tests/test_config_variants.py | 2 +- 25 files changed, 1483 insertions(+), 665 deletions(-) create mode 100644 docs/plans/2026-07-11-encounter-order-custom-imports.md create mode 100644 docs/plans/2026-07-11-reusable-custom-type-codecs.md delete mode 100644 src/Config.dhall delete mode 100644 src/Interpret.dhall delete mode 100644 src/Structures/CustomKind.dhall diff --git a/CHANGELOG.md b/CHANGELOG.md index f98fe04..b55b2bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,45 @@ # Upcoming +- `buildLookup` (`Interpreters/Project.dhall`) and, with it, this generator's + last dependency on pgn's fork-only `Text/equal` builtin are removed from + `src/`: custom-type decode/encode now dispatches through named + `_decode`/`_encode` methods generated onto each custom type's own Python + class (`CompositeModule.dhall`/`EnumModule.dhall`), called by name from + every reference site, instead of resolving classification and fields via a + project-wide structural search (`grep -rn "Text/equal" src` now returns + only two explanatory comments, zero invocations). Array (dims > 0) + decode/encode is built at the call site (`Member.dhall`/ + `ParamsMember.dhall`) instead of a third per-type method, delegating only + the per-element transform to `_decode`/`_encode`: an earlier draft this + session added a per-type `_decode_array` to `EnumModule.dhall`, but it + could not express `elementIsNullable` (a per-column fact, not a per-type + one) and silently broke nullable-element enum-array decode and + enum-array param encode — both working, corpus-exercised paths — caught + by the final whole-branch review and fixed before merge. Behavior change: + because the call site is now kind-uniform, a 1-D composite-array column + or param is no longer rejected at Dhall-generation time the way it used + to be, and — unlike the `_decode_array` design it replaces — no longer + depends on `basedpyright strict` catching a missing method either, since + `_decode`/`_encode` genuinely exist on a composite class too. **This path + has not been exercised against real Postgres, and `tests/golden/` has NOT + been regenerated for this change this session** — the composite-array + fixture addition, its golden regeneration, and confirming actual Postgres + round-trip behavior are a known, deliberate gap in this commit, deferred + to a follow-up pass on a properly provisioned machine (see + `docs/plans/2026-07-11-reusable-custom-type-codecs.md`). + Separately, a composite field nesting another custom type is *also* no + longer rejected at generation time: the `nestedLookup = Absent` stub that + used to force it down the same loud-fail path is gone (it only existed + to satisfy `Member.run`'s old signature). This is not the same kind of + change as the composite-array case above, though — `CompositeModule.dhall`'s + `_decode`/`_encode` still do a blind flat `cast(tuple[...], src)`/splat, + unchanged by this refactor, and never recurse into the nested type's own + codec, so the field silently decodes/encodes wrong rather than being + caught by a type checker. Because the failure mode is `cast()`, which + suppresses type-checking on its argument by design, this is **not** + expected to be caught by `basedpyright strict`. It is a real, silent + architecture gap, flagged here as an open follow-up design question, not + a shipped or backstopped behavior change. - Migrated the generator's internal dependencies to `gen-contract` v4.0.1 and `gen-sdk` v2.0.0, adopting `Sdk.Sigs` in place of the local `Algebras/` module, and restructured the repository layout to match the diff --git a/DESIGN.md b/DESIGN.md index 79e8a7c..952e4cc 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -125,18 +125,28 @@ those the decode is a `cast(, row[""])` to satisfy strict - enum column: `Mood(cast(str, row["feeling"]))`; nullable guards `None`. - enum array column: element-wise rebuild, - `[Mood(v) for v in cast(list[str], row["..."])]`, with per-element and - outer `None` guards driven by `elementIsNullable` / column nullability. - Requires the enum's TypeInfo registered (section 6). + `[Mood._decode(v) for v in cast(list[str], row["..."])]`, with per-element + and outer `None` guards driven by `elementIsNullable` / column nullability, + built at the reference site in `Member.dhall` rather than a per-type array + method (see below). Requires the enum's TypeInfo registered (section 6). - composite column: psycopg returns a namedtuple once the composite is registered; decode is `TypeName(*cast(tuple[...], row["..."]))` with the exact field types. `Member.run` (`Interpreters/Member.dhall`) builds `decodeExpr` as a `Text -> Text` function next to the type info, so `_rows.py` and `types/` -compose the same logic. A composite array, or an enum array with -`dims > 1`, is unimplemented and fails loudly (`Compiled.report`) rather -than emitting wrong Python. +compose the same logic. Any custom-type array with `dims > 1` is +unimplemented and fails loudly (`Compiled.report`), regardless of kind. For +`dims == 1`, `Member.dhall` builds the list comprehension itself +(`[${typeName}._decode(v) for v in cast(...)]`, with per-element/outer +`None` guards driven by `value.elementIsNullable`/`isNullable`) instead of +calling a per-type array method: a per-type, zero-argument method has no way +to see `elementIsNullable`, a per-*column* fact, so it cannot express it. +This makes the branch kind-uniform: a 1-D composite-array column now +type-checks and attempts a real decode the same way an enum-array column +does, instead of being rejected at Dhall-generation time the way it used +to be. See section 12 for why that changed and for the currently-unverified +state of that path. --- @@ -226,10 +236,15 @@ no partial output for the affected query under `Fail`. Add a mapping to The param side (`Interpreters/ParamsMember.dhall`) carries the same loud-fail contract for bind shapes psycopg cannot adapt faithfully. It -rejects, rather than silently mis-binds: - -- a `json`/`jsonb` ARRAY param (`Jsonb` wraps a scalar, not element-wise). -- a composite ARRAY param (psycopg cannot adapt the dataclass array). +still rejects a `json`/`jsonb` ARRAY param (`Jsonb` wraps a scalar, not +element-wise). A composite ARRAY param is no longer rejected here the way +it used to be (see section 12): encode branches on `Natural/isZero +value.dims`, calling `._encode()` for a scalar custom-type param and +building `[x._encode() for x in ]` (with the same +`elementIsNullable`/outer-nullable guards as decode) for an array one, so a +composite-array param now type-checks and attempts a genuine per-element +encode rather than depending on `basedpyright strict` to catch a missing +method. ### Arrays and nullability @@ -276,12 +291,34 @@ composite (a single `value: str | None` field) exists specifically to cover this, exercised as both a param and a `RETURNING` result column in the same statement, with a round-trip test. -Composite fields cannot themselves reference another custom type. -`CustomType.dhall` hardcodes the nested-member lookup to `Absent` -(`nestedLookup`), so a composite nesting a composite (or an enum) fails the -same loud-fail path as any other unresolvable reference rather than -guessing a decode. Widening this is possible but has not been needed by any -project this generator has shipped against yet. +Composite fields nesting another custom type used to be explicitly +rejected: `CustomType.dhall` called `Member.run` with a `nestedLookup` stub +hardcoded to `Absent`, forcing the same loud-fail path as any other +unresolvable reference rather than guessing a decode. That stub is gone — +it existed only to satisfy `Member.run`'s old signature, and was deleted +along with `buildLookup` (section 12) — but its removal does not make this +case work; it only removed the one thing that used to reject it at +generation time. `Member.run` does compute a named-codec `decodeExpr` +(`${typeName}._decode(...)`) for a `Custom`-typed field, but +`CustomType.dhall`'s Composite branch never threads it anywhere: it maps +each member down to a flat `{fieldName, fieldType}` pair (the `Field` shape +`Templates/CompositeModule.dhall` takes) and discards `decodeExpr` +entirely. `CompositeModule.dhall`'s `_decode`/`_encode` — unchanged by this +refactor — render a single blind `${typeName}(*cast(tuple[...], src))` +splat and a flat `(self.field1, ...)` tuple; neither ever calls a nested +field's own `_decode`/`_encode`. So a composite field whose own type is +another custom type still does not decode/encode correctly at +runtime — it is just no longer *rejected* at generation time the way it +used to be. Unlike the composite-array case (section 12), this is **not** +expected to be caught by `basedpyright strict`: `cast()` exists +specifically to suppress type-checking on its argument, so the checker +sees exactly the annotated field type and raises nothing. This is a real, +silent architecture gap introduced by this refactor — flagged here as an +open follow-up design question (should `CustomType.dhall` thread a +member's own `decodeExpr` through to `CompositeModule.dhall`, or does +`_decode` need to become field-aware instead of a blind tuple cast?), not +something fixed in this commit and not on the same footing as the +composite-array case's deferred-but-backstopped behavior change. --- @@ -383,20 +420,27 @@ overwritten on every run. Do not hand-edit it. ```dhall let Sdk = ./Deps/Sdk.dhall -let Config = ./Config.dhall +let OnUnsupported = ./Structures/OnUnsupported.dhall + +let ProjectInterpreter = ./Interpreters/Project.dhall + +let Config = { packageName : Optional Text, emitSync : Optional Bool, onUnsupported : Optional OnUnsupported.Mode } -let interpret = ./Interpret.dhall +let Config/default = { packageName = None Text, emitSync = None Bool, onUnsupported = None OnUnsupported.Mode } -in Sdk.Sigs.generator Config Config/default interpret +in Sdk.Sigs.generator Config Config/default ProjectInterpreter.run ``` `Sdk.Sigs.generator` has signature `\(Config : Type) -> \(defaultConfig : Config) -> \(interpret : Config -> Contract.Project -> Contract.Output) -> ...`; it curries `interpret` against `defaultConfig` whenever the user config is absent and hands -the result to gen-contract's `Contract.module`. `Interpret.dhall` folds the -optional user config into the internal interpreter config and calls -`Interpreters/Project.dhall`, which traverses queries and custom types and -assembles the file list (`Contract.Output`). +the result to gen-contract's `Contract.module`. `Config` is passed straight +through to `Interpreters/Project.dhall` as that interpreter's own `Config` -- +there is no separate config type or resolve step in between, matching every +other gen (java.gen, rust.gen, haskell.gen). `Project.run` folds the optional +user config into the fully-resolved internal config itself (see "Config flow" +below), traverses queries and custom types, and assembles the file list +(`Contract.Output`). `src/` mirrors a typical pgn gen-sdk generator, Python-flavored: `Interpreters/` assembles data, `Templates/` renders it to Python text. The interpreter/template @@ -405,13 +449,10 @@ algebra signatures themselves live in gen-sdk's `Sdk.Sigs` (`interpreter.dhall`/ ```text src/ - package.dhall # Sdk.Sigs.generator Config Config/default interpret (entry handed to gen-sdk) - Config.dhall # user config TYPE: { packageName, emitSync, onUnsupported } - Interpret.dhall # derive interpreter Config from user Config, call Project.run + package.dhall # Config, Config/default, Sdk.Sigs.generator Config Config/default ProjectInterpreter.run Deps/ # pinned remote imports: gen-sdk, gen-contract, lude, dhall Prelude Structures/ Surface.dhall # async/sync token table (section 4) - CustomKind.dhall # Lookup : Name -> < Enum | Composite | Absent > + composite fields ImportSet.dhall # per-module import flags + combine PyIdent.dhall # sanitize names that become Python identifiers (keyword -> name_) OnUnsupported.dhall # < Fail | Skip > (section 11) @@ -438,29 +479,39 @@ src/ EnumModule.dhall / CompositeModule.dhall / TypesInit.dhall / InitModule.dhall ``` -The cross-cutting dependency is the custom-type lookup: `Scalar`/`Value`/ -`Primitive` stop at "Custom + Name"; `Project.run` builds a -`CustomKind.Lookup : Name -> < Enum | Composite | Absent >` from the -(post-Skip-filter) custom types and threads it to `Query.run` -> -`Result`/`ResultColumns`/`ParamsMember`/`Member`. The lookup compares by -`name.inSnakeCase` (the column carries a distinct `Name` occurrence, so -record identity cannot be relied on) and carries the type's alphabetical -index so per-module custom-type import blocks stay sorted. Section 12 -covers why this one comparison still needs a Dhall fork builtin. +The cross-cutting dependency used to be a project-wide custom-type lookup: +`Project.run` built a `CustomKind.Lookup : Name -> < Enum | Composite | +Absent >` from the (post-Skip-filter) custom types and threaded it to +`Query.run` -> `Result`/`ResultColumns`/`ParamsMember`/`Member`. That +lookup, and `Structures/CustomKind.dhall` itself, are deleted. `Scalar`/ +`Value`/`Primitive` still stop at "Custom + Name", but `Member.dhall`/ +`ParamsMember.dhall` now resolve a `Custom` reference by calling the +generated class's `_decode`/`_encode` method directly, keyed off +`name.inPascalCase` — no project-wide search, no classification step +threaded through the query pipeline. Array (dims > 0) decode/encode stays +local to the call site rather than becoming a third per-type method, +because `elementIsNullable` is a per-column fact a per-type method cannot +see. Section 12 covers the removal and the behavior change it introduced. +The old lookup's alphabetical index, used to keep per-module custom-type +import blocks sorted and deduped, is also gone: `ImportSet.dhall` now +renders custom-type imports in encounter order, unsorted and undeduped +(see the comment at its top). ### Config flow -`Config.dhall` is the user-facing type `{ packageName : Optional Text, -emitSync : Optional Bool, onUnsupported : Optional OnUnsupported.Mode }`. -`compile.dhall` folds the optional user config, and each of its optional -fields, into the internal interpreter Config `{ packageName, importName, -emitAsync = True, emitSync, onUnsupported }`, with `packageName` falling -back to the project name in kebab case, `emitSync` to `False`, and -`onUnsupported` to `Fail`. `importName` = `packageName` with `-` -> `_`. -`emitAsync` is always `True`. A project's artifact config can therefore omit -`config:` entirely, supply `config: {}`, or set any subset of the three -keys; see the README's Config reference for the decode semantics pgn itself -applies before this fold ever runs. +`package.dhall`'s `Config` is the user-facing type `{ packageName : Optional +Text, emitSync : Optional Bool, onUnsupported : Optional OnUnsupported.Mode +}`, passed straight through to `Interpreters/Project.dhall` as its own +`Config` -- there is no separate config type or resolve step in between. +`Project.run` folds the optional config, and each of its optional fields, +into the fully-resolved `ResolvedConfig` it passes to every interpreter +below it: `{ packageName, importName, emitSync, onUnsupported }`, with +`packageName` falling back to the project name in kebab case, `emitSync` to +`False`, and `onUnsupported` to `Fail`. `importName` = `packageName` with +`-` -> `_`. A project's artifact config can therefore omit `config:` +entirely, supply `config: {}`, or set any subset of the three keys; see the +README's Config reference for the decode semantics pgn itself applies +before `Project.run` ever sees the value. --- @@ -474,13 +525,18 @@ aborts the whole `Project.run` non-zero via `Lude.Compiled.report`. drop the smallest self-consistent unit and keep the rest of the project generating. `Project.dhall`'s `run` computes, once, whether each custom type and each query would have compiled cleanly (`typeSucceeds` / the `keep` -field of a per-query `QueryCheck`), filters the failing ones out before -building the custom-type lookup and the final query list, and separately -collects a `Report` per dropped unit into `combined.warnings`. A query that -references a skipped custom type resolves that reference to `Absent` -through the (already-filtered) lookup and fails its own compile the same -way any other unsupported shape would, which is what makes the drop cascade -without any special-cased "type X depends on type Y" bookkeeping. +field of a per-query `QueryCheck`), filters the failing ones out to produce +the final surviving custom-type and query lists, and separately collects a +`Report` per dropped unit into `combined.warnings`. + +Whether a query referencing a *specific* skipped custom type still cascades +into its own compile failure is worth re-checking rather than assumed: the +project-wide custom-type lookup this cascade used to route through is gone +(section 12), and neither `Member.dhall` nor `ParamsMember.dhall` currently +take the surviving custom-type list as an input to cross-check a +`customRef` name against. This is unrelated to the `Text/equal` removal and +out of scope for this pass; flagged here only so the Skip-mode cascade +isn't assumed unchanged without someone verifying it. The precedent for this shape is java.gen, an earlier gen-sdk generator for a different target language: it skips unconditionally and silently (an @@ -496,34 +552,108 @@ generator is already ready for that; it isn't waiting on it. --- -## 12. Forked-Dhall (`Text/equal`) dependency risk - -ACCEPTED RISK, recorded so nobody is surprised. pgn is a prebuilt binary -that embeds a FORKED Dhall providing a `Text/equal` builtin, which is not -part of the upstream Dhall standard Prelude. This generator uses it in -exactly one place left: `Interpreters/Project.dhall`'s `buildLookup`, to -match a custom type by its snake-case name while building the -`CustomKind.Lookup`. Everywhere else that used to need name equality -(keyword sanitizing in `PyIdent.dhall`) has since been rewritten against a -`Text/replace`-based trick that needs no fork builtin at all; section 13 -covers how. - -`buildLookup`'s use is harder to remove the same way: it returns a -structural `TypeKind` value, not `Text`, so the replace-based marker trick -(built for sanitizing a `Text -> Text` name) doesn't carry over as-is. -gen-sdk's own `Fixtures` module relies on the same builtin, so a Dhall -evaluator without it can't even typecheck gen-sdk's full package entry -point, only the narrower `module.dhall`/`Project.dhall` imports this -generator pins directly. The clean fix is upstream: a `kind` tag or a -`Natural` index carried directly on `Scalar.Custom`, so the lookup becomes -an equality-free structural match. That is a planned ask against gen-sdk, -not something this generator can do unilaterally. - -Consequence: end-to-end regeneration requires the pgn binary (for its -embedded fork Dhall), a live Postgres, and currently-live remote Dhall -imports (`Deps/*` resolve gen-sdk, lude, and the Prelude over the network, -pinned by sha256). There is no pure-upstream-dhall path to reproduce the -output today. +## 12. Forked-Dhall (`Text/equal`) dependency: removed from this generator + +RESOLVED, not an accepted risk anymore. `Interpreters/Project.dhall`'s +`buildLookup` was this generator's last consumer of pgn's FORKED-Dhall-only +`Text/equal` builtin (not part of the upstream Dhall standard Prelude): it +matched a custom type by its snake-case name while building a +`CustomKind.Lookup : Name -> < Enum | Composite | Absent >`, threaded +through `Query.run` so `Result`/`ResultColumns`/`ParamsMember`/`Member` +could classify a `Custom` reference and pull its fields. `buildLookup` and +`CustomKind.Lookup` usage are removed in commit `ffdd9bd`; the now-empty +`Structures/CustomKind.dhall` file itself is deleted separately in commit +`ab8f4df`. + +In their place, `CompositeModule.dhall`/`EnumModule.dhall` now emit a +`_decode`/`_encode` method directly onto each generated custom type's +Python class, covering the scalar (`dims == 0`) case. `Member.dhall` and +`ParamsMember.dhall` call it by name off `name.inPascalCase` at the +reference site (e.g. `${typeName}._decode(src)`) instead of resolving +classification/fields via a project-wide name search. No name-equality +comparison is needed at all anymore, so there's nothing left for +`Text/equal` to do here. `grep -rn "Text/equal" src` confirms this: it +returns exactly two hits, both explanatory comments about why a mechanism +does *not* use the builtin (`Structures/ImportSet.dhall:7`, +`Structures/PyIdent.dhall:48`), zero actual invocations anywhere in `src/`. + +Array (dims > 0) decode/encode is deliberately NOT a third per-type method. +An earlier draft this session gave `EnumModule.dhall` a `_decode_array` +staticmethod, but the final whole-branch review caught that a per-type, +zero-argument array codec cannot express `elementIsNullable` — an +`Optional`-array-settings field that varies per *column*, not per type (a +`moods: list[Mood | None] | None` column needs a different per-element guard +than a `list[Mood]` column of the same enum). That method hardcoded the +non-nullable-element shape unconditionally, silently breaking +nullable-element enum-array decode (a runtime crash on any `NULL` array +element) and, symmetrically, `ParamsMember.dhall`'s unconditional +`${field}._encode()` broke enum-array param encode (calling `._encode()` on +a `list`). Both were working, corpus-exercised paths before this refactor. +The fix moves array handling back to the call site, exactly where it lived +before this refactor: `Member.dhall`'s dims==1 branch and +`ParamsMember.dhall`'s `Natural/isZero value.dims` branch build the list +comprehension locally, reading `value.elementIsNullable`/`value.dims` off +the column- or param-local `Value.Output`, and delegate only the +per-element transform to `${typeName}._decode(v)` / `x._encode()`. This +restores exact parity with the pre-refactor `enumArrayDecode`/array-param +behavior for enums (verified against +`tests/golden/src/specimen_client/_generated/_rows.py`'s `moods` column and +`statements/insert_specimen.py`'s `moods` param — same shape, just calling +`._decode`/`._encode` per element instead of the enum constructor/bare +pass-through). + +A behavior change worth flagging, now unavoidable rather than accidental: +because the call site's array branch is kind-uniform (the same +`${typeName}._decode(v)`/`x._encode()` call per element regardless of +whether `typeName` is an enum or a composite), a 1-D composite-array column +or param is no longer rejected at Dhall-generation time the way it used to +be (the old "Array of a composite type is not supported" reports are +gone), and — unlike the `_decode_array` design it replaces — no longer +depends on `basedpyright strict` catching a missing method either, since +`CompositeModule.dhall`'s `_decode`/`_encode` genuinely exist. A +composite-array column/param now type-checks and attempts a real +per-element decode/encode. **This path remains unverified against real +Postgres either way** — composite arrays were never tested before this +refactor (they were rejected outright at generation time) and aren't +tested now (same deferred gap, just no longer expected to fail statically). +Adding that fixture, regenerating golden, and confirming actual Postgres +round-trip behavior for a composite-array column/param are deferred to a +follow-up pass on a properly provisioned machine; see +`docs/plans/2026-07-11-reusable-custom-type-codecs.md` for the original +design decision (Option A, chosen deliberately) and the deferred task list. + +A second, related change here: `CustomType.dhall`'s Composite branch used +to call `Member.run` with a `nestedLookup` stub hardcoded to `Absent`, +which forced a composite field nesting another custom type down the same +loud-fail path as any unresolvable reference (see section 5). That stub is +gone — it existed only to satisfy `Member.run`'s old signature, and was +deleted along with `buildLookup` — but, unlike the composite-array case +just above, this is not "the same behavior change, just unexercised." +`CompositeModule.dhall`'s `_decode`/`_encode` do a blind flat +`cast(tuple[...], src)`/splat and never recurse into a nested field's own +codec, unchanged by this refactor; removing the stub only removed the +thing that used to reject a nested custom-type composite field at +generation time, it did not make that field decode/encode correctly. And +because the failure mode runs through `cast()` — which suppresses +type-checking on its argument by design — this is **not** expected to be +caught by `basedpyright strict` the way the composite-array case is. This +is a real, silent architecture gap, not a deferred-but-backstopped +behavior change; see section 5 for detail, and treat it as an open +follow-up design question rather than something covered by the +composite-array follow-up plan. + +This removal is scoped to this generator's own Dhall source. It does not +unblock end-to-end regeneration: `demos/Exhaustive.dhall`/`mise run golden` +still needs the pinned pgn binary (for its embedded fork Dhall) regardless, +because gen-sdk's own `Fixtures` module independently relies on the same +`Text/equal` builtin, and this change doesn't touch gen-sdk. A live +Postgres and currently-live remote Dhall imports (`Deps/*` resolve gen-sdk, +lude, and the Prelude over the network, pinned by sha256) are also still +required for that path. + +Section 13 covers a different, still-in-place mechanism (`PyIdent.dhall`'s +keyword sanitizing), which never depended on `buildLookup` and was already +rewritten off `Text/equal` before this change; it is unaffected either way. --- diff --git a/docs/plans/2026-07-11-encounter-order-custom-imports.md b/docs/plans/2026-07-11-encounter-order-custom-imports.md new file mode 100644 index 0000000..7d1c663 --- /dev/null +++ b/docs/plans/2026-07-11-encounter-order-custom-imports.md @@ -0,0 +1,327 @@ +# Encounter-Order Custom-Type Imports Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Stop requiring a project-wide `order : Natural` to sort and dedupe custom-type import lines in `ImportSet.dhall`. Emit them in encounter order (the order the referencing columns/params are declared) instead, and accept that de-duplicating two references to the *same* custom type within one file is not achievable in vanilla Dhall — verify it isn't actually needed by the current corpus, and document the tradeoff rather than reintroduing a lookup to avoid it. + +**Architecture:** `ImportSet.dhall` today dedupes and alphabetizes custom-type imports by carrying a `dedupKey : Natural` — the type's alphabetical index in `project.customTypes` — through every `CustomImport` value, specifically because Dhall has no `Text` comparison to sort or dedupe on `moduleName`/`className` directly (see the file's own header comment). That `order` value's only source was `buildLookup` (`Interpreters/Project.dhall`), which the companion plan (`2026-07-11-reusable-custom-type-codecs.md`) deletes. Once it's gone, `ImportSet.dhall` has nothing to key on. Rather than re-deriving a Natural surrogate some other way, this plan removes the sort/dedup step and lets `ImportSet.combine`'s existing `List/fold` order (already the query's declared column/param order — a real, already-computed, non-Text-comparison ordering) stand as the output order. + +**Tech Stack:** Dhall (dhall-lang 1.42), Python 3.12 generated output, pytest golden-file harness (`mise run test`, `mise run golden`). + +## Global Constraints + +- **Depends on** `docs/plans/2026-07-11-reusable-custom-type-codecs.md` — that plan's Task 3/4 call `ImportSet.custom customImport` (no `order` argument). Land this plan's Task 1 first, or in the same PR; `ImportSet.customEnum`/`customComposite` (which this plan deletes) are exactly what those tasks stop calling. +- No behavior change to the four already-Natural-keyed stdlib import flags (`uuid`, `datetime`, `date`, `time`, `timedelta`, `decimal`, `jsonb`, `json`, `jsonValue`, `enumArray`) — those are plain `Bool` OR's today and are untouched by this plan. +- Golden fixture output must be regenerated and diffed (`tests/golden/`), not hand-edited. +- Every Dhall file touched must independently type-check: `dhall type --file=`. + +--- + +## Why dedup can't be preserved without reintroducing a lookup + +Worth writing down since it's not obvious and the temptation to "just find a clever `Text/replace` trick" is real — this was checked directly against `PyIdent.dhall`'s working pattern (DESIGN.md section 13) before concluding it doesn't generalize: + +`PyIdent.dhall`'s `sanitizeAgainst` tests a runtime `Text` against a small, **fixed, compile-time-literal** candidate list (the 35 Python keywords) — one known literal at a time, folded. That's why the two-`Text/replace` trick works: one side of every comparison is always a literal. + +Import dedup needs the opposite shape: is this **runtime** `moduleName` (derived from a `Name` that varies per project) equal to any of the **other runtime** `moduleName`s already collected? Both sides are dynamic. No sequence of `Text/replace` calls can decide that, because deciding it requires producing a `Bool` from two non-literal `Text` values, which is exactly the operation Dhall doesn't have (and which pgn's forked `Text/equal` exists to provide). This isn't a missing trick — dedup of dynamically-computed `Text` is unconditionally impossible in vanilla Dhall. The only ways to get it back are: (a) a fork builtin (what we're removing), (b) a pre-assigned `Natural` id per distinct value (what `order` was — sourced from a project-wide search, i.e. `buildLookup`, also being removed), or (c) don't need it. + +This plan takes (c), having checked how much it costs to: + +```bash +grep -rl "^from \.\.types\." tests/golden/src 2>/dev/null | while read f; do + n=$(grep -c "^from \.\.types\." "$f") + [ "$n" -gt 1 ] && echo "$f: $n" +done +``` + +No output — verified during design (see conversation record / re-run before Task 2 to confirm it's still true after Doc 1's regeneration). No file in the current fixture corpus imports the same custom type twice. The risk is real but currently unexercised: if a future query selects the same composite/enum type via two different columns, the generated file will contain two identical `from ..types.X import Y` lines — syntactically valid, harmless to `basedpyright strict` and to Python's import system, just visually redundant. Task 2 adds a corpus case that exercises this on purpose so the tradeoff is documented against real output, not just asserted. + +--- + +## File Structure + +| File | Change | +|---|---| +| `src/Structures/ImportSet.dhall` | Drop `dedupKey` from `CustomImport`; delete `dedupCustoms`, `sortCustoms`, `eqNat`, `leNat`, `sortedCustoms`; collapse `customEnum`/`customComposite` into the existing `custom`; `combine` plain-concatenates `customTypes` instead of deduping. | +| `src/Templates/RowsModule.dhall`, `src/Templates/StatementModule.dhall` | Read `imports.customTypes` directly instead of `ImportSet.sortedCustoms imports`. | +| `tests/golden/` | Regenerate via `mise run golden`; review the (likely negligible) reordering of custom-type import lines from alphabetical to declaration order. | +| `python.gen/DESIGN.md` | Note the ordering change where section 12/13 currently describe the alphabetical-by-`order` scheme. | + +--- + +### Task 1: Simplify `ImportSet.dhall` + +**Files:** +- Modify: `src/Structures/ImportSet.dhall` + +**Interfaces:** +- `CustomImport` loses `dedupKey : Natural` → becomes `{ className : Text, moduleName : Text }`. +- `custom : CustomImport -> Self` — unchanged signature, now the only constructor (no more `customEnum`/`customComposite`). +- `combine : Self -> Self -> Self` — `customTypes` field is now a plain list concatenation. +- `sortedCustoms` is deleted. Callers read `.customTypes` directly. + +- [ ] **Step 1: Replace the file** + +```dhall +let Prelude = ../Deps/Prelude.dhall + +-- A custom-type import line: "from ..types. import ". +-- Emitted in encounter order (the order the referencing columns/params were +-- declared), not sorted. Dhall (upstream) has no Text comparison, so there is +-- no way to alphabetize or dedupe by moduleName/className without either the +-- pgn fork's Text/equal or a project-wide Natural id (previously `order`, +-- sourced from Project.dhall's buildLookup — see +-- docs/plans/2026-07-11-encounter-order-custom-imports.md for why that's +-- gone and why dedup isn't reintroduced some other way). Two references to +-- the same type currently produce two identical lines; harmless to Python +-- and to basedpyright, just not deduped. +let CustomImport = { className : Text, moduleName : Text } + +let Self = + { uuid : Bool + , datetime : Bool + , date : Bool + , time : Bool + , timedelta : Bool + , decimal : Bool + , jsonb : Bool + , json : Bool + , jsonValue : Bool + , enumArray : Bool + , customTypes : List CustomImport + } + +let base = + { uuid = False + , datetime = False + , date = False + , time = False + , timedelta = False + , decimal = False + , jsonb = False + , json = False + , jsonValue = False + , enumArray = False + , customTypes = [] : List CustomImport + } + +let empty + : Self + = base + +let uuid + : Self + = base // { uuid = True } + +let datetime + : Self + = base // { datetime = True } + +let date + : Self + = base // { date = True } + +let time + : Self + = base // { time = True } + +let timedelta + : Self + = base // { timedelta = True } + +let decimal + : Self + = base // { decimal = True } + +let jsonb + : Self + = base // { jsonb = True } + +let json + : Self + = base // { json = True } + +let jsonValue + : Self + = base // { jsonValue = True } + +let enumArray + : Self + = base // { enumArray = True } + +let custom + : CustomImport -> Self + = \(c : CustomImport) -> base // { customTypes = [ c ] } + +let combine = + \(left : Self) -> + \(right : Self) -> + { uuid = left.uuid || right.uuid + , datetime = left.datetime || right.datetime + , date = left.date || right.date + , time = left.time || right.time + , timedelta = left.timedelta || right.timedelta + , decimal = left.decimal || right.decimal + , jsonb = left.jsonb || right.jsonb + , json = left.json || right.json + , jsonValue = left.jsonValue || right.jsonValue + , enumArray = left.enumArray || right.enumArray + , customTypes = left.customTypes # right.customTypes + } + +let combineAll + : List Self -> Self + = \(sets : List Self) -> List/fold Self sets Self combine empty + +in { Type = Self + , CustomImport + , empty + , uuid + , datetime + , date + , time + , timedelta + , decimal + , jsonb + , json + , jsonValue + , enumArray + , custom + , combine + , combineAll + } +``` + +- [ ] **Step 2: Type-check** + +Run: `dhall type --file=src/Structures/ImportSet.dhall` +Expected: prints the record-of-functions signature, no error. + +- [ ] **Step 3: Commit** + +```bash +git add src/Structures/ImportSet.dhall +git commit -m "python.gen: drop order-based sort/dedup from ImportSet, use encounter order" +``` + +--- + +### Task 2: Update the two render call sites + +**Files:** +- Modify: `src/Templates/RowsModule.dhall`, `src/Templates/StatementModule.dhall` + +- [ ] **Step 1: `RowsModule.dhall:58`** + +Change: +```dhall + (ImportSet.sortedCustoms imports) +``` +to: +```dhall + imports.customTypes +``` + +- [ ] **Step 2: `StatementModule.dhall:82`** + +Same change: +```dhall + imports.customTypes +``` + +- [ ] **Step 3: Type-check both** + +Run: `dhall type --file=src/Templates/RowsModule.dhall && dhall type --file=src/Templates/StatementModule.dhall` +Expected: both print their signatures, no error. + +- [ ] **Step 4: Commit** + +```bash +git add src/Templates/RowsModule.dhall src/Templates/StatementModule.dhall +git commit -m "python.gen: render custom-type imports in encounter order" +``` + +--- + +### Task 3: Regenerate golden fixtures and verify the dedup gap directly + +**Files:** +- Modify: `tests/fixture-project/` (temporary, to exercise the dedup gap — see Step 1) +- Regenerate: `tests/golden/` + +- [ ] **Step 1: Confirm today's corpus has no same-type-twice case** + +Run: +```bash +grep -rl "^from \.\.types\." tests/golden/src 2>/dev/null | while read f; do + n=$(grep -c "^from \.\.types\." "$f") + [ "$n" -gt 1 ] && echo "$f: $n" +done +``` +Expected: no output (re-confirms the design-time check above, against the *current* golden tree before this plan's regeneration). + +- [ ] **Step 2: Regenerate golden** + +Run: `mise run golden` +Expected: succeeds. `git diff tests/golden` shows custom-type import lines reordered from alphabetical to declaration order in files with 2+ distinct custom-type imports (e.g. wherever `Mood` and `Point2D` are both imported today — check whether the query's own column order already happens to be alphabetical for that file; if so the diff is empty there and this is confirmed low-risk for the current corpus). + +- [ ] **Step 3: Deliberately add a same-type-twice column to the fixture project** + +Add a query (or extend an existing one) whose result row or param list references the *same* composite or enum type through two different columns/params — e.g. two `mood`-typed columns in one query. This is new fixture surface, not present today; add it under `tests/fixture-project/queries/`. + +Run: `mise run golden` +Expected: succeeds; the regenerated file for that query contains **two** identical `from ..types.mood import Mood` lines (confirm with `grep -c` on the specific file). This is the one visible, accepted consequence of this plan — capture it in the diff review, don't silently let it slip into `tests/golden` unremarked. + +- [ ] **Step 4: Decide whether to keep the same-type-twice fixture case** + +Keeping it in the committed corpus makes the tradeoff a permanent, visible regression test (future readers see the duplicate import and the comment in `ImportSet.dhall` explaining it, instead of being surprised by it later). Removing it keeps the golden diff minimal for this change. Either is fine — this plan recommends **keeping it**, since an accepted-but-invisible tradeoff tends to resurface as a bug report; make the call and note it in the commit message either way. + +- [ ] **Step 5: Run the full test suite** + +Run: `mise run test` +Expected: all pass. Duplicate import lines don't fail `basedpyright strict` (Python tolerates redundant imports) or the golden byte-comparison (it compares against the freshly-committed golden, not some independent expectation). + +- [ ] **Step 6: Commit** + +```bash +git add tests/golden tests/fixture-project +git commit -m "python.gen: regenerate golden fixtures for encounter-order imports" +``` + +--- + +### Task 4: Update `DESIGN.md` + +**Files:** +- Modify: `python.gen/DESIGN.md` + +- [ ] **Step 1: Update section 12/13 cross-references** + +Wherever DESIGN.md currently describes `order`/alphabetical import sorting (it's referenced in passing around sections 12-13 and in code comments already updated by `docs/plans/2026-07-11-reusable-custom-type-codecs.md`'s Task 8), add a short note: custom-type imports are emitted in encounter (declaration) order, not sorted, since the `order` Natural no longer exists once `buildLookup` is gone; same-type-twice references are not deduped (a Dhall limitation, not an oversight — see `src/Structures/ImportSet.dhall`'s header comment for the full reasoning). Point at the fixture case from Task 3 if kept. + +- [ ] **Step 2: Commit** + +```bash +git add python.gen/DESIGN.md +git commit -m "python.gen: document encounter-order custom-type imports in DESIGN.md" +``` + +--- + +## Self-Review + +**Spec coverage:** Task 1 is the actual mechanism change (drop the Natural key, drop sort/dedup). Task 2 fixes the two render call sites that would otherwise reference a deleted `sortedCustoms`. Task 3 regenerates and — importantly — deliberately exercises the one behavior change (duplicate imports for a same-type-twice reference) instead of letting it go unverified. Task 4 keeps DESIGN.md truthful. + +**Placeholder scan:** no TBDs; every step names an exact file, an exact diff, or an exact command with expected output. + +**Dependency on the companion plan:** called out at the top (Global Constraints) and repeated in the companion plan's own self-review — `docs/plans/2026-07-11-reusable-custom-type-codecs.md` Task 3/4 call `ImportSet.custom` with no `order` argument, which only type-checks after this plan's Task 1. Sequence: this plan's Task 1 → companion plan's Tasks 3-6 → this plan's Tasks 2-4 (Task 2 touches templates the companion plan doesn't touch, so it can land anytime after Task 1, but golden regeneration in either plan's Task 8/3 should happen once, after both are code-complete, not twice). + +## Execution Handoff + +Plan complete and saved to `python.gen/docs/plans/2026-07-11-encounter-order-custom-imports.md`. Two execution options: + +**1. Subagent-Driven (recommended)** - dispatch a fresh subagent per task, review between tasks, fast iteration + +**2. Inline Execution** - execute tasks in this session using executing-plans, batch execution with checkpoints + +Which approach? diff --git a/docs/plans/2026-07-11-reusable-custom-type-codecs.md b/docs/plans/2026-07-11-reusable-custom-type-codecs.md new file mode 100644 index 0000000..b2f880c --- /dev/null +++ b/docs/plans/2026-07-11-reusable-custom-type-codecs.md @@ -0,0 +1,633 @@ +# Reusable Custom-Type Codecs Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Delete `buildLookup`/`CustomKind.Lookup` — and with it python.gen's last dependency on pgn's forked `Text/equal` builtin (DESIGN.md section 12) — by generating one `_decode`/`_encode` codec per custom type in `types/.py`, called by name from every reference site, instead of re-deriving decode/encode logic (field types, Composite-vs-Enum classification) at each call site. + +**Architecture:** `CustomType.dhall` already visits every custom type once with its full definition in hand (composite fields or enum variants) and emits `types/.py`. Today it stops at the dataclass/enum class. This plan makes it also emit a `_decode` staticmethod (and, for composites, an `_encode` instance method) on that same class. Every column/param reference site (`Member.dhall`, `ParamsMember.dhall`) currently has to search `project.customTypes` by name to find out whether it's looking at a composite or an enum, because it re-derives the decode/encode expression inline. Once decode/encode is a named method reachable from the `Name` the reference site already carries (`name.inPascalCase`), the reference site just calls it — no search, no classification, no `Text/equal`. + +**Tech Stack:** Dhall (dhall-lang 1.42 vendored via `dhall` CLI; pgn's forked interpreter for the parts still using it), Python 3.12 generated output (psycopg3), basedpyright strict, pytest golden-file harness (`mise run test`, `mise run golden`). + +## Global Constraints + +- No behavior change to non-custom-type (primitive) decode/encode paths. +- Golden fixture output (`tests/golden/`) must be regenerated and diffed, not hand-edited (per `tests/golden/README.md`). +- `mise run test` (pytest, includes `test_generated_passes_basedpyright_strict`) must pass after regeneration. +- Every Dhall file touched must independently type-check: `dhall type --file=`. +- Don't touch `Primitive.dhall`, `Scalar.dhall`, `Value.dhall`, or the `Model`/`Contract` dependency — this plan is scoped to what python.gen can do unilaterally, with the current, unmodified `Scalar = < Primitive : Primitive | Custom : Name >` contract. + +--- + +## Design decision needed before Task 3: array-of-custom-type behavior + +Today, `Member.dhall`'s Custom branch treats arrays differently by kind, because it already knows the kind from `lookup name`: +- **Enum arrays (1-D):** supported — `enumArrayDecode` emits an element-wise list comprehension. +- **Composite arrays:** rejected at generation time — `"Array of a composite type is not supported (element-wise decode is unimplemented)"` (Member.dhall:219) and, for params, `"Array of a composite type as a parameter is not supported"` (ParamsMember.dhall:359). + +Once decode/encode become named methods called unconditionally (no classification at the call site), there's no Dhall-level way to keep rejecting *only* composite arrays without reintroducing some form of lookup. Two ways forward: + +- **A — Recommended: uniform array codec, kind-specific availability.** Every custom type's template *may* define `_decode_array` (a staticmethod that turns the raw array value into a `list[...]`); `EnumModule.dhall` defines it, `CompositeModule.dhall` does not. `Member.dhall` always emits `f"{typeName}._decode_array({src})"` for a 1-D custom-type array, regardless of kind. If that's ever generated against a composite, `basedpyright strict` (already gating `mise run test`, see `tests/golden/README.md`) fails on the missing attribute — a real check, just moved from `pgn generate` time to CI time. Composite-array *params* work the same way: encode always calls `f"{fieldName}._encode()"`; if `fieldName`'s inferred type is `list[Point2D]`, basedpyright flags the missing `._encode` on `list`. This is a genuine, if later, safety net — not a silent hole. The `"Absent"` case (a `customRef` name matching no generated module) degrades the same way: the emitted `from ..types. import ` fails as an unresolved import, also caught by basedpyright strict. +- **B — Preserve today's exact rejection.** Keep a minimal kind signal alive somewhere reachable without a name search — no such source was found during design (see the exploration notes below); the only ones available (project-wide `customTypes` list, or a `Scalar.Custom` contract change) reintroduce either the search or the upstream ask this plan is explicitly trying to avoid. Pick this only if lifting the composite-array restriction is unacceptable without a live Postgres verification first. + +This plan is written for **Option A**. Composite-array support was never tested (DESIGN.md and the fixture corpus have no composite-array column — verified via `grep -rn "^from \.\.types\." tests/golden/`, no file references the same custom type twice, and none of the fixture's composite columns are arrays), so Task 8 includes adding one to the fixture project specifically to exercise this path before it ships. If that Postgres test reveals composite-array decode genuinely doesn't work end-to-end (not just an assumption), fall back to Option B and keep the `"not supported"` report, sourced from a small dedicated check rather than full `buildLookup` (open a follow-up plan; don't block this one on it). + +--- + +## File Structure + +| File | Change | +|---|---| +| `src/Templates/CompositeModule.dhall` | Add `_decode` (staticmethod) and `_encode` (instance method) to the generated dataclass. | +| `src/Templates/EnumModule.dhall` | Add `_decode` and `_decode_array` (staticmethods) and `_encode` (instance method, identity) to the generated `StrEnum`. | +| `src/Interpreters/Member.dhall` | Custom branch calls `{typeName}._decode(...)` / `{typeName}._decode_array(...)` unconditionally; drop the `lookup` parameter and the `Enum`/`Composite`/`Absent` merge. | +| `src/Interpreters/ParamsMember.dhall` | Custom branch calls `{fieldName}._encode()` unconditionally; drop `lookup` and the merge. | +| `src/Interpreters/ResultColumns.dhall`, `src/Interpreters/Result.dhall`, `src/Interpreters/Query.dhall` | Drop the `lookup : CustomKind.Lookup` parameter they only thread through. | +| `src/Interpreters/CustomType.dhall` | Drop `nestedLookup` (nothing left to pass it to). | +| `src/Interpreters/Project.dhall` | Delete `buildLookup`, `IndexedCustomType`, `compositeFields`, `memberPyType`, `lookupConfig` — all exist only to feed `buildLookup`. | +| `src/Structures/CustomKind.dhall` | Delete the file; nothing imports it after the above. | +| `python.gen/DESIGN.md` | Rewrite section 12 (no longer an accepted risk — resolved) and section 13 (unchanged content, but cross-reference updates). | +| `python.gen/docs/upstream-asks.md` | Remove ask 3 (resolved without the upstream change) or mark it withdrawn. | +| `tests/golden/` | Regenerate via `mise run golden`. | + +--- + +### Task 1: Add `_decode`/`_encode` to `CompositeModule.dhall` + +**Files:** +- Modify: `src/Templates/CompositeModule.dhall` + +**Interfaces:** +- Consumes: same `Params = { typeName : Text, extraImports : List Text, fields : List Field }` as today, `Field = { fieldName : Text, fieldType : Text }` — no signature change. +- Produces: the rendered module text now defines `_decode(src: object) -> ""` (staticmethod) and `_encode(self) -> tuple[...]` (instance method) on the dataclass, callable by any generator downstream as `TypeName._decode(x)` / `value._encode()`. + +This was prototyped and verified against `dhall type` and `dhall text` (rendered output checked with `python3 -c "compile(...)"` for both a two-field and a one-field composite — the one-field case needs the trailing-comma tuple, same edge case `compositeBind` already handles today). + +- [ ] **Step 1: Replace the template body** + +```dhall +let Prelude = ../Deps/Prelude.dhall + +let Sdk = ../Deps/Sdk.dhall + +let Field = { fieldName : Text, fieldType : Text } + +-- extraImports are the extra import lines a field type needs (e.g. +-- "from uuid import UUID"). They sit between the dataclass import and the class, +-- separated by one blank line; when empty only the dataclass import is emitted. +let Params = { typeName : Text, extraImports : List Text, fields : List Field } + +let run = + \(params : Params) -> + let fieldLines = + Prelude.Text.concatMapSep + "\n" + Field + ( \(field : Field) -> + " ${field.fieldName}: ${field.fieldType}" + ) + params.fields + + let fieldCount = Prelude.List.length Field params.fields + + let fieldTypesJoined = + Prelude.Text.concatMapSep + ", " + Field + (\(field : Field) -> field.fieldType) + params.fields + + let selfFieldsJoined = + Prelude.Text.concatMapSep + ", " + Field + (\(field : Field) -> "self.${field.fieldName}") + params.fields + + -- Python only treats trailing-comma parens as a 1-tuple; concatMapSep + -- never emits an internal comma for a single-element list, so force one + -- here. Mirrors ParamsMember.dhall's existing compositeBind trick. + let encodeTupleExpr = + if Prelude.Natural.equal fieldCount 1 + then "(${selfFieldsJoined},)" + else "(${selfFieldsJoined})" + + -- _decode/_encode are emitted as literal lines (not a nested multi-line + -- ''...'' block) because Dhall dedents a multi-line literal against its + -- OWN source indentation before splicing it into the outer literal; a + -- nested block loses its intended 4/8-space class-body indentation. + -- Verified against `dhall text` during design. + let codecMethods = + "\n" + ++ " @staticmethod\n" + ++ " def _decode(src: object) -> \"${params.typeName}\":\n" + ++ " return ${params.typeName}(*cast(tuple[${fieldTypesJoined}], src))\n" + ++ "\n" + ++ " def _encode(self) -> tuple[${fieldTypesJoined}]:\n" + ++ " return ${encodeTupleExpr}" + + let imports = + if Prelude.List.null Text params.extraImports + then "from dataclasses import dataclass\nfrom typing import cast" + else '' + from dataclasses import dataclass + from typing import cast + + ${Prelude.Text.concatSep "\n" params.extraImports}'' + + in '' + ${imports} + + + @dataclass(frozen=True, slots=True) + class ${params.typeName}: + """Decoding/encoding this composite requires register_types(conn) first. + + Without per-connection registration psycopg returns the value as a + raw string, which the generated _decode cannot splat into the dataclass. + """ + + ${fieldLines} + ${codecMethods} + '' + +in Sdk.Sigs.template Params run /\ { Field } +``` + +- [ ] **Step 2: Type-check** + +Run: `dhall type --file=src/Templates/CompositeModule.dhall` +Expected: prints the `{ Field : Type, Params : Type, Run : Type, run : ... }` signature, no error. + +- [ ] **Step 3: Spot-render and validate as Python** + +```bash +cat > /tmp/render_composite.dhall <<'EOF' +let CompositeModule = ./src/Templates/CompositeModule.dhall +in CompositeModule.run + { typeName = "Point2D" + , extraImports = [] : List Text + , fields = + [ { fieldName = "x", fieldType = "int" } + , { fieldName = "y", fieldType = "int" } + ] + } +EOF +dhall text --file=/tmp/render_composite.dhall | python3 -c "import sys; compile(sys.stdin.read(), 'point2d.py', 'exec')" && echo OK +``` +Expected: `OK`, and eyeballing the output shows `_decode`/`_encode` indented as class members (4 spaces), not module-level. + +- [ ] **Step 4: Commit** + +```bash +git add src/Templates/CompositeModule.dhall +git commit -m "python.gen: emit _decode/_encode on generated composite dataclasses" +``` + +--- + +### Task 2: Add `_decode`/`_decode_array`/`_encode` to `EnumModule.dhall` + +**Files:** +- Modify: `src/Templates/EnumModule.dhall` + +**Interfaces:** +- Consumes: same `Params = { typeName : Text, variants : List Variant }` — no signature change. +- Produces: `_decode(src: object) -> ""`, `_decode_array(src: object) -> list[""]`, `_encode(self) -> ""` (identity — psycopg binds the `StrEnum` instance directly, matching today's `defaultBind`). + +The scalar/array decode bodies are copied verbatim from `Member.dhall`'s current `enumDecode`/`enumArrayDecode` (Member.dhall:61-82), just moved from "Dhall builds inline Python text at every call site" to "Dhall builds it once, into the class." + +- [ ] **Step 1: Replace the template body** + +```dhall +let Prelude = ../Deps/Prelude.dhall + +let Sdk = ../Deps/Sdk.dhall + +let Variant = { memberName : Text, pgValue : Text } + +let Params = { typeName : Text, variants : List Variant } + +let run = + \(params : Params) -> + -- pgValue is interpolated into a single-line double-quoted Python literal; a + -- label may legally contain a backslash, quote, or control character, so + -- escape them to keep the literal valid and value-equal to the DB label. + -- Order is load-bearing: backslash first (so the escapes added below are not + -- re-escaped), then the control chars, then the closing quote. + let escapeLabel + : Text -> Text + = \(raw : Text) -> + Prelude.Function.composeList + Text + [ Prelude.Text.replace "\\" "\\\\" + , Prelude.Text.replace "\r" "\\r" + , Prelude.Text.replace "\n" "\\n" + , Prelude.Text.replace "\t" "\\t" + , Prelude.Text.replace "\"" "\\\"" + ] + raw + + let memberLines = + Prelude.Text.concatMapSep + "\n" + Variant + ( \(variant : Variant) -> + " ${variant.memberName} = \"${escapeLabel variant.pgValue}\"" + ) + params.variants + + let codecMethods = + "\n" + ++ " @staticmethod\n" + ++ " def _decode(src: object) -> \"${params.typeName}\":\n" + ++ " return ${params.typeName}(cast(str, src))\n" + ++ "\n" + ++ " @staticmethod\n" + ++ " def _decode_array(src: object) -> list[\"${params.typeName}\"]:\n" + ++ " return [\n" + ++ " ${params.typeName}(v)\n" + ++ " for v in cast(list[str], require_array(src))\n" + ++ " ]\n" + ++ "\n" + ++ " def _encode(self) -> \"${params.typeName}\":\n" + ++ " return self" + + in '' + from enum import StrEnum + from typing import cast + + from .._runtime import require_array + + + class ${params.typeName}(StrEnum): + ${memberLines} + ${codecMethods} + '' + +in Sdk.Sigs.template Params run /\ { Variant } +``` + +> `require_array` must already be importable from `.._runtime` relative to `types/.py` — confirm the relative import depth matches `types/`'s actual nesting (one level under the package root per `CustomType.dhall`'s `modulePath = "types/${moduleName}.py"`) before running Step 2; adjust to `.._runtime` vs `..._runtime` to match what `Member.dhall`'s current `enumArrayDecode` assumes at its own call sites (check `ImportSet.dhall`'s handling of `require_array` imports today for the exact existing relative path convention, since this moves an existing import from call sites into the shared module). + +- [ ] **Step 2: Type-check** + +Run: `dhall type --file=src/Templates/EnumModule.dhall` +Expected: signature prints, no error. + +- [ ] **Step 3: Spot-render and validate as Python** + +```bash +cat > /tmp/render_enum.dhall <<'EOF' +let EnumModule = ./src/Templates/EnumModule.dhall +in EnumModule.run + { typeName = "Mood" + , variants = + [ { memberName = "HAPPY", pgValue = "happy" } + , { memberName = "SAD", pgValue = "sad" } + ] + } +EOF +dhall text --file=/tmp/render_enum.dhall | python3 -c "import sys; compile(sys.stdin.read(), 'mood.py', 'exec')" && echo OK +``` +Expected: `OK`. + +- [ ] **Step 4: Commit** + +```bash +git add src/Templates/EnumModule.dhall +git commit -m "python.gen: emit _decode/_decode_array/_encode on generated enum classes" +``` + +--- + +### Task 3: Simplify `Member.dhall`'s Custom branch + +**Files:** +- Modify: `src/Interpreters/Member.dhall` + +**Interfaces:** +- Consumes: `value.scalar.customRef : Optional Model.Name` (unchanged — still comes straight off `Scalar.run`, see `src/Interpreters/Scalar.dhall:44-52`), `value.dims : Natural` (unchanged). +- Produces: `Run = Config -> Input -> Lude.Compiled.Type Output` — **drops the `CustomKind.Lookup` parameter**. Every caller (`ResultColumns.dhall`, `CustomType.dhall`) updates in Task 5/6. + +- [ ] **Step 1: Replace the Custom branch (Member.dhall:129-232) and the `Run` alias (Member.dhall:245)** + +Delete the `merge { Enum = ...; Composite = ...; Absent = ... } (lookup name)` block and the `CustomKind.CompositeField`-typed `compositeDecode` helper (lines 100-116, now dead — the same logic lives in `CompositeModule.dhall`'s `_decode` now). Replace with: + +```dhall + , Custom = + Prelude.Optional.fold + Model.Name + value.scalar.customRef + (Lude.Compiled.Type Output) + ( \(name : Model.Name) -> + let typeName = name.inPascalCase + + let customImport = + { className = typeName, moduleName = name.inSnakeCase } + + let mkOutput = + \(customImports : ImportSet.Type) -> + \(decodeExpr : Text -> Text) -> + { fieldName + , pgName = input.pgName + , pyType + , isNullable = input.isNullable + , imports = + ImportSet.combine baseImports customImports + , decodeExpr + } + + let wrapNullable = + \(call : Text -> Text) -> + \(src : Text) -> + if input.isNullable + then "None if ${src} is None else ${call src}" + else call src + + let dimsIsOne = + Natural/isZero (Natural/subtract 1 value.dims) + + in if Natural/isZero value.dims + then Lude.Compiled.ok + Output + ( mkOutput + (ImportSet.custom customImport) + ( wrapNullable + (\(src : Text) -> "${typeName}._decode(${src})") + ) + ) + else if dimsIsOne + then Lude.Compiled.ok + Output + ( mkOutput + (ImportSet.custom customImport) + ( wrapNullable + ( \(src : Text) -> + "${typeName}._decode_array(${src})" + ) + ) + ) + else Lude.Compiled.report + Output + [ input.pgName, name.inSnakeCase ] + "Array of dimensionality > 1 is not supported" + ) + ( Lude.Compiled.report + Output + [ input.pgName ] + "Custom scalar without a customRef name" + ) +``` + +Note this drops the `Enum`/`Composite` import-set split (`ImportSet.customEnum`/`ImportSet.customComposite`) in favor of one `ImportSet.custom` call — see Doc 2 (`2026-07-11-encounter-order-custom-imports.md`) for why `ImportSet.customEnum`/`customComposite` collapse into a single `ImportSet.custom` once `order` no longer exists. **Land Doc 2 in the same branch as this task, or `dhall type` will fail here** — Task 3 depends on `ImportSet.CustomImport` no longer requiring a `dedupKey`/`order` field. + +- [ ] **Step 2: Drop the `lookup` parameter from `Run`** + +Change: +```dhall +let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output +``` +to: +```dhall +let Run = Config -> Input -> Lude.Compiled.Type Output +``` +and the `run` definition's `\(lookup : CustomKind.Lookup) ->` (Member.dhall:40) — delete that line entirely, since `run` no longer takes it. + +- [ ] **Step 3: Delete the now-dead `CustomKind` import (Member.dhall:9) and the dead `compositeDecode` helper (Member.dhall:100-116)** + +- [ ] **Step 4: Type-check** + +Run: `dhall type --file=src/Interpreters/Member.dhall` +Expected: fails until Task 5 updates `ResultColumns.dhall` (Member's only caller) to stop passing `lookup` — that's fine, type-check `Member.dhall` standalone by temporarily checking `Sdk.Sigs.interpreter Config Input Output run` in isolation, or proceed straight to Task 5 and type-check the pair together. Don't commit Task 3 alone; commit Tasks 3+5+6 together (they're one type-checking unit — Dhall won't let you land a signature change without updating every call site in the same change). + +--- + +### Task 4: Simplify `ParamsMember.dhall`'s Custom branch + +**Files:** +- Modify: `src/Interpreters/ParamsMember.dhall` + +**Interfaces:** +- Consumes: same as Task 3. +- Produces: `Run = Config -> Input -> Lude.Compiled.Type Output` — drops `CustomKind.Lookup`. + +- [ ] **Step 1: Replace the Custom branch (ParamsMember.dhall:311-367)** + +Delete `compositeBind` (lines 258-288, now dead — logic moved into `CompositeModule.dhall`'s `_encode`) and the `merge { Enum = ...; Composite = ...; Absent = ... } (lookup name)` block. Replace with: + +```dhall + in Prelude.Optional.fold + Model.Name + value.scalar.customRef + (Lude.Compiled.Type Output) + ( \(name : Model.Name) -> + let customImport = + { className = name.inPascalCase + , moduleName = name.inSnakeCase + } + + let encodeExpr = + if input.isNullable + then "None if ${fieldName} is None else ${fieldName}._encode()" + else "${fieldName}._encode()" + + in Lude.Compiled.ok + Output + ( mkOutput + (ImportSet.combine value.imports (ImportSet.custom customImport)) + encodeExpr + ) + ) + ( if isJsonArrayParam + then Lude.Compiled.report + Output + [ input.pgName ] + "json/jsonb array as a parameter is not supported" + else Lude.Compiled.ok + Output + (mkOutput value.imports defaultBind) + ) +``` + +This drops the `Natural/isZero value.dims` guard that used to reject composite-array params — per the Design decision above (Option A), an array-of-composite param now generates `{fieldName}._encode()` where `fieldName`'s type is `list[Point2D]`; `list` has no `._encode`, so `basedpyright strict` catches it. Confirm this in Task 8's basedpyright run specifically, not just eyeball it. + +- [ ] **Step 2: Drop the `lookup` parameter from `Run` (ParamsMember.dhall:387) and `run`'s `\(lookup : CustomKind.Lookup) ->` (ParamsMember.dhall:235)** + +- [ ] **Step 3: Delete the dead `CustomKind` import (ParamsMember.dhall:9)** + +- [ ] **Step 4: Type-check together with Task 3** + +Run: `dhall type --file=src/Interpreters/ParamsMember.dhall` +Expected: same caveat as Task 3 Step 4 — its caller (`Query.dhall`) still passes `lookup` until Task 5. + +--- + +### Task 5: Drop `lookup` threading from `ResultColumns.dhall`, `Result.dhall`, `Query.dhall` + +**Files:** +- Modify: `src/Interpreters/ResultColumns.dhall`, `src/Interpreters/Result.dhall`, `src/Interpreters/Query.dhall` + +**Interfaces:** +- Each of these only forwards `lookup` to a callee; none inspect it. Removing it is mechanical. + +- [ ] **Step 1: `ResultColumns.dhall`** — delete `\(lookup : CustomKind.Lookup) ->` (line 62) and change `Member.run config lookup member` (line 76) to `Member.run config member`. Delete the `CustomKind` import (line 9) if nothing else in the file uses it — verify with `grep -n CustomKind src/Interpreters/ResultColumns.dhall` after editing. + +- [ ] **Step 2: `Result.dhall`** — delete both `\(lookup : CustomKind.Lookup) ->` occurrences (lines 66, 93), change `ResultColumns.run config lookup rowClassName columns` (line 89) to `ResultColumns.run config rowClassName columns`, and `rowsOutput config lookup rowClassName` (line 100) to `rowsOutput config rowClassName` — check `rowsOutput`'s own definition for a `lookup` parameter to drop too (it wasn't in the earlier grep excerpt; read the file before editing to confirm). Delete the `CustomKind` import (line 9) if unused after. + +- [ ] **Step 3: `Query.dhall`** — delete `\(lookup : CustomKind.Lookup) ->` (line 140), change `ResultModule.run config lookup rowClassName input.result` (line 156) to drop `lookup`, and `ParamsMember.run config lookup member` (line 173) to `ParamsMember.run config member`. Delete the `CustomKind` import (line 7) if unused after. + +- [ ] **Step 4: Type-check each file standalone** + +Run: `dhall type --file=src/Interpreters/ResultColumns.dhall && dhall type --file=src/Interpreters/Result.dhall && dhall type --file=src/Interpreters/Query.dhall` +Expected: all three print their signatures. `Query.dhall` will still fail until `Project.dhall` (Task 6) stops passing `lookup` to `QueryGen.run` — that's the last link. + +--- + +### Task 6: Delete `buildLookup` and its support code from `Project.dhall` + +**Files:** +- Modify: `src/Interpreters/Project.dhall` + +- [ ] **Step 1: Delete dead helpers** + +Delete, in order (they only exist to feed `buildLookup`, confirmed by re-reading the file top to bottom — nothing else calls `lookupConfig`, `memberPyType`, `compositeFields`, `IndexedCustomType`, or `buildLookup` itself): +- `lookupConfig` (lines 74-80) +- `memberPyType` (lines 85-96) +- `compositeFields` (lines 98-108) +- The local `CompositeField = { fieldName : Text, pyType : Text }` (line 7) — dead once `compositeFields`/`memberPyType` are gone +- `IndexedCustomType` (line 142) +- `buildLookup` (lines 147-181) + +- [ ] **Step 2: Update the call site** + +Find `let lookup = buildLookup effectiveCustomTypes` (line 479) — delete it, and change both call sites that pass `lookup`: +- `QueryGen.run config lookup query` (line 503) → `QueryGen.run config query` +- `(\(query : Model.Query) -> QueryGen.run config lookup query)` (line 524) → `(\(query : Model.Query) -> QueryGen.run config query)` + +- [ ] **Step 3: Delete the dead `CustomKind` import (line 9)** + +- [ ] **Step 4: Type-check the whole chain** + +Run: `dhall type --file=src/Interpreters/Project.dhall` +Expected: prints the module signature with no error — this is the point where Tasks 3-6 all click together (Project → Query → Result/ParamsMember → ResultColumns/Member all now agree on the lookup-free signatures). + +- [ ] **Step 5: Commit Tasks 3-6 together** + +```bash +git add src/Interpreters/Member.dhall src/Interpreters/ParamsMember.dhall \ + src/Interpreters/ResultColumns.dhall src/Interpreters/Result.dhall \ + src/Interpreters/Query.dhall src/Interpreters/Project.dhall +git commit -m "python.gen: call custom-type codecs by name instead of resolving via buildLookup" +``` + +--- + +### Task 7: Drop `nestedLookup` from `CustomType.dhall`, delete `CustomKind.dhall` + +**Files:** +- Modify: `src/Interpreters/CustomType.dhall` +- Delete: `src/Structures/CustomKind.dhall` + +- [ ] **Step 1: `CustomType.dhall`** — delete `nestedLookup` (lines 51-53) and its comment, and change `MemberGen.run config nestedLookup m` (line 127) to `MemberGen.run config m`. Delete the `CustomKind` import (line 11). + +- [ ] **Step 2: Confirm nothing else references `CustomKind`** + +Run: `grep -rln "CustomKind" src` +Expected: no output. + +- [ ] **Step 3: Delete the file** + +```bash +git rm src/Structures/CustomKind.dhall +``` + +- [ ] **Step 4: Type-check the full package entry point** + +Run: `dhall type --file=src/package.dhall` +Expected: prints the top-level module signature, no error. (This is the first point a full-package check is meaningful — earlier steps only checked individual interpreter files.) + +- [ ] **Step 5: Commit** + +```bash +git add src/Interpreters/CustomType.dhall +git commit -m "python.gen: delete CustomKind.dhall, the last remnant of buildLookup" +``` + +--- + +### Task 8: Regenerate golden fixtures, verify, update docs + +**Files:** +- Modify: `tests/fixture-project/` (add a composite-array column per the Design decision above) +- Regenerate: `tests/golden/` +- Modify: `python.gen/DESIGN.md`, `python.gen/docs/upstream-asks.md`, `python.gen/CHANGELOG.md` + +- [ ] **Step 1: Add a composite-array test column to the fixture project** + +Find the fixture's composite-type column definitions under `tests/fixture-project/` (query/table SQL referencing a composite type, e.g. the `Point2D`-typed column used in `insert_specimen`/`get_specimen` per the golden output). Add one query or column that selects/inserts an *array* of that composite type — this is the first real exercise of the Option A fallback path (`basedpyright strict` should catch it if `_decode_array`/`_encode` aren't defined on `Point2D`, since Task 1 deliberately didn't add them to `CompositeModule.dhall`). + +Confirm the expected failure mode first: + +Run: `mise run golden` (needs `PGN_TEST_DATABASE_URL` pointing at a live Postgres — see `tests/golden/README.md`) +Expected: generation succeeds (no Dhall-level rejection — that's the point of Option A), but the regenerated file calls `Point2D._decode_array(...)` or `list[Point2D]._encode()`-shaped code that doesn't exist. + +Run: `mise run test` +Expected: `test_generated_passes_basedpyright_strict` FAILS, citing the missing attribute. This confirms the safety net from the Design decision actually fires. Once confirmed, either: + - revert the fixture addition (if you don't want composite arrays in the committed golden corpus yet), or + - implement `_decode_array`/an array-aware `_encode` on `CompositeModule.dhall` for real and keep the fixture (a follow-up, out of this plan's scope — flag it, don't scope-creep this task). + +- [ ] **Step 2: Remove the composite-array addition (unless implementing it for real per Step 1)** + +- [ ] **Step 3: Regenerate golden for real** + +Run: `mise run golden` +Expected: succeeds, rewrites `tests/golden/src/specimen_client/_generated/**` and both facades. + +- [ ] **Step 4: Review the diff** + +Run: `git diff tests/golden` +Expected: every composite/enum decode/encode call site now reads `TypeName._decode(...)`/`TypeName._decode_array(...)`/`value._encode()` instead of the old inlined `cast(tuple[...], ...)`/`(x.a, x.b)` expressions; `types/point_2_d.py`, `types/mood.py`, `types/tag_value.py` each gain the new methods. No unrelated files change. + +- [ ] **Step 5: Run the full test suite** + +Run: `mise run test` +Expected: all pass, including `test_generated_passes_basedpyright_strict`. + +- [ ] **Step 6: Update `DESIGN.md`** + +Rewrite section 12 (`## 12. Forked-Dhall (Text/equal) dependency risk`) — it's no longer an accepted risk for python.gen; state plainly that `buildLookup` is gone and `Text/equal` is no longer used anywhere in this generator's own Dhall source (grep to confirm: `grep -rn "Text/equal" src` returns nothing). Keep a short note that `demos/Exhaustive.dhall`/`mise run golden` still needs the pinned pgn binary regardless, because `gen-sdk`'s own `Fixtures` module independently uses the fork builtin — this plan doesn't touch that, and it isn't blocked on it. Cross-reference section 13 (unchanged — `PyIdent.dhall`'s trick is unrelated and still in place). + +- [ ] **Step 7: Update `docs/upstream-asks.md`** + +Remove ask 3 (`## 3. gen-sdk: kind tag or Natural index on Scalar.Custom`) or mark it explicitly withdrawn with one line explaining why (`buildLookup`'s only consumer was resolved locally by generating named codecs instead of resolving structural type info by search — see `docs/plans/2026-07-11-reusable-custom-type-codecs.md`). Don't delete the file's other two asks (pragma parsing, warnings printing) — they're unrelated and still open. + +- [ ] **Step 8: Update `CHANGELOG.md`** + +Add an entry under the appropriate section describing the behavior change from the Design decision (composite-array columns/params no longer rejected at generation time; verify at basedpyright-strict time instead) if Step 1's fixture addition was kept, or note it as a documented-but-untested path if reverted. + +- [ ] **Step 9: Commit** + +```bash +git add tests/golden tests/fixture-project python.gen/DESIGN.md python.gen/docs/upstream-asks.md python.gen/CHANGELOG.md +git commit -m "python.gen: regenerate golden fixtures for reusable custom-type codecs" +``` + +--- + +## Self-Review + +**Spec coverage:** Task 1-2 build the reusable codecs (the actual "fix the root cause"). Tasks 3-4 make the two reference sites (decode, encode) call them. Tasks 5-7 remove the now-dead plumbing (`lookup` threading, `buildLookup`, `CustomKind.dhall`) so nothing is left half-migrated. Task 8 proves it against the real toolchain and updates the two docs (`DESIGN.md`, `upstream-asks.md`) that currently assert this is unfixable — both would otherwise go stale and mislead the next reader. + +**Open question carried forward, not silently resolved:** the array-of-composite behavior change (Design decision, Option A vs B) is a real product decision, flagged explicitly rather than picked unilaterally in the diff. Task 8 Step 1 is designed to surface the actual runtime behavior (does `basedpyright strict` really catch it, does composite-array decode actually work against real Postgres) before committing to either branch. + +**Dependency on the companion plan:** Task 3 explicitly calls out that it needs `docs/plans/2026-07-11-encounter-order-custom-imports.md` (`ImportSet.dhall`'s `order`/`dedupKey` removal) landed first or alongside — `ImportSet.customEnum`/`customComposite` currently *require* an `order : Natural` that only `buildLookup` produced. Implement that plan first, or fold both into one PR; don't land this plan's Task 3 against the unmodified `ImportSet.dhall`. + +## Execution Handoff + +Plan complete and saved to `python.gen/docs/plans/2026-07-11-reusable-custom-type-codecs.md`. Two execution options: + +**1. Subagent-Driven (recommended)** - dispatch a fresh subagent per task, review between tasks, fast iteration + +**2. Inline Execution** - execute tasks in this session using executing-plans, batch execution with checkpoints + +Which approach? diff --git a/docs/upstream-asks.md b/docs/upstream-asks.md index 44c3417..8d10b37 100644 --- a/docs/upstream-asks.md +++ b/docs/upstream-asks.md @@ -102,6 +102,18 @@ and the new stderr output only appears when warnings are non-empty. ## 3. gen-sdk: `kind` tag or `Natural` index on `Scalar.Custom` +**WITHDRAWN.** `buildLookup`'s only consumer of the fork-only `Text/equal` +builtin was resolved locally, with no gen-sdk contract change needed: +custom-type decode/encode now dispatches through named +`_decode`/`_decode_array`/`_encode` methods generated onto each custom +type's own Python class (`CompositeModule.dhall`/`EnumModule.dhall`), +called by name from every reference site, instead of resolving +classification/fields via a project-wide structural search. `buildLookup` +and `Structures/CustomKind.dhall` are deleted; see +`docs/plans/2026-07-11-reusable-custom-type-codecs.md` and DESIGN.md +section 12. This section is kept for the historical record of why the ask +existed, not as an open request. + ### Motivation This is the ask already planned in DESIGN.md, section 12. The generator's diff --git a/src/Config.dhall b/src/Config.dhall deleted file mode 100644 index 1377127..0000000 --- a/src/Config.dhall +++ /dev/null @@ -1,15 +0,0 @@ --- User-facing config for this generator. `emitSync` adds a parallel sync surface --- (psycopg.Connection) alongside the default async one, so one project can serve --- both an async backend and a sync (Dagster) consumer from shared Row types. --- `onUnsupported` picks Fail (default, abort loudly) or Skip (drop the --- unsupported statement/type and its dependents, with a warning) when a query --- or custom type hits a PG shape the generator cannot render; see --- Structures/OnUnsupported.dhall. All fields are Optional so a project may omit --- the whole config block or any subset of its keys; Interpret.dhall supplies the --- defaults. -let OnUnsupported = ./Structures/OnUnsupported.dhall - -in { packageName : Optional Text - , emitSync : Optional Bool - , onUnsupported : Optional OnUnsupported.Mode - } : Type diff --git a/src/Deps/Lude.dhall b/src/Deps/Lude.dhall index 30f1f5a..a4b5116 100644 --- a/src/Deps/Lude.dhall +++ b/src/Deps/Lude.dhall @@ -1,3 +1,2 @@ -https://raw.githubusercontent.com/codemine-io/lude.dhall/v5.1.0/src/package.dhall - sha256:46b527b071eba96a17e76b4bc5774645714dd5b4355974d221e705aa7c126e77 - as Source +https://raw.githubusercontent.com/codemine-io/lude.dhall/v5.2.0/src/package.dhall + sha256:b04ff9a38c8be087dfacfe76d057fc5272afec60cac28c9e6c1b78cade7ff6ef diff --git a/src/Deps/Prelude.dhall b/src/Deps/Prelude.dhall index 54107fc..76e2a7a 100644 --- a/src/Deps/Prelude.dhall +++ b/src/Deps/Prelude.dhall @@ -1,3 +1,2 @@ https://raw.githubusercontent.com/dhall-lang/dhall-lang/v23.1.0/Prelude/package.dhall sha256:931cbfae9d746c4611b07633ab1e547637ab4ba138b16bf65ef1b9ad66a60b7f - as Source diff --git a/src/Deps/Sdk.dhall b/src/Deps/Sdk.dhall index 5602c43..18f9fce 100644 --- a/src/Deps/Sdk.dhall +++ b/src/Deps/Sdk.dhall @@ -1,3 +1,2 @@ https://raw.githubusercontent.com/pgenie-io/gen-sdk/v2.0.0/src/package.dhall sha256:b9def6ab1179bc4aaae7fc6e91977f094f75934cd5755175c294a9e97ca71b15 - as Source diff --git a/src/Interpret.dhall b/src/Interpret.dhall deleted file mode 100644 index de4642f..0000000 --- a/src/Interpret.dhall +++ /dev/null @@ -1,54 +0,0 @@ -let Contract = ./Deps/Contract.dhall - -let Prelude = ./Deps/Prelude.dhall - -let Config = ./Config.dhall - -let OnUnsupported = ./Structures/OnUnsupported.dhall - -let ProjectInterpreter = ./Interpreters/Project.dhall - --- Entry point handed to gen-sdk's Sdk.Sigs.generator as `interpret`. Each --- field of Config is independently Optional, so a project may omit the --- whole config block (Sdk.Sigs.generator substitutes an all-None --- defaultConfig, see package.dhall) or any subset of its keys; `defaults` --- collects every fallback in one place (packageName from the project name in --- kebab case, emitSync off, onUnsupported Fail). The async surface is always --- emitted; emitSync adds the sync mirror. -in \(config : Config) -> - \(project : Contract.Project) -> - let defaults = - { packageName = project.name.inKebabCase - , emitSync = False - , onUnsupported = OnUnsupported.Mode.Fail - } - - let packageName = - Prelude.Optional.fold - Text - config.packageName - Text - (\(t : Text) -> t) - defaults.packageName - - let emitSync = - Prelude.Optional.fold - Bool - config.emitSync - Bool - (\(b : Bool) -> b) - defaults.emitSync - - let onUnsupported = - Prelude.Optional.fold - OnUnsupported.Mode - config.onUnsupported - OnUnsupported.Mode - (\(m : OnUnsupported.Mode) -> m) - defaults.onUnsupported - - let importName = Prelude.Text.replace "-" "_" packageName - - let interpreterConfig = { packageName, importName, emitSync, onUnsupported } - - in ProjectInterpreter.run interpreterConfig project diff --git a/src/Interpreters/CustomType.dhall b/src/Interpreters/CustomType.dhall index 676f9ab..abdf94f 100644 --- a/src/Interpreters/CustomType.dhall +++ b/src/Interpreters/CustomType.dhall @@ -8,8 +8,6 @@ let Sdk = ../Deps/Sdk.dhall let ImportSet = ../Structures/ImportSet.dhall -let CustomKind = ../Structures/CustomKind.dhall - let OnUnsupported = ../Structures/OnUnsupported.dhall let MemberGen = ./Member.dhall @@ -44,14 +42,6 @@ let Output = , kind : TypeKind } --- Composite fields could in principle reference other custom types, but pgn never --- nests customs in our corpus and CustomType.run does not receive the project --- lookup (Project.run threads it only to queries). Resolving any nested custom to --- Absent makes Member.run fail loudly instead of guessing a type. -let nestedLookup - : CustomKind.Lookup - = \(_ : Model.Name) -> CustomKind.TypeKind.Absent - -- Render the stdlib/runtime imports a composite field type needs, in a fixed -- order so output stays byte-stable. Reads only the standard flags; nested custom -- types are out of Wave 2 scope (no composite in the corpus references one). @@ -123,9 +113,7 @@ let run = = Lude.Compiled.traverseList Model.Member MemberGen.Output - ( \(m : Model.Member) -> - MemberGen.run config nestedLookup m - ) + ( \(m : Model.Member) -> MemberGen.run config m ) members let assemble = diff --git a/src/Interpreters/Member.dhall b/src/Interpreters/Member.dhall index 8d2d27c..c4e8273 100644 --- a/src/Interpreters/Member.dhall +++ b/src/Interpreters/Member.dhall @@ -4,9 +4,9 @@ let Prelude = ../Deps/Prelude.dhall let Model = ../Deps/Contract.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Sdk = ../Deps/Sdk.dhall -let CustomKind = ../Structures/CustomKind.dhall +let ImportSet = ../Structures/ImportSet.dhall let PyIdent = ../Structures/PyIdent.dhall @@ -37,7 +37,6 @@ let Output = let run = \(config : Config) -> - \(lookup : CustomKind.Lookup) -> \(input : Input) -> -- Result-column / composite-field name becomes a dataclass field and decode -- kwarg, so a keyword-named column must be sanitized; row["..."] keeps the @@ -58,63 +57,6 @@ let run = let passthroughDecode = \(src : Text) -> "cast(${castTarget}, ${src})" - let enumDecode = - \(enumName : Text) -> - \(src : Text) -> - let call = "${enumName}(cast(str, ${src}))" - - in if input.isNullable - then "None if ${src} is None else ${call}" - else call - - -- psycopg returns an enum array as a list of text, so each element - -- is rebuilt into the StrEnum. The cast pins the iterable's element - -- type; the per-element None guard mirrors elementIsNullable and the - -- outer None guard mirrors a nullable column. - let enumArrayDecode = - \(enumName : Text) -> - \(src : Text) -> - let elemCast = - if value.elementIsNullable - then "list[str | None]" - else "list[str]" - - let elemDecode = - if value.elementIsNullable - then "None if v is None else ${enumName}(v)" - else "${enumName}(v)" - - -- require_array fails loudly if the array came back as - -- text (the connection did not register the enum type) - -- instead of iterating a string into bogus members. - let elements = - "[${elemDecode} for v in cast(${elemCast}, require_array(${src}))]" - - in if input.isNullable - then "None if ${src} is None else ${elements}" - else elements - - -- Composites are registered per connection (see register_types), - -- so psycopg returns a namedtuple. The fixed-length tuple cast - -- with each field's exact pyType lets the splat satisfy strict. - let compositeDecode = - \(typeName : Text) -> - \(fields : List CustomKind.CompositeField) -> - \(src : Text) -> - let fieldTypes = - Prelude.Text.concatMapSep - ", " - CustomKind.CompositeField - (\(f : CustomKind.CompositeField) -> f.pyType) - fields - - let call = - "${typeName}(*cast(tuple[${fieldTypes}], ${src}))" - - in if input.isNullable - then "None if ${src} is None else ${call}" - else call - in merge { Passthrough = Lude.Compiled.ok @@ -135,11 +77,7 @@ let run = let typeName = name.inPascalCase let customImport = - \(order : Natural) -> - { className = typeName - , moduleName = name.inSnakeCase - , order - } + { className = typeName, moduleName = name.inSnakeCase } let mkOutput = \(customImports : ImportSet.Type) -> @@ -149,81 +87,57 @@ let run = , pyType , isNullable = input.isNullable , imports = - ImportSet.combine - baseImports - customImports + ImportSet.combine baseImports customImports , decodeExpr } - -- A 1-D enum array decodes element-wise; a scalar - -- custom (dims == 0) keeps the single-value decode. - -- Composite arrays and dims > 1 are unimplemented, - -- so fail loudly rather than emit wrong Python. + let wrapNullable = + \(call : Text -> Text) -> + \(src : Text) -> + if input.isNullable + then "None if ${src} is None else ${call src}" + else call src + let dimsIsOne = Natural/isZero (Natural/subtract 1 value.dims) - in merge - { Enum = - \(order : Natural) -> - let enumImport = - ImportSet.customEnum - (customImport order) - - in if Natural/isZero value.dims - then Lude.Compiled.ok - Output - ( mkOutput - enumImport - (enumDecode typeName) - ) - else if dimsIsOne - then Lude.Compiled.ok - Output - ( mkOutput - ( ImportSet.combine - enumImport - ImportSet.enumArray - ) - (enumArrayDecode typeName) - ) - else Lude.Compiled.report - Output - [ input.pgName - , name.inSnakeCase - ] - "Array of an enum with dimensionality > 1 is not supported" - , Composite = - \ ( composite - : { fields : - List CustomKind.CompositeField - , order : Natural - } - ) -> - if Natural/isZero value.dims - then Lude.Compiled.ok - Output - ( mkOutput - ( ImportSet.customComposite - (customImport composite.order) - ) - ( compositeDecode - typeName - composite.fields + in if Natural/isZero value.dims + then Lude.Compiled.ok + Output + ( mkOutput + (ImportSet.custom customImport) + ( wrapNullable + (\(src : Text) -> "${typeName}._decode(${src})") + ) + ) + else if dimsIsOne + then let elemCast = + if value.elementIsNullable + then "list[str | None]" + else "list[str]" + + let elemDecode = + if value.elementIsNullable + then "None if v is None else ${typeName}._decode(v)" + else "${typeName}._decode(v)" + + in Lude.Compiled.ok + Output + ( mkOutput + ( ImportSet.combine + (ImportSet.custom customImport) + ImportSet.enumArray + ) + ( wrapNullable + ( \(src : Text) -> + "[${elemDecode} for v in cast(${elemCast}, require_array(${src}))]" ) ) - else Lude.Compiled.report - Output - [ input.pgName - , name.inSnakeCase - ] - "Array of a composite type is not supported (element-wise decode is unimplemented)" - , Absent = - Lude.Compiled.report + ) + else Lude.Compiled.report Output - [ name.inSnakeCase ] - "Custom type not found in project customTypes" - } - (lookup name) + [ input.pgName, name.inSnakeCase ] + "Array of dimensionality > 1 is not supported" ) ( Lude.Compiled.report Output @@ -242,6 +156,4 @@ let run = in Lude.Compiled.flatMap Value.Output Output buildOutput compiledValue -let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output - -in { Input, Output, Run, run } +in Sdk.Sigs.interpreter Config Input Output run diff --git a/src/Interpreters/ParamsMember.dhall b/src/Interpreters/ParamsMember.dhall index 2a4b469..457631d 100644 --- a/src/Interpreters/ParamsMember.dhall +++ b/src/Interpreters/ParamsMember.dhall @@ -4,9 +4,9 @@ let Prelude = ../Deps/Prelude.dhall let Model = ../Deps/Contract.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Sdk = ../Deps/Sdk.dhall -let CustomKind = ../Structures/CustomKind.dhall +let ImportSet = ../Structures/ImportSet.dhall let OnUnsupported = ../Structures/OnUnsupported.dhall @@ -232,7 +232,6 @@ let isJsonArray = let run = \(config : Config) -> - \(lookup : CustomKind.Lookup) -> \(input : Input) -> let fieldName = pySafeName input.name.inSnakeCase @@ -253,40 +252,6 @@ let run = then wrapJson "Jsonb" else if needsJsonImport then wrapJson "Json" else fieldName - -- psycopg binds a bare tuple to an anonymous composite, but cannot adapt - -- a dataclass, so a composite param is converted to its field tuple. - let compositeBind = - \(fields : List CustomKind.CompositeField) -> - let joinedFields = - Prelude.Text.concatMapSep - ", " - CustomKind.CompositeField - ( \(f : CustomKind.CompositeField) -> - "${fieldName}.${f.fieldName}" - ) - fields - - -- concatMapSep never emits a separator for a single-element list, so - -- a one-field composite would render "(x.f)": parens around a bare - -- expression, not a tuple. Python only treats trailing-comma parens - -- as a 1-tuple, so force it for exactly one field; concatMapSep - -- already inserts the internal comma for two or more. - let trailingComma = - if Prelude.Natural.equal - ( Prelude.List.length - CustomKind.CompositeField - fields - ) - 1 - then "," - else "" - - let tupleExpr = "(" ++ joinedFields ++ trailingComma ++ ")" - - in if input.isNullable - then "None if ${fieldName} is None else ${tupleExpr}" - else tupleExpr - let buildOutput = \(value : Value.Output) -> let pyType = @@ -314,56 +279,36 @@ let run = (Lude.Compiled.Type Output) ( \(name : Model.Name) -> let customImport = - \(order : Natural) -> - { className = name.inPascalCase - , moduleName = name.inSnakeCase - , order - } - - in merge - { Enum = - \(order : Natural) -> - Lude.Compiled.ok - Output - ( mkOutput - ( ImportSet.combine - value.imports - ( ImportSet.customEnum - (customImport order) - ) - ) - defaultBind - ) - , Composite = - \ ( composite - : { fields : - List CustomKind.CompositeField - , order : Natural - } - ) -> - if Natural/isZero value.dims - then Lude.Compiled.ok - Output - ( mkOutput - ( ImportSet.combine - value.imports - ( ImportSet.customComposite - (customImport composite.order) - ) - ) - (compositeBind composite.fields) - ) - else Lude.Compiled.report - Output - [ input.pgName, name.inSnakeCase ] - "Array of a composite type as a parameter is not supported" - , Absent = - Lude.Compiled.report - Output - [ name.inSnakeCase ] - "Custom type not found in project customTypes" + { className = name.inPascalCase + , moduleName = name.inSnakeCase } - (lookup name) + + let scalarEncode = + if input.isNullable + then "None if ${fieldName} is None else ${fieldName}._encode()" + else "${fieldName}._encode()" + + let arrayElemEncode = + if value.elementIsNullable + then "None if x is None else x._encode()" + else "x._encode()" + + let arrayEncode = + let base = "[${arrayElemEncode} for x in ${fieldName}]" + + in if input.isNullable + then "None if ${fieldName} is None else ${base}" + else base + + let encodeExpr = + if Natural/isZero value.dims then scalarEncode else arrayEncode + + in Lude.Compiled.ok + Output + ( mkOutput + (ImportSet.combine value.imports (ImportSet.custom customImport)) + encodeExpr + ) ) ( if isJsonArrayParam then Lude.Compiled.report @@ -384,6 +329,4 @@ let run = in Lude.Compiled.flatMap Value.Output Output buildOutput compiledValue -let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output - -in { Input, Output, Run, run } +in Sdk.Sigs.interpreter Config Input Output run diff --git a/src/Interpreters/Project.dhall b/src/Interpreters/Project.dhall index d4fa6d6..f142a99 100644 --- a/src/Interpreters/Project.dhall +++ b/src/Interpreters/Project.dhall @@ -6,12 +6,6 @@ let Model = ../Deps/Contract.dhall let Sdk = ../Deps/Sdk.dhall -let CustomKind = ../Structures/CustomKind.dhall - -let PyIdent = ../Structures/PyIdent.dhall - -let Value = ./Value.dhall - let QueryGen = ./Query.dhall let CustomTypeGen = ./CustomType.dhall @@ -38,7 +32,20 @@ let OnUnsupported = ../Structures/OnUnsupported.dhall let Report = { path : List Text, message : Text } +-- The generator's public Config: every field is independently Optional, so a +-- project may omit the whole `config:` block or any subset of its keys. +-- `run` below resolves the fallbacks itself (packageName from the project +-- name, emitSync off, onUnsupported Fail); there is no separate config type +-- or resolve step between package.dhall and here. let Config = + { packageName : Optional Text + , emitSync : Optional Bool + , onUnsupported : Optional OnUnsupported.Mode + } + +-- The fully-resolved shape every downstream interpreter (Query, CustomType, +-- and everything below them) actually declares as its own `Config`. +let ResolvedConfig = { packageName : Text , importName : Text , emitSync : Bool @@ -69,44 +76,6 @@ let withHeader = \(file : Lude.File.Type) -> { path = file.path, content = generatedHeader ++ file.content } --- Internal interpreter config (importName etc.) is only needed to satisfy --- Value.run's signature; rendering a composite field's pyType does not read it. -let lookupConfig - : Config - = { packageName = "" - , importName = "" - , emitSync = False - , onUnsupported = OnUnsupported.Mode.Fail - } - --- Render a composite member's Python type purely. The Err branch is unreachable --- for a valid composite (CustomType.run fails the whole generation first on any --- unsupported field type), so the fallback string is dead. -let memberPyType = - \(member : Model.Member) -> - let valuePyType = - merge - { Ok = - \(wrapped : { value : Value.Output, warnings : List Report }) -> - wrapped.value.pyType - , Err = \(_ : Report) -> "object" - } - (Value.run lookupConfig member.value) - - in valuePyType ++ (if member.isNullable then " | None" else "") - -let compositeFields = - \(members : List Model.Member) -> - Prelude.List.map - Model.Member - CustomKind.CompositeField - ( \(member : Model.Member) -> - { fieldName = PyIdent.pySafeName member.name.inSnakeCase - , pyType = memberPyType member - } - ) - members - -- Schema-qualified type names for psycopg CompositeInfo.fetch (search-path -- safe). Reads CustomTypeGen.Output (already a combineOutputs parameter, and -- already the post-Skip-filter surviving set), not Model.CustomType, so @@ -136,58 +105,18 @@ let enumPgNames = customTypes ) --- A custom type referenced by a Scalar.Custom name resolves by matching its --- snake-case key. The query/column carries a distinct Name occurrence, so the --- lookup compares by inSnakeCase rather than relying on record identity. -let IndexedCustomType = { index : Natural, value : Model.CustomType } - --- pgn emits customTypes alphabetically by name, so the list index is the type's --- alphabetical order. The import renderer sorts on it to keep the per-module --- `from ..types.X import Y` block alphabetical independent of column order. -let buildLookup = - \(customTypes : List Model.CustomType) -> - Prelude.List.fold - IndexedCustomType - (Prelude.List.indexed Model.CustomType customTypes) - CustomKind.Lookup - ( \(entry : IndexedCustomType) -> - \(rest : CustomKind.Lookup) -> - \(name : Model.Name) -> - let ct = entry.value - - -- Text/equal is a pgn embedded-Dhall builtin, absent from the Dhall - -- standard 23.1 Prelude. PyIdent.dhall's replace-trick cannot stand in - -- here: this branch returns a structural TypeKind, not Text. gen-sdk's - -- Fixtures module relies on the same builtin; a kind tag or a Natural - -- index on Scalar.Custom is a planned upstream ask. - in if Text/equal name.inSnakeCase ct.name.inSnakeCase - then merge - { Composite = - \(members : List Model.Member) -> - CustomKind.TypeKind.Composite - { fields = compositeFields members - , order = entry.index - } - , Enum = - \(_ : List Model.EnumVariant) -> - CustomKind.TypeKind.Enum entry.index - , Domain = - \(_ : Model.Value) -> CustomKind.TypeKind.Absent - } - ct.definition - else rest name - ) - (\(_ : Model.Name) -> CustomKind.TypeKind.Absent) - let combineOutputs = - \(config : Config) -> + \(config : ResolvedConfig) -> \(input : Input) -> - \(queries : List QueryGen.Output) -> -- Already the post-Skip-filter surviving set (see `run`); equal to - -- input.customTypes' compiled outputs verbatim when nothing was skipped + -- input.queries' compiled outputs verbatim when nothing was skipped -- (including every Fail-mode run, since Fail never drops anything). - -- Facade/typesInit/register entries are built from this, not - -- input.customTypes, so a skipped type leaves no dangling export. + -- Facade/statement/row entries are built from this, not input.queries, + -- so a skipped query leaves no dangling export. + \(queries : List QueryGen.Output) -> + -- Same as above, but for customTypes. Facade/typesInit/register + -- entries are built from this, not input.customTypes, so a skipped + -- type leaves no dangling export. \(customTypes : List CustomTypeGen.Output) -> -- The generator emits the _generated subtree plus the package-root -- __init__.py facade. The rest of the shell (pyproject.toml, py.typed) is @@ -423,21 +352,54 @@ let combineOutputs = -- warning list. Custom types use a pair of plain functions instead of an -- equivalent record (see `typeSucceeds`/`typeWarning` below) purely because -- that was the faster shape empirically for the type side, and the query --- side re-uses `queryChecks` because `lookup` (built from the surviving --- custom types) threads into every query's Member/ParamsMember resolution: --- calling QueryGen.run config lookup query from more than one place in this --- function (once to decide keep/drop, again to render, again for a --- warning -- each a fresh, separate call site in the source) measurably --- multiplies Dhall's normalization cost per extra call site, confirmed by --- bisection against `pgn generate` wall time (a few seconds regressed to --- minutes with three call sites; this file keeps it to two: one to build --- `queryChecks`, one for the final render). +-- side re-uses `queryChecks` because calling QueryGen.run config query from +-- more than one place in this function (once to decide keep/drop, again to +-- render, again for a warning -- each a fresh, separate call site in the +-- source) measurably multiplies Dhall's normalization cost per extra call +-- site, confirmed by bisection against `pgn generate` wall time (a few +-- seconds regressed to minutes with three call sites; this file keeps it to +-- two: one to build `queryChecks`, one for the final render). let QueryCheck = { query : Model.Query, keep : Bool, warning : Optional Report } let run = \(config : Config) -> \(input : Input) -> - let skip = merge { Fail = False, Skip = True } config.onUnsupported + -- config's fields are independently Optional (a project may omit the + -- whole config: block or any subset of its keys); the fallbacks are + -- resolved here rather than in a separate config type or resolve + -- step, since this is the root interpreter and Config above is + -- exactly the generator's public Config. + let packageName = + Prelude.Optional.fold + Text + config.packageName + Text + (\(t : Text) -> t) + input.name.inKebabCase + + let emitSync = + Prelude.Optional.fold + Bool + config.emitSync + Bool + (\(b : Bool) -> b) + False + + let onUnsupported = + Prelude.Optional.fold + OnUnsupported.Mode + config.onUnsupported + OnUnsupported.Mode + (\(m : OnUnsupported.Mode) -> m) + OnUnsupported.Mode.Fail + + let importName = Prelude.Text.replace "-" "_" packageName + + let resolvedConfig + : ResolvedConfig + = { packageName, importName, emitSync, onUnsupported } + + let skip = merge { Fail = False, Skip = True } resolvedConfig.onUnsupported let typeSucceeds : Model.CustomType -> Bool @@ -448,7 +410,7 @@ let run = True , Err = \(_ : Report) -> False } - (CustomTypeGen.run config ct) + (CustomTypeGen.run resolvedConfig ct) -- Nested under the type's own name so the warning names the type -- that failed, not just the inner member/column that triggered it @@ -463,7 +425,7 @@ let run = , Err = \(err : Report) -> Some { path = [ ct.name.inSnakeCase ] # err.path, message = err.message } } - (CustomTypeGen.run config ct) + (CustomTypeGen.run resolvedConfig ct) -- A skipped custom type resolves to Absent for any query that -- references it, and that query's own Member/ParamsMember @@ -476,8 +438,6 @@ let run = then Prelude.List.filter Model.CustomType typeSucceeds input.customTypes else input.customTypes - let lookup = buildLookup effectiveCustomTypes - -- Fail mode: identical to the pre-Skip code (traverseList straight -- over input.customTypes), so its error message/path is unchanged. let typesForCombine @@ -485,7 +445,7 @@ let run = = Lude.Compiled.traverseList Model.CustomType CustomTypeGen.Output - (\(ct : Model.CustomType) -> CustomTypeGen.run config ct) + (\(ct : Model.CustomType) -> CustomTypeGen.run resolvedConfig ct) effectiveCustomTypes let queryChecks @@ -500,7 +460,7 @@ let run = { query, keep = True, warning = None Report } , Err = \(err : Report) -> { query, keep = False, warning = Some err } } - (QueryGen.run config lookup query) + (QueryGen.run resolvedConfig query) ) input.queries @@ -521,7 +481,7 @@ let run = = Lude.Compiled.traverseList Model.Query QueryGen.Output - (\(query : Model.Query) -> QueryGen.run config lookup query) + (\(query : Model.Query) -> QueryGen.run resolvedConfig query) effectiveQueries let skipWarnings @@ -541,7 +501,7 @@ let run = (List QueryGen.Output) (List CustomTypeGen.Output) Output - (combineOutputs config input) + (combineOutputs resolvedConfig input) queriesForCombine typesForCombine diff --git a/src/Interpreters/Query.dhall b/src/Interpreters/Query.dhall index cce3fcd..b3e86cd 100644 --- a/src/Interpreters/Query.dhall +++ b/src/Interpreters/Query.dhall @@ -2,9 +2,9 @@ let Prelude = ../Deps/Prelude.dhall let Lude = ../Deps/Lude.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Sdk = ../Deps/Sdk.dhall -let CustomKind = ../Structures/CustomKind.dhall +let ImportSet = ../Structures/ImportSet.dhall let PyIdent = ../Structures/PyIdent.dhall @@ -137,7 +137,6 @@ let render = let run = \(config : Config) -> - \(lookup : CustomKind.Lookup) -> \(input : Input) -> let rowClassName = input.name.inPascalCase ++ "Row" @@ -153,7 +152,7 @@ let run = ( Compiled.nest ResultModule.Output "result" - (ResultModule.run config lookup rowClassName input.result) + (ResultModule.run (config /\ { rowClassName }) input.result) ) ( Compiled.nest QueryFragmentsModule.Output @@ -170,11 +169,11 @@ let run = Compiled.nest ParamsMember.Output member.pgName - (ParamsMember.run config lookup member) + (ParamsMember.run config member) ) input.params ) ) ) -in { Input, Output, run } +in Sdk.Sigs.interpreter Config Input Output run diff --git a/src/Interpreters/Result.dhall b/src/Interpreters/Result.dhall index 5d652a0..b20190d 100644 --- a/src/Interpreters/Result.dhall +++ b/src/Interpreters/Result.dhall @@ -4,9 +4,9 @@ let Lude = ../Deps/Lude.dhall let Model = ../Deps/Contract.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Sdk = ../Deps/Sdk.dhall -let CustomKind = ../Structures/CustomKind.dhall +let ImportSet = ../Structures/ImportSet.dhall let OnUnsupported = ../Structures/OnUnsupported.dhall @@ -14,11 +14,17 @@ let ResultColumns = ./ResultColumns.dhall let Compiled = Lude.Compiled +-- rowClassName is supplied by the caller (Query.dhall derives it from the +-- query's own name) rather than living on Model.Result, so it rides on this +-- interpreter's own local Config instead of widening Input away from +-- Model.Result. ResultColumns below does not need it, so it is projected +-- back down to the narrower shared shape at that call site. let Config = { packageName : Text , importName : Text , emitSync : Bool , onUnsupported : OnUnsupported.Mode + , rowClassName : Text } let Input = Model.Result @@ -63,10 +69,8 @@ let cardinalityShape let rowsOutput = \(config : Config) -> - \(lookup : CustomKind.Lookup) -> - \(rowClassName : Text) -> \(rows : Model.ResultRows) -> - let shape = cardinalityShape rows.cardinality rowClassName + let shape = cardinalityShape rows.cardinality config.rowClassName let columns = Prelude.NonEmpty.toList Model.Member rows.columns @@ -78,7 +82,7 @@ let rowsOutput = { returnType = shape.returnType , helperName = shape.helperName , rowClass = Some - { name = rowClassName + { name = config.rowClassName , fieldsBlock = cols.fieldsBlock , decodeBlock = cols.decodeBlock } @@ -86,19 +90,20 @@ let rowsOutput = , callsDecode = True } ) - (ResultColumns.run config lookup rowClassName columns) + ( ResultColumns.run + config.{ packageName, importName, emitSync, onUnsupported } + columns + ) let run = \(config : Config) -> - \(lookup : CustomKind.Lookup) -> - \(rowClassName : Text) -> \(input : Input) -> merge { Void = Compiled.ok Output (noResult "None" "execute_void") , RowsAffected = Compiled.ok Output (noResult "int" "execute_rows_affected") - , Rows = rowsOutput config lookup rowClassName + , Rows = rowsOutput config } input -in { Input, Output, RowClass, run } +in Sdk.Sigs.interpreter Config Input Output run /\ { RowClass } diff --git a/src/Interpreters/ResultColumns.dhall b/src/Interpreters/ResultColumns.dhall index e0bbfaa..6a589a2 100644 --- a/src/Interpreters/ResultColumns.dhall +++ b/src/Interpreters/ResultColumns.dhall @@ -4,9 +4,9 @@ let Lude = ../Deps/Lude.dhall let Model = ../Deps/Contract.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Sdk = ../Deps/Sdk.dhall -let CustomKind = ../Structures/CustomKind.dhall +let ImportSet = ../Structures/ImportSet.dhall let OnUnsupported = ../Structures/OnUnsupported.dhall @@ -59,8 +59,6 @@ let assemble let run = \(config : Config) -> - \(lookup : CustomKind.Lookup) -> - \(rowClassName : Text) -> \(input : Input) -> Compiled.map (List Member.Output) @@ -73,9 +71,9 @@ let run = Compiled.nest Member.Output member.pgName - (Member.run config lookup member) + (Member.run config member) ) input ) -in { Input, Output, run } +in Sdk.Sigs.interpreter Config Input Output run diff --git a/src/Structures/CustomKind.dhall b/src/Structures/CustomKind.dhall deleted file mode 100644 index f2c4a1b..0000000 --- a/src/Structures/CustomKind.dhall +++ /dev/null @@ -1,28 +0,0 @@ -let Model = ../Deps/Contract.dhall - --- A composite field as the decode/encode sites need it: the Python attribute --- name and the rendered Python type (already nullability-applied). Threaded so --- a column decode can build `Point2D(*cast(tuple[x, y], src))` and a param --- encode can build `(value.x, value.y)` without re-reading the customType. -let CompositeField = { fieldName : Text, pyType : Text } - --- Classification of a custom type referenced by a Scalar.Custom name. Composite --- carries its field list so callers render per-field decode/encode. Absent means --- the name did not resolve against project.customTypes, which is a model --- inconsistency the caller turns into a Compiled error. --- Enum and Composite both carry `order`, the type's alphabetical position in --- project.customTypes; the import renderer sorts on it so the per-module --- `from ..types.X import Y` block stays alphabetical regardless of column order. --- NOTE: union is named TypeKind because `Kind` is a reserved Dhall keyword. -let TypeKind = - < Enum : Natural - | Composite : { fields : List CompositeField, order : Natural } - | Absent - > - --- Resolves a custom-type Name to its TypeKind. Project.run builds this from --- project.customTypes and threads it into Member/ParamsMember (and onward to --- Result/ResultColumns) so those interpreters can pick the right decode/encode. -let Lookup = Model.Name -> TypeKind - -in { TypeKind, Lookup, CompositeField } diff --git a/src/Structures/ImportSet.dhall b/src/Structures/ImportSet.dhall index 04fc239..b5c7b81 100644 --- a/src/Structures/ImportSet.dhall +++ b/src/Structures/ImportSet.dhall @@ -1,11 +1,16 @@ let Prelude = ../Deps/Prelude.dhall -- A custom-type import line: "from ..types. import ". --- dedupKey identifies the type for deduplication and orders the rendered import --- block. Dhall (upstream) has no Text comparison, so the caller assigns each --- distinct custom type its alphabetical position in project.customTypes as the --- key; dedup keys on it and the render step sorts ascending to stay alphabetical. -let CustomImport = { className : Text, moduleName : Text, dedupKey : Natural } +-- Emitted in encounter order (the order the referencing columns/params were +-- declared), not sorted. Dhall (upstream) has no Text comparison, so there is +-- no way to alphabetize or dedupe by moduleName/className without either the +-- pgn fork's Text/equal or a project-wide Natural id (previously `order`, +-- sourced from Project.dhall's buildLookup — see +-- docs/plans/2026-07-11-encounter-order-custom-imports.md for why that's +-- gone and why dedup isn't reintroduced some other way). Two references to +-- the same type currently produce two identical lines; harmless to Python +-- and to basedpyright, just not deduped. +let CustomImport = { className : Text, moduleName : Text } let Self = { uuid : Bool @@ -83,97 +88,6 @@ let custom : CustomImport -> Self = \(c : CustomImport) -> base // { customTypes = [ c ] } --- Named-type imports dedup and sort on `order`, the type's alphabetical index in --- project.customTypes. Two references to the same type carry the same order and --- collapse; distinct types keep distinct keys so the render step can sort them. -let customEnum - : { className : Text, moduleName : Text, order : Natural } -> Self - = \(c : { className : Text, moduleName : Text, order : Natural }) -> - custom - { className = c.className, moduleName = c.moduleName, dedupKey = c.order } - -let customComposite - : { className : Text, moduleName : Text, order : Natural } -> Self - = \(c : { className : Text, moduleName : Text, order : Natural }) -> - custom - { className = c.className, moduleName = c.moduleName, dedupKey = c.order } - -let eqNat = - \(a : Natural) -> - \(b : Natural) -> - Natural/isZero (Natural/subtract a b) - && Natural/isZero (Natural/subtract b a) - -let dedupCustoms - : List CustomImport -> List CustomImport - = \(items : List CustomImport) -> - let State = { seen : List Natural, acc : List CustomImport } - - let step = - \(item : CustomImport) -> - \(state : State) -> - let known = - Prelude.List.any - Natural - (\(k : Natural) -> eqNat k item.dedupKey) - state.seen - - in if known - then state - else { seen = state.seen # [ item.dedupKey ] - , acc = state.acc # [ item ] - } - - in ( List/fold - CustomImport - items - State - step - { seen = [] : List Natural, acc = [] : List CustomImport } - ).acc - -let leNat = \(a : Natural) -> \(b : Natural) -> Natural/isZero (Natural/subtract b a) - --- Insertion sort over custom imports by ascending dedupKey (the alphabetical --- order). The list is tiny (one or two types per module), so the cost is moot --- and the result is a stable alphabetical import block. -let sortCustoms - : List CustomImport -> List CustomImport - = \(items : List CustomImport) -> - let insert = - \(item : CustomImport) -> - \(acc : List CustomImport) -> - let State = { placed : Bool, out : List CustomImport } - - -- foldLeft so the scan runs left-to-right and `item` lands before - -- the FIRST element that is >= it (a right List/fold visits the - -- list back-to-front and would insert before the last such - -- element, mis-ordering blocks of 3+ types). - let step = - \(state : State) -> - \(cur : CustomImport) -> - if state.placed - then { placed = True, out = state.out # [ cur ] } - else if leNat item.dedupKey cur.dedupKey - then { placed = True - , out = state.out # [ item, cur ] - } - else { placed = False, out = state.out # [ cur ] } - - let folded = - Prelude.List.foldLeft - CustomImport - acc - State - step - { placed = False, out = [] : List CustomImport } - - in if folded.placed - then folded.out - else folded.out # [ item ] - - in List/fold CustomImport items (List CustomImport) insert ([] : List CustomImport) - let combine = \(left : Self) -> \(right : Self) -> @@ -187,17 +101,13 @@ let combine = , json = left.json || right.json , jsonValue = left.jsonValue || right.jsonValue , enumArray = left.enumArray || right.enumArray - , customTypes = dedupCustoms (left.customTypes # right.customTypes) + , customTypes = left.customTypes # right.customTypes } let combineAll : List Self -> Self = \(sets : List Self) -> List/fold Self sets Self combine empty -let sortedCustoms - : Self -> List CustomImport - = \(self : Self) -> sortCustoms self.customTypes - in { Type = Self , CustomImport , empty @@ -212,9 +122,6 @@ in { Type = Self , jsonValue , enumArray , custom - , customEnum - , customComposite , combine , combineAll - , sortedCustoms } diff --git a/src/Templates/CompositeModule.dhall b/src/Templates/CompositeModule.dhall index b1ab3e0..32c4ee5 100644 --- a/src/Templates/CompositeModule.dhall +++ b/src/Templates/CompositeModule.dhall @@ -20,11 +20,50 @@ let run = ) params.fields + let fieldCount = Prelude.List.length Field params.fields + + let fieldTypesJoined = + Prelude.Text.concatMapSep + ", " + Field + (\(field : Field) -> field.fieldType) + params.fields + + let selfFieldsJoined = + Prelude.Text.concatMapSep + ", " + Field + (\(field : Field) -> "self.${field.fieldName}") + params.fields + + -- Python only treats trailing-comma parens as a 1-tuple; concatMapSep + -- never emits an internal comma for a single-element list, so force one + -- here. Mirrors ParamsMember.dhall's existing compositeBind trick. + let encodeTupleExpr = + if Prelude.Natural.equal fieldCount 1 + then "(${selfFieldsJoined},)" + else "(${selfFieldsJoined})" + + -- _decode/_encode are emitted as literal lines (not a nested multi-line + -- ''...'' block) because Dhall dedents a multi-line literal against its + -- OWN source indentation before splicing it into the outer literal; a + -- nested block loses its intended 4/8-space class-body indentation. + -- Verified against `dhall text` during design. + let codecMethods = + "\n" + ++ " @staticmethod\n" + ++ " def _decode(src: object) -> \"${params.typeName}\":\n" + ++ " return ${params.typeName}(*cast(tuple[${fieldTypesJoined}], src))\n" + ++ "\n" + ++ " def _encode(self) -> tuple[${fieldTypesJoined}]:\n" + ++ " return ${encodeTupleExpr}" + let imports = if Prelude.List.null Text params.extraImports - then "from dataclasses import dataclass" + then "from dataclasses import dataclass\nfrom typing import cast" else '' from dataclasses import dataclass + from typing import cast ${Prelude.Text.concatSep "\n" params.extraImports}'' @@ -41,6 +80,7 @@ let run = """ ${fieldLines} + ${codecMethods} '' in Sdk.Sigs.template Params run /\ { Field } diff --git a/src/Templates/EnumModule.dhall b/src/Templates/EnumModule.dhall index 1ebf486..04c4bfe 100644 --- a/src/Templates/EnumModule.dhall +++ b/src/Templates/EnumModule.dhall @@ -35,12 +35,23 @@ let run = ) params.variants + let codecMethods = + "\n" + ++ " @staticmethod\n" + ++ " def _decode(src: object) -> \"${params.typeName}\":\n" + ++ " return ${params.typeName}(cast(str, src))\n" + ++ "\n" + ++ " def _encode(self) -> \"${params.typeName}\":\n" + ++ " return self" + in '' from enum import StrEnum + from typing import cast class ${params.typeName}(StrEnum): ${memberLines} + ${codecMethods} '' in Sdk.Sigs.template Params run /\ { Variant } diff --git a/src/Templates/RowsModule.dhall b/src/Templates/RowsModule.dhall index 18da31b..b1d5b13 100644 --- a/src/Templates/RowsModule.dhall +++ b/src/Templates/RowsModule.dhall @@ -55,7 +55,7 @@ let customImportLines ( \(c : ImportSet.CustomImport) -> "from .types." ++ c.moduleName ++ " import " ++ c.className ) - (ImportSet.sortedCustoms imports) + imports.customTypes let renderImports : ImportSet.Type -> Text diff --git a/src/Templates/StatementModule.dhall b/src/Templates/StatementModule.dhall index f2a6f61..325e21f 100644 --- a/src/Templates/StatementModule.dhall +++ b/src/Templates/StatementModule.dhall @@ -79,7 +79,7 @@ let customImportLines ( \(c : ImportSet.CustomImport) -> "from ${typesPrefix}.${c.moduleName} import ${c.className}" ) - (ImportSet.sortedCustoms imports) + imports.customTypes let rowsImportLine : Params -> List Text diff --git a/src/package.dhall b/src/package.dhall index 3d07cf7..5161367 100644 --- a/src/package.dhall +++ b/src/package.dhall @@ -2,7 +2,22 @@ let Sdk = ./Deps/Sdk.dhall let OnUnsupported = ./Structures/OnUnsupported.dhall -let Config = ./Config.dhall +let ProjectInterpreter = ./Interpreters/Project.dhall + +-- User-facing config for this generator. `emitSync` adds a parallel sync +-- surface (psycopg.Connection) alongside the default async one, so one +-- project can serve both an async backend and a sync (Dagster) consumer from +-- shared Row types. `onUnsupported` picks Fail (default, abort loudly) or +-- Skip (drop the unsupported statement/type and its dependents, with a +-- warning) when a query or custom type hits a PG shape the generator cannot +-- render; see Structures/OnUnsupported.dhall. All fields are Optional so a +-- project may omit the whole config block or any subset of its keys; +-- Interpreters/Project.dhall's `run` supplies the defaults. +let Config = + { packageName : Optional Text + , emitSync : Optional Bool + , onUnsupported : Optional OnUnsupported.Mode + } : Type let Config/default : Config @@ -11,6 +26,4 @@ let Config/default , onUnsupported = None OnUnsupported.Mode } -let interpret = ./Interpret.dhall - -in Sdk.Sigs.generator Config Config/default interpret +in Sdk.Sigs.generator Config Config/default ProjectInterpreter.run diff --git a/tests/test_config_variants.py b/tests/test_config_variants.py index 9b23d18..e0d3ecf 100644 --- a/tests/test_config_variants.py +++ b/tests/test_config_variants.py @@ -66,7 +66,7 @@ def test_absent_config_key_defaults_both_fields(generated_tree: Path) -> None: def test_unknown_config_key_is_ignored_not_rejected(generated_tree: Path) -> None: - """An extra key not in Config.dhall (bogusField) does not fail generation.""" + """An extra key not in the generator's Config type (bogusField) does not fail generation.""" package = _package_dir(generated_tree, "python-unknown-key") assert package.name == "unknown_key_client" assert not (package / "sync").exists() From 76255508f57f17230e1a8f153810ca234db3582b Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sun, 12 Jul 2026 06:29:44 +0300 Subject: [PATCH 11/21] Drop the plans --- ...26-07-11-encounter-order-custom-imports.md | 327 --------- .../2026-07-11-reusable-custom-type-codecs.md | 633 ------------------ 2 files changed, 960 deletions(-) delete mode 100644 docs/plans/2026-07-11-encounter-order-custom-imports.md delete mode 100644 docs/plans/2026-07-11-reusable-custom-type-codecs.md diff --git a/docs/plans/2026-07-11-encounter-order-custom-imports.md b/docs/plans/2026-07-11-encounter-order-custom-imports.md deleted file mode 100644 index 7d1c663..0000000 --- a/docs/plans/2026-07-11-encounter-order-custom-imports.md +++ /dev/null @@ -1,327 +0,0 @@ -# Encounter-Order Custom-Type Imports Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Stop requiring a project-wide `order : Natural` to sort and dedupe custom-type import lines in `ImportSet.dhall`. Emit them in encounter order (the order the referencing columns/params are declared) instead, and accept that de-duplicating two references to the *same* custom type within one file is not achievable in vanilla Dhall — verify it isn't actually needed by the current corpus, and document the tradeoff rather than reintroduing a lookup to avoid it. - -**Architecture:** `ImportSet.dhall` today dedupes and alphabetizes custom-type imports by carrying a `dedupKey : Natural` — the type's alphabetical index in `project.customTypes` — through every `CustomImport` value, specifically because Dhall has no `Text` comparison to sort or dedupe on `moduleName`/`className` directly (see the file's own header comment). That `order` value's only source was `buildLookup` (`Interpreters/Project.dhall`), which the companion plan (`2026-07-11-reusable-custom-type-codecs.md`) deletes. Once it's gone, `ImportSet.dhall` has nothing to key on. Rather than re-deriving a Natural surrogate some other way, this plan removes the sort/dedup step and lets `ImportSet.combine`'s existing `List/fold` order (already the query's declared column/param order — a real, already-computed, non-Text-comparison ordering) stand as the output order. - -**Tech Stack:** Dhall (dhall-lang 1.42), Python 3.12 generated output, pytest golden-file harness (`mise run test`, `mise run golden`). - -## Global Constraints - -- **Depends on** `docs/plans/2026-07-11-reusable-custom-type-codecs.md` — that plan's Task 3/4 call `ImportSet.custom customImport` (no `order` argument). Land this plan's Task 1 first, or in the same PR; `ImportSet.customEnum`/`customComposite` (which this plan deletes) are exactly what those tasks stop calling. -- No behavior change to the four already-Natural-keyed stdlib import flags (`uuid`, `datetime`, `date`, `time`, `timedelta`, `decimal`, `jsonb`, `json`, `jsonValue`, `enumArray`) — those are plain `Bool` OR's today and are untouched by this plan. -- Golden fixture output must be regenerated and diffed (`tests/golden/`), not hand-edited. -- Every Dhall file touched must independently type-check: `dhall type --file=`. - ---- - -## Why dedup can't be preserved without reintroducing a lookup - -Worth writing down since it's not obvious and the temptation to "just find a clever `Text/replace` trick" is real — this was checked directly against `PyIdent.dhall`'s working pattern (DESIGN.md section 13) before concluding it doesn't generalize: - -`PyIdent.dhall`'s `sanitizeAgainst` tests a runtime `Text` against a small, **fixed, compile-time-literal** candidate list (the 35 Python keywords) — one known literal at a time, folded. That's why the two-`Text/replace` trick works: one side of every comparison is always a literal. - -Import dedup needs the opposite shape: is this **runtime** `moduleName` (derived from a `Name` that varies per project) equal to any of the **other runtime** `moduleName`s already collected? Both sides are dynamic. No sequence of `Text/replace` calls can decide that, because deciding it requires producing a `Bool` from two non-literal `Text` values, which is exactly the operation Dhall doesn't have (and which pgn's forked `Text/equal` exists to provide). This isn't a missing trick — dedup of dynamically-computed `Text` is unconditionally impossible in vanilla Dhall. The only ways to get it back are: (a) a fork builtin (what we're removing), (b) a pre-assigned `Natural` id per distinct value (what `order` was — sourced from a project-wide search, i.e. `buildLookup`, also being removed), or (c) don't need it. - -This plan takes (c), having checked how much it costs to: - -```bash -grep -rl "^from \.\.types\." tests/golden/src 2>/dev/null | while read f; do - n=$(grep -c "^from \.\.types\." "$f") - [ "$n" -gt 1 ] && echo "$f: $n" -done -``` - -No output — verified during design (see conversation record / re-run before Task 2 to confirm it's still true after Doc 1's regeneration). No file in the current fixture corpus imports the same custom type twice. The risk is real but currently unexercised: if a future query selects the same composite/enum type via two different columns, the generated file will contain two identical `from ..types.X import Y` lines — syntactically valid, harmless to `basedpyright strict` and to Python's import system, just visually redundant. Task 2 adds a corpus case that exercises this on purpose so the tradeoff is documented against real output, not just asserted. - ---- - -## File Structure - -| File | Change | -|---|---| -| `src/Structures/ImportSet.dhall` | Drop `dedupKey` from `CustomImport`; delete `dedupCustoms`, `sortCustoms`, `eqNat`, `leNat`, `sortedCustoms`; collapse `customEnum`/`customComposite` into the existing `custom`; `combine` plain-concatenates `customTypes` instead of deduping. | -| `src/Templates/RowsModule.dhall`, `src/Templates/StatementModule.dhall` | Read `imports.customTypes` directly instead of `ImportSet.sortedCustoms imports`. | -| `tests/golden/` | Regenerate via `mise run golden`; review the (likely negligible) reordering of custom-type import lines from alphabetical to declaration order. | -| `python.gen/DESIGN.md` | Note the ordering change where section 12/13 currently describe the alphabetical-by-`order` scheme. | - ---- - -### Task 1: Simplify `ImportSet.dhall` - -**Files:** -- Modify: `src/Structures/ImportSet.dhall` - -**Interfaces:** -- `CustomImport` loses `dedupKey : Natural` → becomes `{ className : Text, moduleName : Text }`. -- `custom : CustomImport -> Self` — unchanged signature, now the only constructor (no more `customEnum`/`customComposite`). -- `combine : Self -> Self -> Self` — `customTypes` field is now a plain list concatenation. -- `sortedCustoms` is deleted. Callers read `.customTypes` directly. - -- [ ] **Step 1: Replace the file** - -```dhall -let Prelude = ../Deps/Prelude.dhall - --- A custom-type import line: "from ..types. import ". --- Emitted in encounter order (the order the referencing columns/params were --- declared), not sorted. Dhall (upstream) has no Text comparison, so there is --- no way to alphabetize or dedupe by moduleName/className without either the --- pgn fork's Text/equal or a project-wide Natural id (previously `order`, --- sourced from Project.dhall's buildLookup — see --- docs/plans/2026-07-11-encounter-order-custom-imports.md for why that's --- gone and why dedup isn't reintroduced some other way). Two references to --- the same type currently produce two identical lines; harmless to Python --- and to basedpyright, just not deduped. -let CustomImport = { className : Text, moduleName : Text } - -let Self = - { uuid : Bool - , datetime : Bool - , date : Bool - , time : Bool - , timedelta : Bool - , decimal : Bool - , jsonb : Bool - , json : Bool - , jsonValue : Bool - , enumArray : Bool - , customTypes : List CustomImport - } - -let base = - { uuid = False - , datetime = False - , date = False - , time = False - , timedelta = False - , decimal = False - , jsonb = False - , json = False - , jsonValue = False - , enumArray = False - , customTypes = [] : List CustomImport - } - -let empty - : Self - = base - -let uuid - : Self - = base // { uuid = True } - -let datetime - : Self - = base // { datetime = True } - -let date - : Self - = base // { date = True } - -let time - : Self - = base // { time = True } - -let timedelta - : Self - = base // { timedelta = True } - -let decimal - : Self - = base // { decimal = True } - -let jsonb - : Self - = base // { jsonb = True } - -let json - : Self - = base // { json = True } - -let jsonValue - : Self - = base // { jsonValue = True } - -let enumArray - : Self - = base // { enumArray = True } - -let custom - : CustomImport -> Self - = \(c : CustomImport) -> base // { customTypes = [ c ] } - -let combine = - \(left : Self) -> - \(right : Self) -> - { uuid = left.uuid || right.uuid - , datetime = left.datetime || right.datetime - , date = left.date || right.date - , time = left.time || right.time - , timedelta = left.timedelta || right.timedelta - , decimal = left.decimal || right.decimal - , jsonb = left.jsonb || right.jsonb - , json = left.json || right.json - , jsonValue = left.jsonValue || right.jsonValue - , enumArray = left.enumArray || right.enumArray - , customTypes = left.customTypes # right.customTypes - } - -let combineAll - : List Self -> Self - = \(sets : List Self) -> List/fold Self sets Self combine empty - -in { Type = Self - , CustomImport - , empty - , uuid - , datetime - , date - , time - , timedelta - , decimal - , jsonb - , json - , jsonValue - , enumArray - , custom - , combine - , combineAll - } -``` - -- [ ] **Step 2: Type-check** - -Run: `dhall type --file=src/Structures/ImportSet.dhall` -Expected: prints the record-of-functions signature, no error. - -- [ ] **Step 3: Commit** - -```bash -git add src/Structures/ImportSet.dhall -git commit -m "python.gen: drop order-based sort/dedup from ImportSet, use encounter order" -``` - ---- - -### Task 2: Update the two render call sites - -**Files:** -- Modify: `src/Templates/RowsModule.dhall`, `src/Templates/StatementModule.dhall` - -- [ ] **Step 1: `RowsModule.dhall:58`** - -Change: -```dhall - (ImportSet.sortedCustoms imports) -``` -to: -```dhall - imports.customTypes -``` - -- [ ] **Step 2: `StatementModule.dhall:82`** - -Same change: -```dhall - imports.customTypes -``` - -- [ ] **Step 3: Type-check both** - -Run: `dhall type --file=src/Templates/RowsModule.dhall && dhall type --file=src/Templates/StatementModule.dhall` -Expected: both print their signatures, no error. - -- [ ] **Step 4: Commit** - -```bash -git add src/Templates/RowsModule.dhall src/Templates/StatementModule.dhall -git commit -m "python.gen: render custom-type imports in encounter order" -``` - ---- - -### Task 3: Regenerate golden fixtures and verify the dedup gap directly - -**Files:** -- Modify: `tests/fixture-project/` (temporary, to exercise the dedup gap — see Step 1) -- Regenerate: `tests/golden/` - -- [ ] **Step 1: Confirm today's corpus has no same-type-twice case** - -Run: -```bash -grep -rl "^from \.\.types\." tests/golden/src 2>/dev/null | while read f; do - n=$(grep -c "^from \.\.types\." "$f") - [ "$n" -gt 1 ] && echo "$f: $n" -done -``` -Expected: no output (re-confirms the design-time check above, against the *current* golden tree before this plan's regeneration). - -- [ ] **Step 2: Regenerate golden** - -Run: `mise run golden` -Expected: succeeds. `git diff tests/golden` shows custom-type import lines reordered from alphabetical to declaration order in files with 2+ distinct custom-type imports (e.g. wherever `Mood` and `Point2D` are both imported today — check whether the query's own column order already happens to be alphabetical for that file; if so the diff is empty there and this is confirmed low-risk for the current corpus). - -- [ ] **Step 3: Deliberately add a same-type-twice column to the fixture project** - -Add a query (or extend an existing one) whose result row or param list references the *same* composite or enum type through two different columns/params — e.g. two `mood`-typed columns in one query. This is new fixture surface, not present today; add it under `tests/fixture-project/queries/`. - -Run: `mise run golden` -Expected: succeeds; the regenerated file for that query contains **two** identical `from ..types.mood import Mood` lines (confirm with `grep -c` on the specific file). This is the one visible, accepted consequence of this plan — capture it in the diff review, don't silently let it slip into `tests/golden` unremarked. - -- [ ] **Step 4: Decide whether to keep the same-type-twice fixture case** - -Keeping it in the committed corpus makes the tradeoff a permanent, visible regression test (future readers see the duplicate import and the comment in `ImportSet.dhall` explaining it, instead of being surprised by it later). Removing it keeps the golden diff minimal for this change. Either is fine — this plan recommends **keeping it**, since an accepted-but-invisible tradeoff tends to resurface as a bug report; make the call and note it in the commit message either way. - -- [ ] **Step 5: Run the full test suite** - -Run: `mise run test` -Expected: all pass. Duplicate import lines don't fail `basedpyright strict` (Python tolerates redundant imports) or the golden byte-comparison (it compares against the freshly-committed golden, not some independent expectation). - -- [ ] **Step 6: Commit** - -```bash -git add tests/golden tests/fixture-project -git commit -m "python.gen: regenerate golden fixtures for encounter-order imports" -``` - ---- - -### Task 4: Update `DESIGN.md` - -**Files:** -- Modify: `python.gen/DESIGN.md` - -- [ ] **Step 1: Update section 12/13 cross-references** - -Wherever DESIGN.md currently describes `order`/alphabetical import sorting (it's referenced in passing around sections 12-13 and in code comments already updated by `docs/plans/2026-07-11-reusable-custom-type-codecs.md`'s Task 8), add a short note: custom-type imports are emitted in encounter (declaration) order, not sorted, since the `order` Natural no longer exists once `buildLookup` is gone; same-type-twice references are not deduped (a Dhall limitation, not an oversight — see `src/Structures/ImportSet.dhall`'s header comment for the full reasoning). Point at the fixture case from Task 3 if kept. - -- [ ] **Step 2: Commit** - -```bash -git add python.gen/DESIGN.md -git commit -m "python.gen: document encounter-order custom-type imports in DESIGN.md" -``` - ---- - -## Self-Review - -**Spec coverage:** Task 1 is the actual mechanism change (drop the Natural key, drop sort/dedup). Task 2 fixes the two render call sites that would otherwise reference a deleted `sortedCustoms`. Task 3 regenerates and — importantly — deliberately exercises the one behavior change (duplicate imports for a same-type-twice reference) instead of letting it go unverified. Task 4 keeps DESIGN.md truthful. - -**Placeholder scan:** no TBDs; every step names an exact file, an exact diff, or an exact command with expected output. - -**Dependency on the companion plan:** called out at the top (Global Constraints) and repeated in the companion plan's own self-review — `docs/plans/2026-07-11-reusable-custom-type-codecs.md` Task 3/4 call `ImportSet.custom` with no `order` argument, which only type-checks after this plan's Task 1. Sequence: this plan's Task 1 → companion plan's Tasks 3-6 → this plan's Tasks 2-4 (Task 2 touches templates the companion plan doesn't touch, so it can land anytime after Task 1, but golden regeneration in either plan's Task 8/3 should happen once, after both are code-complete, not twice). - -## Execution Handoff - -Plan complete and saved to `python.gen/docs/plans/2026-07-11-encounter-order-custom-imports.md`. Two execution options: - -**1. Subagent-Driven (recommended)** - dispatch a fresh subagent per task, review between tasks, fast iteration - -**2. Inline Execution** - execute tasks in this session using executing-plans, batch execution with checkpoints - -Which approach? diff --git a/docs/plans/2026-07-11-reusable-custom-type-codecs.md b/docs/plans/2026-07-11-reusable-custom-type-codecs.md deleted file mode 100644 index b2f880c..0000000 --- a/docs/plans/2026-07-11-reusable-custom-type-codecs.md +++ /dev/null @@ -1,633 +0,0 @@ -# Reusable Custom-Type Codecs Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Delete `buildLookup`/`CustomKind.Lookup` — and with it python.gen's last dependency on pgn's forked `Text/equal` builtin (DESIGN.md section 12) — by generating one `_decode`/`_encode` codec per custom type in `types/.py`, called by name from every reference site, instead of re-deriving decode/encode logic (field types, Composite-vs-Enum classification) at each call site. - -**Architecture:** `CustomType.dhall` already visits every custom type once with its full definition in hand (composite fields or enum variants) and emits `types/.py`. Today it stops at the dataclass/enum class. This plan makes it also emit a `_decode` staticmethod (and, for composites, an `_encode` instance method) on that same class. Every column/param reference site (`Member.dhall`, `ParamsMember.dhall`) currently has to search `project.customTypes` by name to find out whether it's looking at a composite or an enum, because it re-derives the decode/encode expression inline. Once decode/encode is a named method reachable from the `Name` the reference site already carries (`name.inPascalCase`), the reference site just calls it — no search, no classification, no `Text/equal`. - -**Tech Stack:** Dhall (dhall-lang 1.42 vendored via `dhall` CLI; pgn's forked interpreter for the parts still using it), Python 3.12 generated output (psycopg3), basedpyright strict, pytest golden-file harness (`mise run test`, `mise run golden`). - -## Global Constraints - -- No behavior change to non-custom-type (primitive) decode/encode paths. -- Golden fixture output (`tests/golden/`) must be regenerated and diffed, not hand-edited (per `tests/golden/README.md`). -- `mise run test` (pytest, includes `test_generated_passes_basedpyright_strict`) must pass after regeneration. -- Every Dhall file touched must independently type-check: `dhall type --file=`. -- Don't touch `Primitive.dhall`, `Scalar.dhall`, `Value.dhall`, or the `Model`/`Contract` dependency — this plan is scoped to what python.gen can do unilaterally, with the current, unmodified `Scalar = < Primitive : Primitive | Custom : Name >` contract. - ---- - -## Design decision needed before Task 3: array-of-custom-type behavior - -Today, `Member.dhall`'s Custom branch treats arrays differently by kind, because it already knows the kind from `lookup name`: -- **Enum arrays (1-D):** supported — `enumArrayDecode` emits an element-wise list comprehension. -- **Composite arrays:** rejected at generation time — `"Array of a composite type is not supported (element-wise decode is unimplemented)"` (Member.dhall:219) and, for params, `"Array of a composite type as a parameter is not supported"` (ParamsMember.dhall:359). - -Once decode/encode become named methods called unconditionally (no classification at the call site), there's no Dhall-level way to keep rejecting *only* composite arrays without reintroducing some form of lookup. Two ways forward: - -- **A — Recommended: uniform array codec, kind-specific availability.** Every custom type's template *may* define `_decode_array` (a staticmethod that turns the raw array value into a `list[...]`); `EnumModule.dhall` defines it, `CompositeModule.dhall` does not. `Member.dhall` always emits `f"{typeName}._decode_array({src})"` for a 1-D custom-type array, regardless of kind. If that's ever generated against a composite, `basedpyright strict` (already gating `mise run test`, see `tests/golden/README.md`) fails on the missing attribute — a real check, just moved from `pgn generate` time to CI time. Composite-array *params* work the same way: encode always calls `f"{fieldName}._encode()"`; if `fieldName`'s inferred type is `list[Point2D]`, basedpyright flags the missing `._encode` on `list`. This is a genuine, if later, safety net — not a silent hole. The `"Absent"` case (a `customRef` name matching no generated module) degrades the same way: the emitted `from ..types. import ` fails as an unresolved import, also caught by basedpyright strict. -- **B — Preserve today's exact rejection.** Keep a minimal kind signal alive somewhere reachable without a name search — no such source was found during design (see the exploration notes below); the only ones available (project-wide `customTypes` list, or a `Scalar.Custom` contract change) reintroduce either the search or the upstream ask this plan is explicitly trying to avoid. Pick this only if lifting the composite-array restriction is unacceptable without a live Postgres verification first. - -This plan is written for **Option A**. Composite-array support was never tested (DESIGN.md and the fixture corpus have no composite-array column — verified via `grep -rn "^from \.\.types\." tests/golden/`, no file references the same custom type twice, and none of the fixture's composite columns are arrays), so Task 8 includes adding one to the fixture project specifically to exercise this path before it ships. If that Postgres test reveals composite-array decode genuinely doesn't work end-to-end (not just an assumption), fall back to Option B and keep the `"not supported"` report, sourced from a small dedicated check rather than full `buildLookup` (open a follow-up plan; don't block this one on it). - ---- - -## File Structure - -| File | Change | -|---|---| -| `src/Templates/CompositeModule.dhall` | Add `_decode` (staticmethod) and `_encode` (instance method) to the generated dataclass. | -| `src/Templates/EnumModule.dhall` | Add `_decode` and `_decode_array` (staticmethods) and `_encode` (instance method, identity) to the generated `StrEnum`. | -| `src/Interpreters/Member.dhall` | Custom branch calls `{typeName}._decode(...)` / `{typeName}._decode_array(...)` unconditionally; drop the `lookup` parameter and the `Enum`/`Composite`/`Absent` merge. | -| `src/Interpreters/ParamsMember.dhall` | Custom branch calls `{fieldName}._encode()` unconditionally; drop `lookup` and the merge. | -| `src/Interpreters/ResultColumns.dhall`, `src/Interpreters/Result.dhall`, `src/Interpreters/Query.dhall` | Drop the `lookup : CustomKind.Lookup` parameter they only thread through. | -| `src/Interpreters/CustomType.dhall` | Drop `nestedLookup` (nothing left to pass it to). | -| `src/Interpreters/Project.dhall` | Delete `buildLookup`, `IndexedCustomType`, `compositeFields`, `memberPyType`, `lookupConfig` — all exist only to feed `buildLookup`. | -| `src/Structures/CustomKind.dhall` | Delete the file; nothing imports it after the above. | -| `python.gen/DESIGN.md` | Rewrite section 12 (no longer an accepted risk — resolved) and section 13 (unchanged content, but cross-reference updates). | -| `python.gen/docs/upstream-asks.md` | Remove ask 3 (resolved without the upstream change) or mark it withdrawn. | -| `tests/golden/` | Regenerate via `mise run golden`. | - ---- - -### Task 1: Add `_decode`/`_encode` to `CompositeModule.dhall` - -**Files:** -- Modify: `src/Templates/CompositeModule.dhall` - -**Interfaces:** -- Consumes: same `Params = { typeName : Text, extraImports : List Text, fields : List Field }` as today, `Field = { fieldName : Text, fieldType : Text }` — no signature change. -- Produces: the rendered module text now defines `_decode(src: object) -> ""` (staticmethod) and `_encode(self) -> tuple[...]` (instance method) on the dataclass, callable by any generator downstream as `TypeName._decode(x)` / `value._encode()`. - -This was prototyped and verified against `dhall type` and `dhall text` (rendered output checked with `python3 -c "compile(...)"` for both a two-field and a one-field composite — the one-field case needs the trailing-comma tuple, same edge case `compositeBind` already handles today). - -- [ ] **Step 1: Replace the template body** - -```dhall -let Prelude = ../Deps/Prelude.dhall - -let Sdk = ../Deps/Sdk.dhall - -let Field = { fieldName : Text, fieldType : Text } - --- extraImports are the extra import lines a field type needs (e.g. --- "from uuid import UUID"). They sit between the dataclass import and the class, --- separated by one blank line; when empty only the dataclass import is emitted. -let Params = { typeName : Text, extraImports : List Text, fields : List Field } - -let run = - \(params : Params) -> - let fieldLines = - Prelude.Text.concatMapSep - "\n" - Field - ( \(field : Field) -> - " ${field.fieldName}: ${field.fieldType}" - ) - params.fields - - let fieldCount = Prelude.List.length Field params.fields - - let fieldTypesJoined = - Prelude.Text.concatMapSep - ", " - Field - (\(field : Field) -> field.fieldType) - params.fields - - let selfFieldsJoined = - Prelude.Text.concatMapSep - ", " - Field - (\(field : Field) -> "self.${field.fieldName}") - params.fields - - -- Python only treats trailing-comma parens as a 1-tuple; concatMapSep - -- never emits an internal comma for a single-element list, so force one - -- here. Mirrors ParamsMember.dhall's existing compositeBind trick. - let encodeTupleExpr = - if Prelude.Natural.equal fieldCount 1 - then "(${selfFieldsJoined},)" - else "(${selfFieldsJoined})" - - -- _decode/_encode are emitted as literal lines (not a nested multi-line - -- ''...'' block) because Dhall dedents a multi-line literal against its - -- OWN source indentation before splicing it into the outer literal; a - -- nested block loses its intended 4/8-space class-body indentation. - -- Verified against `dhall text` during design. - let codecMethods = - "\n" - ++ " @staticmethod\n" - ++ " def _decode(src: object) -> \"${params.typeName}\":\n" - ++ " return ${params.typeName}(*cast(tuple[${fieldTypesJoined}], src))\n" - ++ "\n" - ++ " def _encode(self) -> tuple[${fieldTypesJoined}]:\n" - ++ " return ${encodeTupleExpr}" - - let imports = - if Prelude.List.null Text params.extraImports - then "from dataclasses import dataclass\nfrom typing import cast" - else '' - from dataclasses import dataclass - from typing import cast - - ${Prelude.Text.concatSep "\n" params.extraImports}'' - - in '' - ${imports} - - - @dataclass(frozen=True, slots=True) - class ${params.typeName}: - """Decoding/encoding this composite requires register_types(conn) first. - - Without per-connection registration psycopg returns the value as a - raw string, which the generated _decode cannot splat into the dataclass. - """ - - ${fieldLines} - ${codecMethods} - '' - -in Sdk.Sigs.template Params run /\ { Field } -``` - -- [ ] **Step 2: Type-check** - -Run: `dhall type --file=src/Templates/CompositeModule.dhall` -Expected: prints the `{ Field : Type, Params : Type, Run : Type, run : ... }` signature, no error. - -- [ ] **Step 3: Spot-render and validate as Python** - -```bash -cat > /tmp/render_composite.dhall <<'EOF' -let CompositeModule = ./src/Templates/CompositeModule.dhall -in CompositeModule.run - { typeName = "Point2D" - , extraImports = [] : List Text - , fields = - [ { fieldName = "x", fieldType = "int" } - , { fieldName = "y", fieldType = "int" } - ] - } -EOF -dhall text --file=/tmp/render_composite.dhall | python3 -c "import sys; compile(sys.stdin.read(), 'point2d.py', 'exec')" && echo OK -``` -Expected: `OK`, and eyeballing the output shows `_decode`/`_encode` indented as class members (4 spaces), not module-level. - -- [ ] **Step 4: Commit** - -```bash -git add src/Templates/CompositeModule.dhall -git commit -m "python.gen: emit _decode/_encode on generated composite dataclasses" -``` - ---- - -### Task 2: Add `_decode`/`_decode_array`/`_encode` to `EnumModule.dhall` - -**Files:** -- Modify: `src/Templates/EnumModule.dhall` - -**Interfaces:** -- Consumes: same `Params = { typeName : Text, variants : List Variant }` — no signature change. -- Produces: `_decode(src: object) -> ""`, `_decode_array(src: object) -> list[""]`, `_encode(self) -> ""` (identity — psycopg binds the `StrEnum` instance directly, matching today's `defaultBind`). - -The scalar/array decode bodies are copied verbatim from `Member.dhall`'s current `enumDecode`/`enumArrayDecode` (Member.dhall:61-82), just moved from "Dhall builds inline Python text at every call site" to "Dhall builds it once, into the class." - -- [ ] **Step 1: Replace the template body** - -```dhall -let Prelude = ../Deps/Prelude.dhall - -let Sdk = ../Deps/Sdk.dhall - -let Variant = { memberName : Text, pgValue : Text } - -let Params = { typeName : Text, variants : List Variant } - -let run = - \(params : Params) -> - -- pgValue is interpolated into a single-line double-quoted Python literal; a - -- label may legally contain a backslash, quote, or control character, so - -- escape them to keep the literal valid and value-equal to the DB label. - -- Order is load-bearing: backslash first (so the escapes added below are not - -- re-escaped), then the control chars, then the closing quote. - let escapeLabel - : Text -> Text - = \(raw : Text) -> - Prelude.Function.composeList - Text - [ Prelude.Text.replace "\\" "\\\\" - , Prelude.Text.replace "\r" "\\r" - , Prelude.Text.replace "\n" "\\n" - , Prelude.Text.replace "\t" "\\t" - , Prelude.Text.replace "\"" "\\\"" - ] - raw - - let memberLines = - Prelude.Text.concatMapSep - "\n" - Variant - ( \(variant : Variant) -> - " ${variant.memberName} = \"${escapeLabel variant.pgValue}\"" - ) - params.variants - - let codecMethods = - "\n" - ++ " @staticmethod\n" - ++ " def _decode(src: object) -> \"${params.typeName}\":\n" - ++ " return ${params.typeName}(cast(str, src))\n" - ++ "\n" - ++ " @staticmethod\n" - ++ " def _decode_array(src: object) -> list[\"${params.typeName}\"]:\n" - ++ " return [\n" - ++ " ${params.typeName}(v)\n" - ++ " for v in cast(list[str], require_array(src))\n" - ++ " ]\n" - ++ "\n" - ++ " def _encode(self) -> \"${params.typeName}\":\n" - ++ " return self" - - in '' - from enum import StrEnum - from typing import cast - - from .._runtime import require_array - - - class ${params.typeName}(StrEnum): - ${memberLines} - ${codecMethods} - '' - -in Sdk.Sigs.template Params run /\ { Variant } -``` - -> `require_array` must already be importable from `.._runtime` relative to `types/.py` — confirm the relative import depth matches `types/`'s actual nesting (one level under the package root per `CustomType.dhall`'s `modulePath = "types/${moduleName}.py"`) before running Step 2; adjust to `.._runtime` vs `..._runtime` to match what `Member.dhall`'s current `enumArrayDecode` assumes at its own call sites (check `ImportSet.dhall`'s handling of `require_array` imports today for the exact existing relative path convention, since this moves an existing import from call sites into the shared module). - -- [ ] **Step 2: Type-check** - -Run: `dhall type --file=src/Templates/EnumModule.dhall` -Expected: signature prints, no error. - -- [ ] **Step 3: Spot-render and validate as Python** - -```bash -cat > /tmp/render_enum.dhall <<'EOF' -let EnumModule = ./src/Templates/EnumModule.dhall -in EnumModule.run - { typeName = "Mood" - , variants = - [ { memberName = "HAPPY", pgValue = "happy" } - , { memberName = "SAD", pgValue = "sad" } - ] - } -EOF -dhall text --file=/tmp/render_enum.dhall | python3 -c "import sys; compile(sys.stdin.read(), 'mood.py', 'exec')" && echo OK -``` -Expected: `OK`. - -- [ ] **Step 4: Commit** - -```bash -git add src/Templates/EnumModule.dhall -git commit -m "python.gen: emit _decode/_decode_array/_encode on generated enum classes" -``` - ---- - -### Task 3: Simplify `Member.dhall`'s Custom branch - -**Files:** -- Modify: `src/Interpreters/Member.dhall` - -**Interfaces:** -- Consumes: `value.scalar.customRef : Optional Model.Name` (unchanged — still comes straight off `Scalar.run`, see `src/Interpreters/Scalar.dhall:44-52`), `value.dims : Natural` (unchanged). -- Produces: `Run = Config -> Input -> Lude.Compiled.Type Output` — **drops the `CustomKind.Lookup` parameter**. Every caller (`ResultColumns.dhall`, `CustomType.dhall`) updates in Task 5/6. - -- [ ] **Step 1: Replace the Custom branch (Member.dhall:129-232) and the `Run` alias (Member.dhall:245)** - -Delete the `merge { Enum = ...; Composite = ...; Absent = ... } (lookup name)` block and the `CustomKind.CompositeField`-typed `compositeDecode` helper (lines 100-116, now dead — the same logic lives in `CompositeModule.dhall`'s `_decode` now). Replace with: - -```dhall - , Custom = - Prelude.Optional.fold - Model.Name - value.scalar.customRef - (Lude.Compiled.Type Output) - ( \(name : Model.Name) -> - let typeName = name.inPascalCase - - let customImport = - { className = typeName, moduleName = name.inSnakeCase } - - let mkOutput = - \(customImports : ImportSet.Type) -> - \(decodeExpr : Text -> Text) -> - { fieldName - , pgName = input.pgName - , pyType - , isNullable = input.isNullable - , imports = - ImportSet.combine baseImports customImports - , decodeExpr - } - - let wrapNullable = - \(call : Text -> Text) -> - \(src : Text) -> - if input.isNullable - then "None if ${src} is None else ${call src}" - else call src - - let dimsIsOne = - Natural/isZero (Natural/subtract 1 value.dims) - - in if Natural/isZero value.dims - then Lude.Compiled.ok - Output - ( mkOutput - (ImportSet.custom customImport) - ( wrapNullable - (\(src : Text) -> "${typeName}._decode(${src})") - ) - ) - else if dimsIsOne - then Lude.Compiled.ok - Output - ( mkOutput - (ImportSet.custom customImport) - ( wrapNullable - ( \(src : Text) -> - "${typeName}._decode_array(${src})" - ) - ) - ) - else Lude.Compiled.report - Output - [ input.pgName, name.inSnakeCase ] - "Array of dimensionality > 1 is not supported" - ) - ( Lude.Compiled.report - Output - [ input.pgName ] - "Custom scalar without a customRef name" - ) -``` - -Note this drops the `Enum`/`Composite` import-set split (`ImportSet.customEnum`/`ImportSet.customComposite`) in favor of one `ImportSet.custom` call — see Doc 2 (`2026-07-11-encounter-order-custom-imports.md`) for why `ImportSet.customEnum`/`customComposite` collapse into a single `ImportSet.custom` once `order` no longer exists. **Land Doc 2 in the same branch as this task, or `dhall type` will fail here** — Task 3 depends on `ImportSet.CustomImport` no longer requiring a `dedupKey`/`order` field. - -- [ ] **Step 2: Drop the `lookup` parameter from `Run`** - -Change: -```dhall -let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output -``` -to: -```dhall -let Run = Config -> Input -> Lude.Compiled.Type Output -``` -and the `run` definition's `\(lookup : CustomKind.Lookup) ->` (Member.dhall:40) — delete that line entirely, since `run` no longer takes it. - -- [ ] **Step 3: Delete the now-dead `CustomKind` import (Member.dhall:9) and the dead `compositeDecode` helper (Member.dhall:100-116)** - -- [ ] **Step 4: Type-check** - -Run: `dhall type --file=src/Interpreters/Member.dhall` -Expected: fails until Task 5 updates `ResultColumns.dhall` (Member's only caller) to stop passing `lookup` — that's fine, type-check `Member.dhall` standalone by temporarily checking `Sdk.Sigs.interpreter Config Input Output run` in isolation, or proceed straight to Task 5 and type-check the pair together. Don't commit Task 3 alone; commit Tasks 3+5+6 together (they're one type-checking unit — Dhall won't let you land a signature change without updating every call site in the same change). - ---- - -### Task 4: Simplify `ParamsMember.dhall`'s Custom branch - -**Files:** -- Modify: `src/Interpreters/ParamsMember.dhall` - -**Interfaces:** -- Consumes: same as Task 3. -- Produces: `Run = Config -> Input -> Lude.Compiled.Type Output` — drops `CustomKind.Lookup`. - -- [ ] **Step 1: Replace the Custom branch (ParamsMember.dhall:311-367)** - -Delete `compositeBind` (lines 258-288, now dead — logic moved into `CompositeModule.dhall`'s `_encode`) and the `merge { Enum = ...; Composite = ...; Absent = ... } (lookup name)` block. Replace with: - -```dhall - in Prelude.Optional.fold - Model.Name - value.scalar.customRef - (Lude.Compiled.Type Output) - ( \(name : Model.Name) -> - let customImport = - { className = name.inPascalCase - , moduleName = name.inSnakeCase - } - - let encodeExpr = - if input.isNullable - then "None if ${fieldName} is None else ${fieldName}._encode()" - else "${fieldName}._encode()" - - in Lude.Compiled.ok - Output - ( mkOutput - (ImportSet.combine value.imports (ImportSet.custom customImport)) - encodeExpr - ) - ) - ( if isJsonArrayParam - then Lude.Compiled.report - Output - [ input.pgName ] - "json/jsonb array as a parameter is not supported" - else Lude.Compiled.ok - Output - (mkOutput value.imports defaultBind) - ) -``` - -This drops the `Natural/isZero value.dims` guard that used to reject composite-array params — per the Design decision above (Option A), an array-of-composite param now generates `{fieldName}._encode()` where `fieldName`'s type is `list[Point2D]`; `list` has no `._encode`, so `basedpyright strict` catches it. Confirm this in Task 8's basedpyright run specifically, not just eyeball it. - -- [ ] **Step 2: Drop the `lookup` parameter from `Run` (ParamsMember.dhall:387) and `run`'s `\(lookup : CustomKind.Lookup) ->` (ParamsMember.dhall:235)** - -- [ ] **Step 3: Delete the dead `CustomKind` import (ParamsMember.dhall:9)** - -- [ ] **Step 4: Type-check together with Task 3** - -Run: `dhall type --file=src/Interpreters/ParamsMember.dhall` -Expected: same caveat as Task 3 Step 4 — its caller (`Query.dhall`) still passes `lookup` until Task 5. - ---- - -### Task 5: Drop `lookup` threading from `ResultColumns.dhall`, `Result.dhall`, `Query.dhall` - -**Files:** -- Modify: `src/Interpreters/ResultColumns.dhall`, `src/Interpreters/Result.dhall`, `src/Interpreters/Query.dhall` - -**Interfaces:** -- Each of these only forwards `lookup` to a callee; none inspect it. Removing it is mechanical. - -- [ ] **Step 1: `ResultColumns.dhall`** — delete `\(lookup : CustomKind.Lookup) ->` (line 62) and change `Member.run config lookup member` (line 76) to `Member.run config member`. Delete the `CustomKind` import (line 9) if nothing else in the file uses it — verify with `grep -n CustomKind src/Interpreters/ResultColumns.dhall` after editing. - -- [ ] **Step 2: `Result.dhall`** — delete both `\(lookup : CustomKind.Lookup) ->` occurrences (lines 66, 93), change `ResultColumns.run config lookup rowClassName columns` (line 89) to `ResultColumns.run config rowClassName columns`, and `rowsOutput config lookup rowClassName` (line 100) to `rowsOutput config rowClassName` — check `rowsOutput`'s own definition for a `lookup` parameter to drop too (it wasn't in the earlier grep excerpt; read the file before editing to confirm). Delete the `CustomKind` import (line 9) if unused after. - -- [ ] **Step 3: `Query.dhall`** — delete `\(lookup : CustomKind.Lookup) ->` (line 140), change `ResultModule.run config lookup rowClassName input.result` (line 156) to drop `lookup`, and `ParamsMember.run config lookup member` (line 173) to `ParamsMember.run config member`. Delete the `CustomKind` import (line 7) if unused after. - -- [ ] **Step 4: Type-check each file standalone** - -Run: `dhall type --file=src/Interpreters/ResultColumns.dhall && dhall type --file=src/Interpreters/Result.dhall && dhall type --file=src/Interpreters/Query.dhall` -Expected: all three print their signatures. `Query.dhall` will still fail until `Project.dhall` (Task 6) stops passing `lookup` to `QueryGen.run` — that's the last link. - ---- - -### Task 6: Delete `buildLookup` and its support code from `Project.dhall` - -**Files:** -- Modify: `src/Interpreters/Project.dhall` - -- [ ] **Step 1: Delete dead helpers** - -Delete, in order (they only exist to feed `buildLookup`, confirmed by re-reading the file top to bottom — nothing else calls `lookupConfig`, `memberPyType`, `compositeFields`, `IndexedCustomType`, or `buildLookup` itself): -- `lookupConfig` (lines 74-80) -- `memberPyType` (lines 85-96) -- `compositeFields` (lines 98-108) -- The local `CompositeField = { fieldName : Text, pyType : Text }` (line 7) — dead once `compositeFields`/`memberPyType` are gone -- `IndexedCustomType` (line 142) -- `buildLookup` (lines 147-181) - -- [ ] **Step 2: Update the call site** - -Find `let lookup = buildLookup effectiveCustomTypes` (line 479) — delete it, and change both call sites that pass `lookup`: -- `QueryGen.run config lookup query` (line 503) → `QueryGen.run config query` -- `(\(query : Model.Query) -> QueryGen.run config lookup query)` (line 524) → `(\(query : Model.Query) -> QueryGen.run config query)` - -- [ ] **Step 3: Delete the dead `CustomKind` import (line 9)** - -- [ ] **Step 4: Type-check the whole chain** - -Run: `dhall type --file=src/Interpreters/Project.dhall` -Expected: prints the module signature with no error — this is the point where Tasks 3-6 all click together (Project → Query → Result/ParamsMember → ResultColumns/Member all now agree on the lookup-free signatures). - -- [ ] **Step 5: Commit Tasks 3-6 together** - -```bash -git add src/Interpreters/Member.dhall src/Interpreters/ParamsMember.dhall \ - src/Interpreters/ResultColumns.dhall src/Interpreters/Result.dhall \ - src/Interpreters/Query.dhall src/Interpreters/Project.dhall -git commit -m "python.gen: call custom-type codecs by name instead of resolving via buildLookup" -``` - ---- - -### Task 7: Drop `nestedLookup` from `CustomType.dhall`, delete `CustomKind.dhall` - -**Files:** -- Modify: `src/Interpreters/CustomType.dhall` -- Delete: `src/Structures/CustomKind.dhall` - -- [ ] **Step 1: `CustomType.dhall`** — delete `nestedLookup` (lines 51-53) and its comment, and change `MemberGen.run config nestedLookup m` (line 127) to `MemberGen.run config m`. Delete the `CustomKind` import (line 11). - -- [ ] **Step 2: Confirm nothing else references `CustomKind`** - -Run: `grep -rln "CustomKind" src` -Expected: no output. - -- [ ] **Step 3: Delete the file** - -```bash -git rm src/Structures/CustomKind.dhall -``` - -- [ ] **Step 4: Type-check the full package entry point** - -Run: `dhall type --file=src/package.dhall` -Expected: prints the top-level module signature, no error. (This is the first point a full-package check is meaningful — earlier steps only checked individual interpreter files.) - -- [ ] **Step 5: Commit** - -```bash -git add src/Interpreters/CustomType.dhall -git commit -m "python.gen: delete CustomKind.dhall, the last remnant of buildLookup" -``` - ---- - -### Task 8: Regenerate golden fixtures, verify, update docs - -**Files:** -- Modify: `tests/fixture-project/` (add a composite-array column per the Design decision above) -- Regenerate: `tests/golden/` -- Modify: `python.gen/DESIGN.md`, `python.gen/docs/upstream-asks.md`, `python.gen/CHANGELOG.md` - -- [ ] **Step 1: Add a composite-array test column to the fixture project** - -Find the fixture's composite-type column definitions under `tests/fixture-project/` (query/table SQL referencing a composite type, e.g. the `Point2D`-typed column used in `insert_specimen`/`get_specimen` per the golden output). Add one query or column that selects/inserts an *array* of that composite type — this is the first real exercise of the Option A fallback path (`basedpyright strict` should catch it if `_decode_array`/`_encode` aren't defined on `Point2D`, since Task 1 deliberately didn't add them to `CompositeModule.dhall`). - -Confirm the expected failure mode first: - -Run: `mise run golden` (needs `PGN_TEST_DATABASE_URL` pointing at a live Postgres — see `tests/golden/README.md`) -Expected: generation succeeds (no Dhall-level rejection — that's the point of Option A), but the regenerated file calls `Point2D._decode_array(...)` or `list[Point2D]._encode()`-shaped code that doesn't exist. - -Run: `mise run test` -Expected: `test_generated_passes_basedpyright_strict` FAILS, citing the missing attribute. This confirms the safety net from the Design decision actually fires. Once confirmed, either: - - revert the fixture addition (if you don't want composite arrays in the committed golden corpus yet), or - - implement `_decode_array`/an array-aware `_encode` on `CompositeModule.dhall` for real and keep the fixture (a follow-up, out of this plan's scope — flag it, don't scope-creep this task). - -- [ ] **Step 2: Remove the composite-array addition (unless implementing it for real per Step 1)** - -- [ ] **Step 3: Regenerate golden for real** - -Run: `mise run golden` -Expected: succeeds, rewrites `tests/golden/src/specimen_client/_generated/**` and both facades. - -- [ ] **Step 4: Review the diff** - -Run: `git diff tests/golden` -Expected: every composite/enum decode/encode call site now reads `TypeName._decode(...)`/`TypeName._decode_array(...)`/`value._encode()` instead of the old inlined `cast(tuple[...], ...)`/`(x.a, x.b)` expressions; `types/point_2_d.py`, `types/mood.py`, `types/tag_value.py` each gain the new methods. No unrelated files change. - -- [ ] **Step 5: Run the full test suite** - -Run: `mise run test` -Expected: all pass, including `test_generated_passes_basedpyright_strict`. - -- [ ] **Step 6: Update `DESIGN.md`** - -Rewrite section 12 (`## 12. Forked-Dhall (Text/equal) dependency risk`) — it's no longer an accepted risk for python.gen; state plainly that `buildLookup` is gone and `Text/equal` is no longer used anywhere in this generator's own Dhall source (grep to confirm: `grep -rn "Text/equal" src` returns nothing). Keep a short note that `demos/Exhaustive.dhall`/`mise run golden` still needs the pinned pgn binary regardless, because `gen-sdk`'s own `Fixtures` module independently uses the fork builtin — this plan doesn't touch that, and it isn't blocked on it. Cross-reference section 13 (unchanged — `PyIdent.dhall`'s trick is unrelated and still in place). - -- [ ] **Step 7: Update `docs/upstream-asks.md`** - -Remove ask 3 (`## 3. gen-sdk: kind tag or Natural index on Scalar.Custom`) or mark it explicitly withdrawn with one line explaining why (`buildLookup`'s only consumer was resolved locally by generating named codecs instead of resolving structural type info by search — see `docs/plans/2026-07-11-reusable-custom-type-codecs.md`). Don't delete the file's other two asks (pragma parsing, warnings printing) — they're unrelated and still open. - -- [ ] **Step 8: Update `CHANGELOG.md`** - -Add an entry under the appropriate section describing the behavior change from the Design decision (composite-array columns/params no longer rejected at generation time; verify at basedpyright-strict time instead) if Step 1's fixture addition was kept, or note it as a documented-but-untested path if reverted. - -- [ ] **Step 9: Commit** - -```bash -git add tests/golden tests/fixture-project python.gen/DESIGN.md python.gen/docs/upstream-asks.md python.gen/CHANGELOG.md -git commit -m "python.gen: regenerate golden fixtures for reusable custom-type codecs" -``` - ---- - -## Self-Review - -**Spec coverage:** Task 1-2 build the reusable codecs (the actual "fix the root cause"). Tasks 3-4 make the two reference sites (decode, encode) call them. Tasks 5-7 remove the now-dead plumbing (`lookup` threading, `buildLookup`, `CustomKind.dhall`) so nothing is left half-migrated. Task 8 proves it against the real toolchain and updates the two docs (`DESIGN.md`, `upstream-asks.md`) that currently assert this is unfixable — both would otherwise go stale and mislead the next reader. - -**Open question carried forward, not silently resolved:** the array-of-composite behavior change (Design decision, Option A vs B) is a real product decision, flagged explicitly rather than picked unilaterally in the diff. Task 8 Step 1 is designed to surface the actual runtime behavior (does `basedpyright strict` really catch it, does composite-array decode actually work against real Postgres) before committing to either branch. - -**Dependency on the companion plan:** Task 3 explicitly calls out that it needs `docs/plans/2026-07-11-encounter-order-custom-imports.md` (`ImportSet.dhall`'s `order`/`dedupKey` removal) landed first or alongside — `ImportSet.customEnum`/`customComposite` currently *require* an `order : Natural` that only `buildLookup` produced. Implement that plan first, or fold both into one PR; don't land this plan's Task 3 against the unmodified `ImportSet.dhall`. - -## Execution Handoff - -Plan complete and saved to `python.gen/docs/plans/2026-07-11-reusable-custom-type-codecs.md`. Two execution options: - -**1. Subagent-Driven (recommended)** - dispatch a fresh subagent per task, review between tasks, fast iteration - -**2. Inline Execution** - execute tasks in this session using executing-plans, batch execution with checkpoints - -Which approach? From bd42d46cdbf74c2fe0841c773d2989be83af97c4 Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sun, 12 Jul 2026 08:08:42 +0300 Subject: [PATCH 12/21] Add plans --- .../2026-07-12-configurable-sync-output.md | 1367 +++++++++++++++++ ...026-07-12-distribute-rows-per-statement.md | 996 ++++++++++++ 2 files changed, 2363 insertions(+) create mode 100644 docs/plans/2026-07-12-configurable-sync-output.md create mode 100644 docs/plans/2026-07-12-distribute-rows-per-statement.md diff --git a/docs/plans/2026-07-12-configurable-sync-output.md b/docs/plans/2026-07-12-configurable-sync-output.md new file mode 100644 index 0000000..5c047b4 --- /dev/null +++ b/docs/plans/2026-07-12-configurable-sync-output.md @@ -0,0 +1,1367 @@ +# Configurable Sync Output Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the additive `emitSync : Optional Bool` config knob (which always emits async and optionally bolts a second `sync/` tree on top) with an exclusive `sync : Optional Bool` knob, default `False`, that selects exactly one surface — async or sync — and emits it at the same unified file paths either way, so flipping the flag never changes the shape of the output tree or any import path, only file contents. + +**Architecture:** `Interpreters/Project.dhall` picks one `Surface.Type` value (`Surface.async` when `config.sync` is `False`, `Surface.sync` when `True`) and threads it through a single render pass instead of today's two passes (always-async + conditionally-additive-sync). `Interpreters/Query.dhall`'s per-query `Output` collapses its `asyncModulePath`/`asyncContent`/`syncModulePath`/`syncContent` two-surface duplication down to one `modulePath`/`content` pair. `Structures/Surface.dhall`'s import-depth tokens (`rowsImport`, `corePrefix`, `typesPrefix`) collapse to the same value for both surfaces, since sync statement modules no longer live one package deeper under `sync/statements/` — they live at the exact path async statements use today. `_rows.py` and `types/` are untouched by this plan (still surface-agnostic, still emitted once); splitting `_rows.py` per statement is a separate, later plan (`docs/plans/2026-07-12-distribute-rows-per-statement.md`) that depends on this one landing first. + +**Tech Stack:** Dhall (dhall-lang 1.42 vendored via `dhall` CLI; pgn's forked interpreter for the parts still using it), Python 3.12 generated output (psycopg3), basedpyright strict, pytest golden-file harness (`mise run test`, `mise run golden`), pgn 0.9.1 (mise-pinned). + +## Global Constraints + +- No behavior change to param/result type mapping, custom-type codecs, or SQL rendering — this plan is scoped to the sync/async surface-selection mechanism only. +- Default behavior when `config.sync` is omitted must be byte-identical to today's `emitSync` omitted/false default (async only, same paths) — verified by golden diff. +- Golden fixture output (`tests/golden/`, new `tests/golden_sync/`) must be regenerated via `mise run golden`, not hand-edited (per `tests/golden/README.md`). +- `mise run test` (pytest, includes `test_generated_passes_basedpyright_strict`) must pass after regeneration, for both the async and sync golden trees. +- Every Dhall file touched must independently type-check via the whole-generator check: `dhall type --file=src/package.dhall`. +- This is a breaking config change (the `emitSync` key stops existing). Per this repo's convention (see `CHANGELOG.md`'s "Upcoming" section and the "Drop the plans" precedent), document it there — no deprecation shim, no dual-key transition period. +- CI's memory-reduction script (`.github/workflows/ci.yml`, "Reduce the fixture project to the python artifact") truncates `tests/fixture-project/project1.pgn.yaml` at the `# The variants below` marker. Any new artifact that must run in CI (not just locally) has to be placed *before* that marker. + +--- + +## File Structure + +| File | Change | +|---|---| +| `src/package.dhall` | Public `Config`/`Config/default`: rename `emitSync` field to `sync`; rewrite doc comment. | +| `src/Interpreters/Project.dhall` | `Config`/`ResolvedConfig`: rename field. `run`: rename resolve binding. `combineOutputs`: replace always-async-plus-optional-sync emission with single-surface emission at unified paths. | +| `src/Interpreters/Query.dhall` | `Config`: rename field. `Output`: collapse `asyncModulePath`/`asyncContent`/`syncModulePath`/`syncContent` to `modulePath`/`content`. `render`: pick one surface, call `StatementModule.run` once instead of twice. | +| `src/Structures/Surface.dhall` | Collapse `sync`'s `rowsImport`/`corePrefix`/`typesPrefix` to the same depth as `async`'s (no more extra-dot nesting). Rewrite header comment. | +| `src/Templates/RuntimeModule.dhall` | Update the `syncContent` doc comment (no longer describes a `sync/_runtime.py` path). | +| `src/Interpreters/{Value,ResultColumns,Member,Scalar,QueryFragments,ParamsMember,CustomType,Primitive,Result}.dhall` | Mechanical rename: local `Config` type's `emitSync : Bool` field → `sync : Bool` (each is a narrowing projection of `ResolvedConfig`, never itself branches on the value except `Result.dhall`'s field-selection literal). | +| `demos/Exhaustive.dhall` | Rename `emitSync = Some True` → `sync = Some True`. | +| `tests/fixture-project/project1.pgn.yaml` | Main `python:` artifact: drop `emitSync: true` (default `sync: false` going forward). Add new `python-sync:` artifact (`sync: true`), placed before the CI truncation marker. Rename `emitSync` → `sync` in the `python-sync-only` and `python-null` variants. | +| `tests/golden_sync/` (new) | Hand-written shell (`pyproject.toml`, `README.md`, `src/specimen_sync_client/py.typed`) mirroring `tests/golden/`'s shell, for the new `python-sync` artifact's committed golden tree. | +| `tests/_harness.py` | Add `GOLDEN_DIR_SYNC` path constant. | +| `tests/conftest.py` | Add a `full_package_sync` fixture mirroring `full_package`, sourced from the `python_sync` artifact directory and `GOLDEN_DIR_SYNC`. | +| `tests/test_generated.py` | Drop `SYNC_FACADE_INIT` and the two-facade loop in `test_generated_matches_golden`. Add `test_generated_sync_matches_golden` and `test_generated_sync_passes_basedpyright_strict`. Delete `test_roundtrip_sync_and_cross_surface_identity` (cross-surface identity no longer exists once surfaces are exclusive); replace with `test_roundtrip_sync_surface` against `full_package_sync`. Move `test_roundtrip_single_field_composite_sync` onto `full_package_sync` with unified (non-`.sync.`) import paths. | +| `tests/test_config_variants.py` | Replace the `(package / "sync").exists()` presence check (no longer meaningful once paths are unified) with a content-based signal read from `_generated/_runtime.py`. Rename `emitSync` → `sync` throughout prose and assertions. | +| `mise.toml` | `golden` task: also generate and rsync the `python-sync` artifact into `tests/golden_sync/`. | +| `README.md` | Config reference table, quickstart example, "Key features," "Using the generated code" section. | +| `DESIGN.md` | Sections 1 ("What the generator emits"), 3 (runtime import example), 4 ("Surface mechanism"), 9 (facade), 10 (Config flow) — rewrite to describe exclusive single-surface selection at unified paths. | +| `CHANGELOG.md` | New "Upcoming" entry documenting the breaking config change. | + +--- + +### Task 1: Rename `emitSync` to `sync` across the Config-threading chain (no behavior change) + +Pure rename first, isolated from the behavior change in Task 2, so it is independently verifiable: after this task, generation is byte-identical to before, just keyed on a renamed config field. + +**Files:** +- Modify: `src/package.dhall` +- Modify: `src/Interpreters/Project.dhall` +- Modify: `src/Interpreters/Query.dhall` +- Modify: `src/Interpreters/Value.dhall`, `ResultColumns.dhall`, `Member.dhall`, `Scalar.dhall`, `QueryFragments.dhall`, `ParamsMember.dhall`, `CustomType.dhall`, `Primitive.dhall`, `Result.dhall` +- Modify: `demos/Exhaustive.dhall` + +**Interfaces:** +- Produces: every interpreter's `Config` type now has a `sync : Bool` field (was `emitSync : Bool`); `Project.dhall`'s public `Config`/`ResolvedConfig` and `src/package.dhall`'s `Config`/`Config/default` match. Task 2 reads `config.sync` / `resolvedConfig.sync`. + +- [ ] **Step 1: Rename in `src/package.dhall`** + +```dhall +let Sdk = ./Deps/Sdk.dhall + +let OnUnsupported = ./Structures/OnUnsupported.dhall + +let ProjectInterpreter = ./Interpreters/Project.dhall + +-- User-facing config for this generator. `sync` picks which single surface +-- the generator emits: `False` (default) emits the async surface +-- (psycopg.AsyncConnection); `True` emits the sync surface +-- (psycopg.Connection) instead, at the exact same paths — flipping this +-- flag never changes the output tree's shape or any import path, only file +-- contents. `onUnsupported` picks Fail (default, abort loudly) or Skip (drop +-- the unsupported statement/type and its dependents, with a warning) when a +-- query or custom type hits a PG shape the generator cannot render; see +-- Structures/OnUnsupported.dhall. All fields are Optional so a project may +-- omit the whole config block or any subset of its keys; +-- Interpreters/Project.dhall's `run` supplies the defaults. +let Config = + { packageName : Optional Text + , sync : Optional Bool + , onUnsupported : Optional OnUnsupported.Mode + } : Type + +let Config/default + : Config + = { packageName = None Text + , sync = None Bool + , onUnsupported = None OnUnsupported.Mode + } + +in Sdk.Sigs.generator Config Config/default ProjectInterpreter.run +``` + +- [ ] **Step 2: Rename in `src/Interpreters/Project.dhall`** + +Replace lines 35-53 (the `Config` and `ResolvedConfig` declarations): + +```dhall +-- The generator's public Config: every field is independently Optional, so a +-- project may omit the whole `config:` block or any subset of its keys. +-- `run` below resolves the fallbacks itself (packageName from the project +-- name, sync off (async), onUnsupported Fail); there is no separate config +-- type or resolve step between package.dhall and here. +let Config = + { packageName : Optional Text + , sync : Optional Bool + , onUnsupported : Optional OnUnsupported.Mode + } + +-- The fully-resolved shape every downstream interpreter (Query, CustomType, +-- and everything below them) actually declares as its own `Config`. +let ResolvedConfig = + { packageName : Text + , importName : Text + , sync : Bool + , onUnsupported : OnUnsupported.Mode + } +``` + +Replace the `emitSync` resolve binding in `run` (around line 380-386): + +```dhall + let sync = + Prelude.Optional.fold + Bool + config.sync + Bool + (\(b : Bool) -> b) + False +``` + +And its use in building `resolvedConfig` (around line 398-400): + +```dhall + let resolvedConfig + : ResolvedConfig + = { packageName, importName, sync, onUnsupported } +``` + +Leave the rest of `combineOutputs`'s body as-is for this task — Task 2 rewrites that function's structure anyway — but rename the two literal occurrences of the field name inside it so Step 7's type-check passes. Line 272 (doc comment): + +```dhall + -- config.sync. It reuses the shared `_rows.py` and `types/`, so only +``` + +Line 325 (the conditional gating `syncFiles`): + +```dhall + let syncFiles = + if config.sync + then [ syncSubpackageInit +``` + +- [ ] **Step 3: Rename in `src/Interpreters/Query.dhall`** + +Line 28 (`Config` record): `, emitSync : Bool` → `, sync : Bool`. + +Line 40 comment ("emitted only when config.emitSync") → "emitted only when config.sync" (Task 2 rewrites this comment fully; this step is the minimal rename). + +- [ ] **Step 4: Rename the narrowing `Config` field in the nine passthrough interpreters** + +Each of these files declares a local `Config` type that is a narrowing projection of `ResolvedConfig`, purely so later Dhall record-selection syntax can pick the field out by name. None of them branch on the value except `Result.dhall`'s selection literal (next step). In each file, change the line `, emitSync : Bool` to `, sync : Bool`: + +| File | Line | +|---|---| +| `src/Interpreters/Value.dhall` | 18 | +| `src/Interpreters/ResultColumns.dhall` | 20 | +| `src/Interpreters/Member.dhall` | 20 | +| `src/Interpreters/Scalar.dhall` | 16 | +| `src/Interpreters/QueryFragments.dhall` | 16 | +| `src/Interpreters/ParamsMember.dhall` | 18 | +| `src/Interpreters/CustomType.dhall` | 22 | +| `src/Interpreters/Primitive.dhall` | 14 | +| `src/Interpreters/Result.dhall` | 25 | + +- [ ] **Step 5: Rename the field-selection literal in `src/Interpreters/Result.dhall`** + +Line 94: + +```dhall + config.{ packageName, importName, sync, onUnsupported } +``` + +- [ ] **Step 6: Rename in `demos/Exhaustive.dhall`** + +Line 29: + +```dhall + { packageName = None Text + , sync = Some True + , onUnsupported = Some OnUnsupported.Mode.Skip + } +``` + +- [ ] **Step 7: Type-check the whole generator** + +Run: `cd python.gen && dhall type --file=src/package.dhall` +Expected: prints the generator's function type, no errors. (This alone proves every renamed `Config` record still lines up across every call site — Dhall's structural typing would reject a mismatched field name.) + +- [ ] **Step 8: Confirm no `emitSync` references remain in `src/`** + +Run: `cd python.gen && grep -rn "emitSync" src/ demos/` +Expected: no output. + +- [ ] **Step 9: Commit** + +```bash +git add src/package.dhall src/Interpreters/Project.dhall src/Interpreters/Query.dhall \ + src/Interpreters/Value.dhall src/Interpreters/ResultColumns.dhall src/Interpreters/Member.dhall \ + src/Interpreters/Scalar.dhall src/Interpreters/QueryFragments.dhall src/Interpreters/ParamsMember.dhall \ + src/Interpreters/CustomType.dhall src/Interpreters/Primitive.dhall src/Interpreters/Result.dhall \ + demos/Exhaustive.dhall +git commit -m "python.gen: rename emitSync config field to sync (no behavior change)" +``` + +--- + +### Task 2: Make `sync` exclusive — unify output paths, single-surface emission + +**Files:** +- Modify: `src/Structures/Surface.dhall` +- Modify: `src/Interpreters/Query.dhall` +- Modify: `src/Interpreters/Project.dhall` +- Modify: `src/Templates/RuntimeModule.dhall` (comment only) +- Modify: `src/Templates/FacadeModule.dhall` (drop dead surface-selection params) + +**Interfaces:** +- Consumes: `resolvedConfig.sync : Bool` (from Task 1). +- Produces: `Query.dhall`'s `Output` record now has `modulePath : Text` and `content : Text` (replacing the four async/sync-prefixed fields) — any code outside this task that referenced `query.asyncModulePath` etc. must be updated in this same task, since Dhall's structural typing will fail closed on the old field names. + +- [ ] **Step 1: Collapse the import-depth tokens in `src/Structures/Surface.dhall`** + +Sync statement modules no longer live one package deeper (`sync/statements/`) — they live at the exact same `statements/` path async uses. Both surfaces now reach `_rows`, `_core`, and `types/` at the same relative depth. + +```dhall +-- A code-generation surface: the async or sync flavour of a statement module. +-- Exactly one surface is emitted per generate (Interpreters/Project.dhall +-- picks async or sync from config.sync and renders everything at the same +-- unified paths), so the two Surface values differ only in the tokens that +-- vary between `async def`/`def`, `AsyncConnection`/`Connection`, and +-- `await `/``. Both reach the shared `_rows`/`_core`/`types` modules at the +-- same relative import depth, since neither surface is nested under a +-- surface-named subdirectory. +let Surface = + { defKeyword : Text + , connType : Text + , awaitKw : Text + , rowsImport : Text + , corePrefix : Text + , typesPrefix : Text + } + +let async + : Surface + = { defKeyword = "async def" + , connType = "AsyncConnection" + , awaitKw = "await " + , rowsImport = ".._rows" + , corePrefix = ".._core" + , typesPrefix = "..types" + } + +let sync + : Surface + = { defKeyword = "def" + , connType = "Connection" + , awaitKw = "" + , rowsImport = ".._rows" + , corePrefix = ".._core" + , typesPrefix = "..types" + } + +in { Type = Surface, async, sync } +``` + +- [ ] **Step 2: Collapse `Query.dhall`'s per-surface duplication** + +Replace the `Output` record (lines 42-51): + +```dhall +-- A query contributes a shared Row (assembled into `_rows.py` by Project) plus +-- one thin statement module, rendered for whichever surface config.sync picked. +let Output = + { functionName : Text + , rowClassName : Optional Text + , rowDef : Optional RowsModule.RowDef + , rowImports : ImportSet.Type + , modulePath : Text + , content : Text + } +``` + +Replace the tail of `render` (from `let mkModule = ...` through the end of the function, lines 112-136): + +```dhall + let surface = if config.sync then Surface.sync else Surface.async + + let content = + StatementModule.run + { functionName + , returnType = result.returnType + , helperName = result.helperName + , callsDecode = result.callsDecode + , sqlLiteral = fragments.sqlLiteral + , rowClassName + , decodeName + , paramSigLines + , paramDictEntries + , imports = paramImports + , surface + } + + in { functionName + , rowClassName + , rowDef + , rowImports = result.imports + , modulePath = "statements/${functionName}.py" + , content + } +``` + +- [ ] **Step 3: Rewrite `Project.dhall`'s `combineOutputs` for single-surface emission** + +Replace the whole body from the `let asyncFacade = ...` binding (line 157) through `let allFiles = ...` (line 347) with: + +```dhall + let surface = if config.sync then Surface.sync else Surface.async + + let facade = + { path = packagePrefix ++ "__init__.py" + , content = + FacadeModule.run + { generatedPrefix = "._generated" + , statementsPath = "statements" + , statements = facadeStatements + , types = facadeTypes + } + } + + -- Surface-agnostic; performs no I/O, so exactly one copy is emitted + -- regardless of surface. Both runtime bodies re-export from it. + let coreModule = + { path = srcPrefix ++ "_core.py", content = CoreModule.run {=} } + + let runtimeModule = + { path = srcPrefix ++ "_runtime.py" + , content = + if config.sync then RuntimeModule.runSync {=} else RuntimeModule.run {=} + } + + let statementsInit = + { path = srcPrefix ++ "statements/__init__.py" + , content = + InitModule.run { docstring = "Generated SQL statements." } + } + + -- The shared Row dataclasses + decode functions, imported by the + -- statement modules of whichever surface was selected. + let rowDefs = + Prelude.List.concatMap + QueryGen.Output + RowsModule.RowDef + ( \(query : QueryGen.Output) -> + Prelude.Optional.toList RowsModule.RowDef query.rowDef + ) + queries + + let rowImports = + ImportSet.combineAll + ( Prelude.List.map + QueryGen.Output + ImportSet.Type + (\(query : QueryGen.Output) -> query.rowImports) + queries + ) + + let rowsFiles = + if Prelude.List.null RowsModule.RowDef rowDefs + then [] : List Lude.File.Type + else [ { path = srcPrefix ++ "_rows.py" + , content = + RowsModule.run { rows = rowDefs, imports = rowImports } + } + ] + + let statementFiles = + Prelude.List.map + QueryGen.Output + Lude.File.Type + ( \(query : QueryGen.Output) -> + { path = srcPrefix ++ query.modulePath + , content = query.content + } + ) + queries + + let typeFiles = + Prelude.List.map + CustomTypeGen.Output + Lude.File.Type + ( \(ct : CustomTypeGen.Output) -> + { path = srcPrefix ++ ct.modulePath + , content = ct.moduleContent + } + ) + customTypes + + let typesInitExports = + Prelude.List.map + CustomTypeGen.Output + TypesInit.Export + (\(ct : CustomTypeGen.Output) -> { moduleName = ct.moduleName, typeName = ct.typeName }) + customTypes + + let typesInitFiles = + if Prelude.List.null CustomTypeGen.Output customTypes + then [] : List Lude.File.Type + else [ { path = srcPrefix ++ "types/__init__.py" + , content = TypesInit.run { exports = typesInitExports } + } + ] + + let compositeNames = compositePgNames customTypes + + let enumNames = enumPgNames customTypes + + let hasCustomRegistration = + Prelude.Bool.not + ( Prelude.Bool.and + [ Prelude.List.null Text compositeNames + , Prelude.List.null Text enumNames + ] + ) + + let registerFiles = + if hasCustomRegistration + then [ { path = srcPrefix ++ "_register.py" + , content = + RegisterModule.run { compositeNames, enumNames, surface } + } + ] + else [] : List Lude.File.Type + + let staticFiles = + [ facade, topInit, coreModule, runtimeModule, statementsInit ] + + let allFiles = + staticFiles + # registerFiles + # rowsFiles + # typesInitFiles + # typeFiles + # statementFiles + + in Prelude.List.map Lude.File.Type Lude.File.Type withHeader allFiles + : Output +``` + +Note `topInit` (the `_generated/__init__.py` facade docstring file, defined earlier in the function and unchanged) is still referenced in `staticFiles` — leave its definition (around line 128-135 in the original) exactly as-is; only the block below it changes. + +- [ ] **Step 4: Drop `FacadeModule.dhall`'s dead surface-selection parameters** + +`generatedPrefix` and `statementsPath` exist only because the old design called `FacadeModule.run` twice with different values (once for the async facade at the package root, once for the sync facade one level down at `sync/__init__.py`). Now there is exactly one facade, always at the package root, so both become constants — hardcode them and drop them from `Params`. + +Replace the header comment and `Params` declaration (lines 13-24): + +```dhall +-- A statement's public surface: the function and, when the query returns rows, +-- its frozen Row dataclass. functionName doubles as the leaf module name. +let StatementExport = + { functionName : Text, rowClassName : Optional Text } + +-- A custom type re-exported from types/: the leaf module name plus the class. +let TypeExport = { moduleName : Text, className : Text } + +-- The facade always lives at the package root, importing from `_generated` +-- (prefix `._generated`) and `_generated/statements` (statementsPath +-- `statements`) — both constant now that exactly one surface is emitted per +-- generate (Interpreters/Project.dhall picks it via config.sync). +let generatedPrefix = "._generated" + +let statementsPath = "statements" + +let Params = + { statements : List StatementExport + , types : List TypeExport + } +``` + +Replace every remaining `params.generatedPrefix` with `generatedPrefix` and every `params.statementsPath` with `statementsPath` in the rest of the file (the `runtimeBlock`, `typeBlock`, `rowBlock`, and `statementBlock` bindings inside `run`). + +Then, back in `Interpreters/Project.dhall`, simplify the `facade` binding from Step 3 above to drop the now-nonexistent fields: + +```dhall + let facade = + { path = packagePrefix ++ "__init__.py" + , content = + FacadeModule.run + { statements = facadeStatements, types = facadeTypes } + } +``` + +- [ ] **Step 5: Update the `syncContent` doc comment in `src/Templates/RuntimeModule.dhall`** + +Replace the comment at lines 84-88: + +```dhall +-- The sync surface's body, selected in place of `content` at the same +-- `_runtime.py` path when config.sync is True (Interpreters/Project.dhall). +-- The five helpers are the same shape with `def`/`Connection`/`with`/no- +-- `await`. JsonValue/NoRowError/require_array are re-exported from _core so +-- both surfaces share one canonical identity rather than two equal-but- +-- distinct definitions. +``` + +- [ ] **Step 6: Type-check the whole generator** + +Run: `cd python.gen && dhall type --file=src/package.dhall` +Expected: prints the generator's function type, no errors. + +- [ ] **Step 7: Delete the stale freeze file so the next `pgn generate` re-resolves the working tree** + +Run: `rm -f tests/fixture-project/freeze1.pgn.yaml` +Expected: file removed (pgn's freeze cache does not know the generator source changed underneath a stable relative path; a stale freeze would silently keep emitting the old two-tree output). + +- [ ] **Step 8: Commit** + +```bash +git add src/Structures/Surface.dhall src/Interpreters/Query.dhall src/Interpreters/Project.dhall \ + src/Templates/RuntimeModule.dhall src/Templates/FacadeModule.dhall +git commit -m "python.gen: make sync/async mutually exclusive at unified output paths" +``` + +(This task intentionally does not update `tests/fixture-project/project1.pgn.yaml` yet — the main `python:` artifact still has `sync: true` from before Task 1's rename, so at this point in the plan, generating against it produces sync-surface content at the paths async used to occupy, and `tests/golden/` has not been refreshed yet, so `test_generated_matches_golden` and the sync-specific round-trip tests are expected to fail until Task 3 and Task 4 land. This is fine mid-plan; the full suite is the gate at the end, not after every task.) + +--- + +### Task 3: Update the fixture project config + +**Files:** +- Modify: `tests/fixture-project/project1.pgn.yaml` + +**Interfaces:** +- Produces: a `python-sync` artifact (packageName `specimen-sync-client`) alongside the existing `python` artifact (now `sync` omitted, i.e. async by default), both ahead of the CI truncation marker. Task 4/5 consume `artifacts/python_sync` and the `specimen_sync_client` package name. + +- [ ] **Step 1: Rewrite `tests/fixture-project/project1.pgn.yaml`** + +```yaml +space: python-gen +name: fixture +version: 0.0.0 +postgres: 18 +artifacts: + python: + gen: ../../src/package.dhall + config: + packageName: specimen-client + python-sync: + gen: ../../src/package.dhall + config: + packageName: specimen-sync-client + sync: true + # The variants below pin pgn's actual YAML->Dhall decode semantics for the + # Optional Config knobs (undocumented by pgn itself); see + # tests/test_config_variants.py for the assertions. + python-name-only: + gen: ../../src/package.dhall + config: + packageName: name-only-client + python-sync-only: + gen: ../../src/package.dhall + config: + sync: true + python-empty: + gen: ../../src/package.dhall + config: {} + python-bare: + gen: ../../src/package.dhall + python-unknown-key: + gen: ../../src/package.dhall + config: + packageName: unknown-key-client + bogusField: 1 + python-null: + gen: ../../src/package.dhall + config: + packageName: null-client + sync: null +``` + +Note `python:` drops the `emitSync: true` key entirely (was: `packageName: specimen-client`, `emitSync: true`) — the primary golden fixture now represents the default (`sync` omitted → async), matching what most of `tests/test_generated.py` exercises. `python-sync` is new and sits *before* the `# The variants below` marker, so CI's truncation step (`.github/workflows/ci.yml`) keeps it alongside `python`. + +- [ ] **Step 2: Commit** + +```bash +git add tests/fixture-project/project1.pgn.yaml +git commit -m "python.gen: add python-sync fixture artifact, drop emitSync from main artifact" +``` + +--- + +### Task 4: Add the `tests/golden_sync/` shell and wire up golden regeneration for both artifacts + +**Files:** +- Create: `tests/golden_sync/pyproject.toml` +- Create: `tests/golden_sync/README.md` +- Create: `tests/golden_sync/src/specimen_sync_client/py.typed` +- Modify: `tests/_harness.py` +- Modify: `mise.toml` +- Modify: `tests/golden/pyproject.toml` (drop the now-absent `emitSync`-driven `sync/` package reference — see Step 5) + +**Interfaces:** +- Produces: `GOLDEN_DIR_SYNC` (a `Path` constant in `tests/_harness.py`), importable by `tests/conftest.py` and `tests/test_generated.py`. + +- [ ] **Step 1: Create `tests/golden_sync/pyproject.toml`** + +```toml +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "specimen-sync-client" +version = "0.0.0" +requires-python = ">=3.12" +dependencies = ["psycopg>=3.2"] + +[tool.hatch.build.targets.wheel] +packages = ["src/specimen_sync_client"] + +[tool.ruff] +exclude = ["src/specimen_sync_client/_generated"] +``` + +- [ ] **Step 2: Create `tests/golden_sync/src/specimen_sync_client/py.typed`** + +Empty file (PEP 561 marker, matching `tests/golden/src/specimen_client/py.typed`). + +```bash +mkdir -p tests/golden_sync/src/specimen_sync_client +touch tests/golden_sync/src/specimen_sync_client/py.typed +``` + +- [ ] **Step 3: Create `tests/golden_sync/README.md`** + +```markdown +# Golden files (sync surface) + +The sync-surface counterpart of `tests/golden/`: a committed full fixture +package generated with `config: { sync: true }`, package name +`specimen_sync_client`. Same structure and purpose as `tests/golden/README.md` +describes for the async (default) surface — the hand-written shell here +(`pyproject.toml`, `py.typed`) plus the generated `_generated/` subtree and +the generated package-root facade `src/specimen_sync_client/__init__.py`. + +`test_generated_sync_matches_golden` regenerates the `python-sync` fixture +artifact into a temp tree and asserts every file under the fresh +`_generated/` plus the facade equals its golden twin here, both ways (no +missing, no extra). `test_generated_sync_passes_basedpyright_strict` runs +basedpyright strict on this full golden package. + +## Updating the golden + +Run only when a generator change legitimately alters the sync surface's +output, and review the resulting diff before committing. + +```bash +mise run golden +``` + +`mise run golden` refreshes both `tests/golden/` (the `python` artifact) and +this directory (the `python-sync` artifact) in one pass — see `mise.toml`. +``` + +- [ ] **Step 4: Add `GOLDEN_DIR_SYNC` to `tests/_harness.py`** + +Line 28, immediately after `GOLDEN_DIR = HERE / "golden"`: + +```python +GOLDEN_DIR = HERE / "golden" +GOLDEN_DIR_SYNC = HERE / "golden_sync" +``` + +- [ ] **Step 5: Confirm `tests/golden/pyproject.toml` needs no change** + +`tests/golden/pyproject.toml`'s `[tool.ruff] exclude` and `packages` entries already reference only `src/specimen_client` (not a `sync/` sub-path), so this file needs no edit — the sync surface's shell lives entirely under the new `tests/golden_sync/`, a sibling directory, not nested inside the existing golden tree. (Listed in File Structure above out of caution; this step is a no-op confirmation, not an edit.) + +- [ ] **Step 6: Rewrite the `golden` task in `mise.toml`** + +Replace the `[tasks.golden]` block: + +```toml +# Regenerates the "python" and "python-sync" artifacts: a full 7-artifact +# generate peaks at ~31 GB RSS (memory goes to normalizing the generator +# closure per artifact, see ci.yml), so this keeps only the two artifacts +# golden actually needs instead of all 7. +[tasks.golden] +description = "Refresh tests/golden and tests/golden_sync from the working-tree src/" +run = ''' +#!/usr/bin/env bash +set -euo pipefail +root="$(pwd)" +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT +cp -R "$root/tests/fixture-project/." "$tmp/fixture" +rm -f "$tmp/fixture/freeze1.pgn.yaml" +rm -rf "$tmp/fixture/artifacts" +python3 - "$tmp/fixture/project1.pgn.yaml" "$root/src/package.dhall" <<'EOF' +import sys +from pathlib import Path + +p = Path(sys.argv[1]) +head, sep, _ = p.read_text().partition(" # The variants below") +assert sep, "variants marker not found in project1.pgn.yaml" +_ = p.write_text(head.rstrip().replace("../../src/package.dhall", sys.argv[2]) + "\n") +EOF +cd "$tmp/fixture" +pgn --database-url "${PGN_TEST_DATABASE_URL:-postgresql://postgres:postgres@localhost:5432/postgres?sslmode=disable}" generate +rsync -a --delete artifacts/python/src/specimen_client/_generated/ "$root/tests/golden/src/specimen_client/_generated/" +cp artifacts/python/src/specimen_client/__init__.py "$root/tests/golden/src/specimen_client/__init__.py" +rsync -a --delete artifacts/python_sync/src/specimen_sync_client/_generated/ "$root/tests/golden_sync/src/specimen_sync_client/_generated/" +cp artifacts/python_sync/src/specimen_sync_client/__init__.py "$root/tests/golden_sync/src/specimen_sync_client/__init__.py" +echo "golden refreshed; review with: git diff tests/golden tests/golden_sync" +''' +``` + +Note the `cp .../sync/__init__.py` line from the original task is gone (no sync facade sub-path exists anymore); the sync surface's facade is now at the same top-level `__init__.py` position as async's, just in the sibling `python_sync` artifact directory. + +- [ ] **Step 7: Run the golden refresh** + +Run: `cd python.gen && mise run golden` +Expected: exits 0, prints `golden refreshed; review with: git diff tests/golden tests/golden_sync`. Needs a reachable Postgres (`PGN_TEST_DATABASE_URL`, default `postgresql://postgres:postgres@localhost:5432/postgres?sslmode=disable`). + +- [ ] **Step 8: Review the diff** + +Run: `cd python.gen && git status tests/golden tests/golden_sync && git diff tests/golden` +Expected: `tests/golden/src/specimen_client/_generated/sync/` and `tests/golden/src/specimen_client/sync/__init__.py` are deleted (14 files: `sync/__init__.py`, `sync/_register.py`, `sync/_runtime.py`, `sync/statements/__init__.py`, 10 `sync/statements/*.py`, package-root `sync/__init__.py`). `tests/golden_sync/src/specimen_sync_client/_generated/` is new and structurally mirrors what `tests/golden/src/specimen_client/_generated/` looked like *before* this plan minus the `sync/` nesting, with `def`/`Connection`/no-`await` content instead of `async def`/`AsyncConnection`/`await `. No other content in either tree should have changed (params, SQL, custom types, row shapes are untouched by this plan). + +- [ ] **Step 9: Commit** + +```bash +git add tests/golden tests/golden_sync tests/_harness.py mise.toml +git commit -m "python.gen: add tests/golden_sync, regenerate both golden trees for exclusive sync" +``` + +--- + +### Task 5: Update the pytest harness for two independent golden trees + +**Files:** +- Modify: `tests/conftest.py` +- Modify: `tests/test_generated.py` + +**Interfaces:** +- Consumes: `GOLDEN_DIR_SYNC` (Task 4), `artifacts/python_sync` (Task 3), package name `specimen_sync_client`. +- Produces: `full_package_sync` fixture (mirrors `full_package`), usable by any test needing an importable sync-surface package. + +- [ ] **Step 1: Add a `generated_tree_sync`-adjacent path helper and `full_package_sync` fixture to `tests/conftest.py`** + +Add after the existing `full_package` fixture (end of file): + +```python +@pytest.fixture(scope="session") +def full_package_sync(generated_tree: Path, tmp_path_factory: pytest.TempPathFactory) -> Path: + """The sync-surface counterpart of `full_package`. + + `generated_tree` already ran `pgn generate` against the full project + (all artifacts, including `python-sync`), so this reuses that one + subprocess call rather than invoking pgn again: it just points at the + sibling `python_sync` artifact directory instead of `python`. + """ + generated_tree_sync = generated_tree.parent.parent / "artifacts" / "python_sync" + root = tmp_path_factory.mktemp("pkg-sync") + shell_src = GOLDEN_DIR_SYNC / "src" + generated_src = generated_tree_sync / "src" + _ = shutil.copytree(shell_src, root / "src", ignore=shutil.ignore_patterns("_generated", "__init__.py")) + for pkg_dir in generated_src.iterdir(): + dest_pkg = root / "src" / pkg_dir.name + _ = shutil.copytree(pkg_dir / "_generated", dest_pkg / "_generated") + _ = shutil.copy2(pkg_dir / "__init__.py", dest_pkg / "__init__.py") + return root +``` + +Add `GOLDEN_DIR_SYNC` to the existing `from tests._harness import (...)` block at the top of the file: + +```python +from tests._harness import ( + FIXTURE_PROJECT, + GOLDEN_DIR, + GOLDEN_DIR_SYNC, + HERE, + SRC_DIR, + admin_database_url, + effective_database_name, + run_pgn, +) +``` + +- [ ] **Step 2: Update `tests/test_generated.py`'s module-level constants** + +Replace lines 33-39: + +```python +# The generator emits the /_generated subtree plus the package-root +# __init__.py facade; the rest of the shell (pyproject.toml, py.typed) is +# hand-written and lives in golden as a committed fixture, not produced by +# generate. +GENERATED_SUBTREE = Path("src/specimen_client/_generated") +FACADE_INIT = Path("src/specimen_client/__init__.py") +GENERATED_SUBTREE_SYNC = Path("src/specimen_sync_client/_generated") +FACADE_INIT_SYNC = Path("src/specimen_sync_client/__init__.py") +``` + +Update the import line to include `GOLDEN_DIR_SYNC`: + +```python +from tests._harness import FIXTURE_PROJECT, GOLDEN_DIR, GOLDEN_DIR_SYNC, HERE, ensure_droppable, run_pgn +``` + +- [ ] **Step 3: Simplify `test_generated_matches_golden`, add its sync counterpart** + +Replace the `for facade in (FACADE_INIT, SYNC_FACADE_INIT):` loop (lines 116-118) with a single check: + +```python + if (generated_tree / FACADE_INIT).read_text() != (GOLDEN_DIR / FACADE_INIT).read_text(): + mismatched.append(str(FACADE_INIT)) +``` + +Add a new test after `test_generated_matches_golden`: + +```python +def test_generated_sync_matches_golden(generated_tree: Path) -> None: + """Sync-surface counterpart of test_generated_matches_golden. + + generated_tree points at the "python" artifact; the sync surface's + output lives in the sibling "python_sync" artifact directory produced by + the same pgn generate call (see the generated_tree fixture). + """ + generated_tree_sync = generated_tree.parent.parent / "artifacts" / "python_sync" + produced_root = generated_tree_sync / GENERATED_SUBTREE_SYNC + golden_root = GOLDEN_DIR_SYNC / GENERATED_SUBTREE_SYNC + + produced = _relative_files(produced_root) + golden = _relative_files(golden_root) + + missing = sorted(str(p) for p in golden - produced) + extra = sorted(str(p) for p in produced - golden) + assert not missing, f"golden files not produced by the generator: {missing}" + assert not extra, f"generator emitted files absent from golden: {extra}" + + mismatched: list[str] = [] + for rel in sorted(produced, key=str): + if (produced_root / rel).read_text() != (golden_root / rel).read_text(): + mismatched.append(str(rel)) + + if (generated_tree_sync / FACADE_INIT_SYNC).read_text() != (GOLDEN_DIR_SYNC / FACADE_INIT_SYNC).read_text(): + mismatched.append(str(FACADE_INIT_SYNC)) + + assert not mismatched, ( + "generated sync output drifted from golden in: " + + ", ".join(mismatched) + + "\nupdate via: mise run golden (see tests/golden_sync/README.md)" + ) +``` + +- [ ] **Step 4: Add `test_generated_sync_passes_basedpyright_strict`** + +Add after `test_generated_passes_basedpyright_strict`: + +```python +def test_generated_sync_passes_basedpyright_strict(tmp_path: Path) -> None: + """basedpyright strict on the full sync-surface golden package.""" + config = tmp_path / "pyrightconfig.json" + _ = config.write_text( + json.dumps( + { + "pythonVersion": "3.12", + "typeCheckingMode": "strict", + "include": [str(GOLDEN_DIR_SYNC / "src")], + "venvPath": str(HARNESS_ROOT), + "venv": ".venv", + "reportMissingModuleSource": False, + } + ) + ) + result = subprocess.run( + ["basedpyright", "--project", str(config), "--outputjson"], + capture_output=True, + text=True, + ) + + if not result.stdout.strip(): + pytest.fail(f"basedpyright produced no JSON (exit {result.returncode}):\n{result.stderr}") + + summary = json.loads(result.stdout)["summary"] + assert summary["filesAnalyzed"] > 0, f"basedpyright analyzed no files; bad include path?\n{result.stdout}" + assert summary["errorCount"] == 0 and summary["warningCount"] == 0, ( + f"basedpyright strict reported issues: {summary}\n{result.stdout}" + ) +``` + +- [ ] **Step 5: Delete `test_roundtrip_sync_and_cross_surface_identity`, replace with `test_roundtrip_sync_surface`** + +Delete the whole function (lines 392-479 in the original — cross-surface identity is not a meaningful concept once each surface is its own independent generate with its own `_rows.py`/package). Replace with: + +```python +def test_roundtrip_sync_surface(full_package_sync: Path, roundtrip_db: str) -> None: + """The sync surface, generated independently with config.sync = true. + + Drives the generated sync functions (psycopg.Connection, no await) end + to end, at the same unified paths the async surface uses (no `.sync.` + sub-path) — this package was generated standalone, not alongside async. + """ + _apply_migrations(roundtrip_db) + import_module = _import_client_sync(full_package_sync) + + register = import_module("specimen_sync_client._generated._register") + mood_mod = import_module("specimen_sync_client._generated.types.mood") + point_mod = import_module("specimen_sync_client._generated.types.point_2_d") + insert = import_module("specimen_sync_client._generated.statements.insert_specimen") + get = import_module("specimen_sync_client._generated.statements.get_specimen") + by_moods = import_module("specimen_sync_client._generated.statements.list_specimens_by_moods") + bump = import_module("specimen_sync_client._generated.statements.bump_specimen_revision") + + Mood = mood_mod.Mood + Point2D = point_mod.Point2D + + conn = psycopg.connect(roundtrip_db, autocommit=True) + try: + register.register_types(conn) + + inserted = insert.insert_specimen( + conn, + doc_jsonb={"k": "v", "n": 1}, + feeling=Mood.HAPPY, + origin=Point2D(x=1.5, y=2.5), + flag=True, + small=1, + medium=2, + large=3, + ratio=0.5, + precise=0.25, + title="alpha", + code="C-1", + letter="x", + born_on=date(2020, 1, 2), + amount=Decimal("12.34"), + blob=b"\x00\x01", + doc_json=[1, 2, 3], + maybe_text=None, + maybe_int=None, + maybe_uuid=None, + maybe_ts=None, + maybe_num=None, + tags=["a", None, "b"], + related_ids=None, + grid=None, + moods=[Mood.HAPPY, None, Mood.SAD], + ) + assert isinstance(inserted.feeling, Mood) + assert inserted.feeling is Mood.HAPPY + assert isinstance(inserted.origin, Point2D) + assert inserted.doc_jsonb == {"k": "v", "n": 1} + assert inserted.moods == [Mood.HAPPY, None, Mood.SAD] + assert inserted.moods is not None + assert inserted.moods[0] is Mood.HAPPY + + specimen_id = inserted.id + hit = get.get_specimen(conn, id=specimen_id) + assert hit is not None + assert hit.id == specimen_id + assert get.get_specimen(conn, id=specimen_id + 10_000) is None + + mood_rows = by_moods.list_specimens_by_moods(conn, moods=[Mood.HAPPY]) + assert [r.id for r in mood_rows] == [specimen_id] + assert by_moods.list_specimens_by_moods(conn, moods=[Mood.SAD]) == [] + + affected = bump.bump_specimen_revision(conn, id=specimen_id) + assert affected == 1 + bumped = get.get_specimen(conn, id=specimen_id) + assert bumped is not None + assert bumped.rev == 2 + finally: + conn.close() +``` + +Add the `_import_client_sync` helper next to `_import_client` (same shape, different module-name prefix so `sys.modules` cache invalidation targets the right package): + +```python +def _import_client_sync(full_package_sync: Path): # noqa: ANN202 - dynamic module set + src = str(full_package_sync / "src") + if src not in sys.path: + sys.path.insert(0, src) + for name in list(sys.modules): + if name == "specimen_sync_client" or name.startswith("specimen_sync_client."): + del sys.modules[name] + return importlib.import_module +``` + +- [ ] **Step 6: Move `test_roundtrip_single_field_composite_sync` onto the sync-only package** + +Replace the function body: + +```python +def test_roundtrip_single_field_composite_sync(full_package_sync: Path, roundtrip_db: str) -> None: + """Sync-surface counterpart of test_roundtrip_single_field_composite.""" + _apply_migrations(roundtrip_db) + import_module = _import_client_sync(full_package_sync) + + register = import_module("specimen_sync_client._generated._register") + tag_mod = import_module("specimen_sync_client._generated.types.tag_value") + insert = import_module("specimen_sync_client._generated.statements.insert_tagged_item") + get = import_module("specimen_sync_client._generated.statements.get_tagged_item") + + TagValue = tag_mod.TagValue + + conn = psycopg.connect(roundtrip_db, autocommit=True) + try: + register.register_types(conn) + + inserted = insert.insert_tagged_item(conn, name="widget", tag=TagValue(value="blue")) + assert isinstance(inserted.tag, TagValue) + assert inserted.tag == TagValue(value="blue") + + hit = get.get_tagged_item(conn, id=inserted.id) + assert hit is not None + assert hit.tag == TagValue(value="blue") + finally: + conn.close() +``` + +- [ ] **Step 7: Run the generated-output test module** + +Run: `cd python.gen && mise run test -- tests/test_generated.py -v` +Expected: all tests pass, including the new `test_generated_sync_matches_golden`, `test_generated_sync_passes_basedpyright_strict`, `test_roundtrip_sync_surface`, and `test_roundtrip_single_field_composite_sync`. + +- [ ] **Step 8: Commit** + +```bash +git add tests/conftest.py tests/test_generated.py +git commit -m "python.gen: split sync-surface tests onto their own golden fixture" +``` + +--- + +### Task 6: Fix the config-variant tests' sync-detection signal + +**Files:** +- Modify: `tests/test_config_variants.py` + +**Interfaces:** +- Consumes: `_generated/_runtime.py` content (Task 2 made `def fetch_many`/`async def fetch_many` the sync/async tell, since there is no more `sync/` subdirectory to check for existence). + +- [ ] **Step 1: Rewrite `tests/test_config_variants.py`** + +```python +"""Pins pgn's actual YAML->Dhall decode semantics for the Optional Config knobs. + +pgn's decode behavior for record types is undocumented, so each artifact in +project1.pgn.yaml drives a different subset of `config` keys through the same +compile.dhall and the assertions below record what pgn 0.6.5 was observed to do, +not a documented contract. These variants are pinned by the directory/package +name and which surface (sync or async) they produce, not a full golden tree. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.skipif( + os.environ.get("HARNESS_CI_REDUCED") == "1", + reason=( + "variant decode semantics are pinned locally; CI runs the reduced " + "single-artifact project because pgn generate of all 7 artifacts " + "exhausts GitHub-hosted runner memory" + ), +) + + +def _artifact_src(generated_tree: Path, artifact_key: str) -> Path: + # generated_tree is /artifacts/python; artifact directories are + # named after the artifact key with "-" replaced by "_" (pgn's own doing, + # independent of the Dhall config below). + project_root = generated_tree.parent.parent + artifact_dir = artifact_key.replace("-", "_") + return project_root / "artifacts" / artifact_dir / "src" + + +def _package_dir(generated_tree: Path, artifact_key: str) -> Path: + src = _artifact_src(generated_tree, artifact_key) + packages = [p for p in src.iterdir() if p.is_dir()] + assert len(packages) == 1, f"expected exactly one package under {src}, found {packages}" + return packages[0] + + +def _is_sync(package: Path) -> bool: + """Whether the generated package's runtime module is the sync surface. + + There is no `sync/` subdirectory to check anymore (config.sync selects + which content is rendered at the *same* `_runtime.py` path); the async + body defines `async def fetch_many`, the sync body defines `def + fetch_many` with no `async`. + """ + runtime = (package / "_generated" / "_runtime.py").read_text() + return "async def fetch_many" not in runtime + + +def test_name_only_config_derives_package_and_defaults_sync_off(generated_tree: Path) -> None: + package = _package_dir(generated_tree, "python-name-only") + assert package.name == "name_only_client" + assert not _is_sync(package) + + +def test_sync_only_config_defaults_package_name_from_project(generated_tree: Path) -> None: + package = _package_dir(generated_tree, "python-sync-only") + assert package.name == "fixture" + assert _is_sync(package) + + +def test_empty_config_object_defaults_both_fields(generated_tree: Path) -> None: + package = _package_dir(generated_tree, "python-empty") + assert package.name == "fixture" + assert not _is_sync(package) + + +def test_absent_config_key_defaults_both_fields(generated_tree: Path) -> None: + """No `config:` key at all decodes the same as `config: {}` (None Config).""" + package = _package_dir(generated_tree, "python-bare") + assert package.name == "fixture" + assert not _is_sync(package) + + +def test_unknown_config_key_is_ignored_not_rejected(generated_tree: Path) -> None: + """An extra key not in the generator's Config type (bogusField) does not fail generation.""" + package = _package_dir(generated_tree, "python-unknown-key") + assert package.name == "unknown_key_client" + assert not _is_sync(package) + + +def test_null_value_decodes_as_absent_field(generated_tree: Path) -> None: + """`sync: null` decodes to None, same as omitting the key (default False).""" + package = _package_dir(generated_tree, "python-null") + assert package.name == "null_client" + assert not _is_sync(package) +``` + +- [ ] **Step 2: Run the config-variant tests** + +Run: `cd python.gen && mise run test -- tests/test_config_variants.py -v` +Expected: all six tests pass (unless `HARNESS_CI_REDUCED=1` is set locally, in which case they skip — matches existing behavior). + +- [ ] **Step 3: Commit** + +```bash +git add tests/test_config_variants.py +git commit -m "python.gen: fix config-variant sync detection for unified output paths" +``` + +--- + +### Task 7: Update documentation + +**Files:** +- Modify: `README.md` +- Modify: `DESIGN.md` +- Modify: `CHANGELOG.md` + +- [ ] **Step 1: Update `README.md`'s config reference table** + +Replace the table at lines 76-80: + +```markdown +| key | type | default | +| --------------- | ------------------ | ------------------------------ | +| `packageName` | `Text` | the project name, kebab-cased | +| `sync` | `Bool` | `False` | +| `onUnsupported` | `"Fail" \| "Skip"` | `"Fail"` | +``` + +- [ ] **Step 2: Update the quickstart example (lines 36-42)** + +```yaml +artifacts: + python: + gen: https://raw.githubusercontent.com/slavashvets/python.gen/master/src/package.dhall + config: + packageName: my-db-client + sync: true +``` + +- [ ] **Step 3: Update "Key features" (line 21-23)** + +```markdown +- **Async or sync client over psycopg 3.** `config.sync` (default `False`) + picks exactly one surface — the async client (`psycopg.AsyncConnection`) or + the sync one (`psycopg.Connection`) — at the same output paths either way; + install `psycopg[binary]` for the fast C implementation. +``` + +- [ ] **Step 4: Update "Using the generated code" (lines 220-240)** + +Replace the closing paragraph and code block: + +```markdown +## Using the generated code + +Import from the flat facade, not from `_generated` directly: + +```python +from my_db_client import get_specimen, GetSpecimenRow, Mood +``` + +The facade re-exports every Row dataclass, every enum/composite, and every +statement function with `X as X` markers and a matching `__all__`. If the +project has composites or enums, call `register_types(conn)` once per +connection before relying on native composite decode or enum-array results; +scalar enums do not need it. + +With `sync: true`, the entire client is generated against +`psycopg.Connection` instead — same import path, same facade shape, only the +function signatures and I/O become synchronous: + +```python +from my_db_client import get_specimen, GetSpecimenRow, Mood + +def handler(conn): + row = get_specimen(conn, id=42) +``` + +A project that needs both an async backend and a sync consumer (e.g. a +Dagster pipeline) generates two artifacts pointed at this same `gen:`, one +with `sync: true` and one without, each with its own `packageName` — not one +artifact with both surfaces bundled together. +``` + +- [ ] **Step 5: Update `DESIGN.md` sections describing the surface mechanism** + +Rewrite the "What the generator emits" / "What the generator adds when `emitSync`" split (roughly lines 43-71) into a single section describing unconditional, surface-selected emission: + +```markdown +### What the generator emits + +``` +src//__init__.py # generated facade (re-exports everything) +src//_generated/__init__.py +src//_generated/_core.py # surface-agnostic: JsonValue, NoRowError, require_array +src//_generated/_runtime.py # I/O helpers for whichever surface config.sync picked +src//_generated/_register.py # only if composites or enums +src//_generated/_rows.py # shared Row dataclasses + decode functions +src//_generated/statements/__init__.py +src//_generated/statements/.py # one thin wrapper per query, def or async def +src//_generated/types/__init__.py # only if custom types exist +src//_generated/types/.py +``` + +Exactly one surface is emitted per generate, picked by `config.sync` +(`False`, the default, emits async; `True` emits sync) — the tree shape and +every import path are identical either way; only the statement modules' +`def`/`async def`, `Connection`/`AsyncConnection`, and `_runtime.py`'s body +change. A project that needs both surfaces generates two artifacts against +this same `gen:` with different `packageName`s, one with `sync: true` and +one without. +``` + +Rewrite the "Surface mechanism" section (roughly lines 192-217) to drop the "emits both, gated by config.emitSync" framing: + +```markdown +## 4. Surface mechanism (async / sync) + +```dhall +{ defKeyword : Text -- "async def" | "def" +, connType : Text -- "AsyncConnection" | "Connection" +, awaitKw : Text -- "await " | "" +, rowsImport : Text -- ".._rows" (same depth for both surfaces) +, corePrefix : Text -- ".._core" (same depth for both surfaces) +, typesPrefix : Text -- "..types" (same depth for both surfaces) +} +``` + +`Surface.async` and `Surface.sync` are the two values. `Interpreters/ +Project.dhall` picks one — `if config.sync then Surface.sync else +Surface.async` — and threads it through a single render pass: +`Interpreters/Query.dhall` calls the statement-module template once per +query (not twice), and `Interpreters/Project.dhall` picks the matching +`_runtime.py` body and, when custom types exist, the matching `_register.py` +body. `_rows.py` and `types/` are surface-agnostic and always render exactly +once, so switching `config.sync` costs only the thin I/O wrappers. +``` + +Update the facade section (roughly lines 400-409) to remove the two-facade description: + +```markdown +- every statement function from `_generated/statements/`. + +The facade lives at `/__init__.py` (prefix `._generated`, statements +under `statements`) regardless of which surface `config.sync` selected. +``` + +Update the Config-flow section's example (roughly lines 427-429, 503-509) to use `sync` in place of `emitSync`, matching Task 1/2's rename, e.g.: + +```dhall +let Config = { packageName : Optional Text, sync : Optional Bool, onUnsupported : Optional OnUnsupported.Mode } + +let Config/default = { packageName = None Text, sync = None Bool, onUnsupported = None OnUnsupported.Mode } +``` + +And the prose right below the second code block (roughly lines 500-514, "### Config flow"): replace every `emitSync` with `sync`, and `` `emitSync` to `False` `` with `` `sync` to `False` ``. + +Also update the "Generator decomposition" file-tree block (roughly lines 450-480): `Structures/Surface.dhall`'s comment changes from "async/sync token table (section 4)" to "async/sync token table, both surfaces at the same import depth (section 4)"; `Interpreters/Query.dhall`'s comment changes from "assemble one query: shared rowDef + async/sync statement modules" to "assemble one query: shared rowDef + one statement module for whichever surface config.sync picked". + +- [ ] **Step 6: Add a `CHANGELOG.md` "Upcoming" entry** + +Add to the top of the `# Upcoming` section: + +```markdown +- **Breaking:** `emitSync` is gone. In its place, `sync : Optional Bool` + (default `False`) picks exactly one surface per generate — async or sync — + emitted at the same unified paths either way (no more `sync/` subdirectory, + no more second package-root facade). Previously `emitSync: true` added a + second, nested sync tree alongside the always-emitted async one; a project + that needs both surfaces now generates two artifacts against this same + `gen:` with different `packageName`s, one with `sync: true` and one + without. See `docs/plans/2026-07-12-configurable-sync-output.md` for the + full rationale and migration shape. `tests/golden_sync/` is a new committed + golden fixture (`specimen_sync_client`) exercising the sync surface + end-to-end (basedpyright strict + round-trip), alongside the existing + `tests/golden/` (`specimen_client`, now async-only). +``` + +- [ ] **Step 7: Commit** + +```bash +git add README.md DESIGN.md CHANGELOG.md +git commit -m "python.gen: document the exclusive sync/async config change" +``` + +--- + +### Task 8: Full verification pass + +**Files:** none (verification only). + +- [ ] **Step 1: Run the full harness** + +Run: `cd python.gen && mise run test` +Expected: all tests pass (allow ~10 minutes; most of it is pgn's own type inference against Postgres). + +- [ ] **Step 2: Confirm no stray `emitSync` references anywhere in the repo** + +Run: `cd python.gen && grep -rn "emitSync" . --include="*.dhall" --include="*.md" --include="*.py" --include="*.yaml" --include="*.toml" | grep -v "^CHANGELOG.md:"` +Expected: no output (the `CHANGELOG.md` exclusion is because older, already-shipped entries legitimately still mention the old field name as history — only today's new entry from Task 7 should be free of it going forward, and that new entry doesn't mention `emitSync` at all so the grep with the exclusion is really just a safety margin, not an expected hit). + +- [ ] **Step 3: Confirm both golden trees are internally consistent with the new fixture config** + +Run: `cd python.gen && git status --short tests/golden tests/golden_sync` +Expected: clean (everything from Task 4's regeneration already committed). + +- [ ] **Step 4: Manually skim one generated statement file from each surface** + +Run: `head -30 tests/golden/src/specimen_client/_generated/statements/get_specimen.py tests/golden_sync/src/specimen_sync_client/_generated/statements/get_specimen.py` +Expected: the async file shows `from psycopg import AsyncConnection` and `async def get_specimen(...)`; the sync file shows `from psycopg import Connection` and `def get_specimen(...)` (no `async`), both importing from `.._rows`/`.._core` (not `..._rows`/`..._core`) — confirming Task 2's depth collapse took effect. diff --git a/docs/plans/2026-07-12-distribute-rows-per-statement.md b/docs/plans/2026-07-12-distribute-rows-per-statement.md new file mode 100644 index 0000000..6e909eb --- /dev/null +++ b/docs/plans/2026-07-12-distribute-rows-per-statement.md @@ -0,0 +1,996 @@ +# Distribute Rows Per Statement Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Delete the shared `_rows.py` god-module. Each query's Row dataclass and `decode_` function move verbatim into that query's own statement module (`statements/.py`), since there is already a strict 1:1 relationship between a query and its Row (no cross-statement sharing, no deduplication — confirmed empirically: 227 queries, 227 distinct Row classes, one dataclass and one decode function per query, always named after the query itself). + +**Architecture:** `Templates/RowsModule.dhall` is deleted; its `RowDef` type and `renderRow` function move into `Templates/StatementModule.dhall`, the file's sole remaining consumer. `StatementModule.dhall`'s `Params` gains `rowDef : Optional RowDef` (replacing the old `rowClassName`-only field, which existed just to build an `import ... from _rows` line that no longer exists) and renders the Row class + decode function inline, directly above the `SQL = ...` constant. `Interpreters/Query.dhall` merges its param-side and result-side `ImportSet`s into one combined set per statement file (previously kept separate: param imports fed the statement module, result/row imports fed a project-wide accumulator for `_rows.py`). `Interpreters/Project.dhall` drops the `rowDefs`/`rowImports`/`rowsFiles` machinery entirely. `Templates/FacadeModule.dhall`'s per-statement import line grows an optional Row-class alias, so the facade imports a query's Row class from that query's own statement module instead of a shared `_rows` import block. + +**Tech Stack:** Dhall (dhall-lang 1.42 vendored via `dhall` CLI; pgn's forked interpreter for the parts still using it), Python 3.12 generated output (psycopg3), basedpyright strict, pytest golden-file harness (`mise run test`, `mise run golden`), pgn 0.9.1 (mise-pinned). + +## Global Constraints + +- **Depends on `docs/plans/2026-07-12-configurable-sync-output.md` landing first.** This plan assumes exactly one statement tree per generate at unified paths (`statements/.py`, no `sync/` nesting) — it touches the same statement-module template that plan collapsed to a single surface. Do not start this plan until that one is merged and both `tests/golden/` and `tests/golden_sync/` are green. +- No change to decode LOGIC (the `cast(...)` expressions, enum/composite decode bodies, field ordering) — this plan only relocates already-correct rendering code from one file to another. `Interpreters/Member.dhall`, `Result.dhall`, `ResultColumns.dhall` are untouched. +- Row class and decode function names are unchanged (`SelectMood000Row`, `decode_select_mood_0_0_0`, etc.) — this is a pure code-motion plan, not a renaming plan (renaming to something shorter given the new file-local context, e.g. `Row`/`decode`, is an explicitly deferred, separate concern). +- Golden fixture output (`tests/golden/`, `tests/golden_sync/`) must be regenerated via `mise run golden`, not hand-edited. +- `mise run test` must pass after regeneration, for both golden trees. +- Every Dhall file touched must independently type-check via `dhall type --file=src/package.dhall`. +- This is a breaking change for any consumer importing `from ._generated._rows import X` directly. The README already tells consumers not to do that ("Import from the flat facade, not from `_generated` directly"), so this is "you were told" breakage, not a silent one — still worth a CHANGELOG line since `_rows.py` is a real, previously-stable module path someone could have depended on anyway. + +--- + +## File Structure + +| File | Change | +|---|---| +| `src/Templates/StatementModule.dhall` | Absorb `RowDef`/`renderRow` from `RowsModule.dhall`. `Params`: replace `rowClassName : Optional Text` with `rowDef : Optional RowDef`. Drop `rowsImportLine`. Add row-driven stdlib imports (`Mapping`, `dataclass`, `cast`) and a `require_array` import line, both gated on whether the query has a row. Splice the rendered row block between the import section and the `SQL = ...` constant. | +| `src/Templates/RowsModule.dhall` | Delete. | +| `src/Interpreters/Query.dhall` | Drop the `RowsModule` import; merge param and result `ImportSet`s into one; retarget the row's type to `StatementModule.RowDef`; drop `rowDef`/`rowImports` from `Output` (both become `render`-internal only). | +| `src/Interpreters/Project.dhall` | Drop the `RowsModule` import and the `rowDefs`/`rowImports`/`rowsFiles` bindings; drop `# rowsFiles` from `allFiles`. | +| `src/Templates/FacadeModule.dhall` | Drop the separate `_rows` import block (`rowBlock`); fold each statement's Row-class alias onto its own statement import line. | +| `tests/test_unsupported_types.py` | Drop the `_rows.py` orphan-reference check and the `_rows` import-smoke-test line (row content is now private to each statement file, already covered by the existing "skipped statement file doesn't exist" checks). | +| `tests/golden/`, `tests/golden_sync/` | Regenerate via `mise run golden`; `_rows.py` disappears from both, its content redistributed into each `statements/*.py` file. | +| `README.md` | "Using the generated code" no longer needs updating (already says "import from the flat facade," unaffected by where Rows physically live) — no change needed; confirmed in Task 3. | +| `DESIGN.md` | Section "## 2. Shared type layer: `_rows.py` and `types/`" rewritten (rows are no longer shared/centralized); "Generated package layout" tree (as left by the sync-output plan) drops the `_rows.py` line; "Generator decomposition" file tree drops the `RowsModule.dhall` line and updates `StatementModule.dhall`'s description; the byte-offset example referencing `tests/golden/.../_rows.py`'s `moods` column is repointed at the new location. | +| `CHANGELOG.md` | New "Upcoming" entry documenting the removal of `_rows.py`. | + +--- + +### Task 1: Move `RowDef`/`renderRow` into `StatementModule.dhall` and render the row inline + +**Files:** +- Modify: `src/Templates/StatementModule.dhall` +- Delete: `src/Templates/RowsModule.dhall` + +**Interfaces:** +- Produces: `StatementModule.RowDef = { className : Text, fieldsBlock : Text, decodeBlock : Text, decodeName : Text }` (moved verbatim from `RowsModule.RowDef`) and `StatementModule.Params.rowDef : Optional RowDef` (replacing `rowClassName : Optional Text`). Task 2 (`Query.dhall`) is the consumer. + +- [ ] **Step 1: Read `src/Templates/RowsModule.dhall`'s `RowDef` and `renderRow` one more time to confirm the exact text being moved** + +Run: `cat src/Templates/RowsModule.dhall` +Expected (for reference — this exact text is being relocated, not rewritten): + +```dhall +let RowDef = + { className : Text + , fieldsBlock : Text + , decodeBlock : Text + , decodeName : Text + } +``` + +```dhall +let renderRow + : RowDef -> Text + = \(row : RowDef) -> + "@dataclass(frozen=True, slots=True)\n" + ++ "class " + ++ row.className + ++ ":\n" + ++ indentAll 4 row.fieldsBlock + ++ "\n\n\n" + ++ "def " + ++ row.decodeName + ++ "(row: Mapping[str, object]) -> " + ++ row.className + ++ ":\n" + ++ " return " + ++ row.className + ++ "(\n" + ++ indentAll 8 row.decodeBlock + ++ "\n )" +``` + +- [ ] **Step 2: Rewrite `src/Templates/StatementModule.dhall` in full** + +```dhall +let Prelude = ../Deps/Prelude.dhall + +let Lude = ../Deps/Lude.dhall + +let Sdk = ../Deps/Sdk.dhall + +let ImportSet = ../Structures/ImportSet.dhall + +let Surface = ../Structures/Surface.dhall + +-- A query's frozen Row dataclass plus its module-level decode function, +-- rendered directly into that query's own statement module (there is a +-- strict 1:1 relationship between a query and its Row — no cross-statement +-- sharing — so co-locating them costs nothing and removes the need for a +-- shared _rows.py import). +let RowDef = + { className : Text + , fieldsBlock : Text + , decodeBlock : Text + , decodeName : Text + } + +-- Prefix every line (including the first) with `n` spaces, leaving blank lines +-- untouched so trailing whitespace never appears. +let indentAll + : Natural -> Text -> Text + = \(n : Natural) -> + \(text : Text) -> + let pad = Prelude.Text.replicate n " " + + in pad ++ Lude.Text.indentNonEmpty n text + +let renderRow + : RowDef -> Text + = \(row : RowDef) -> + "@dataclass(frozen=True, slots=True)\n" + ++ "class " + ++ row.className + ++ ":\n" + ++ indentAll 4 row.fieldsBlock + ++ "\n\n\n" + ++ "def " + ++ row.decodeName + ++ "(row: Mapping[str, object]) -> " + ++ row.className + ++ ":\n" + ++ " return " + ++ row.className + ++ "(\n" + ++ indentAll 8 row.decodeBlock + ++ "\n )" + +-- "" when the query returns no rows (Void/RowsAffected); otherwise the +-- rendered class + decode function, framed with the same "\n\n\n" (two +-- blank lines) PEP8 spacing _rows.py used between consecutive row defs, on +-- both sides — matching the two-blank-lines-before/after convention for a +-- top-level class or function. +let renderRowBlock + : Optional RowDef -> Text + = \(rowDef : Optional RowDef) -> + merge + { None = "" + , Some = \(row : RowDef) -> "\n" ++ renderRow row ++ "\n\n\n" + } + rowDef + +let hasRow + : Optional RowDef -> Bool + = \(rowDef : Optional RowDef) -> + merge { None = False, Some = \(_ : RowDef) -> True } rowDef + +-- A statement module is the thin per-surface I/O wrapper: it renders its own +-- Row dataclass and decode function (when the query returns rows) and one +-- `async def`/`def` over a connection. `imports` carries BOTH the parameter +-- type imports and the result-column imports merged into one set +-- (Interpreters/Query.dhall combines them), since both now live in this one +-- file. +let Params = + { functionName : Text + , returnType : Text + , helperName : Text + , callsDecode : Bool + , sqlLiteral : Text + , rowDef : Optional RowDef + , decodeName : Text + , paramSigLines : List Text + , paramDictEntries : List Text + , imports : ImportSet.Type + , surface : Surface.Type + } + +let importLineIf + : Bool -> Text -> List Text + = \(cond : Bool) -> \(line : Text) -> if cond then [ line ] else [] : List Text + +-- "from datetime import ..." collapses the four datetime members into one line. +let datetimeImport + : ImportSet.Type -> List Text + = \(imports : ImportSet.Type) -> + let names = + importLineIf imports.date "date" + # importLineIf imports.datetime "datetime" + # importLineIf imports.time "time" + # importLineIf imports.timedelta "timedelta" + + in if Prelude.Bool.not (Prelude.List.null Text names) + then [ "from datetime import " ++ Prelude.Text.concatSep ", " names ] + else [] : List Text + +-- The I/O helper (fetch_*/execute_*) always comes from _runtime; JsonValue and +-- require_array, when used, come from _core via the surface's corePrefix. +let runtimeImport + : Text -> Text + = \(helperName : Text) -> "from .._runtime import " ++ helperName + +let coreImport + : Text -> Bool -> List Text + = \(corePrefix : Text) -> + \(jsonValue : Bool) -> + if jsonValue + then [ "from ${corePrefix} import JsonValue" ] + else [] : List Text + +let customImportLines + : Text -> ImportSet.Type -> List Text + = \(typesPrefix : Text) -> + \(imports : ImportSet.Type) -> + Prelude.List.map + ImportSet.CustomImport + Text + ( \(c : ImportSet.CustomImport) -> + "from ${typesPrefix}.${c.moduleName} import ${c.className}" + ) + imports.customTypes + +let renderImports + : Params -> Text + = \(params : Params) -> + let imports = params.imports + + let rowIsPresent = hasRow params.rowDef + + let stdlibBlock = + importLineIf rowIsPresent "from collections.abc import Mapping" + # importLineIf rowIsPresent "from dataclasses import dataclass" + # datetimeImport imports + # importLineIf imports.decimal "from decimal import Decimal" + # importLineIf rowIsPresent "from typing import cast" + # importLineIf imports.uuid "from uuid import UUID" + + let psycopgBlock = + [ "from psycopg import ${params.surface.connType}" ] + # importLineIf + imports.json + "from psycopg.types.json import Json" + # importLineIf + imports.jsonb + "from psycopg.types.json import Jsonb" + + let localBlock = + coreImport params.surface.corePrefix imports.jsonValue + # importLineIf + imports.enumArray + "from ${params.surface.corePrefix} import require_array" + # [ runtimeImport params.helperName ] + # customImportLines params.surface.typesPrefix imports + + let groups = + [ [ "from __future__ import annotations" ] + , stdlibBlock + , psycopgBlock + , localBlock + ] + + let nonEmptyGroups = + Prelude.List.filter + (List Text) + ( \(g : List Text) -> + Prelude.Bool.not (Prelude.List.null Text g) + ) + groups + + in Prelude.Text.concatMapSep + "\n\n" + (List Text) + (\(g : List Text) -> Prelude.Text.concatSep "\n" g) + nonEmptyGroups + +let renderSignature + : Params -> Text + = \(params : Params) -> + let hasParams = + Prelude.Bool.not (Prelude.List.null Text params.paramSigLines) + + let kwMarker = if hasParams then " *,\n" else "" + + let paramBlock = + Prelude.Text.concatMap + Text + (\(line : Text) -> " " ++ line ++ ",\n") + params.paramSigLines + + in params.surface.defKeyword + ++ " " + ++ params.functionName + ++ "(\n" + ++ " conn: ${params.surface.connType}[object],\n" + ++ kwMarker + ++ paramBlock + ++ ") -> " + ++ params.returnType + ++ ":" + +-- Emit the dict multi-line with a magic trailing comma so ruff keeps it +-- expanded at any width, which keeps the generated file format-stable. +let renderParamsDict + : Params -> Text + = \(params : Params) -> + if Prelude.List.null Text params.paramDictEntries + then "params: dict[str, object] = {}" + else "params: dict[str, object] = {\n" + ++ Prelude.Text.concatMap + Text + (\(entry : Text) -> " " ++ entry ++ ",\n") + params.paramDictEntries + ++ "}" + +let renderCall + : Params -> Text + = \(params : Params) -> + let await = params.surface.awaitKw + + in if params.callsDecode + then "return ${await}${params.helperName}(conn, _SQL, params, ${params.decodeName})" + else "return ${await}${params.helperName}(conn, _SQL, params)" + +in Sdk.Sigs.template + Params + ( \(params : Params) -> + renderImports params + ++ "\n\n" + ++ renderRowBlock params.rowDef + -- The leading backslash after the opening quotes keeps the first SQL + -- line flush (no blank line); the newline before the closing quotes is + -- the only deviation from the raw text, a harmless trailing newline for + -- psycopg. + ++ "SQL = \"\"\"\\\n" + ++ params.sqlLiteral + -- Encode once at import; the helpers take bytes so each call skips a + -- per-query str->bytes allocation (psycopg auto-prepare keys on the + -- bytes value, so equal bytes still hit the prepared-statement cache). + ++ "\n\"\"\"\n\n_SQL = SQL.encode()\n\n\n" + ++ renderSignature params + ++ "\n" + ++ indentAll 4 (renderParamsDict params) + ++ "\n" + ++ indentAll 4 (renderCall params) + ++ "\n" + ) + /\ { RowDef } +``` + +Note the final `/\ { RowDef }` — `Query.dhall` (Task 2) needs to reference `StatementModule.RowDef` as a type, the same way it used to reference `RowsModule.RowDef`, so `RowDef` must be exported from the module's record the same way `RowsModule.dhall` did (`Sdk.Sigs.template Params run /\ { RowDef }`). + +- [ ] **Step 3: Delete `src/Templates/RowsModule.dhall`** + +Run: `git rm src/Templates/RowsModule.dhall` + +- [ ] **Step 4: Confirm the file is self-contained** + +Run: `cd python.gen && grep -n "RowsModule" src/Templates/StatementModule.dhall` +Expected: no output (the new file must not reference the deleted module). + +- [ ] **Step 5: Commit** + +```bash +git add src/Templates/StatementModule.dhall +git commit -m "python.gen: move RowDef/renderRow into StatementModule, render rows inline" +``` + +(This task alone does not yet type-check the whole generator — `Query.dhall` and `Project.dhall` still reference the now-deleted `RowsModule` and the old `Params.rowClassName` field. Task 2 fixes that; run `dhall type --file=src/package.dhall` at the end of Task 2, not this one.) + +--- + +### Task 2: Merge imports and retarget `Query.dhall` and `Project.dhall` + +**Files:** +- Modify: `src/Interpreters/Query.dhall` +- Modify: `src/Interpreters/Project.dhall` + +**Interfaces:** +- Consumes: `StatementModule.RowDef` (Task 1). +- Produces: `Query.dhall`'s `Output` shrinks to `{ functionName : Text, rowClassName : Optional Text, modulePath : Text, content : Text }` (drops `rowDef` and `rowImports`, both now `render`-internal only). `Project.dhall`'s `combineOutputs` no longer builds `_rows.py`. + +- [ ] **Step 1: Rewrite `src/Interpreters/Query.dhall` in full** + +```dhall +let Lude = ../Deps/Lude.dhall + +let Prelude = ../Deps/Prelude.dhall + +let Model = ../Deps/Contract.dhall + +let Sdk = ../Deps/Sdk.dhall + +let ImportSet = ../Structures/ImportSet.dhall + +let PyIdent = ../Structures/PyIdent.dhall + +let Surface = ../Structures/Surface.dhall + +let OnUnsupported = ../Structures/OnUnsupported.dhall + +let ResultModule = ./Result.dhall + +let QueryFragmentsModule = ./QueryFragments.dhall + +let ParamsMember = ./ParamsMember.dhall + +let StatementModule = ../Templates/StatementModule.dhall + +let Config = + { packageName : Text + , importName : Text + , sync : Bool + , onUnsupported : OnUnsupported.Mode + } + +let Compiled = Lude.Compiled + +let Input = Model.Query + +-- A query renders to one thin statement module: its own Row dataclass and +-- decode function (when it returns rows) plus the I/O wrapper for whichever +-- surface config.sync picked. rowClassName is still surfaced here (not just +-- internal to the rendered content) because Project.dhall's facade needs the +-- name to build the re-export line; the Row's full definition does not +-- leave this module. +let Output = + { functionName : Text + , rowClassName : Optional Text + , modulePath : Text + , content : Text + } + +let render = + \(config : Config) -> + \(input : Input) -> + \(result : ResultModule.Output) -> + \(fragments : QueryFragmentsModule.Output) -> + \(params : List ParamsMember.Output) -> + -- The function name is also the module filename and the facade import name; + -- a query named like a Python keyword would emit `def class(...)`, a module + -- `class.py`, and `from ... import class` (all SyntaxErrors), so sanitize it + -- like params and result columns. SQL/dict/row lookups key off raw names. + let functionName = PyIdent.pySafeName input.name.inSnakeCase + + let decodeName = "decode_${functionName}" + + let paramSigLines = + Prelude.List.map + ParamsMember.Output + Text + (\(p : ParamsMember.Output) -> p.fieldName ++ ": " ++ p.pyType) + params + + let paramDictEntries = + Prelude.List.map + ParamsMember.Output + Text + ( \(p : ParamsMember.Output) -> + "\"" ++ p.pgName ++ "\": " ++ p.bindExpr + ) + params + + let paramImports = + ImportSet.combineAll + ( Prelude.List.map + ParamsMember.Output + ImportSet.Type + (\(p : ParamsMember.Output) -> p.imports) + params + ) + + let rowClassName = + Prelude.Optional.map + ResultModule.RowClass + Text + (\(rc : ResultModule.RowClass) -> rc.name) + result.rowClass + + let rowDef = + Prelude.Optional.map + ResultModule.RowClass + StatementModule.RowDef + ( \(rc : ResultModule.RowClass) -> + { className = rc.name + , fieldsBlock = rc.fieldsBlock + , decodeBlock = rc.decodeBlock + , decodeName + } + ) + result.rowClass + + -- The Row's own imports (JsonValue, Decimal, custom types, ...) and + -- the parameters' imports both land in this one file now, so they + -- merge into a single ImportSet instead of flowing to two separate + -- consumers (the statement module and, formerly, _rows.py). + let mergedImports = ImportSet.combine paramImports result.imports + + let surface = if config.sync then Surface.sync else Surface.async + + let content = + StatementModule.run + { functionName + , returnType = result.returnType + , helperName = result.helperName + , callsDecode = result.callsDecode + , sqlLiteral = fragments.sqlLiteral + , rowDef + , decodeName + , paramSigLines + , paramDictEntries + , imports = mergedImports + , surface + } + + in { functionName + , rowClassName + , modulePath = "statements/${functionName}.py" + , content + } + +let run = + \(config : Config) -> + \(input : Input) -> + let rowClassName = input.name.inPascalCase ++ "Row" + + in Compiled.nest + Output + input.srcPath + ( Compiled.map3 + ResultModule.Output + QueryFragmentsModule.Output + (List ParamsMember.Output) + Output + (render config input) + ( Compiled.nest + ResultModule.Output + "result" + (ResultModule.run (config /\ { rowClassName }) input.result) + ) + ( Compiled.nest + QueryFragmentsModule.Output + "sql" + (QueryFragmentsModule.run config input.fragments) + ) + ( Compiled.nest + (List ParamsMember.Output) + "params" + ( Compiled.traverseList + Model.Member + ParamsMember.Output + ( \(member : Model.Member) -> + Compiled.nest + ParamsMember.Output + member.pgName + (ParamsMember.run config member) + ) + input.params + ) + ) + ) + +in Sdk.Sigs.interpreter Config Input Output run +``` + +- [ ] **Step 2: Drop the `RowsModule` import and the `_rows.py` machinery from `src/Interpreters/Project.dhall`** + +Delete line 25 (`let RowsModule = ../Templates/RowsModule.dhall`). + +Delete the `rowDefs`, `rowImports`, and `rowsFiles` bindings from `combineOutputs` (these sat between the `runtimeModule`/`statementsInit` bindings and `statementFiles`, per the sync-output plan's Task 2 Step 3 rewrite): + +```dhall + let rowDefs = + Prelude.List.concatMap + QueryGen.Output + RowsModule.RowDef + ( \(query : QueryGen.Output) -> + Prelude.Optional.toList RowsModule.RowDef query.rowDef + ) + queries + + let rowImports = + ImportSet.combineAll + ( Prelude.List.map + QueryGen.Output + ImportSet.Type + (\(query : QueryGen.Output) -> query.rowImports) + queries + ) + + let rowsFiles = + if Prelude.List.null RowsModule.RowDef rowDefs + then [] : List Lude.File.Type + else [ { path = srcPrefix ++ "_rows.py" + , content = + RowsModule.run { rows = rowDefs, imports = rowImports } + } + ] + +``` + +(all three bindings, deleted in full — `statementFiles` becomes the very next binding after `statementsInit`.) + +Update `allFiles` to drop the now-nonexistent `rowsFiles`: + +```dhall + let allFiles = + staticFiles + # registerFiles + # typesInitFiles + # typeFiles + # statementFiles +``` + +- [ ] **Step 3: Type-check the whole generator** + +Run: `cd python.gen && dhall type --file=src/package.dhall` +Expected: prints the generator's function type, no errors. + +- [ ] **Step 4: Confirm no `RowsModule` references remain** + +Run: `cd python.gen && grep -rn "RowsModule" src/` +Expected: no output. + +- [ ] **Step 5: Commit** + +```bash +git add src/Interpreters/Query.dhall src/Interpreters/Project.dhall +git commit -m "python.gen: merge row and param imports per statement, delete _rows.py emission" +``` + +--- + +### Task 3: Update `FacadeModule.dhall`'s Row re-export + +**Files:** +- Modify: `src/Templates/FacadeModule.dhall` +- Modify: `src/Templates/CoreModule.dhall` (comment only) + +- [ ] **Step 0: Fix the stale `_rows.py` mention in `src/Templates/CoreModule.dhall`'s header comment** + +Line 9, "and _rows.py, the statement modules, and the facades import these" → "and the statement modules and facades import these" (the full comment, lines 3-10): + +```dhall +-- The surface-agnostic core of a generated package, emitted once at +-- _generated/_core.py. It owns the names shared by every module and by both the +-- async and sync surfaces: the JsonValue alias, the NoRowError/DecodeError +-- exceptions, and the require_array decode guard. It performs no I/O, so there is +-- exactly one copy regardless of surface. The two _runtime.py modules re-export +-- JsonValue/NoRowError/require_array from here so off-contract imports keep +-- working, and the statement modules and facades import these names from +-- _core directly. +``` + +- [ ] **Step 1: Fold the Row-class alias onto each statement's own import line** + +Replace the body of `run` (everything from `let runtimeBlock = ...` through `let allNames = ...`, i.e. lines 49-112 of the pre-this-plan file): + +```dhall +let run = + \(params : Params) -> + let runtimeBlock = + "from ${generatedPrefix}._core import " + ++ Prelude.Text.concatMapSep + ", " + Text + alias + runtimeNames + + let typeBlock = + Prelude.Text.concatMapSep + "\n" + TypeExport + ( \(t : TypeExport) -> + "from ${generatedPrefix}.types.${t.moduleName} import ${alias t.className}" + ) + params.types + + let rows = rowNames params.statements + + -- A query's Row class now lives in that query's own statement + -- module (there is no shared _rows module anymore), so the row + -- alias, when present, rides on the same import line as the + -- statement function itself: "from ...statements.fn import fn as + -- fn, RowCls as RowCls" — one line per statement, not two separate + -- import blocks. + let statementBlock = + Prelude.Text.concatMapSep + "\n" + StatementExport + ( \(s : StatementExport) -> + let rowSuffix = + merge + { None = "", Some = \(row : Text) -> ", ${alias row}" } + s.rowClassName + + in "from ${generatedPrefix}.${statementsPath}.${s.functionName} import " + ++ alias s.functionName + ++ rowSuffix + ) + params.statements + + let importGroups = + [ runtimeBlock ] + # ( if Prelude.List.null TypeExport params.types + then [] : List Text + else [ typeBlock ] + ) + # ( if Prelude.List.null StatementExport params.statements + then [] : List Text + else [ statementBlock ] + ) + + let importSection = Prelude.Text.concatSep "\n\n" importGroups + + let allNames = + runtimeNames + # Prelude.List.map + TypeExport + Text + (\(t : TypeExport) -> t.className) + params.types + # rows + # Prelude.List.map + StatementExport + Text + (\(s : StatementExport) -> s.functionName) + params.statements + + let allEntries = + Prelude.Text.concatMap + Text + (\(name : Text) -> " \"${name}\",\n") + allNames + + in '' + ${importSection} + + __all__ = [ + ${allEntries}] + '' +``` + +Note `rowNames`/`rowNames params.statements` (the `rows` binding) is still needed — it still feeds `allNames` for `__all__` — only the separate `rowBlock` import section it used to also build is gone. Leave the `rowNames` function definition itself untouched. + +- [ ] **Step 2: Type-check the whole generator** + +Run: `cd python.gen && dhall type --file=src/package.dhall` +Expected: prints the generator's function type, no errors. + +- [ ] **Step 3: Commit** + +```bash +git add src/Templates/FacadeModule.dhall src/Templates/CoreModule.dhall +git commit -m "python.gen: facade imports each Row class from its own statement module" +``` + +--- + +### Task 4: Regenerate both golden trees and fix the unsupported-types test + +**Files:** +- Modify: `tests/test_unsupported_types.py` +- Regenerate: `tests/golden/`, `tests/golden_sync/` + +**Interfaces:** +- Consumes: `mise run golden` (unchanged mechanism from the sync-output plan's Task 4 — that plan already made it regenerate both trees in one pass). + +- [ ] **Step 1: Delete the stale freeze file** + +Run: `rm -f tests/fixture-project/freeze1.pgn.yaml` + +- [ ] **Step 2: Update `tests/test_unsupported_types.py`** + +Delete the `rows = (src / "_rows.py").read_text()` line and its assertion. Before: + +```python + facade = (package_src / "__init__.py").read_text() + rows = (src / "_rows.py").read_text() + register = (src / "_register.py").read_text() + types_init = (src / "types" / "__init__.py").read_text() + for orphan in ( + "probe_unsupported", + "probe_json_array", + "probe_nested_composite", + "wrapped_point", + "WrappedPoint", + ): + assert orphan not in facade, f"facade references skipped {orphan}" + assert orphan not in rows, f"_rows references skipped {orphan}" + assert orphan not in register, f"_register references skipped {orphan}" + assert orphan not in types_init, f"types/__init__ references skipped {orphan}" +``` + +After: + +```python + facade = (package_src / "__init__.py").read_text() + register = (src / "_register.py").read_text() + types_init = (src / "types" / "__init__.py").read_text() + for orphan in ( + "probe_unsupported", + "probe_json_array", + "probe_nested_composite", + "wrapped_point", + "WrappedPoint", + ): + assert orphan not in facade, f"facade references skipped {orphan}" + assert orphan not in register, f"_register references skipped {orphan}" + assert orphan not in types_init, f"types/__init__ references skipped {orphan}" +``` + +(A skipped query's Row now lives only inside that query's own statement file, which the adjacent assertion at line 171 — `assert not (src / "statements" / f"{name}.py").exists()` — already proves never got written; there is no longer a shared file where an orphaned Row reference could leak.) + +Delete the `importlib.import_module("fixture._generated._rows")` line. Before: + +```python + importlib.import_module("fixture") + importlib.import_module("fixture._generated._register") + importlib.import_module("fixture._generated._rows") + for name in kept_statements: + importlib.import_module(f"fixture._generated.statements.{name}") +``` + +After: + +```python + importlib.import_module("fixture") + importlib.import_module("fixture._generated._register") + for name in kept_statements: + importlib.import_module(f"fixture._generated.statements.{name}") +``` + +- [ ] **Step 3: Run the golden refresh** + +Run: `cd python.gen && mise run golden` +Expected: exits 0. Needs a reachable Postgres (`PGN_TEST_DATABASE_URL`). + +- [ ] **Step 4: Review the diff** + +Run: `cd python.gen && git status tests/golden tests/golden_sync && git diff --stat tests/golden tests/golden_sync` +Expected: `tests/golden/src/specimen_client/_generated/_rows.py` and `tests/golden_sync/src/specimen_sync_client/_generated/_rows.py` are deleted. Every `_generated/statements/*.py` file in both trees grows (gains its own Row dataclass + decode function, plus the new `Mapping`/`dataclass`/`typing.cast` imports where it returns rows). `src/specimen_client/__init__.py` and `src/specimen_sync_client/__init__.py` (the facades) change: each statement's import line grows a `, RowCls as RowCls` suffix, and the old separate `from ._generated._rows import (...)` block disappears. No SQL, param, or decode-expression content should differ from before this plan — only file placement and import statements. + +- [ ] **Step 5: Spot-check one regenerated statement file** + +Run: `cat tests/golden/src/specimen_client/_generated/statements/get_specimen.py` +Expected: imports (including `from collections.abc import Mapping`, `from dataclasses import dataclass`, `from typing import cast`), then `@dataclass(frozen=True, slots=True)\nclass GetSpecimenRow:`, then `def decode_get_specimen(row: Mapping[str, object]) -> GetSpecimenRow:`, then two blank lines, then `SQL = """...`, then `async def get_specimen(...)`. No `from .._rows import` line anywhere in the file. + +- [ ] **Step 6: Commit** + +```bash +git add tests/golden tests/golden_sync tests/test_unsupported_types.py +git commit -m "python.gen: regenerate golden trees with rows distributed per statement" +``` + +--- + +### Task 5: Run the full pytest harness + +**Files:** none (verification only — Tasks 1-4 already touched every file the harness exercises; this task's job is to confirm they cohere). + +- [ ] **Step 1: Run the full harness** + +Run: `cd python.gen && mise run test` +Expected: all tests pass, including `test_generated_matches_golden`, `test_generated_sync_matches_golden`, both basedpyright-strict tests, all round-trip tests, and `test_unsupported_types.py`'s skip-mode tests. + +- [ ] **Step 2: Confirm no stray `_rows`/`RowsModule` references remain in source or tests** + +Run: `cd python.gen && grep -rln "_rows\|RowsModule" src/ tests/*.py demos/ 2>/dev/null` +Expected: no output. + +- [ ] **Step 3: Commit if Steps 1-2 required any fixes** + +If the full-suite run surfaced anything Tasks 1-4 missed, fix it here and commit; otherwise this task is a clean pass-through and needs no commit of its own. + +--- + +### Task 6: Update documentation + +**Files:** +- Modify: `DESIGN.md` +- Modify: `CHANGELOG.md` + +- [ ] **Step 1: Rewrite `DESIGN.md`'s "Generated package layout" tree** + +Starting from the tree the sync-output plan (`docs/plans/2026-07-12-configurable-sync-output.md`, Task 7 Step 5) already left behind, drop the `_rows.py` line: + +```text +src//__init__.py # generated facade (re-exports everything) +src//_generated/__init__.py +src//_generated/_core.py # surface-agnostic: JsonValue, NoRowError, require_array +src//_generated/_runtime.py # I/O helpers for whichever surface config.sync picked +src//_generated/_register.py # only if composites or enums +src//_generated/statements/__init__.py +src//_generated/statements/.py # one wrapper per query: its own Row + decode + def, def or async def +src//_generated/types/__init__.py # only if custom types exist +src//_generated/types/.py +``` + +- [ ] **Step 2: Rewrite `## 2. Shared type layer: `_rows.py` and `types/`` (renumber/retitle to "## 2. Per-statement rows and the shared `types/` layer")** + +```markdown +## 2. Per-statement rows and the shared `types/` layer + +Decode is LOCAL to each statement now, not centralized. For each query that +returns rows, its own statement module holds one +`@dataclass(frozen=True, slots=True)` Row plus a module-level +`decode_(row: Mapping[str, object]) -> ` function, immediately +above the `SQL = ...` constant. There is a strict 1:1 relationship between a +query and its Row (no two queries share a Row class, even when their result +shapes are identical), so co-locating them costs nothing: + +```python +# statements/get_specimen.py +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import cast + +from psycopg import AsyncConnection + +from .._runtime import fetch_single + + +@dataclass(frozen=True, slots=True) +class GetSpecimenRow: + ... + + +def decode_get_specimen(row: Mapping[str, object]) -> GetSpecimenRow: + return GetSpecimenRow(...) + + +SQL = """...""" + +_SQL = SQL.encode() + + +async def get_specimen(conn: AsyncConnection[object], *, id: int) -> GetSpecimenRow | None: + ... + return await fetch_single(conn, _SQL, params, decode_get_specimen) +``` + +`Templates/StatementModule.dhall` renders both the Row and the wrapper in one +pass; `Interpreters/Query.dhall` merges the parameter-side and result-side +`ImportSet`s into one set per file, since both now live in the same module. +The package-root facade re-exports every Row from its own statement module +(`from ._generated.statements.get_specimen import get_specimen as +get_specimen, GetSpecimenRow as GetSpecimenRow`), not from a shared module. + +`types/.py` still holds the enum / composite class, unchanged by this +— custom types genuinely are shared across every query that references them, +unlike Rows. A statement module's custom-type imports (whether the type +appears in a param or a result column) all use the same `${typesPrefix}` +prefix (`..types`) now, since both originate from the same file. +``` + +- [ ] **Step 3: Update the "Generator decomposition" file tree** + +Drop the `RowsModule.dhall` line and update `StatementModule.dhall`'s description: + +```text + Templates/ + CoreModule.dhall # shared _core.py: JsonValue, NoRowError/DecodeError, require_array + RuntimeModule.dhall # async + sync _runtime.py bodies + StatementModule.dhall # one wrapper per query: its own Row dataclass + decode fn, plus the I/O def + RegisterModule.dhall # _register.py (per surface) + FacadeModule.dhall # the flat package-root facade + EnumModule.dhall / CompositeModule.dhall / TypesInit.dhall / InitModule.dhall +``` + +- [ ] **Step 4: Repoint the byte-offset example that referenced `_rows.py`** + +Run: `grep -n "_rows.py" DESIGN.md` and update the surrounding sentence (around what was originally line 600, describing the `moods` column) to reference `tests/golden/src/specimen_client/_generated/statements/list_specimens_by_moods.py` (or whichever statement file the example was illustrating) instead of `_rows.py`. + +- [ ] **Step 5: Add a `CHANGELOG.md` "Upcoming" entry** + +```markdown +- **Breaking:** `_rows.py` is gone. Each query's Row dataclass and + `decode_` function now live directly in that query's own statement + module (`statements/.py`), rendered immediately above the + `SQL = ...` constant, instead of in one shared, project-wide file. This + only affects code importing `from ._generated._rows import ...` + directly — the README has always said to import from the flat facade + instead, and the facade's re-exported names (`GetSpecimenRow`, + `decode_get_specimen`, etc.) are unchanged. See + `docs/plans/2026-07-12-distribute-rows-per-statement.md` for the full + rationale. +``` + +- [ ] **Step 6: Commit** + +```bash +git add DESIGN.md CHANGELOG.md +git commit -m "python.gen: document per-statement row distribution" +``` From c30e2127bf6d2b70e6c5c30c7fa1ac91af99ec5b Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sun, 12 Jul 2026 08:17:53 +0300 Subject: [PATCH 13/21] python.gen: rename emitSync config field to sync (no behavior change) --- demos/Exhaustive.dhall | 2 +- src/Interpreters/CustomType.dhall | 2 +- src/Interpreters/Member.dhall | 2 +- src/Interpreters/ParamsMember.dhall | 2 +- src/Interpreters/Primitive.dhall | 2 +- src/Interpreters/Project.dhall | 16 ++++++++-------- src/Interpreters/Query.dhall | 4 ++-- src/Interpreters/QueryFragments.dhall | 2 +- src/Interpreters/Result.dhall | 4 ++-- src/Interpreters/ResultColumns.dhall | 2 +- src/Interpreters/Scalar.dhall | 2 +- src/Interpreters/Value.dhall | 2 +- src/Templates/RuntimeModule.dhall | 2 +- src/package.dhall | 22 ++++++++++++---------- 14 files changed, 34 insertions(+), 32 deletions(-) diff --git a/demos/Exhaustive.dhall b/demos/Exhaustive.dhall index 35e8f28..584f943 100644 --- a/demos/Exhaustive.dhall +++ b/demos/Exhaustive.dhall @@ -26,7 +26,7 @@ let project = Sdk.Fixtures.Exhaustive let config = Some { packageName = None Text - , emitSync = Some True + , sync = Some True , onUnsupported = Some OnUnsupported.Mode.Skip } diff --git a/src/Interpreters/CustomType.dhall b/src/Interpreters/CustomType.dhall index abdf94f..dbde176 100644 --- a/src/Interpreters/CustomType.dhall +++ b/src/Interpreters/CustomType.dhall @@ -19,7 +19,7 @@ let CompositeModule = ../Templates/CompositeModule.dhall let Config = { packageName : Text , importName : Text - , emitSync : Bool + , sync : Bool , onUnsupported : OnUnsupported.Mode } diff --git a/src/Interpreters/Member.dhall b/src/Interpreters/Member.dhall index c4e8273..a9acd0a 100644 --- a/src/Interpreters/Member.dhall +++ b/src/Interpreters/Member.dhall @@ -17,7 +17,7 @@ let Value = ./Value.dhall let Config = { packageName : Text , importName : Text - , emitSync : Bool + , sync : Bool , onUnsupported : OnUnsupported.Mode } diff --git a/src/Interpreters/ParamsMember.dhall b/src/Interpreters/ParamsMember.dhall index 457631d..6d7b794 100644 --- a/src/Interpreters/ParamsMember.dhall +++ b/src/Interpreters/ParamsMember.dhall @@ -15,7 +15,7 @@ let Value = ./Value.dhall let Config = { packageName : Text , importName : Text - , emitSync : Bool + , sync : Bool , onUnsupported : OnUnsupported.Mode } diff --git a/src/Interpreters/Primitive.dhall b/src/Interpreters/Primitive.dhall index 052ef1b..d58e07f 100644 --- a/src/Interpreters/Primitive.dhall +++ b/src/Interpreters/Primitive.dhall @@ -11,7 +11,7 @@ let OnUnsupported = ../Structures/OnUnsupported.dhall let Config = { packageName : Text , importName : Text - , emitSync : Bool + , sync : Bool , onUnsupported : OnUnsupported.Mode } diff --git a/src/Interpreters/Project.dhall b/src/Interpreters/Project.dhall index f142a99..e2dc47b 100644 --- a/src/Interpreters/Project.dhall +++ b/src/Interpreters/Project.dhall @@ -35,11 +35,11 @@ let Report = { path : List Text, message : Text } -- The generator's public Config: every field is independently Optional, so a -- project may omit the whole `config:` block or any subset of its keys. -- `run` below resolves the fallbacks itself (packageName from the project --- name, emitSync off, onUnsupported Fail); there is no separate config type +-- name, sync off, onUnsupported Fail); there is no separate config type -- or resolve step between package.dhall and here. let Config = { packageName : Optional Text - , emitSync : Optional Bool + , sync : Optional Bool , onUnsupported : Optional OnUnsupported.Mode } @@ -48,7 +48,7 @@ let Config = let ResolvedConfig = { packageName : Text , importName : Text - , emitSync : Bool + , sync : Bool , onUnsupported : OnUnsupported.Mode } @@ -269,7 +269,7 @@ let combineOutputs = else [] : List Lude.File.Type -- The sync surface mirrors the async one under `sync/`, gated on - -- config.emitSync. It reuses the shared `_rows.py` and `types/`, so only + -- config.sync. It reuses the shared `_rows.py` and `types/`, so only -- the I/O wrappers (statements, runtime, register) and the sync facade -- are sync-specific. let syncStatementFiles = @@ -322,7 +322,7 @@ let combineOutputs = } let syncFiles = - if config.emitSync + if config.sync then [ syncSubpackageInit , syncRuntime , syncStatementsInit @@ -377,10 +377,10 @@ let run = (\(t : Text) -> t) input.name.inKebabCase - let emitSync = + let sync = Prelude.Optional.fold Bool - config.emitSync + config.sync Bool (\(b : Bool) -> b) False @@ -397,7 +397,7 @@ let run = let resolvedConfig : ResolvedConfig - = { packageName, importName, emitSync, onUnsupported } + = { packageName, importName, sync, onUnsupported } let skip = merge { Fail = False, Skip = True } resolvedConfig.onUnsupported diff --git a/src/Interpreters/Query.dhall b/src/Interpreters/Query.dhall index b3e86cd..caa7c5e 100644 --- a/src/Interpreters/Query.dhall +++ b/src/Interpreters/Query.dhall @@ -25,7 +25,7 @@ let StatementModule = ../Templates/StatementModule.dhall let Config = { packageName : Text , importName : Text - , emitSync : Bool + , sync : Bool , onUnsupported : OnUnsupported.Mode } @@ -37,7 +37,7 @@ let Input = Model.Query -- A query contributes a shared Row (assembled into `_rows.py` by Project) plus a -- thin statement module per surface. asyncModule is always emitted; syncModule is --- emitted only when config.emitSync. rowImports are the result-column imports, +-- emitted only when config.sync. rowImports are the result-column imports, -- folded into `_rows.py`. let Output = { functionName : Text diff --git a/src/Interpreters/QueryFragments.dhall b/src/Interpreters/QueryFragments.dhall index 7813900..a8a3cf0 100644 --- a/src/Interpreters/QueryFragments.dhall +++ b/src/Interpreters/QueryFragments.dhall @@ -13,7 +13,7 @@ let OnUnsupported = ../Structures/OnUnsupported.dhall let Config = { packageName : Text , importName : Text - , emitSync : Bool + , sync : Bool , onUnsupported : OnUnsupported.Mode } diff --git a/src/Interpreters/Result.dhall b/src/Interpreters/Result.dhall index b20190d..a54bd6e 100644 --- a/src/Interpreters/Result.dhall +++ b/src/Interpreters/Result.dhall @@ -22,7 +22,7 @@ let Compiled = Lude.Compiled let Config = { packageName : Text , importName : Text - , emitSync : Bool + , sync : Bool , onUnsupported : OnUnsupported.Mode , rowClassName : Text } @@ -91,7 +91,7 @@ let rowsOutput = } ) ( ResultColumns.run - config.{ packageName, importName, emitSync, onUnsupported } + config.{ packageName, importName, sync, onUnsupported } columns ) diff --git a/src/Interpreters/ResultColumns.dhall b/src/Interpreters/ResultColumns.dhall index 6a589a2..e6be05e 100644 --- a/src/Interpreters/ResultColumns.dhall +++ b/src/Interpreters/ResultColumns.dhall @@ -17,7 +17,7 @@ let Compiled = Lude.Compiled let Config = { packageName : Text , importName : Text - , emitSync : Bool + , sync : Bool , onUnsupported : OnUnsupported.Mode } diff --git a/src/Interpreters/Scalar.dhall b/src/Interpreters/Scalar.dhall index 63ec788..78939f4 100644 --- a/src/Interpreters/Scalar.dhall +++ b/src/Interpreters/Scalar.dhall @@ -13,7 +13,7 @@ let Primitive = ./Primitive.dhall let Config = { packageName : Text , importName : Text - , emitSync : Bool + , sync : Bool , onUnsupported : OnUnsupported.Mode } diff --git a/src/Interpreters/Value.dhall b/src/Interpreters/Value.dhall index e1c8d6a..b9ff26f 100644 --- a/src/Interpreters/Value.dhall +++ b/src/Interpreters/Value.dhall @@ -15,7 +15,7 @@ let Scalar = ./Scalar.dhall let Config = { packageName : Text , importName : Text - , emitSync : Bool + , sync : Bool , onUnsupported : OnUnsupported.Mode } diff --git a/src/Templates/RuntimeModule.dhall b/src/Templates/RuntimeModule.dhall index baa56bc..fa7e3f0 100644 --- a/src/Templates/RuntimeModule.dhall +++ b/src/Templates/RuntimeModule.dhall @@ -81,7 +81,7 @@ let content = _ = await cur.execute(sql, params) '' --- The sync mirror, emitted at _generated/sync/_runtime.py when emitSync. The +-- The sync mirror, emitted at _generated/sync/_runtime.py when config.sync is True. The -- five helpers are the same shape with `def`/`Connection`/`with`/no-`await`. -- JsonValue/NoRowError/require_array are re-exported from _core (two levels up) -- so both surfaces share one canonical identity rather than two equal-but- diff --git a/src/package.dhall b/src/package.dhall index 5161367..b3c282e 100644 --- a/src/package.dhall +++ b/src/package.dhall @@ -4,25 +4,27 @@ let OnUnsupported = ./Structures/OnUnsupported.dhall let ProjectInterpreter = ./Interpreters/Project.dhall --- User-facing config for this generator. `emitSync` adds a parallel sync --- surface (psycopg.Connection) alongside the default async one, so one --- project can serve both an async backend and a sync (Dagster) consumer from --- shared Row types. `onUnsupported` picks Fail (default, abort loudly) or --- Skip (drop the unsupported statement/type and its dependents, with a --- warning) when a query or custom type hits a PG shape the generator cannot --- render; see Structures/OnUnsupported.dhall. All fields are Optional so a --- project may omit the whole config block or any subset of its keys; +-- User-facing config for this generator. `sync` picks which single surface +-- the generator emits: `False` (default) emits the async surface +-- (psycopg.AsyncConnection); `True` emits the sync surface +-- (psycopg.Connection) instead, at the exact same paths — flipping this +-- flag never changes the output tree's shape or any import path, only file +-- contents. `onUnsupported` picks Fail (default, abort loudly) or Skip (drop +-- the unsupported statement/type and its dependents, with a warning) when a +-- query or custom type hits a PG shape the generator cannot render; see +-- Structures/OnUnsupported.dhall. All fields are Optional so a project may +-- omit the whole config block or any subset of its keys; -- Interpreters/Project.dhall's `run` supplies the defaults. let Config = { packageName : Optional Text - , emitSync : Optional Bool + , sync : Optional Bool , onUnsupported : Optional OnUnsupported.Mode } : Type let Config/default : Config = { packageName = None Text - , emitSync = None Bool + , sync = None Bool , onUnsupported = None OnUnsupported.Mode } From be513908512a07dd93c6e11da570ab0c4d210556 Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sun, 12 Jul 2026 08:32:42 +0300 Subject: [PATCH 14/21] python.gen: make sync/async mutually exclusive at unified output paths --- src/Interpreters/Project.dhall | 104 ++++++------------------------ src/Interpreters/Query.dhall | 49 +++++++------- src/Structures/Surface.dhall | 24 +++---- src/Templates/FacadeModule.dhall | 25 ++++--- src/Templates/RuntimeModule.dhall | 11 ++-- 5 files changed, 69 insertions(+), 144 deletions(-) diff --git a/src/Interpreters/Project.dhall b/src/Interpreters/Project.dhall index e2dc47b..cbdbcaa 100644 --- a/src/Interpreters/Project.dhall +++ b/src/Interpreters/Project.dhall @@ -154,24 +154,25 @@ let combineOutputs = ) customTypes - let asyncFacade = + let surface = if config.sync then Surface.sync else Surface.async + + let facade = { path = packagePrefix ++ "__init__.py" , content = FacadeModule.run - { generatedPrefix = "._generated" - , statementsPath = "statements" - , statements = facadeStatements - , types = facadeTypes - } + { statements = facadeStatements, types = facadeTypes } } -- Surface-agnostic; performs no I/O, so exactly one copy is emitted - -- regardless of surface. Both _runtime.py modules re-export from it. + -- regardless of surface. Both runtime bodies re-export from it. let coreModule = { path = srcPrefix ++ "_core.py", content = CoreModule.run {=} } let runtimeModule = - { path = srcPrefix ++ "_runtime.py", content = RuntimeModule.run {=} } + { path = srcPrefix ++ "_runtime.py" + , content = + if config.sync then RuntimeModule.runSync {=} else RuntimeModule.run {=} + } let statementsInit = { path = srcPrefix ++ "statements/__init__.py" @@ -179,9 +180,8 @@ let combineOutputs = InitModule.run { docstring = "Generated SQL statements." } } - -- The shared Row dataclasses + decode functions, imported by both the - -- async and sync statement modules so the two surfaces share one set of - -- types (cross-surface identity). + -- The shared Row dataclasses + decode functions, imported by the + -- statement modules of whichever surface was selected. let rowDefs = Prelude.List.concatMap QueryGen.Output @@ -209,13 +209,13 @@ let combineOutputs = } ] - let asyncStatementFiles = + let statementFiles = Prelude.List.map QueryGen.Output Lude.File.Type ( \(query : QueryGen.Output) -> - { path = srcPrefix ++ query.asyncModulePath - , content = query.asyncContent + { path = srcPrefix ++ query.modulePath + , content = query.content } ) queries @@ -262,87 +262,21 @@ let combineOutputs = if hasCustomRegistration then [ { path = srcPrefix ++ "_register.py" , content = - RegisterModule.run - { compositeNames, enumNames, surface = Surface.async } + RegisterModule.run { compositeNames, enumNames, surface } } ] else [] : List Lude.File.Type - -- The sync surface mirrors the async one under `sync/`, gated on - -- config.sync. It reuses the shared `_rows.py` and `types/`, so only - -- the I/O wrappers (statements, runtime, register) and the sync facade - -- are sync-specific. - let syncStatementFiles = - Prelude.List.map - QueryGen.Output - Lude.File.Type - ( \(query : QueryGen.Output) -> - { path = srcPrefix ++ query.syncModulePath - , content = query.syncContent - } - ) - queries - - let syncSubpackageInit = - { path = srcPrefix ++ "sync/__init__.py" - , content = - InitModule.run { docstring = "Generated sync database client." } - } - - let syncStatementsInit = - { path = srcPrefix ++ "sync/statements/__init__.py" - , content = - InitModule.run { docstring = "Generated SQL statements (sync)." } - } - - let syncRuntime = - { path = srcPrefix ++ "sync/_runtime.py" - , content = RuntimeModule.runSync {=} - } - - let syncRegisterFiles = - if hasCustomRegistration - then [ { path = srcPrefix ++ "sync/_register.py" - , content = - RegisterModule.run - { compositeNames, enumNames, surface = Surface.sync } - } - ] - else [] : List Lude.File.Type - - let syncFacade = - { path = packagePrefix ++ "sync/__init__.py" - , content = - FacadeModule.run - { generatedPrefix = ".._generated" - , statementsPath = "sync.statements" - , statements = facadeStatements - , types = facadeTypes - } - } - - let syncFiles = - if config.sync - then [ syncSubpackageInit - , syncRuntime - , syncStatementsInit - , syncFacade - ] - # syncRegisterFiles - # syncStatementFiles - else [] : List Lude.File.Type - - let asyncStaticFiles = - [ asyncFacade, topInit, coreModule, runtimeModule, statementsInit ] + let staticFiles = + [ facade, topInit, coreModule, runtimeModule, statementsInit ] let allFiles = - asyncStaticFiles + staticFiles # registerFiles # rowsFiles # typesInitFiles # typeFiles - # asyncStatementFiles - # syncFiles + # statementFiles in Prelude.List.map Lude.File.Type Lude.File.Type withHeader allFiles : Output diff --git a/src/Interpreters/Query.dhall b/src/Interpreters/Query.dhall index caa7c5e..e5e3aa7 100644 --- a/src/Interpreters/Query.dhall +++ b/src/Interpreters/Query.dhall @@ -35,19 +35,15 @@ let Model = ../Deps/Contract.dhall let Input = Model.Query --- A query contributes a shared Row (assembled into `_rows.py` by Project) plus a --- thin statement module per surface. asyncModule is always emitted; syncModule is --- emitted only when config.sync. rowImports are the result-column imports, --- folded into `_rows.py`. +-- A query contributes a shared Row (assembled into `_rows.py` by Project) plus +-- one thin statement module, rendered for whichever surface config.sync picked. let Output = { functionName : Text , rowClassName : Optional Text , rowDef : Optional RowsModule.RowDef , rowImports : ImportSet.Type - , asyncModulePath : Text - , asyncContent : Text - , syncModulePath : Text - , syncContent : Text + , modulePath : Text + , content : Text } let render = @@ -109,30 +105,29 @@ let render = ) result.rowClass - let mkModule = - \(surface : Surface.Type) -> - StatementModule.run - { functionName - , returnType = result.returnType - , helperName = result.helperName - , callsDecode = result.callsDecode - , sqlLiteral = fragments.sqlLiteral - , rowClassName - , decodeName - , paramSigLines - , paramDictEntries - , imports = paramImports - , surface - } + let surface = if config.sync then Surface.sync else Surface.async + + let content = + StatementModule.run + { functionName + , returnType = result.returnType + , helperName = result.helperName + , callsDecode = result.callsDecode + , sqlLiteral = fragments.sqlLiteral + , rowClassName + , decodeName + , paramSigLines + , paramDictEntries + , imports = paramImports + , surface + } in { functionName , rowClassName , rowDef , rowImports = result.imports - , asyncModulePath = "statements/${functionName}.py" - , asyncContent = mkModule Surface.async - , syncModulePath = "sync/statements/${functionName}.py" - , syncContent = mkModule Surface.sync + , modulePath = "statements/${functionName}.py" + , content } let run = diff --git a/src/Structures/Surface.dhall b/src/Structures/Surface.dhall index 50019bc..fc8552e 100644 --- a/src/Structures/Surface.dhall +++ b/src/Structures/Surface.dhall @@ -1,15 +1,11 @@ -- A code-generation surface: the async or sync flavour of a statement module. --- Backend (async) and ingest (sync) share one project, one generate, and the --- same Row dataclasses + enums; only the I/O wrapper differs per surface. The --- fields are the exact tokens that vary between `async def`/`def`, --- `AsyncConnection`/`Connection`, `await `/``, and the relative-import depth of --- the shared `_rows`/`types` modules (sync statement modules sit one package --- deeper under `sync/statements/`, so they reach the shared modules with an --- extra dot). The runtime import is `.._runtime` for both surfaces (async from --- `statements/`, sync from `sync/statements/`), so it needs no field here. --- corePrefix mirrors rowsImport's depth: statement modules import JsonValue from --- `_core` directly, so sync reaches it with an extra dot (`..._core`) exactly --- like it reaches `_rows` (`..._rows`). +-- Exactly one surface is emitted per generate (Interpreters/Project.dhall +-- picks async or sync from config.sync and renders everything at the same +-- unified paths), so the two Surface values differ only in the tokens that +-- vary between `async def`/`def`, `AsyncConnection`/`Connection`, and +-- `await `/``. Both reach the shared `_rows`/`_core`/`types` modules at the +-- same relative import depth, since neither surface is nested under a +-- surface-named subdirectory. let Surface = { defKeyword : Text , connType : Text @@ -34,9 +30,9 @@ let sync = { defKeyword = "def" , connType = "Connection" , awaitKw = "" - , rowsImport = "..._rows" - , corePrefix = "..._core" - , typesPrefix = "...types" + , rowsImport = ".._rows" + , corePrefix = ".._core" + , typesPrefix = "..types" } in { Type = Surface, async, sync } diff --git a/src/Templates/FacadeModule.dhall b/src/Templates/FacadeModule.dhall index 18d6f2e..b1b0d44 100644 --- a/src/Templates/FacadeModule.dhall +++ b/src/Templates/FacadeModule.dhall @@ -10,16 +10,15 @@ let StatementExport = -- A custom type re-exported from types/: the leaf module name plus the class. let TypeExport = { moduleName : Text, className : Text } --- generatedPrefix and statementsPath select the surface: the async facade lives --- at the package root (prefix `._generated`, statements under `statements`); the --- sync facade lives one level down at `sync/__init__.py` (prefix `.._generated`, --- statements under `sync.statements`). Both re-export the SAME Row dataclasses --- from `_rows` and the SAME enums/composites from `types`, so the two surfaces --- share one set of types. +-- The facade always lives at the package root, importing from `_generated` +-- (prefix `._generated`) and `_generated/statements` (statementsPath +-- `statements`) — both constant now that exactly one surface is emitted per +-- generate (Interpreters/Project.dhall picks it via config.sync). +let generatedPrefix = "._generated" +let statementsPath = "statements" + let Params = - { generatedPrefix : Text - , statementsPath : Text - , statements : List StatementExport + { statements : List StatementExport , types : List TypeExport } @@ -48,7 +47,7 @@ let runtimeNames = [ "JsonValue", "NoRowError" ] let run = \(params : Params) -> let runtimeBlock = - "from ${params.generatedPrefix}._core import " + "from ${generatedPrefix}._core import " ++ Prelude.Text.concatMapSep ", " Text @@ -60,14 +59,14 @@ let run = "\n" TypeExport ( \(t : TypeExport) -> - "from ${params.generatedPrefix}.types.${t.moduleName} import ${alias t.className}" + "from ${generatedPrefix}.types.${t.moduleName} import ${alias t.className}" ) params.types let rows = rowNames params.statements let rowBlock = - "from ${params.generatedPrefix}._rows import (\n" + "from ${generatedPrefix}._rows import (\n" ++ Prelude.Text.concatMap Text (\(name : Text) -> " ${alias name},\n") @@ -79,7 +78,7 @@ let run = "\n" StatementExport ( \(s : StatementExport) -> - "from ${params.generatedPrefix}.${params.statementsPath}.${s.functionName} import ${alias s.functionName}" + "from ${generatedPrefix}.${statementsPath}.${s.functionName} import ${alias s.functionName}" ) params.statements diff --git a/src/Templates/RuntimeModule.dhall b/src/Templates/RuntimeModule.dhall index fa7e3f0..e13a2d0 100644 --- a/src/Templates/RuntimeModule.dhall +++ b/src/Templates/RuntimeModule.dhall @@ -81,10 +81,11 @@ let content = _ = await cur.execute(sql, params) '' --- The sync mirror, emitted at _generated/sync/_runtime.py when config.sync is True. The --- five helpers are the same shape with `def`/`Connection`/`with`/no-`await`. --- JsonValue/NoRowError/require_array are re-exported from _core (two levels up) --- so both surfaces share one canonical identity rather than two equal-but- +-- The sync surface's body, selected in place of `content` at the same +-- `_runtime.py` path when config.sync is True (Interpreters/Project.dhall). +-- The five helpers are the same shape with `def`/`Connection`/`with`/no- +-- `await`. JsonValue/NoRowError/require_array are re-exported from _core so +-- both surfaces share one canonical identity rather than two equal-but- -- distinct definitions. let syncContent = '' @@ -96,7 +97,7 @@ let syncContent = from psycopg import Connection from psycopg.rows import dict_row - from .._core import JsonValue as JsonValue, NoRowError as NoRowError, require_array as require_array + from ._core import JsonValue as JsonValue, NoRowError as NoRowError, require_array as require_array _T = TypeVar("_T") _Row = Mapping[str, object] From cf6b165b84eb7e410a37a806972938c35104ff50 Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sun, 12 Jul 2026 08:36:12 +0300 Subject: [PATCH 15/21] python.gen: add python-sync fixture artifact, drop emitSync from main artifact --- tests/fixture-project/project1.pgn.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/fixture-project/project1.pgn.yaml b/tests/fixture-project/project1.pgn.yaml index ff75999..c6fc885 100644 --- a/tests/fixture-project/project1.pgn.yaml +++ b/tests/fixture-project/project1.pgn.yaml @@ -7,7 +7,11 @@ artifacts: gen: ../../src/package.dhall config: packageName: specimen-client - emitSync: true + python-sync: + gen: ../../src/package.dhall + config: + packageName: specimen-sync-client + sync: true # The variants below pin pgn's actual YAML->Dhall decode semantics for the # Optional Config knobs (undocumented by pgn itself); see # tests/test_config_variants.py for the assertions. @@ -18,7 +22,7 @@ artifacts: python-sync-only: gen: ../../src/package.dhall config: - emitSync: true + sync: true python-empty: gen: ../../src/package.dhall config: {} @@ -33,4 +37,4 @@ artifacts: gen: ../../src/package.dhall config: packageName: null-client - emitSync: null + sync: null From fe5f82b84be8d04c5436fdb18928f5cc883ef910 Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sun, 12 Jul 2026 08:57:59 +0300 Subject: [PATCH 16/21] python.gen: add tests/golden_sync, regenerate both golden trees for exclusive sync --- mise.toml | 11 +- tests/_harness.py | 1 + .../src/specimen_client/_generated/_rows.py | 33 +- .../_generated/statements/insert_specimen.py | 7 +- .../statements/insert_tagged_item.py | 2 +- .../statements/list_specimens_by_feeling.py | 2 +- .../statements/list_specimens_by_moods.py | 2 +- .../specimen_client/_generated/types/mood.py | 8 + .../_generated/types/point_2_d.py | 8 + .../_generated/types/tag_value.py | 8 + .../src/specimen_client/sync/__init__.py | 63 ---- tests/golden_sync/README.md | 26 ++ tests/golden_sync/pyproject.toml | 15 + .../src/specimen_sync_client/__init__.py | 63 ++++ .../_generated}/__init__.py | 2 +- .../specimen_sync_client/_generated/_core.py | 35 ++ .../_generated}/_register.py | 0 .../specimen_sync_client/_generated/_rows.py | 303 ++++++++++++++++++ .../_generated}/_runtime.py | 2 +- .../_generated/statements}/__init__.py | 2 +- .../statements/bump_specimen_revision.py | 0 .../_generated}/statements/get_specimen.py | 2 +- .../_generated}/statements/get_tagged_item.py | 2 +- .../_generated}/statements/insert_specimen.py | 15 +- .../statements/insert_tagged_item.py | 6 +- .../statements/list_specimens_by_class.py | 2 +- .../statements/list_specimens_by_feeling.py | 6 +- .../statements/list_specimens_by_ids.py | 2 +- .../statements/list_specimens_by_moods.py | 6 +- .../list_specimens_keyword_column.py | 2 +- .../statements/search_specimens.py | 4 +- .../_generated/types/__init__.py | 7 + .../_generated/types/mood.py | 19 ++ .../_generated/types/point_2_d.py | 25 ++ .../_generated/types/tag_value.py | 24 ++ .../src/specimen_sync_client/py.typed | 0 36 files changed, 605 insertions(+), 110 deletions(-) delete mode 100644 tests/golden/src/specimen_client/sync/__init__.py create mode 100644 tests/golden_sync/README.md create mode 100644 tests/golden_sync/pyproject.toml create mode 100644 tests/golden_sync/src/specimen_sync_client/__init__.py rename tests/{golden/src/specimen_client/_generated/sync/statements => golden_sync/src/specimen_sync_client/_generated}/__init__.py (73%) create mode 100644 tests/golden_sync/src/specimen_sync_client/_generated/_core.py rename tests/{golden/src/specimen_client/_generated/sync => golden_sync/src/specimen_sync_client/_generated}/_register.py (100%) create mode 100644 tests/golden_sync/src/specimen_sync_client/_generated/_rows.py rename tests/{golden/src/specimen_client/_generated/sync => golden_sync/src/specimen_sync_client/_generated}/_runtime.py (94%) rename tests/{golden/src/specimen_client/_generated/sync => golden_sync/src/specimen_sync_client/_generated/statements}/__init__.py (80%) rename tests/{golden/src/specimen_client/_generated/sync => golden_sync/src/specimen_sync_client/_generated}/statements/bump_specimen_revision.py (100%) rename tests/{golden/src/specimen_client/_generated/sync => golden_sync/src/specimen_sync_client/_generated}/statements/get_specimen.py (93%) rename tests/{golden/src/specimen_client/_generated/sync => golden_sync/src/specimen_sync_client/_generated}/statements/get_tagged_item.py (91%) rename tests/{golden/src/specimen_client/_generated/sync => golden_sync/src/specimen_sync_client/_generated}/statements/insert_specimen.py (89%) rename tests/{golden/src/specimen_client/_generated/sync => golden_sync/src/specimen_sync_client/_generated}/statements/insert_tagged_item.py (84%) rename tests/{golden/src/specimen_client/_generated/sync => golden_sync/src/specimen_sync_client/_generated}/statements/list_specimens_by_class.py (92%) rename tests/{golden/src/specimen_client/_generated/sync => golden_sync/src/specimen_sync_client/_generated}/statements/list_specimens_by_feeling.py (79%) rename tests/{golden/src/specimen_client/_generated/sync => golden_sync/src/specimen_sync_client/_generated}/statements/list_specimens_by_ids.py (90%) rename tests/{golden/src/specimen_client/_generated/sync => golden_sync/src/specimen_sync_client/_generated}/statements/list_specimens_by_moods.py (78%) rename tests/{golden/src/specimen_client/_generated/sync => golden_sync/src/specimen_sync_client/_generated}/statements/list_specimens_keyword_column.py (91%) rename tests/{golden/src/specimen_client/_generated/sync => golden_sync/src/specimen_sync_client/_generated}/statements/search_specimens.py (92%) create mode 100644 tests/golden_sync/src/specimen_sync_client/_generated/types/__init__.py create mode 100644 tests/golden_sync/src/specimen_sync_client/_generated/types/mood.py create mode 100644 tests/golden_sync/src/specimen_sync_client/_generated/types/point_2_d.py create mode 100644 tests/golden_sync/src/specimen_sync_client/_generated/types/tag_value.py create mode 100644 tests/golden_sync/src/specimen_sync_client/py.typed diff --git a/mise.toml b/mise.toml index 68903bf..3855c94 100644 --- a/mise.toml +++ b/mise.toml @@ -37,8 +37,12 @@ run = "bench/generate.sh without" # see ci.yml), a single-artifact one at ~10 GB. The temp copy guarantees a # fresh resolve of the working-tree src/ (no stale freeze) and keeps pgn's # scratch files out of the repo. +# Regenerates the "python" and "python-sync" artifacts: a full 7-artifact +# generate peaks at ~31 GB RSS (memory goes to normalizing the generator +# closure per artifact, see ci.yml), so this keeps only the two artifacts +# golden actually needs instead of all 7. [tasks.golden] -description = "Refresh tests/golden from the working-tree src/ (single-artifact run)" +description = "Refresh tests/golden and tests/golden_sync from the working-tree src/" run = ''' #!/usr/bin/env bash set -euo pipefail @@ -61,6 +65,7 @@ cd "$tmp/fixture" pgn --database-url "${PGN_TEST_DATABASE_URL:-postgresql://postgres:postgres@localhost:5432/postgres?sslmode=disable}" generate rsync -a --delete artifacts/python/src/specimen_client/_generated/ "$root/tests/golden/src/specimen_client/_generated/" cp artifacts/python/src/specimen_client/__init__.py "$root/tests/golden/src/specimen_client/__init__.py" -cp artifacts/python/src/specimen_client/sync/__init__.py "$root/tests/golden/src/specimen_client/sync/__init__.py" -echo "golden refreshed; review with: git diff tests/golden" +rsync -a --delete artifacts/python_sync/src/specimen_sync_client/_generated/ "$root/tests/golden_sync/src/specimen_sync_client/_generated/" +cp artifacts/python_sync/src/specimen_sync_client/__init__.py "$root/tests/golden_sync/src/specimen_sync_client/__init__.py" +echo "golden refreshed; review with: git diff tests/golden tests/golden_sync" ''' diff --git a/tests/_harness.py b/tests/_harness.py index 9e5ffbd..9b90292 100644 --- a/tests/_harness.py +++ b/tests/_harness.py @@ -26,6 +26,7 @@ SRC_DIR = HERE.parent / "src" FIXTURE_PROJECT = HERE / "fixture-project" GOLDEN_DIR = HERE / "golden" +GOLDEN_DIR_SYNC = HERE / "golden_sync" # pgn creates its own temp database from this admin URL; we never write into the # target database itself. Default points at a local Postgres on the standard port; diff --git a/tests/golden/src/specimen_client/_generated/_rows.py b/tests/golden/src/specimen_client/_generated/_rows.py index e5e586d..b413258 100644 --- a/tests/golden/src/specimen_client/_generated/_rows.py +++ b/tests/golden/src/specimen_client/_generated/_rows.py @@ -16,6 +16,15 @@ from .types.mood import Mood from .types.point_2_d import Point2D from .types.tag_value import TagValue +from .types.mood import Mood +from .types.mood import Mood +from .types.point_2_d import Point2D +from .types.tag_value import TagValue +from .types.mood import Mood +from .types.point_2_d import Point2D +from .types.mood import Mood +from .types.mood import Mood +from .types.mood import Mood @dataclass(frozen=True, slots=True) @@ -79,8 +88,8 @@ def decode_get_specimen(row: Mapping[str, object]) -> GetSpecimenRow: tags=cast(list[str | None], row["tags"]), related_ids=cast(list[UUID | None] | None, row["related_ids"]), grid=cast(list[int | None] | None, row["grid"]), - feeling=Mood(cast(str, row["feeling"])), - origin=None if row["origin"] is None else Point2D(*cast(tuple[float | None, float | None], row["origin"])), + feeling=Mood._decode(row["feeling"]), + origin=None if row["origin"] is None else Point2D._decode(row["origin"]), label=cast(str, row["label"]), rev=cast(int, row["rev"]), meta=cast(JsonValue, row["meta"]), @@ -98,7 +107,7 @@ def decode_get_tagged_item(row: Mapping[str, object]) -> GetTaggedItemRow: return GetTaggedItemRow( id=cast(int, row["id"]), name=cast(str, row["name"]), - tag=TagValue(*cast(tuple[str | None], row["tag"])), + tag=TagValue._decode(row["tag"]), ) @@ -164,9 +173,9 @@ def decode_insert_specimen(row: Mapping[str, object]) -> InsertSpecimenRow: tags=cast(list[str | None], row["tags"]), related_ids=cast(list[UUID | None] | None, row["related_ids"]), grid=cast(list[int | None] | None, row["grid"]), - feeling=Mood(cast(str, row["feeling"])), - moods=None if row["moods"] is None else [None if v is None else Mood(v) for v in cast(list[str | None], require_array(row["moods"]))], - origin=None if row["origin"] is None else Point2D(*cast(tuple[float | None, float | None], row["origin"])), + feeling=Mood._decode(row["feeling"]), + moods=None if row["moods"] is None else [None if v is None else Mood._decode(v) for v in cast(list[str | None], require_array(row["moods"]))], + origin=None if row["origin"] is None else Point2D._decode(row["origin"]), label=cast(str, row["label"]), rev=cast(int, row["rev"]), meta=cast(JsonValue, row["meta"]), @@ -184,7 +193,7 @@ def decode_insert_tagged_item(row: Mapping[str, object]) -> InsertTaggedItemRow: return InsertTaggedItemRow( id=cast(int, row["id"]), name=cast(str, row["name"]), - tag=TagValue(*cast(tuple[str | None], row["tag"])), + tag=TagValue._decode(row["tag"]), ) @@ -218,11 +227,11 @@ def decode_list_specimens_by_feeling(row: Mapping[str, object]) -> ListSpecimens return ListSpecimensByFeelingRow( id=cast(int, row["id"]), pub_id=cast(UUID, row["pub_id"]), - feeling=Mood(cast(str, row["feeling"])), + feeling=Mood._decode(row["feeling"]), title=cast(str, row["title"]), label=cast(str, row["label"]), rev=cast(int, row["rev"]), - origin=None if row["origin"] is None else Point2D(*cast(tuple[float | None, float | None], row["origin"])), + origin=None if row["origin"] is None else Point2D._decode(row["origin"]), tags=cast(list[str | None], row["tags"]), meta=cast(JsonValue, row["meta"]), ) @@ -240,7 +249,7 @@ def decode_list_specimens_by_ids(row: Mapping[str, object]) -> ListSpecimensById return ListSpecimensByIdsRow( id=cast(int, row["id"]), pub_id=cast(UUID, row["pub_id"]), - feeling=Mood(cast(str, row["feeling"])), + feeling=Mood._decode(row["feeling"]), title=cast(str, row["title"]), ) @@ -258,8 +267,8 @@ def decode_list_specimens_by_moods(row: Mapping[str, object]) -> ListSpecimensBy return ListSpecimensByMoodsRow( id=cast(int, row["id"]), pub_id=cast(UUID, row["pub_id"]), - feeling=Mood(cast(str, row["feeling"])), - moods=None if row["moods"] is None else [None if v is None else Mood(v) for v in cast(list[str | None], require_array(row["moods"]))], + feeling=Mood._decode(row["feeling"]), + moods=None if row["moods"] is None else [None if v is None else Mood._decode(v) for v in cast(list[str | None], require_array(row["moods"]))], title=cast(str, row["title"]), ) diff --git a/tests/golden/src/specimen_client/_generated/statements/insert_specimen.py b/tests/golden/src/specimen_client/_generated/statements/insert_specimen.py index f97eb70..9f1b2e5 100644 --- a/tests/golden/src/specimen_client/_generated/statements/insert_specimen.py +++ b/tests/golden/src/specimen_client/_generated/statements/insert_specimen.py @@ -16,6 +16,7 @@ from .._rows import InsertSpecimenRow, decode_insert_specimen from .._runtime import fetch_single from ..types.mood import Mood +from ..types.mood import Mood from ..types.point_2_d import Point2D SQL = """\ @@ -110,8 +111,8 @@ async def insert_specimen( "tags": tags, "related_ids": related_ids, "grid": grid, - "feeling": feeling, - "moods": moods, - "origin": None if origin is None else (origin.x, origin.y), + "feeling": feeling._encode(), + "moods": None if moods is None else [None if x is None else x._encode() for x in moods], + "origin": None if origin is None else origin._encode(), } return await fetch_single(conn, _SQL, params, decode_insert_specimen) diff --git a/tests/golden/src/specimen_client/_generated/statements/insert_tagged_item.py b/tests/golden/src/specimen_client/_generated/statements/insert_tagged_item.py index b2fff1c..912594e 100644 --- a/tests/golden/src/specimen_client/_generated/statements/insert_tagged_item.py +++ b/tests/golden/src/specimen_client/_generated/statements/insert_tagged_item.py @@ -29,6 +29,6 @@ async def insert_tagged_item( ) -> InsertTaggedItemRow: params: dict[str, object] = { "name": name, - "tag": (tag.value,), + "tag": tag._encode(), } return await fetch_single(conn, _SQL, params, decode_insert_tagged_item) diff --git a/tests/golden/src/specimen_client/_generated/statements/list_specimens_by_feeling.py b/tests/golden/src/specimen_client/_generated/statements/list_specimens_by_feeling.py index 2173d01..73699d7 100644 --- a/tests/golden/src/specimen_client/_generated/statements/list_specimens_by_feeling.py +++ b/tests/golden/src/specimen_client/_generated/statements/list_specimens_by_feeling.py @@ -28,6 +28,6 @@ async def list_specimens_by_feeling( feeling: Mood | None, ) -> list[ListSpecimensByFeelingRow]: params: dict[str, object] = { - "feeling": feeling, + "feeling": None if feeling is None else feeling._encode(), } return await fetch_many(conn, _SQL, params, decode_list_specimens_by_feeling) diff --git a/tests/golden/src/specimen_client/_generated/statements/list_specimens_by_moods.py b/tests/golden/src/specimen_client/_generated/statements/list_specimens_by_moods.py index 644d601..1d5ba33 100644 --- a/tests/golden/src/specimen_client/_generated/statements/list_specimens_by_moods.py +++ b/tests/golden/src/specimen_client/_generated/statements/list_specimens_by_moods.py @@ -28,6 +28,6 @@ async def list_specimens_by_moods( moods: list[Mood | None] | None, ) -> list[ListSpecimensByMoodsRow]: params: dict[str, object] = { - "moods": moods, + "moods": None if moods is None else [None if x is None else x._encode() for x in moods], } return await fetch_many(conn, _SQL, params, decode_list_specimens_by_moods) diff --git a/tests/golden/src/specimen_client/_generated/types/mood.py b/tests/golden/src/specimen_client/_generated/types/mood.py index ece63e0..a91c901 100644 --- a/tests/golden/src/specimen_client/_generated/types/mood.py +++ b/tests/golden/src/specimen_client/_generated/types/mood.py @@ -3,9 +3,17 @@ # SPDX-License-Identifier: MIT-0 from enum import StrEnum +from typing import cast class Mood(StrEnum): HAPPY = "happy" SAD = "sad" MEH = "meh" + + @staticmethod + def _decode(src: object) -> "Mood": + return Mood(cast(str, src)) + + def _encode(self) -> "Mood": + return self diff --git a/tests/golden/src/specimen_client/_generated/types/point_2_d.py b/tests/golden/src/specimen_client/_generated/types/point_2_d.py index 1874b55..9bf2915 100644 --- a/tests/golden/src/specimen_client/_generated/types/point_2_d.py +++ b/tests/golden/src/specimen_client/_generated/types/point_2_d.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: MIT-0 from dataclasses import dataclass +from typing import cast @dataclass(frozen=True, slots=True) @@ -15,3 +16,10 @@ class Point2D: x: float | None y: float | None + + @staticmethod + def _decode(src: object) -> "Point2D": + return Point2D(*cast(tuple[float | None, float | None], src)) + + def _encode(self) -> tuple[float | None, float | None]: + return (self.x, self.y) diff --git a/tests/golden/src/specimen_client/_generated/types/tag_value.py b/tests/golden/src/specimen_client/_generated/types/tag_value.py index 13814f9..bfdbbfc 100644 --- a/tests/golden/src/specimen_client/_generated/types/tag_value.py +++ b/tests/golden/src/specimen_client/_generated/types/tag_value.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: MIT-0 from dataclasses import dataclass +from typing import cast @dataclass(frozen=True, slots=True) @@ -14,3 +15,10 @@ class TagValue: """ value: str | None + + @staticmethod + def _decode(src: object) -> "TagValue": + return TagValue(*cast(tuple[str | None], src)) + + def _encode(self) -> tuple[str | None]: + return (self.value,) diff --git a/tests/golden/src/specimen_client/sync/__init__.py b/tests/golden/src/specimen_client/sync/__init__.py deleted file mode 100644 index 18b75c1..0000000 --- a/tests/golden/src/specimen_client/sync/__init__.py +++ /dev/null @@ -1,63 +0,0 @@ -# @generated by python.gen (pGenie). DO NOT EDIT. -# SPDX-FileCopyrightText: 2026 Viacheslav Shvets -# SPDX-License-Identifier: MIT-0 - -from .._generated._core import JsonValue as JsonValue, NoRowError as NoRowError - -from .._generated.types.mood import Mood as Mood -from .._generated.types.point_2_d import Point2D as Point2D -from .._generated.types.tag_value import TagValue as TagValue - -from .._generated._rows import ( - GetSpecimenRow as GetSpecimenRow, - GetTaggedItemRow as GetTaggedItemRow, - InsertSpecimenRow as InsertSpecimenRow, - InsertTaggedItemRow as InsertTaggedItemRow, - ListSpecimensByClassRow as ListSpecimensByClassRow, - ListSpecimensByFeelingRow as ListSpecimensByFeelingRow, - ListSpecimensByIdsRow as ListSpecimensByIdsRow, - ListSpecimensByMoodsRow as ListSpecimensByMoodsRow, - ListSpecimensKeywordColumnRow as ListSpecimensKeywordColumnRow, - SearchSpecimensRow as SearchSpecimensRow, -) - -from .._generated.sync.statements.bump_specimen_revision import bump_specimen_revision as bump_specimen_revision -from .._generated.sync.statements.get_specimen import get_specimen as get_specimen -from .._generated.sync.statements.get_tagged_item import get_tagged_item as get_tagged_item -from .._generated.sync.statements.insert_specimen import insert_specimen as insert_specimen -from .._generated.sync.statements.insert_tagged_item import insert_tagged_item as insert_tagged_item -from .._generated.sync.statements.list_specimens_by_class import list_specimens_by_class as list_specimens_by_class -from .._generated.sync.statements.list_specimens_by_feeling import list_specimens_by_feeling as list_specimens_by_feeling -from .._generated.sync.statements.list_specimens_by_ids import list_specimens_by_ids as list_specimens_by_ids -from .._generated.sync.statements.list_specimens_by_moods import list_specimens_by_moods as list_specimens_by_moods -from .._generated.sync.statements.list_specimens_keyword_column import list_specimens_keyword_column as list_specimens_keyword_column -from .._generated.sync.statements.search_specimens import search_specimens as search_specimens - -__all__ = [ - "JsonValue", - "NoRowError", - "Mood", - "Point2D", - "TagValue", - "GetSpecimenRow", - "GetTaggedItemRow", - "InsertSpecimenRow", - "InsertTaggedItemRow", - "ListSpecimensByClassRow", - "ListSpecimensByFeelingRow", - "ListSpecimensByIdsRow", - "ListSpecimensByMoodsRow", - "ListSpecimensKeywordColumnRow", - "SearchSpecimensRow", - "bump_specimen_revision", - "get_specimen", - "get_tagged_item", - "insert_specimen", - "insert_tagged_item", - "list_specimens_by_class", - "list_specimens_by_feeling", - "list_specimens_by_ids", - "list_specimens_by_moods", - "list_specimens_keyword_column", - "search_specimens", -] diff --git a/tests/golden_sync/README.md b/tests/golden_sync/README.md new file mode 100644 index 0000000..6f6d14b --- /dev/null +++ b/tests/golden_sync/README.md @@ -0,0 +1,26 @@ +# Golden files (sync surface) + +The sync-surface counterpart of `tests/golden/`: a committed full fixture +package generated with `config: { sync: true }`, package name +`specimen_sync_client`. Same structure and purpose as `tests/golden/README.md` +describes for the async (default) surface — the hand-written shell here +(`pyproject.toml`, `py.typed`) plus the generated `_generated/` subtree and +the generated package-root facade `src/specimen_sync_client/__init__.py`. + +`test_generated_sync_matches_golden` regenerates the `python-sync` fixture +artifact into a temp tree and asserts every file under the fresh +`_generated/` plus the facade equals its golden twin here, both ways (no +missing, no extra). `test_generated_sync_passes_basedpyright_strict` runs +basedpyright strict on this full golden package. + +## Updating the golden + +Run only when a generator change legitimately alters the sync surface's +output, and review the resulting diff before committing. + +```bash +mise run golden +``` + +`mise run golden` refreshes both `tests/golden/` (the `python` artifact) and +this directory (the `python-sync` artifact) in one pass — see `mise.toml`. \ No newline at end of file diff --git a/tests/golden_sync/pyproject.toml b/tests/golden_sync/pyproject.toml new file mode 100644 index 0000000..26a00bf --- /dev/null +++ b/tests/golden_sync/pyproject.toml @@ -0,0 +1,15 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "specimen-sync-client" +version = "0.0.0" +requires-python = ">=3.12" +dependencies = ["psycopg>=3.2"] + +[tool.hatch.build.targets.wheel] +packages = ["src/specimen_sync_client"] + +[tool.ruff] +exclude = ["src/specimen_sync_client/_generated"] \ No newline at end of file diff --git a/tests/golden_sync/src/specimen_sync_client/__init__.py b/tests/golden_sync/src/specimen_sync_client/__init__.py new file mode 100644 index 0000000..c34dbc1 --- /dev/null +++ b/tests/golden_sync/src/specimen_sync_client/__init__.py @@ -0,0 +1,63 @@ +# @generated by python.gen (pGenie). DO NOT EDIT. +# SPDX-FileCopyrightText: 2026 Viacheslav Shvets +# SPDX-License-Identifier: MIT-0 + +from ._generated._core import JsonValue as JsonValue, NoRowError as NoRowError + +from ._generated.types.mood import Mood as Mood +from ._generated.types.point_2_d import Point2D as Point2D +from ._generated.types.tag_value import TagValue as TagValue + +from ._generated._rows import ( + GetSpecimenRow as GetSpecimenRow, + GetTaggedItemRow as GetTaggedItemRow, + InsertSpecimenRow as InsertSpecimenRow, + InsertTaggedItemRow as InsertTaggedItemRow, + ListSpecimensByClassRow as ListSpecimensByClassRow, + ListSpecimensByFeelingRow as ListSpecimensByFeelingRow, + ListSpecimensByIdsRow as ListSpecimensByIdsRow, + ListSpecimensByMoodsRow as ListSpecimensByMoodsRow, + ListSpecimensKeywordColumnRow as ListSpecimensKeywordColumnRow, + SearchSpecimensRow as SearchSpecimensRow, +) + +from ._generated.statements.bump_specimen_revision import bump_specimen_revision as bump_specimen_revision +from ._generated.statements.get_specimen import get_specimen as get_specimen +from ._generated.statements.get_tagged_item import get_tagged_item as get_tagged_item +from ._generated.statements.insert_specimen import insert_specimen as insert_specimen +from ._generated.statements.insert_tagged_item import insert_tagged_item as insert_tagged_item +from ._generated.statements.list_specimens_by_class import list_specimens_by_class as list_specimens_by_class +from ._generated.statements.list_specimens_by_feeling import list_specimens_by_feeling as list_specimens_by_feeling +from ._generated.statements.list_specimens_by_ids import list_specimens_by_ids as list_specimens_by_ids +from ._generated.statements.list_specimens_by_moods import list_specimens_by_moods as list_specimens_by_moods +from ._generated.statements.list_specimens_keyword_column import list_specimens_keyword_column as list_specimens_keyword_column +from ._generated.statements.search_specimens import search_specimens as search_specimens + +__all__ = [ + "JsonValue", + "NoRowError", + "Mood", + "Point2D", + "TagValue", + "GetSpecimenRow", + "GetTaggedItemRow", + "InsertSpecimenRow", + "InsertTaggedItemRow", + "ListSpecimensByClassRow", + "ListSpecimensByFeelingRow", + "ListSpecimensByIdsRow", + "ListSpecimensByMoodsRow", + "ListSpecimensKeywordColumnRow", + "SearchSpecimensRow", + "bump_specimen_revision", + "get_specimen", + "get_tagged_item", + "insert_specimen", + "insert_tagged_item", + "list_specimens_by_class", + "list_specimens_by_feeling", + "list_specimens_by_ids", + "list_specimens_by_moods", + "list_specimens_keyword_column", + "search_specimens", +] diff --git a/tests/golden/src/specimen_client/_generated/sync/statements/__init__.py b/tests/golden_sync/src/specimen_sync_client/_generated/__init__.py similarity index 73% rename from tests/golden/src/specimen_client/_generated/sync/statements/__init__.py rename to tests/golden_sync/src/specimen_sync_client/_generated/__init__.py index 3f0a22b..3117013 100644 --- a/tests/golden/src/specimen_client/_generated/sync/statements/__init__.py +++ b/tests/golden_sync/src/specimen_sync_client/_generated/__init__.py @@ -2,6 +2,6 @@ # SPDX-FileCopyrightText: 2026 Viacheslav Shvets # SPDX-License-Identifier: MIT-0 -"""Generated SQL statements (sync).""" +"""Generated database client for specimen-sync-client.""" __all__: list[str] = [] diff --git a/tests/golden_sync/src/specimen_sync_client/_generated/_core.py b/tests/golden_sync/src/specimen_sync_client/_generated/_core.py new file mode 100644 index 0000000..8927292 --- /dev/null +++ b/tests/golden_sync/src/specimen_sync_client/_generated/_core.py @@ -0,0 +1,35 @@ +# @generated by python.gen (pGenie). DO NOT EDIT. +# SPDX-FileCopyrightText: 2026 Viacheslav Shvets +# SPDX-License-Identifier: MIT-0 + +"""Shared types and decode helpers, surface-agnostic; no I/O.""" + +from __future__ import annotations + +from typing import cast + +type JsonValue = None | bool | int | float | str | list["JsonValue"] | dict[str, "JsonValue"] + + +class NoRowError(RuntimeError): + """A single-row query returned no rows.""" + + +class DecodeError(RuntimeError): + """A row value failed to decode into its target type.""" + + +def require_array(value: object) -> list[object]: + """Guard an enum-array column decode. + + psycopg returns an enum array as a Python list only when the enum type is + registered on the connection (register_types); without it the value comes + back as the raw array text, which would iterate into bogus members. Fail + clearly instead. + """ + if isinstance(value, list): + return cast(list[object], value) + raise RuntimeError( + "enum array decoded as text; call register_types() on the connection " + "before decoding enum-array columns" + ) diff --git a/tests/golden/src/specimen_client/_generated/sync/_register.py b/tests/golden_sync/src/specimen_sync_client/_generated/_register.py similarity index 100% rename from tests/golden/src/specimen_client/_generated/sync/_register.py rename to tests/golden_sync/src/specimen_sync_client/_generated/_register.py diff --git a/tests/golden_sync/src/specimen_sync_client/_generated/_rows.py b/tests/golden_sync/src/specimen_sync_client/_generated/_rows.py new file mode 100644 index 0000000..b413258 --- /dev/null +++ b/tests/golden_sync/src/specimen_sync_client/_generated/_rows.py @@ -0,0 +1,303 @@ +# @generated by python.gen (pGenie). DO NOT EDIT. +# SPDX-FileCopyrightText: 2026 Viacheslav Shvets +# SPDX-License-Identifier: MIT-0 + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import date, datetime +from decimal import Decimal +from typing import cast +from uuid import UUID + +from ._core import JsonValue +from ._core import require_array +from .types.mood import Mood +from .types.point_2_d import Point2D +from .types.tag_value import TagValue +from .types.mood import Mood +from .types.mood import Mood +from .types.point_2_d import Point2D +from .types.tag_value import TagValue +from .types.mood import Mood +from .types.point_2_d import Point2D +from .types.mood import Mood +from .types.mood import Mood +from .types.mood import Mood + + +@dataclass(frozen=True, slots=True) +class GetSpecimenRow: + id: int + pub_id: UUID + flag: bool + small: int + medium: int + large: int + ratio: float + precise: float + title: str + code: str + letter: str + born_on: date + created_at: datetime + amount: Decimal + blob: bytes + doc_json: JsonValue + doc_jsonb: JsonValue + maybe_text: str | None + maybe_int: int | None + maybe_uuid: UUID | None + maybe_ts: datetime | None + maybe_num: Decimal | None + tags: list[str | None] + related_ids: list[UUID | None] | None + grid: list[int | None] | None + feeling: Mood + origin: Point2D | None + label: str + rev: int + meta: JsonValue + + +def decode_get_specimen(row: Mapping[str, object]) -> GetSpecimenRow: + return GetSpecimenRow( + id=cast(int, row["id"]), + pub_id=cast(UUID, row["pub_id"]), + flag=cast(bool, row["flag"]), + small=cast(int, row["small"]), + medium=cast(int, row["medium"]), + large=cast(int, row["large"]), + ratio=cast(float, row["ratio"]), + precise=cast(float, row["precise"]), + title=cast(str, row["title"]), + code=cast(str, row["code"]), + letter=cast(str, row["letter"]), + born_on=cast(date, row["born_on"]), + created_at=cast(datetime, row["created_at"]), + amount=cast(Decimal, row["amount"]), + blob=cast(bytes, row["blob"]), + doc_json=cast(JsonValue, row["doc_json"]), + doc_jsonb=cast(JsonValue, row["doc_jsonb"]), + maybe_text=cast(str | None, row["maybe_text"]), + maybe_int=cast(int | None, row["maybe_int"]), + maybe_uuid=cast(UUID | None, row["maybe_uuid"]), + maybe_ts=cast(datetime | None, row["maybe_ts"]), + maybe_num=cast(Decimal | None, row["maybe_num"]), + tags=cast(list[str | None], row["tags"]), + related_ids=cast(list[UUID | None] | None, row["related_ids"]), + grid=cast(list[int | None] | None, row["grid"]), + feeling=Mood._decode(row["feeling"]), + origin=None if row["origin"] is None else Point2D._decode(row["origin"]), + label=cast(str, row["label"]), + rev=cast(int, row["rev"]), + meta=cast(JsonValue, row["meta"]), + ) + + +@dataclass(frozen=True, slots=True) +class GetTaggedItemRow: + id: int + name: str + tag: TagValue + + +def decode_get_tagged_item(row: Mapping[str, object]) -> GetTaggedItemRow: + return GetTaggedItemRow( + id=cast(int, row["id"]), + name=cast(str, row["name"]), + tag=TagValue._decode(row["tag"]), + ) + + +@dataclass(frozen=True, slots=True) +class InsertSpecimenRow: + id: int + pub_id: UUID + flag: bool + small: int + medium: int + large: int + ratio: float + precise: float + title: str + code: str + letter: str + born_on: date + created_at: datetime + amount: Decimal + blob: bytes + doc_json: JsonValue + doc_jsonb: JsonValue + maybe_text: str | None + maybe_int: int | None + maybe_uuid: UUID | None + maybe_ts: datetime | None + maybe_num: Decimal | None + tags: list[str | None] + related_ids: list[UUID | None] | None + grid: list[int | None] | None + feeling: Mood + moods: list[Mood | None] | None + origin: Point2D | None + label: str + rev: int + meta: JsonValue + + +def decode_insert_specimen(row: Mapping[str, object]) -> InsertSpecimenRow: + return InsertSpecimenRow( + id=cast(int, row["id"]), + pub_id=cast(UUID, row["pub_id"]), + flag=cast(bool, row["flag"]), + small=cast(int, row["small"]), + medium=cast(int, row["medium"]), + large=cast(int, row["large"]), + ratio=cast(float, row["ratio"]), + precise=cast(float, row["precise"]), + title=cast(str, row["title"]), + code=cast(str, row["code"]), + letter=cast(str, row["letter"]), + born_on=cast(date, row["born_on"]), + created_at=cast(datetime, row["created_at"]), + amount=cast(Decimal, row["amount"]), + blob=cast(bytes, row["blob"]), + doc_json=cast(JsonValue, row["doc_json"]), + doc_jsonb=cast(JsonValue, row["doc_jsonb"]), + maybe_text=cast(str | None, row["maybe_text"]), + maybe_int=cast(int | None, row["maybe_int"]), + maybe_uuid=cast(UUID | None, row["maybe_uuid"]), + maybe_ts=cast(datetime | None, row["maybe_ts"]), + maybe_num=cast(Decimal | None, row["maybe_num"]), + tags=cast(list[str | None], row["tags"]), + related_ids=cast(list[UUID | None] | None, row["related_ids"]), + grid=cast(list[int | None] | None, row["grid"]), + feeling=Mood._decode(row["feeling"]), + moods=None if row["moods"] is None else [None if v is None else Mood._decode(v) for v in cast(list[str | None], require_array(row["moods"]))], + origin=None if row["origin"] is None else Point2D._decode(row["origin"]), + label=cast(str, row["label"]), + rev=cast(int, row["rev"]), + meta=cast(JsonValue, row["meta"]), + ) + + +@dataclass(frozen=True, slots=True) +class InsertTaggedItemRow: + id: int + name: str + tag: TagValue + + +def decode_insert_tagged_item(row: Mapping[str, object]) -> InsertTaggedItemRow: + return InsertTaggedItemRow( + id=cast(int, row["id"]), + name=cast(str, row["name"]), + tag=TagValue._decode(row["tag"]), + ) + + +@dataclass(frozen=True, slots=True) +class ListSpecimensByClassRow: + id: int + title: str + + +def decode_list_specimens_by_class(row: Mapping[str, object]) -> ListSpecimensByClassRow: + return ListSpecimensByClassRow( + id=cast(int, row["id"]), + title=cast(str, row["title"]), + ) + + +@dataclass(frozen=True, slots=True) +class ListSpecimensByFeelingRow: + id: int + pub_id: UUID + feeling: Mood + title: str + label: str + rev: int + origin: Point2D | None + tags: list[str | None] + meta: JsonValue + + +def decode_list_specimens_by_feeling(row: Mapping[str, object]) -> ListSpecimensByFeelingRow: + return ListSpecimensByFeelingRow( + id=cast(int, row["id"]), + pub_id=cast(UUID, row["pub_id"]), + feeling=Mood._decode(row["feeling"]), + title=cast(str, row["title"]), + label=cast(str, row["label"]), + rev=cast(int, row["rev"]), + origin=None if row["origin"] is None else Point2D._decode(row["origin"]), + tags=cast(list[str | None], row["tags"]), + meta=cast(JsonValue, row["meta"]), + ) + + +@dataclass(frozen=True, slots=True) +class ListSpecimensByIdsRow: + id: int + pub_id: UUID + feeling: Mood + title: str + + +def decode_list_specimens_by_ids(row: Mapping[str, object]) -> ListSpecimensByIdsRow: + return ListSpecimensByIdsRow( + id=cast(int, row["id"]), + pub_id=cast(UUID, row["pub_id"]), + feeling=Mood._decode(row["feeling"]), + title=cast(str, row["title"]), + ) + + +@dataclass(frozen=True, slots=True) +class ListSpecimensByMoodsRow: + id: int + pub_id: UUID + feeling: Mood + moods: list[Mood | None] | None + title: str + + +def decode_list_specimens_by_moods(row: Mapping[str, object]) -> ListSpecimensByMoodsRow: + return ListSpecimensByMoodsRow( + id=cast(int, row["id"]), + pub_id=cast(UUID, row["pub_id"]), + feeling=Mood._decode(row["feeling"]), + moods=None if row["moods"] is None else [None if v is None else Mood._decode(v) for v in cast(list[str | None], require_array(row["moods"]))], + title=cast(str, row["title"]), + ) + + +@dataclass(frozen=True, slots=True) +class ListSpecimensKeywordColumnRow: + id: int + class_: str + + +def decode_list_specimens_keyword_column(row: Mapping[str, object]) -> ListSpecimensKeywordColumnRow: + return ListSpecimensKeywordColumnRow( + id=cast(int, row["id"]), + class_=cast(str, row["class"]), + ) + + +@dataclass(frozen=True, slots=True) +class SearchSpecimensRow: + id: int + title: str + label: str + meta: JsonValue + + +def decode_search_specimens(row: Mapping[str, object]) -> SearchSpecimensRow: + return SearchSpecimensRow( + id=cast(int, row["id"]), + title=cast(str, row["title"]), + label=cast(str, row["label"]), + meta=cast(JsonValue, row["meta"]), + ) diff --git a/tests/golden/src/specimen_client/_generated/sync/_runtime.py b/tests/golden_sync/src/specimen_sync_client/_generated/_runtime.py similarity index 94% rename from tests/golden/src/specimen_client/_generated/sync/_runtime.py rename to tests/golden_sync/src/specimen_sync_client/_generated/_runtime.py index 312a249..ced0afa 100644 --- a/tests/golden/src/specimen_client/_generated/sync/_runtime.py +++ b/tests/golden_sync/src/specimen_sync_client/_generated/_runtime.py @@ -10,7 +10,7 @@ from psycopg import Connection from psycopg.rows import dict_row -from .._core import JsonValue as JsonValue, NoRowError as NoRowError, require_array as require_array +from ._core import JsonValue as JsonValue, NoRowError as NoRowError, require_array as require_array _T = TypeVar("_T") _Row = Mapping[str, object] diff --git a/tests/golden/src/specimen_client/_generated/sync/__init__.py b/tests/golden_sync/src/specimen_sync_client/_generated/statements/__init__.py similarity index 80% rename from tests/golden/src/specimen_client/_generated/sync/__init__.py rename to tests/golden_sync/src/specimen_sync_client/_generated/statements/__init__.py index 47a8df4..3c3ae7f 100644 --- a/tests/golden/src/specimen_client/_generated/sync/__init__.py +++ b/tests/golden_sync/src/specimen_sync_client/_generated/statements/__init__.py @@ -2,6 +2,6 @@ # SPDX-FileCopyrightText: 2026 Viacheslav Shvets # SPDX-License-Identifier: MIT-0 -"""Generated sync database client.""" +"""Generated SQL statements.""" __all__: list[str] = [] diff --git a/tests/golden/src/specimen_client/_generated/sync/statements/bump_specimen_revision.py b/tests/golden_sync/src/specimen_sync_client/_generated/statements/bump_specimen_revision.py similarity index 100% rename from tests/golden/src/specimen_client/_generated/sync/statements/bump_specimen_revision.py rename to tests/golden_sync/src/specimen_sync_client/_generated/statements/bump_specimen_revision.py diff --git a/tests/golden/src/specimen_client/_generated/sync/statements/get_specimen.py b/tests/golden_sync/src/specimen_sync_client/_generated/statements/get_specimen.py similarity index 93% rename from tests/golden/src/specimen_client/_generated/sync/statements/get_specimen.py rename to tests/golden_sync/src/specimen_sync_client/_generated/statements/get_specimen.py index 748445d..d6a598b 100644 --- a/tests/golden/src/specimen_client/_generated/sync/statements/get_specimen.py +++ b/tests/golden_sync/src/specimen_sync_client/_generated/statements/get_specimen.py @@ -6,7 +6,7 @@ from psycopg import Connection -from ..._rows import GetSpecimenRow, decode_get_specimen +from .._rows import GetSpecimenRow, decode_get_specimen from .._runtime import fetch_optional SQL = """\ diff --git a/tests/golden/src/specimen_client/_generated/sync/statements/get_tagged_item.py b/tests/golden_sync/src/specimen_sync_client/_generated/statements/get_tagged_item.py similarity index 91% rename from tests/golden/src/specimen_client/_generated/sync/statements/get_tagged_item.py rename to tests/golden_sync/src/specimen_sync_client/_generated/statements/get_tagged_item.py index 08757e6..29c06f3 100644 --- a/tests/golden/src/specimen_client/_generated/sync/statements/get_tagged_item.py +++ b/tests/golden_sync/src/specimen_sync_client/_generated/statements/get_tagged_item.py @@ -6,7 +6,7 @@ from psycopg import Connection -from ..._rows import GetTaggedItemRow, decode_get_tagged_item +from .._rows import GetTaggedItemRow, decode_get_tagged_item from .._runtime import fetch_optional SQL = """\ diff --git a/tests/golden/src/specimen_client/_generated/sync/statements/insert_specimen.py b/tests/golden_sync/src/specimen_sync_client/_generated/statements/insert_specimen.py similarity index 89% rename from tests/golden/src/specimen_client/_generated/sync/statements/insert_specimen.py rename to tests/golden_sync/src/specimen_sync_client/_generated/statements/insert_specimen.py index e8ea5f5..455abe9 100644 --- a/tests/golden/src/specimen_client/_generated/sync/statements/insert_specimen.py +++ b/tests/golden_sync/src/specimen_sync_client/_generated/statements/insert_specimen.py @@ -12,11 +12,12 @@ from psycopg.types.json import Json from psycopg.types.json import Jsonb -from ..._core import JsonValue -from ..._rows import InsertSpecimenRow, decode_insert_specimen +from .._core import JsonValue +from .._rows import InsertSpecimenRow, decode_insert_specimen from .._runtime import fetch_single -from ...types.mood import Mood -from ...types.point_2_d import Point2D +from ..types.mood import Mood +from ..types.mood import Mood +from ..types.point_2_d import Point2D SQL = """\ -- single row: insert ... returning the full type surface. @@ -110,8 +111,8 @@ def insert_specimen( "tags": tags, "related_ids": related_ids, "grid": grid, - "feeling": feeling, - "moods": moods, - "origin": None if origin is None else (origin.x, origin.y), + "feeling": feeling._encode(), + "moods": None if moods is None else [None if x is None else x._encode() for x in moods], + "origin": None if origin is None else origin._encode(), } return fetch_single(conn, _SQL, params, decode_insert_specimen) diff --git a/tests/golden/src/specimen_client/_generated/sync/statements/insert_tagged_item.py b/tests/golden_sync/src/specimen_sync_client/_generated/statements/insert_tagged_item.py similarity index 84% rename from tests/golden/src/specimen_client/_generated/sync/statements/insert_tagged_item.py rename to tests/golden_sync/src/specimen_sync_client/_generated/statements/insert_tagged_item.py index 6d1f35a..c02fcd2 100644 --- a/tests/golden/src/specimen_client/_generated/sync/statements/insert_tagged_item.py +++ b/tests/golden_sync/src/specimen_sync_client/_generated/statements/insert_tagged_item.py @@ -6,9 +6,9 @@ from psycopg import Connection -from ..._rows import InsertTaggedItemRow, decode_insert_tagged_item +from .._rows import InsertTaggedItemRow, decode_insert_tagged_item from .._runtime import fetch_single -from ...types.tag_value import TagValue +from ..types.tag_value import TagValue SQL = """\ -- single row: insert exercising a single-field composite as a parameter and @@ -29,6 +29,6 @@ def insert_tagged_item( ) -> InsertTaggedItemRow: params: dict[str, object] = { "name": name, - "tag": (tag.value,), + "tag": tag._encode(), } return fetch_single(conn, _SQL, params, decode_insert_tagged_item) diff --git a/tests/golden/src/specimen_client/_generated/sync/statements/list_specimens_by_class.py b/tests/golden_sync/src/specimen_sync_client/_generated/statements/list_specimens_by_class.py similarity index 92% rename from tests/golden/src/specimen_client/_generated/sync/statements/list_specimens_by_class.py rename to tests/golden_sync/src/specimen_sync_client/_generated/statements/list_specimens_by_class.py index a5309e4..eb32126 100644 --- a/tests/golden/src/specimen_client/_generated/sync/statements/list_specimens_by_class.py +++ b/tests/golden_sync/src/specimen_sync_client/_generated/statements/list_specimens_by_class.py @@ -6,7 +6,7 @@ from psycopg import Connection -from ..._rows import ListSpecimensByClassRow, decode_list_specimens_by_class +from .._rows import ListSpecimensByClassRow, decode_list_specimens_by_class from .._runtime import fetch_many SQL = """\ diff --git a/tests/golden/src/specimen_client/_generated/sync/statements/list_specimens_by_feeling.py b/tests/golden_sync/src/specimen_sync_client/_generated/statements/list_specimens_by_feeling.py similarity index 79% rename from tests/golden/src/specimen_client/_generated/sync/statements/list_specimens_by_feeling.py rename to tests/golden_sync/src/specimen_sync_client/_generated/statements/list_specimens_by_feeling.py index 4b5f2cd..449865a 100644 --- a/tests/golden/src/specimen_client/_generated/sync/statements/list_specimens_by_feeling.py +++ b/tests/golden_sync/src/specimen_sync_client/_generated/statements/list_specimens_by_feeling.py @@ -6,9 +6,9 @@ from psycopg import Connection -from ..._rows import ListSpecimensByFeelingRow, decode_list_specimens_by_feeling +from .._rows import ListSpecimensByFeelingRow, decode_list_specimens_by_feeling from .._runtime import fetch_many -from ...types.mood import Mood +from ..types.mood import Mood SQL = """\ -- many: select with order by. Enum parameter ($feeling). @@ -28,6 +28,6 @@ def list_specimens_by_feeling( feeling: Mood | None, ) -> list[ListSpecimensByFeelingRow]: params: dict[str, object] = { - "feeling": feeling, + "feeling": None if feeling is None else feeling._encode(), } return fetch_many(conn, _SQL, params, decode_list_specimens_by_feeling) diff --git a/tests/golden/src/specimen_client/_generated/sync/statements/list_specimens_by_ids.py b/tests/golden_sync/src/specimen_sync_client/_generated/statements/list_specimens_by_ids.py similarity index 90% rename from tests/golden/src/specimen_client/_generated/sync/statements/list_specimens_by_ids.py rename to tests/golden_sync/src/specimen_sync_client/_generated/statements/list_specimens_by_ids.py index 94c6299..0512032 100644 --- a/tests/golden/src/specimen_client/_generated/sync/statements/list_specimens_by_ids.py +++ b/tests/golden_sync/src/specimen_sync_client/_generated/statements/list_specimens_by_ids.py @@ -8,7 +8,7 @@ from psycopg import Connection -from ..._rows import ListSpecimensByIdsRow, decode_list_specimens_by_ids +from .._rows import ListSpecimensByIdsRow, decode_list_specimens_by_ids from .._runtime import fetch_many SQL = """\ diff --git a/tests/golden/src/specimen_client/_generated/sync/statements/list_specimens_by_moods.py b/tests/golden_sync/src/specimen_sync_client/_generated/statements/list_specimens_by_moods.py similarity index 78% rename from tests/golden/src/specimen_client/_generated/sync/statements/list_specimens_by_moods.py rename to tests/golden_sync/src/specimen_sync_client/_generated/statements/list_specimens_by_moods.py index 5aaebd0..32f03ea 100644 --- a/tests/golden/src/specimen_client/_generated/sync/statements/list_specimens_by_moods.py +++ b/tests/golden_sync/src/specimen_sync_client/_generated/statements/list_specimens_by_moods.py @@ -6,9 +6,9 @@ from psycopg import Connection -from ..._rows import ListSpecimensByMoodsRow, decode_list_specimens_by_moods +from .._rows import ListSpecimensByMoodsRow, decode_list_specimens_by_moods from .._runtime import fetch_many -from ...types.mood import Mood +from ..types.mood import Mood SQL = """\ -- many: enum array parameter via = any($moods::mood[]); returns the enum array column. @@ -28,6 +28,6 @@ def list_specimens_by_moods( moods: list[Mood | None] | None, ) -> list[ListSpecimensByMoodsRow]: params: dict[str, object] = { - "moods": moods, + "moods": None if moods is None else [None if x is None else x._encode() for x in moods], } return fetch_many(conn, _SQL, params, decode_list_specimens_by_moods) diff --git a/tests/golden/src/specimen_client/_generated/sync/statements/list_specimens_keyword_column.py b/tests/golden_sync/src/specimen_sync_client/_generated/statements/list_specimens_keyword_column.py similarity index 91% rename from tests/golden/src/specimen_client/_generated/sync/statements/list_specimens_keyword_column.py rename to tests/golden_sync/src/specimen_sync_client/_generated/statements/list_specimens_keyword_column.py index 1b36708..0a7b949 100644 --- a/tests/golden/src/specimen_client/_generated/sync/statements/list_specimens_keyword_column.py +++ b/tests/golden_sync/src/specimen_sync_client/_generated/statements/list_specimens_keyword_column.py @@ -6,7 +6,7 @@ from psycopg import Connection -from ..._rows import ListSpecimensKeywordColumnRow, decode_list_specimens_keyword_column +from .._rows import ListSpecimensKeywordColumnRow, decode_list_specimens_keyword_column from .._runtime import fetch_many SQL = """\ diff --git a/tests/golden/src/specimen_client/_generated/sync/statements/search_specimens.py b/tests/golden_sync/src/specimen_sync_client/_generated/statements/search_specimens.py similarity index 92% rename from tests/golden/src/specimen_client/_generated/sync/statements/search_specimens.py rename to tests/golden_sync/src/specimen_sync_client/_generated/statements/search_specimens.py index 73006c1..42bb069 100644 --- a/tests/golden/src/specimen_client/_generated/sync/statements/search_specimens.py +++ b/tests/golden_sync/src/specimen_sync_client/_generated/statements/search_specimens.py @@ -7,8 +7,8 @@ from psycopg import Connection from psycopg.types.json import Jsonb -from ..._core import JsonValue -from ..._rows import SearchSpecimensRow, decode_search_specimens +from .._core import JsonValue +from .._rows import SearchSpecimensRow, decode_search_specimens from .._runtime import fetch_many SQL = """\ diff --git a/tests/golden_sync/src/specimen_sync_client/_generated/types/__init__.py b/tests/golden_sync/src/specimen_sync_client/_generated/types/__init__.py new file mode 100644 index 0000000..c8a6ba8 --- /dev/null +++ b/tests/golden_sync/src/specimen_sync_client/_generated/types/__init__.py @@ -0,0 +1,7 @@ +# @generated by python.gen (pGenie). DO NOT EDIT. +# SPDX-FileCopyrightText: 2026 Viacheslav Shvets +# SPDX-License-Identifier: MIT-0 + +from .mood import Mood as Mood +from .point_2_d import Point2D as Point2D +from .tag_value import TagValue as TagValue diff --git a/tests/golden_sync/src/specimen_sync_client/_generated/types/mood.py b/tests/golden_sync/src/specimen_sync_client/_generated/types/mood.py new file mode 100644 index 0000000..a91c901 --- /dev/null +++ b/tests/golden_sync/src/specimen_sync_client/_generated/types/mood.py @@ -0,0 +1,19 @@ +# @generated by python.gen (pGenie). DO NOT EDIT. +# SPDX-FileCopyrightText: 2026 Viacheslav Shvets +# SPDX-License-Identifier: MIT-0 + +from enum import StrEnum +from typing import cast + + +class Mood(StrEnum): + HAPPY = "happy" + SAD = "sad" + MEH = "meh" + + @staticmethod + def _decode(src: object) -> "Mood": + return Mood(cast(str, src)) + + def _encode(self) -> "Mood": + return self diff --git a/tests/golden_sync/src/specimen_sync_client/_generated/types/point_2_d.py b/tests/golden_sync/src/specimen_sync_client/_generated/types/point_2_d.py new file mode 100644 index 0000000..9bf2915 --- /dev/null +++ b/tests/golden_sync/src/specimen_sync_client/_generated/types/point_2_d.py @@ -0,0 +1,25 @@ +# @generated by python.gen (pGenie). DO NOT EDIT. +# SPDX-FileCopyrightText: 2026 Viacheslav Shvets +# SPDX-License-Identifier: MIT-0 + +from dataclasses import dataclass +from typing import cast + + +@dataclass(frozen=True, slots=True) +class Point2D: + """Decoding/encoding this composite requires register_types(conn) first. + + Without per-connection registration psycopg returns the value as a + raw string, which the generated _decode cannot splat into the dataclass. + """ + + x: float | None + y: float | None + + @staticmethod + def _decode(src: object) -> "Point2D": + return Point2D(*cast(tuple[float | None, float | None], src)) + + def _encode(self) -> tuple[float | None, float | None]: + return (self.x, self.y) diff --git a/tests/golden_sync/src/specimen_sync_client/_generated/types/tag_value.py b/tests/golden_sync/src/specimen_sync_client/_generated/types/tag_value.py new file mode 100644 index 0000000..bfdbbfc --- /dev/null +++ b/tests/golden_sync/src/specimen_sync_client/_generated/types/tag_value.py @@ -0,0 +1,24 @@ +# @generated by python.gen (pGenie). DO NOT EDIT. +# SPDX-FileCopyrightText: 2026 Viacheslav Shvets +# SPDX-License-Identifier: MIT-0 + +from dataclasses import dataclass +from typing import cast + + +@dataclass(frozen=True, slots=True) +class TagValue: + """Decoding/encoding this composite requires register_types(conn) first. + + Without per-connection registration psycopg returns the value as a + raw string, which the generated _decode cannot splat into the dataclass. + """ + + value: str | None + + @staticmethod + def _decode(src: object) -> "TagValue": + return TagValue(*cast(tuple[str | None], src)) + + def _encode(self) -> tuple[str | None]: + return (self.value,) diff --git a/tests/golden_sync/src/specimen_sync_client/py.typed b/tests/golden_sync/src/specimen_sync_client/py.typed new file mode 100644 index 0000000..e69de29 From 3008b708e6b1c45743eb3a3603f5397e3ef00935 Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sun, 12 Jul 2026 09:05:02 +0300 Subject: [PATCH 17/21] python.gen: split sync-surface tests onto their own golden fixture --- tests/conftest.py | 22 +++++++ tests/test_generated.py | 133 ++++++++++++++++++++++++++++++---------- 2 files changed, 121 insertions(+), 34 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5bac571..719c227 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,6 +16,7 @@ from tests._harness import ( FIXTURE_PROJECT, GOLDEN_DIR, + GOLDEN_DIR_SYNC, HERE, SRC_DIR, admin_database_url, @@ -114,3 +115,24 @@ def full_package(generated_tree: Path, tmp_path_factory: pytest.TempPathFactory) (dest_pkg / "sync").mkdir(parents=True, exist_ok=True) _ = shutil.copy2(sync_facade, dest_pkg / "sync" / "__init__.py") return root + + +@pytest.fixture(scope="session") +def full_package_sync(generated_tree: Path, tmp_path_factory: pytest.TempPathFactory) -> Path: + """The sync-surface counterpart of `full_package`. + + `generated_tree` already ran `pgn generate` against the full project + (all artifacts, including `python-sync`), so this reuses that one + subprocess call rather than invoking pgn again: it just points at the + sibling `python_sync` artifact directory instead of `python`. + """ + generated_tree_sync = generated_tree.parent.parent / "artifacts" / "python_sync" + root = tmp_path_factory.mktemp("pkg-sync") + shell_src = GOLDEN_DIR_SYNC / "src" + generated_src = generated_tree_sync / "src" + _ = shutil.copytree(shell_src, root / "src", ignore=shutil.ignore_patterns("_generated", "__init__.py")) + for pkg_dir in generated_src.iterdir(): + dest_pkg = root / "src" / pkg_dir.name + _ = shutil.copytree(pkg_dir / "_generated", dest_pkg / "_generated") + _ = shutil.copy2(pkg_dir / "__init__.py", dest_pkg / "__init__.py") + return root diff --git a/tests/test_generated.py b/tests/test_generated.py index b671193..7675c71 100644 --- a/tests/test_generated.py +++ b/tests/test_generated.py @@ -26,7 +26,7 @@ import pytest from psycopg.conninfo import make_conninfo -from tests._harness import FIXTURE_PROJECT, GOLDEN_DIR, HERE, ensure_droppable, run_pgn +from tests._harness import FIXTURE_PROJECT, GOLDEN_DIR, GOLDEN_DIR_SYNC, HERE, ensure_droppable, run_pgn HARNESS_ROOT = HERE.parent @@ -36,7 +36,8 @@ # generate. GENERATED_SUBTREE = Path("src/specimen_client/_generated") FACADE_INIT = Path("src/specimen_client/__init__.py") -SYNC_FACADE_INIT = Path("src/specimen_client/sync/__init__.py") +GENERATED_SUBTREE_SYNC = Path("src/specimen_sync_client/_generated") +FACADE_INIT_SYNC = Path("src/specimen_sync_client/__init__.py") def test_fixture_project_analyses_clean(pgn_bin: str, pgn_admin_url: str, fixture_copy: Path) -> None: @@ -113,9 +114,8 @@ def test_generated_matches_golden(generated_tree: Path) -> None: if (produced_root / rel).read_text() != (golden_root / rel).read_text(): mismatched.append(str(rel)) - for facade in (FACADE_INIT, SYNC_FACADE_INIT): - if (generated_tree / facade).read_text() != (GOLDEN_DIR / facade).read_text(): - mismatched.append(str(facade)) + if (generated_tree / FACADE_INIT).read_text() != (GOLDEN_DIR / FACADE_INIT).read_text(): + mismatched.append(str(FACADE_INIT)) assert not mismatched, ( "generated output drifted from golden in: " @@ -124,6 +124,40 @@ def test_generated_matches_golden(generated_tree: Path) -> None: ) +def test_generated_sync_matches_golden(generated_tree: Path) -> None: + """Sync-surface counterpart of test_generated_matches_golden. + + generated_tree points at the "python" artifact; the sync surface's + output lives in the sibling "python_sync" artifact directory produced by + the same pgn generate call (see the generated_tree fixture). + """ + generated_tree_sync = generated_tree.parent.parent / "artifacts" / "python_sync" + produced_root = generated_tree_sync / GENERATED_SUBTREE_SYNC + golden_root = GOLDEN_DIR_SYNC / GENERATED_SUBTREE_SYNC + + produced = _relative_files(produced_root) + golden = _relative_files(golden_root) + + missing = sorted(str(p) for p in golden - produced) + extra = sorted(str(p) for p in produced - golden) + assert not missing, f"golden files not produced by the generator: {missing}" + assert not extra, f"generator emitted files absent from golden: {extra}" + + mismatched: list[str] = [] + for rel in sorted(produced, key=str): + if (produced_root / rel).read_text() != (golden_root / rel).read_text(): + mismatched.append(str(rel)) + + if (generated_tree_sync / FACADE_INIT_SYNC).read_text() != (GOLDEN_DIR_SYNC / FACADE_INIT_SYNC).read_text(): + mismatched.append(str(FACADE_INIT_SYNC)) + + assert not mismatched, ( + "generated sync output drifted from golden in: " + + ", ".join(mismatched) + + "\nupdate via: mise run golden (see tests/golden_sync/README.md)" + ) + + def test_generated_passes_basedpyright_strict(tmp_path: Path) -> None: """basedpyright strict on the FULL golden package: zero errors and warnings. @@ -166,6 +200,37 @@ def test_generated_passes_basedpyright_strict(tmp_path: Path) -> None: ) +def test_generated_sync_passes_basedpyright_strict(tmp_path: Path) -> None: + """basedpyright strict on the full sync-surface golden package.""" + config = tmp_path / "pyrightconfig.json" + _ = config.write_text( + json.dumps( + { + "pythonVersion": "3.12", + "typeCheckingMode": "strict", + "include": [str(GOLDEN_DIR_SYNC / "src")], + "venvPath": str(HARNESS_ROOT), + "venv": ".venv", + "reportMissingModuleSource": False, + } + ) + ) + result = subprocess.run( + ["basedpyright", "--project", str(config), "--outputjson"], + capture_output=True, + text=True, + ) + + if not result.stdout.strip(): + pytest.fail(f"basedpyright produced no JSON (exit {result.returncode}):\n{result.stderr}") + + summary = json.loads(result.stdout)["summary"] + assert summary["filesAnalyzed"] > 0, f"basedpyright analyzed no files; bad include path?\n{result.stdout}" + assert summary["errorCount"] == 0 and summary["warningCount"] == 0, ( + f"basedpyright strict reported issues: {summary}\n{result.stdout}" + ) + + @pytest.fixture def roundtrip_db(pgn_admin_url: str) -> Iterator[str]: """A uniquely named throwaway database on pg0, dropped on teardown.""" @@ -215,6 +280,16 @@ def _import_client(full_package: Path): # noqa: ANN202 - dynamic module set return importlib.import_module +def _import_client_sync(full_package_sync: Path): # noqa: ANN202 - dynamic module set + src = str(full_package_sync / "src") + if src not in sys.path: + sys.path.insert(0, src) + for name in list(sys.modules): + if name == "specimen_sync_client" or name.startswith("specimen_sync_client."): + del sys.modules[name] + return importlib.import_module + + def test_roundtrip_type_mappings(full_package: Path, roundtrip_db: str) -> None: """INSERT then SELECT through the generated client, asserting the mappings. @@ -389,37 +464,27 @@ def test_require_array_rejects_unregistered_enum_array(full_package: Path) -> No _ = runtime.require_array("{happy,sad}") -def test_roundtrip_sync_and_cross_surface_identity(full_package: Path, roundtrip_db: str) -> None: - """The sync surface decodes through the same shared Row types as async. +def test_roundtrip_sync_surface(full_package_sync: Path, roundtrip_db: str) -> None: + """The sync surface, generated independently with config.sync = true. - Drives the generated sync functions (psycopg.Connection, no await) end to end, - and asserts the Row dataclasses are shared across surfaces: both facades and - both statement modules expose the same class object (defined once in - _generated._rows), so a value typed against one surface is the other's too. + Drives the generated sync functions (psycopg.Connection, no await) end + to end, at the same unified paths the async surface uses (no `.sync.` + sub-path) — this package was generated standalone, not alongside async. """ _apply_migrations(roundtrip_db) - import_module = _import_client(full_package) + import_module = _import_client_sync(full_package_sync) - register = import_module("specimen_client._generated.sync._register") - mood_mod = import_module("specimen_client._generated.types.mood") - point_mod = import_module("specimen_client._generated.types.point_2_d") - insert = import_module("specimen_client._generated.sync.statements.insert_specimen") - get = import_module("specimen_client._generated.sync.statements.get_specimen") - by_moods = import_module("specimen_client._generated.sync.statements.list_specimens_by_moods") - bump = import_module("specimen_client._generated.sync.statements.bump_specimen_revision") + register = import_module("specimen_sync_client._generated._register") + mood_mod = import_module("specimen_sync_client._generated.types.mood") + point_mod = import_module("specimen_sync_client._generated.types.point_2_d") + insert = import_module("specimen_sync_client._generated.statements.insert_specimen") + get = import_module("specimen_sync_client._generated.statements.get_specimen") + by_moods = import_module("specimen_sync_client._generated.statements.list_specimens_by_moods") + bump = import_module("specimen_sync_client._generated.statements.bump_specimen_revision") Mood = mood_mod.Mood Point2D = point_mod.Point2D - # Cross-surface identity: the async facade, the sync facade, the sync - # statement module, and the shared _rows module all expose the same object. - async_facade = import_module("specimen_client") - sync_facade = import_module("specimen_client.sync") - rows_mod = import_module("specimen_client._generated._rows") - assert async_facade.InsertSpecimenRow is sync_facade.InsertSpecimenRow - assert insert.InsertSpecimenRow is rows_mod.InsertSpecimenRow - assert async_facade.InsertSpecimenRow is rows_mod.InsertSpecimenRow - conn = psycopg.connect(roundtrip_db, autocommit=True) try: register.register_types(conn) @@ -518,15 +583,15 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_roundtrip_single_field_composite_sync(full_package: Path, roundtrip_db: str) -> None: +def test_roundtrip_single_field_composite_sync(full_package_sync: Path, roundtrip_db: str) -> None: """Sync-surface counterpart of test_roundtrip_single_field_composite.""" _apply_migrations(roundtrip_db) - import_module = _import_client(full_package) + import_module = _import_client_sync(full_package_sync) - register = import_module("specimen_client._generated.sync._register") - tag_mod = import_module("specimen_client._generated.types.tag_value") - insert = import_module("specimen_client._generated.sync.statements.insert_tagged_item") - get = import_module("specimen_client._generated.sync.statements.get_tagged_item") + register = import_module("specimen_sync_client._generated._register") + tag_mod = import_module("specimen_sync_client._generated.types.tag_value") + insert = import_module("specimen_sync_client._generated.statements.insert_tagged_item") + get = import_module("specimen_sync_client._generated.statements.get_tagged_item") TagValue = tag_mod.TagValue From 5bab734da5014e4c26046ffa1570d1091f117081 Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sun, 12 Jul 2026 09:15:02 +0300 Subject: [PATCH 18/21] python.gen: fix config-variant sync detection for unified output paths --- tests/test_config_variants.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/test_config_variants.py b/tests/test_config_variants.py index e0d3ecf..9a46b07 100644 --- a/tests/test_config_variants.py +++ b/tests/test_config_variants.py @@ -4,7 +4,7 @@ project1.pgn.yaml drives a different subset of `config` keys through the same compile.dhall and the assertions below record what pgn 0.6.5 was observed to do, not a documented contract. These variants are pinned by the directory/package -name and sync-surface presence they produce, not a full golden tree. +name and which surface (sync or async) they produce, not a full golden tree. """ from __future__ import annotations @@ -40,40 +40,52 @@ def _package_dir(generated_tree: Path, artifact_key: str) -> Path: return packages[0] +def _is_sync(package: Path) -> bool: + """Whether the generated package's runtime module is the sync surface. + + There is no `sync/` subdirectory to check anymore (config.sync selects + which content is rendered at the *same* `_runtime.py` path); the async + body defines `async def fetch_many`, the sync body defines `def + fetch_many` with no `async`. + """ + runtime = (package / "_generated" / "_runtime.py").read_text() + return "async def fetch_many" not in runtime + + def test_name_only_config_derives_package_and_defaults_sync_off(generated_tree: Path) -> None: package = _package_dir(generated_tree, "python-name-only") assert package.name == "name_only_client" - assert not (package / "sync").exists() + assert not _is_sync(package) def test_sync_only_config_defaults_package_name_from_project(generated_tree: Path) -> None: package = _package_dir(generated_tree, "python-sync-only") assert package.name == "fixture" - assert (package / "sync" / "__init__.py").is_file() + assert _is_sync(package) def test_empty_config_object_defaults_both_fields(generated_tree: Path) -> None: package = _package_dir(generated_tree, "python-empty") assert package.name == "fixture" - assert not (package / "sync").exists() + assert not _is_sync(package) def test_absent_config_key_defaults_both_fields(generated_tree: Path) -> None: """No `config:` key at all decodes the same as `config: {}` (None Config).""" package = _package_dir(generated_tree, "python-bare") assert package.name == "fixture" - assert not (package / "sync").exists() + assert not _is_sync(package) def test_unknown_config_key_is_ignored_not_rejected(generated_tree: Path) -> None: """An extra key not in the generator's Config type (bogusField) does not fail generation.""" package = _package_dir(generated_tree, "python-unknown-key") assert package.name == "unknown_key_client" - assert not (package / "sync").exists() + assert not _is_sync(package) def test_null_value_decodes_as_absent_field(generated_tree: Path) -> None: - """`emitSync: null` decodes to None, same as omitting the key (default False).""" + """`sync: null` decodes to None, same as omitting the key (default False).""" package = _package_dir(generated_tree, "python-null") assert package.name == "null_client" - assert not (package / "sync").exists() + assert not _is_sync(package) From 28928023edcca13158062cb1f05ea7eb0faecc6c Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sun, 12 Jul 2026 09:25:23 +0300 Subject: [PATCH 19/21] python.gen: document the exclusive sync/async config change --- CHANGELOG.md | 13 ++++++ DESIGN.md | 117 +++++++++++++++++++++------------------------------ README.md | 37 ++++++++++------ 3 files changed, 84 insertions(+), 83 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b55b2bf..a56b709 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Upcoming +- **Breaking:** `emitSync` is gone. In its place, `sync : Optional Bool` + (default `False`) picks exactly one surface per generate — async or sync — + emitted at the same unified paths either way (no more `sync/` subdirectory, + no more second package-root facade). Previously `emitSync: true` added a + second, nested sync tree alongside the always-emitted async one; a project + that needs both surfaces now generates two artifacts against this same + `gen:` with different `packageName`s, one with `sync: true` and one + without. See `docs/plans/2026-07-12-configurable-sync-output.md` for the + full rationale and migration shape. `tests/golden_sync/` is a new committed + golden fixture (`specimen_sync_client`) exercising the sync surface + end-to-end (basedpyright strict + round-trip), alongside the existing + `tests/golden/` (`specimen_client`, now async-only). + - `buildLookup` (`Interpreters/Project.dhall`) and, with it, this generator's last dependency on pgn's fork-only `Text/equal` builtin are removed from `src/`: custom-type decode/encode now dispatches through named diff --git a/DESIGN.md b/DESIGN.md index 952e4cc..734a0b0 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -6,9 +6,10 @@ and golden output under `tests/`. When this doc and the tree disagree, the tree wins. The generator turns a pgn project (queries and custom types, analyzed against -a live PostgreSQL) into a typed Python database client. It emits two parallel -surfaces, async (`psycopg.AsyncConnection`) and sync (`psycopg.Connection`), -over one shared set of Row dataclasses and decode functions. +a live PostgreSQL) into a typed Python database client. It emits exactly one +surface per generate — async (`psycopg.AsyncConnection`) by default, or sync +(`psycopg.Connection`) when `config.sync` is `True` — at the same unified +output paths either way. Constraints that bound every decision below: @@ -30,47 +31,37 @@ artifact's own root. Every `path` is relative to that root and starts with by `_` (`my-db-client` -> `my_db_client`). The generator emits the entire `src//_generated/` subtree plus -the two package-root facade modules (`__init__.py` and, when sync is on, -`sync/__init__.py`). Everything else, a `pyproject.toml` and a `py.typed` -marker, is the consumer's own hand-written shell; the generator never -touches it. +the package-root facade module (`__init__.py`). Everything else, a +`pyproject.toml` and a `py.typed` marker, is the consumer's own hand-written +shell; the generator never touches it. Every emitted `.py` file starts with the exact first line `# @generated by python.gen (pGenie). DO NOT EDIT.` followed by a blank line. The header is prepended once in `Interpreters/Project.dhall` (`withHeader`) so it applies to all modules. -### What the generator emits (async surface, always) +### What the generator emits -```text -src//__init__.py # generated flat facade (re-exports everything) -src//_generated/__init__.py # docstring + __all__: list[str] = [] -src//_generated/_runtime.py # async I/O helpers + JsonValue + NoRowError -src//_generated/_rows.py # SHARED Row dataclasses + decode_ fns -src//_generated/_register.py # only if the project has composites or enums -src//_generated/types/__init__.py # only if customTypes is non-empty -src//_generated/types/.py # one per custom type (enum / composite) -src//_generated/statements/__init__.py -src//_generated/statements/.py # one thin async wrapper per query ``` - -### What the generator adds when `emitSync` - -```text -src//sync/__init__.py # generated sync facade -src//_generated/sync/__init__.py -src//_generated/sync/_runtime.py # sync I/O helpers; re-exports JsonValue/NoRowError -src//_generated/sync/_register.py # only if composites or enums -src//_generated/sync/statements/__init__.py -src//_generated/sync/statements/.py # one thin sync wrapper per query +src//__init__.py # generated facade (re-exports everything) +src//_generated/__init__.py +src//_generated/_core.py # surface-agnostic: JsonValue, NoRowError, require_array +src//_generated/_runtime.py # I/O helpers for whichever surface config.sync picked +src//_generated/_register.py # only if composites or enums +src//_generated/_rows.py # shared Row dataclasses + decode functions +src//_generated/statements/__init__.py +src//_generated/statements/.py # one thin wrapper per query, def or async def +src//_generated/types/__init__.py # only if custom types exist +src//_generated/types/.py ``` -The sync surface reuses the SAME `_rows.py` and `types/` modules. Only the -I/O wrappers (statements, runtime, register) and the sync facade are -sync-specific. Sync statement modules sit one package deeper, so they reach -the shared modules with an extra dot (`..._rows`, `...types`); the async -ones use `.._rows`, `..types`. This dot-depth difference is exactly what -`Structures/Surface.dhall` parameterizes (section 4). +Exactly one surface is emitted per generate, picked by `config.sync` +(`False`, the default, emits async; `True` emits sync) — the tree shape and +every import path are identical either way; only the statement modules' +`def`/`async def`, `Connection`/`AsyncConnection`, and `_runtime.py`'s body +change. A project that needs both surfaces generates two artifacts against +this same `gen:` with different `packageName`s, one with `sync: true` and +one without. ### Naming (all from the pgn `Name` object, never recomputed) @@ -164,8 +155,8 @@ Self-contained, emitted once per surface from `Templates/RuntimeModule.dhall` | `Void` | `execute_void` | `None` | The async runtime defines `JsonValue` and `NoRowError`; the sync runtime -re-exports both from the async one (`from .._runtime import JsonValue, -NoRowError`) so there is one canonical identity. The missing-single-row case +re-exports both from `_core` (`from ._core import JsonValue, NoRowError`) +so there is one canonical identity. The missing-single-row case raises `NoRowError(RuntimeError)`. Result helpers open `conn.cursor(row_factory=dict_row)` and pass each row @@ -191,31 +182,24 @@ narrows at its own boundary. ## 4. Surface mechanism (async / sync) -`Structures/Surface.dhall` is a five-field token table that captures -everything that varies between the two surfaces: - ```dhall { defKeyword : Text -- "async def" | "def" , connType : Text -- "AsyncConnection" | "Connection" , awaitKw : Text -- "await " | "" -, rowsImport : Text -- ".._rows" | "..._rows" -, typesPrefix: Text -- "..types" | "...types" +, rowsImport : Text -- ".._rows" (same depth for both surfaces) +, corePrefix : Text -- ".._core" (same depth for both surfaces) +, typesPrefix : Text -- "..types" (same depth for both surfaces) } ``` -`Surface.async` and `Surface.sync` are the two values. The statement-module -template (`Templates/StatementModule.dhall`) and the register template take -a `surface` field and render `${surface.defKeyword} ${functionName}(... conn: -${surface.connType}[object] ...)`, `return ${surface.awaitKw}${helperName}(...)`, -and the surface-correct relative imports. `Interpreters/Query.dhall` renders -each query twice (`mkModule Surface.async`, `mkModule Surface.sync`) and -returns both an `asyncContent`/`asyncModulePath` and a -`syncContent`/`syncModulePath`. `Interpreters/Project.dhall` always emits the -async files; the sync files are emitted only `if config.emitSync`. - -The type layer (`_rows.py`, `types/`) is rendered once and is -surface-agnostic, so adding the sync surface costs only the thin wrappers, -not a second copy of the types or decode. +`Surface.async` and `Surface.sync` are the two values. `Interpreters/ +Project.dhall` picks one — `if config.sync then Surface.sync else +Surface.async` — and threads it through a single render pass: +`Interpreters/Query.dhall` calls the statement-module template once per +query (not twice), and `Interpreters/Project.dhall` picks the matching +`_runtime.py` body and, when custom types exist, the matching `_register.py` +body. `_rows.py` and `types/` are surface-agnostic and always render exactly +once, so switching `config.sync` costs only the thin I/O wrappers. --- @@ -398,15 +382,10 @@ matching `__all__`: - every custom type from `_generated/types/`, - every Row dataclass from `_generated/_rows`, -- every statement function from `_generated/statements/` (async) or - `_generated/sync/statements/` (sync). +- every statement function from `_generated/statements/`. -So consumers do `from my_db_client import get_specimen, GetSpecimenRow, -Mood` against the flat root, and `from my_db_client.sync import ...` for the -sync surface. The async facade lives at `/__init__.py` (prefix -`._generated`, statements under `statements`); the sync facade lives at -`/sync/__init__.py` (prefix `.._generated`, statements under -`sync.statements`). Both re-export the SAME Rows and enums. +The facade lives at `/__init__.py` (prefix `._generated`, statements +under `statements`) regardless of which surface `config.sync` selected. This file is GENERATED: it carries the `@generated` header and is overwritten on every run. Do not hand-edit it. @@ -424,9 +403,9 @@ let OnUnsupported = ./Structures/OnUnsupported.dhall let ProjectInterpreter = ./Interpreters/Project.dhall -let Config = { packageName : Optional Text, emitSync : Optional Bool, onUnsupported : Optional OnUnsupported.Mode } +let Config = { packageName : Optional Text, sync : Optional Bool, onUnsupported : Optional OnUnsupported.Mode } -let Config/default = { packageName = None Text, emitSync = None Bool, onUnsupported = None OnUnsupported.Mode } +let Config/default = { packageName = None Text, sync = None Bool, onUnsupported = None OnUnsupported.Mode } in Sdk.Sigs.generator Config Config/default ProjectInterpreter.run ``` @@ -452,7 +431,7 @@ src/ package.dhall # Config, Config/default, Sdk.Sigs.generator Config Config/default ProjectInterpreter.run Deps/ # pinned remote imports: gen-sdk, gen-contract, lude, dhall Prelude Structures/ - Surface.dhall # async/sync token table (section 4) + Surface.dhall # async/sync token table, both surfaces at the same import depth (section 4) ImportSet.dhall # per-module import flags + combine PyIdent.dhall # sanitize names that become Python identifiers (keyword -> name_) OnUnsupported.dhall # < Fail | Skip > (section 11) @@ -466,7 +445,7 @@ src/ QueryFragments.dhall # fragments -> escaped, %%-doubled, named-style SQL literal ResultColumns.dhall # Rows columns -> Row field lines + decode body Result.dhall # Void|RowsAffected|Rows -> return type + helper + rowClass - Query.dhall # assemble one query: shared rowDef + async/sync statement modules + Query.dhall # assemble one query: shared rowDef + one statement module for whichever surface config.sync picked Project.dhall # traverse queries+customTypes, assemble all files + facade + header, # apply the Skip filter (section 11) Templates/ @@ -500,13 +479,13 @@ renders custom-type imports in encounter order, unsorted and undeduped ### Config flow `package.dhall`'s `Config` is the user-facing type `{ packageName : Optional -Text, emitSync : Optional Bool, onUnsupported : Optional OnUnsupported.Mode +Text, sync : Optional Bool, onUnsupported : Optional OnUnsupported.Mode }`, passed straight through to `Interpreters/Project.dhall` as its own `Config` -- there is no separate config type or resolve step in between. `Project.run` folds the optional config, and each of its optional fields, into the fully-resolved `ResolvedConfig` it passes to every interpreter -below it: `{ packageName, importName, emitSync, onUnsupported }`, with -`packageName` falling back to the project name in kebab case, `emitSync` to +below it: `{ packageName, importName, sync, onUnsupported }`, with +`packageName` falling back to the project name in kebab case, `sync` to `False`, and `onUnsupported` to `Fail`. `importName` = `packageName` with `-` -> `_`. A project's artifact config can therefore omit `config:` entirely, supply `config: {}`, or set any subset of the three keys; see the diff --git a/README.md b/README.md index 9e9a17f..a72eb9e 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,10 @@ A Dhall-authored code generator for [pgenie](https://github.com/pgenie-io/pgenie pgn reads a project's SQL queries and custom types from a live PostgreSQL, infers parameter and result types, then hands them to this generator, which -emits a typed Python client: one async client (`psycopg.AsyncConnection`), -optionally a parallel sync one (`psycopg.Connection`), sharing a single set -of Row dataclasses, decode functions, and enum/composite types. Generated -code depends on `psycopg>=3.2` and the stdlib only, nothing else. +emits a typed Python client: async (`psycopg.AsyncConnection`) by default, +or sync (`psycopg.Connection`) when `config.sync` is `True`, at the same +unified output paths either way. Generated code depends on `psycopg>=3.2` +and the stdlib only, nothing else. Every emitted file starts with `# @generated by python.gen (pGenie). DO NOT EDIT.` and an `SPDX-License-Identifier: MIT-0` header: the generated code is @@ -18,9 +18,10 @@ yours to use under your own project's terms, no attribution required. ## Key features -- **Async and sync clients over psycopg 3.** One shared set of Row dataclasses, - decode functions, and enum/composite types serves both surfaces; install - `psycopg[binary]` for the fast C implementation. +- **Async or sync client over psycopg 3.** `config.sync` (default `False`) + picks exactly one surface — the async client (`psycopg.AsyncConnection`) or + the sync one (`psycopg.Connection`) — at the same output paths either way; + install `psycopg[binary]` for the fast C implementation. - **Fully typed, end to end.** Parameter and result types are inferred from your live PostgreSQL by preparing the real queries, and every emitted file passes `basedpyright` strict with zero errors or warnings. @@ -38,7 +39,7 @@ artifacts: gen: https://raw.githubusercontent.com/slavashvets/python.gen/master/src/package.dhall config: packageName: my-db-client - emitSync: true + sync: true ``` `master` moves; once a `v0.1.0` tag exists, pin to the tagged raw URL instead @@ -76,7 +77,7 @@ as long as they resolve to the same source. | key | type | default | | --------------- | ------------------ | ------------------------------ | | `packageName` | `Text` | the project name, kebab-cased | -| `emitSync` | `Bool` | `False` | +| `sync` | `Bool` | `False` | | `onUnsupported` | `"Fail" \| "Skip"` | `"Fail"` | All three keys are optional, and so is the `config:` block itself. Decode @@ -87,7 +88,7 @@ generator's own test suite rather than assumed): - a key set to `null` decodes as absent, i.e. its default. - an unknown key is ignored, not rejected as a schema error. - each key falls back to its own default independently, so a partial config - (e.g. only `emitSync`) works field by field. + (e.g. only `sync`) works field by field. `packageName` becomes the Python import name by replacing `-` with `_` (`my-db-client` -> `my_db_client`). @@ -231,14 +232,22 @@ project has composites or enums, call `register_types(conn)` once per connection before relying on native composite decode or enum-array results; scalar enums do not need it. -With `emitSync: true`, a second surface appears under `my_db_client.sync`, -built from the exact same Row types and custom types as the async one, only -the I/O wrappers differ: +With `sync: true`, the entire client is generated against +`psycopg.Connection` instead — same import path, same facade shape, only the +function signatures and I/O become synchronous: ```python -from my_db_client.sync import get_specimen, GetSpecimenRow +from my_db_client import get_specimen, GetSpecimenRow, Mood + +def handler(conn): + row = get_specimen(conn, id=42) ``` +A project that needs both an async backend and a sync consumer (e.g. a +Dagster pipeline) generates two artifacts pointed at this same `gen:`, one +with `sync: true` and one without, each with its own `packageName` — not one +artifact with both surfaces bundled together. + ## Development This repo's own harness (`tests/`) needs a live PostgreSQL: it runs `pgn From a0b527e7c66b5f96bc79dd5f741086b0944aeb3e Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sun, 12 Jul 2026 10:09:00 +0300 Subject: [PATCH 20/21] Add demos to fixtures task --- docs/tasks/demos-to-fixtures.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 docs/tasks/demos-to-fixtures.md diff --git a/docs/tasks/demos-to-fixtures.md b/docs/tasks/demos-to-fixtures.md new file mode 100644 index 0000000..fe5b6dd --- /dev/null +++ b/docs/tasks/demos-to-fixtures.md @@ -0,0 +1,32 @@ +# python.gen: `demos/` → `fixtures/` rename + +Postponed from the cross-generator rename. Apply when ready. + +## Directory rename + +```bash +cd /Users/mojojojo/repos/pgenie/python.gen +git mv demos/ fixtures/ +``` + +## File edits + +| File | Changes | +|---|---| +| `build.bash` | `demos/Exhaustive.dhall` → `fixtures/Exhaustive.dhall`; `regenerate_demo_output` → `regenerate_fixture_output` | +| `.github/workflows/ci.yml` | `demos/Exhaustive.dhall` → `fixtures/Exhaustive.dhall` | +| `.github/scripts/build-contract-shell.sh` | comment: `demos/Exhaustive.dhall` → `fixtures/Exhaustive.dhall` | +| `DESIGN.md` | 3 references: lines 646, 766, 770 — `demos/Exhaustive.dhall` → `fixtures/Exhaustive.dhall` | +| `CHANGELOG.md` | Add Upcoming entry documenting the rename | + +## Cleanup + +```bash +rm -rf .superpowers/sdd/ +``` + +## Left as-is + +- `Sdk.Fixtures.Exhaustive` import (stays) +- `tests/fixture-project/` (stays — separate concept) +- `notes.md` (historical) \ No newline at end of file From 82bd4ade5e159e0df0a3c71d2983fa554301de2e Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sun, 12 Jul 2026 10:09:34 +0300 Subject: [PATCH 21/21] Drop the task doc --- .../2026-07-12-configurable-sync-output.md | 1367 ----------------- 1 file changed, 1367 deletions(-) delete mode 100644 docs/plans/2026-07-12-configurable-sync-output.md diff --git a/docs/plans/2026-07-12-configurable-sync-output.md b/docs/plans/2026-07-12-configurable-sync-output.md deleted file mode 100644 index 5c047b4..0000000 --- a/docs/plans/2026-07-12-configurable-sync-output.md +++ /dev/null @@ -1,1367 +0,0 @@ -# Configurable Sync Output Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace the additive `emitSync : Optional Bool` config knob (which always emits async and optionally bolts a second `sync/` tree on top) with an exclusive `sync : Optional Bool` knob, default `False`, that selects exactly one surface — async or sync — and emits it at the same unified file paths either way, so flipping the flag never changes the shape of the output tree or any import path, only file contents. - -**Architecture:** `Interpreters/Project.dhall` picks one `Surface.Type` value (`Surface.async` when `config.sync` is `False`, `Surface.sync` when `True`) and threads it through a single render pass instead of today's two passes (always-async + conditionally-additive-sync). `Interpreters/Query.dhall`'s per-query `Output` collapses its `asyncModulePath`/`asyncContent`/`syncModulePath`/`syncContent` two-surface duplication down to one `modulePath`/`content` pair. `Structures/Surface.dhall`'s import-depth tokens (`rowsImport`, `corePrefix`, `typesPrefix`) collapse to the same value for both surfaces, since sync statement modules no longer live one package deeper under `sync/statements/` — they live at the exact path async statements use today. `_rows.py` and `types/` are untouched by this plan (still surface-agnostic, still emitted once); splitting `_rows.py` per statement is a separate, later plan (`docs/plans/2026-07-12-distribute-rows-per-statement.md`) that depends on this one landing first. - -**Tech Stack:** Dhall (dhall-lang 1.42 vendored via `dhall` CLI; pgn's forked interpreter for the parts still using it), Python 3.12 generated output (psycopg3), basedpyright strict, pytest golden-file harness (`mise run test`, `mise run golden`), pgn 0.9.1 (mise-pinned). - -## Global Constraints - -- No behavior change to param/result type mapping, custom-type codecs, or SQL rendering — this plan is scoped to the sync/async surface-selection mechanism only. -- Default behavior when `config.sync` is omitted must be byte-identical to today's `emitSync` omitted/false default (async only, same paths) — verified by golden diff. -- Golden fixture output (`tests/golden/`, new `tests/golden_sync/`) must be regenerated via `mise run golden`, not hand-edited (per `tests/golden/README.md`). -- `mise run test` (pytest, includes `test_generated_passes_basedpyright_strict`) must pass after regeneration, for both the async and sync golden trees. -- Every Dhall file touched must independently type-check via the whole-generator check: `dhall type --file=src/package.dhall`. -- This is a breaking config change (the `emitSync` key stops existing). Per this repo's convention (see `CHANGELOG.md`'s "Upcoming" section and the "Drop the plans" precedent), document it there — no deprecation shim, no dual-key transition period. -- CI's memory-reduction script (`.github/workflows/ci.yml`, "Reduce the fixture project to the python artifact") truncates `tests/fixture-project/project1.pgn.yaml` at the `# The variants below` marker. Any new artifact that must run in CI (not just locally) has to be placed *before* that marker. - ---- - -## File Structure - -| File | Change | -|---|---| -| `src/package.dhall` | Public `Config`/`Config/default`: rename `emitSync` field to `sync`; rewrite doc comment. | -| `src/Interpreters/Project.dhall` | `Config`/`ResolvedConfig`: rename field. `run`: rename resolve binding. `combineOutputs`: replace always-async-plus-optional-sync emission with single-surface emission at unified paths. | -| `src/Interpreters/Query.dhall` | `Config`: rename field. `Output`: collapse `asyncModulePath`/`asyncContent`/`syncModulePath`/`syncContent` to `modulePath`/`content`. `render`: pick one surface, call `StatementModule.run` once instead of twice. | -| `src/Structures/Surface.dhall` | Collapse `sync`'s `rowsImport`/`corePrefix`/`typesPrefix` to the same depth as `async`'s (no more extra-dot nesting). Rewrite header comment. | -| `src/Templates/RuntimeModule.dhall` | Update the `syncContent` doc comment (no longer describes a `sync/_runtime.py` path). | -| `src/Interpreters/{Value,ResultColumns,Member,Scalar,QueryFragments,ParamsMember,CustomType,Primitive,Result}.dhall` | Mechanical rename: local `Config` type's `emitSync : Bool` field → `sync : Bool` (each is a narrowing projection of `ResolvedConfig`, never itself branches on the value except `Result.dhall`'s field-selection literal). | -| `demos/Exhaustive.dhall` | Rename `emitSync = Some True` → `sync = Some True`. | -| `tests/fixture-project/project1.pgn.yaml` | Main `python:` artifact: drop `emitSync: true` (default `sync: false` going forward). Add new `python-sync:` artifact (`sync: true`), placed before the CI truncation marker. Rename `emitSync` → `sync` in the `python-sync-only` and `python-null` variants. | -| `tests/golden_sync/` (new) | Hand-written shell (`pyproject.toml`, `README.md`, `src/specimen_sync_client/py.typed`) mirroring `tests/golden/`'s shell, for the new `python-sync` artifact's committed golden tree. | -| `tests/_harness.py` | Add `GOLDEN_DIR_SYNC` path constant. | -| `tests/conftest.py` | Add a `full_package_sync` fixture mirroring `full_package`, sourced from the `python_sync` artifact directory and `GOLDEN_DIR_SYNC`. | -| `tests/test_generated.py` | Drop `SYNC_FACADE_INIT` and the two-facade loop in `test_generated_matches_golden`. Add `test_generated_sync_matches_golden` and `test_generated_sync_passes_basedpyright_strict`. Delete `test_roundtrip_sync_and_cross_surface_identity` (cross-surface identity no longer exists once surfaces are exclusive); replace with `test_roundtrip_sync_surface` against `full_package_sync`. Move `test_roundtrip_single_field_composite_sync` onto `full_package_sync` with unified (non-`.sync.`) import paths. | -| `tests/test_config_variants.py` | Replace the `(package / "sync").exists()` presence check (no longer meaningful once paths are unified) with a content-based signal read from `_generated/_runtime.py`. Rename `emitSync` → `sync` throughout prose and assertions. | -| `mise.toml` | `golden` task: also generate and rsync the `python-sync` artifact into `tests/golden_sync/`. | -| `README.md` | Config reference table, quickstart example, "Key features," "Using the generated code" section. | -| `DESIGN.md` | Sections 1 ("What the generator emits"), 3 (runtime import example), 4 ("Surface mechanism"), 9 (facade), 10 (Config flow) — rewrite to describe exclusive single-surface selection at unified paths. | -| `CHANGELOG.md` | New "Upcoming" entry documenting the breaking config change. | - ---- - -### Task 1: Rename `emitSync` to `sync` across the Config-threading chain (no behavior change) - -Pure rename first, isolated from the behavior change in Task 2, so it is independently verifiable: after this task, generation is byte-identical to before, just keyed on a renamed config field. - -**Files:** -- Modify: `src/package.dhall` -- Modify: `src/Interpreters/Project.dhall` -- Modify: `src/Interpreters/Query.dhall` -- Modify: `src/Interpreters/Value.dhall`, `ResultColumns.dhall`, `Member.dhall`, `Scalar.dhall`, `QueryFragments.dhall`, `ParamsMember.dhall`, `CustomType.dhall`, `Primitive.dhall`, `Result.dhall` -- Modify: `demos/Exhaustive.dhall` - -**Interfaces:** -- Produces: every interpreter's `Config` type now has a `sync : Bool` field (was `emitSync : Bool`); `Project.dhall`'s public `Config`/`ResolvedConfig` and `src/package.dhall`'s `Config`/`Config/default` match. Task 2 reads `config.sync` / `resolvedConfig.sync`. - -- [ ] **Step 1: Rename in `src/package.dhall`** - -```dhall -let Sdk = ./Deps/Sdk.dhall - -let OnUnsupported = ./Structures/OnUnsupported.dhall - -let ProjectInterpreter = ./Interpreters/Project.dhall - --- User-facing config for this generator. `sync` picks which single surface --- the generator emits: `False` (default) emits the async surface --- (psycopg.AsyncConnection); `True` emits the sync surface --- (psycopg.Connection) instead, at the exact same paths — flipping this --- flag never changes the output tree's shape or any import path, only file --- contents. `onUnsupported` picks Fail (default, abort loudly) or Skip (drop --- the unsupported statement/type and its dependents, with a warning) when a --- query or custom type hits a PG shape the generator cannot render; see --- Structures/OnUnsupported.dhall. All fields are Optional so a project may --- omit the whole config block or any subset of its keys; --- Interpreters/Project.dhall's `run` supplies the defaults. -let Config = - { packageName : Optional Text - , sync : Optional Bool - , onUnsupported : Optional OnUnsupported.Mode - } : Type - -let Config/default - : Config - = { packageName = None Text - , sync = None Bool - , onUnsupported = None OnUnsupported.Mode - } - -in Sdk.Sigs.generator Config Config/default ProjectInterpreter.run -``` - -- [ ] **Step 2: Rename in `src/Interpreters/Project.dhall`** - -Replace lines 35-53 (the `Config` and `ResolvedConfig` declarations): - -```dhall --- The generator's public Config: every field is independently Optional, so a --- project may omit the whole `config:` block or any subset of its keys. --- `run` below resolves the fallbacks itself (packageName from the project --- name, sync off (async), onUnsupported Fail); there is no separate config --- type or resolve step between package.dhall and here. -let Config = - { packageName : Optional Text - , sync : Optional Bool - , onUnsupported : Optional OnUnsupported.Mode - } - --- The fully-resolved shape every downstream interpreter (Query, CustomType, --- and everything below them) actually declares as its own `Config`. -let ResolvedConfig = - { packageName : Text - , importName : Text - , sync : Bool - , onUnsupported : OnUnsupported.Mode - } -``` - -Replace the `emitSync` resolve binding in `run` (around line 380-386): - -```dhall - let sync = - Prelude.Optional.fold - Bool - config.sync - Bool - (\(b : Bool) -> b) - False -``` - -And its use in building `resolvedConfig` (around line 398-400): - -```dhall - let resolvedConfig - : ResolvedConfig - = { packageName, importName, sync, onUnsupported } -``` - -Leave the rest of `combineOutputs`'s body as-is for this task — Task 2 rewrites that function's structure anyway — but rename the two literal occurrences of the field name inside it so Step 7's type-check passes. Line 272 (doc comment): - -```dhall - -- config.sync. It reuses the shared `_rows.py` and `types/`, so only -``` - -Line 325 (the conditional gating `syncFiles`): - -```dhall - let syncFiles = - if config.sync - then [ syncSubpackageInit -``` - -- [ ] **Step 3: Rename in `src/Interpreters/Query.dhall`** - -Line 28 (`Config` record): `, emitSync : Bool` → `, sync : Bool`. - -Line 40 comment ("emitted only when config.emitSync") → "emitted only when config.sync" (Task 2 rewrites this comment fully; this step is the minimal rename). - -- [ ] **Step 4: Rename the narrowing `Config` field in the nine passthrough interpreters** - -Each of these files declares a local `Config` type that is a narrowing projection of `ResolvedConfig`, purely so later Dhall record-selection syntax can pick the field out by name. None of them branch on the value except `Result.dhall`'s selection literal (next step). In each file, change the line `, emitSync : Bool` to `, sync : Bool`: - -| File | Line | -|---|---| -| `src/Interpreters/Value.dhall` | 18 | -| `src/Interpreters/ResultColumns.dhall` | 20 | -| `src/Interpreters/Member.dhall` | 20 | -| `src/Interpreters/Scalar.dhall` | 16 | -| `src/Interpreters/QueryFragments.dhall` | 16 | -| `src/Interpreters/ParamsMember.dhall` | 18 | -| `src/Interpreters/CustomType.dhall` | 22 | -| `src/Interpreters/Primitive.dhall` | 14 | -| `src/Interpreters/Result.dhall` | 25 | - -- [ ] **Step 5: Rename the field-selection literal in `src/Interpreters/Result.dhall`** - -Line 94: - -```dhall - config.{ packageName, importName, sync, onUnsupported } -``` - -- [ ] **Step 6: Rename in `demos/Exhaustive.dhall`** - -Line 29: - -```dhall - { packageName = None Text - , sync = Some True - , onUnsupported = Some OnUnsupported.Mode.Skip - } -``` - -- [ ] **Step 7: Type-check the whole generator** - -Run: `cd python.gen && dhall type --file=src/package.dhall` -Expected: prints the generator's function type, no errors. (This alone proves every renamed `Config` record still lines up across every call site — Dhall's structural typing would reject a mismatched field name.) - -- [ ] **Step 8: Confirm no `emitSync` references remain in `src/`** - -Run: `cd python.gen && grep -rn "emitSync" src/ demos/` -Expected: no output. - -- [ ] **Step 9: Commit** - -```bash -git add src/package.dhall src/Interpreters/Project.dhall src/Interpreters/Query.dhall \ - src/Interpreters/Value.dhall src/Interpreters/ResultColumns.dhall src/Interpreters/Member.dhall \ - src/Interpreters/Scalar.dhall src/Interpreters/QueryFragments.dhall src/Interpreters/ParamsMember.dhall \ - src/Interpreters/CustomType.dhall src/Interpreters/Primitive.dhall src/Interpreters/Result.dhall \ - demos/Exhaustive.dhall -git commit -m "python.gen: rename emitSync config field to sync (no behavior change)" -``` - ---- - -### Task 2: Make `sync` exclusive — unify output paths, single-surface emission - -**Files:** -- Modify: `src/Structures/Surface.dhall` -- Modify: `src/Interpreters/Query.dhall` -- Modify: `src/Interpreters/Project.dhall` -- Modify: `src/Templates/RuntimeModule.dhall` (comment only) -- Modify: `src/Templates/FacadeModule.dhall` (drop dead surface-selection params) - -**Interfaces:** -- Consumes: `resolvedConfig.sync : Bool` (from Task 1). -- Produces: `Query.dhall`'s `Output` record now has `modulePath : Text` and `content : Text` (replacing the four async/sync-prefixed fields) — any code outside this task that referenced `query.asyncModulePath` etc. must be updated in this same task, since Dhall's structural typing will fail closed on the old field names. - -- [ ] **Step 1: Collapse the import-depth tokens in `src/Structures/Surface.dhall`** - -Sync statement modules no longer live one package deeper (`sync/statements/`) — they live at the exact same `statements/` path async uses. Both surfaces now reach `_rows`, `_core`, and `types/` at the same relative depth. - -```dhall --- A code-generation surface: the async or sync flavour of a statement module. --- Exactly one surface is emitted per generate (Interpreters/Project.dhall --- picks async or sync from config.sync and renders everything at the same --- unified paths), so the two Surface values differ only in the tokens that --- vary between `async def`/`def`, `AsyncConnection`/`Connection`, and --- `await `/``. Both reach the shared `_rows`/`_core`/`types` modules at the --- same relative import depth, since neither surface is nested under a --- surface-named subdirectory. -let Surface = - { defKeyword : Text - , connType : Text - , awaitKw : Text - , rowsImport : Text - , corePrefix : Text - , typesPrefix : Text - } - -let async - : Surface - = { defKeyword = "async def" - , connType = "AsyncConnection" - , awaitKw = "await " - , rowsImport = ".._rows" - , corePrefix = ".._core" - , typesPrefix = "..types" - } - -let sync - : Surface - = { defKeyword = "def" - , connType = "Connection" - , awaitKw = "" - , rowsImport = ".._rows" - , corePrefix = ".._core" - , typesPrefix = "..types" - } - -in { Type = Surface, async, sync } -``` - -- [ ] **Step 2: Collapse `Query.dhall`'s per-surface duplication** - -Replace the `Output` record (lines 42-51): - -```dhall --- A query contributes a shared Row (assembled into `_rows.py` by Project) plus --- one thin statement module, rendered for whichever surface config.sync picked. -let Output = - { functionName : Text - , rowClassName : Optional Text - , rowDef : Optional RowsModule.RowDef - , rowImports : ImportSet.Type - , modulePath : Text - , content : Text - } -``` - -Replace the tail of `render` (from `let mkModule = ...` through the end of the function, lines 112-136): - -```dhall - let surface = if config.sync then Surface.sync else Surface.async - - let content = - StatementModule.run - { functionName - , returnType = result.returnType - , helperName = result.helperName - , callsDecode = result.callsDecode - , sqlLiteral = fragments.sqlLiteral - , rowClassName - , decodeName - , paramSigLines - , paramDictEntries - , imports = paramImports - , surface - } - - in { functionName - , rowClassName - , rowDef - , rowImports = result.imports - , modulePath = "statements/${functionName}.py" - , content - } -``` - -- [ ] **Step 3: Rewrite `Project.dhall`'s `combineOutputs` for single-surface emission** - -Replace the whole body from the `let asyncFacade = ...` binding (line 157) through `let allFiles = ...` (line 347) with: - -```dhall - let surface = if config.sync then Surface.sync else Surface.async - - let facade = - { path = packagePrefix ++ "__init__.py" - , content = - FacadeModule.run - { generatedPrefix = "._generated" - , statementsPath = "statements" - , statements = facadeStatements - , types = facadeTypes - } - } - - -- Surface-agnostic; performs no I/O, so exactly one copy is emitted - -- regardless of surface. Both runtime bodies re-export from it. - let coreModule = - { path = srcPrefix ++ "_core.py", content = CoreModule.run {=} } - - let runtimeModule = - { path = srcPrefix ++ "_runtime.py" - , content = - if config.sync then RuntimeModule.runSync {=} else RuntimeModule.run {=} - } - - let statementsInit = - { path = srcPrefix ++ "statements/__init__.py" - , content = - InitModule.run { docstring = "Generated SQL statements." } - } - - -- The shared Row dataclasses + decode functions, imported by the - -- statement modules of whichever surface was selected. - let rowDefs = - Prelude.List.concatMap - QueryGen.Output - RowsModule.RowDef - ( \(query : QueryGen.Output) -> - Prelude.Optional.toList RowsModule.RowDef query.rowDef - ) - queries - - let rowImports = - ImportSet.combineAll - ( Prelude.List.map - QueryGen.Output - ImportSet.Type - (\(query : QueryGen.Output) -> query.rowImports) - queries - ) - - let rowsFiles = - if Prelude.List.null RowsModule.RowDef rowDefs - then [] : List Lude.File.Type - else [ { path = srcPrefix ++ "_rows.py" - , content = - RowsModule.run { rows = rowDefs, imports = rowImports } - } - ] - - let statementFiles = - Prelude.List.map - QueryGen.Output - Lude.File.Type - ( \(query : QueryGen.Output) -> - { path = srcPrefix ++ query.modulePath - , content = query.content - } - ) - queries - - let typeFiles = - Prelude.List.map - CustomTypeGen.Output - Lude.File.Type - ( \(ct : CustomTypeGen.Output) -> - { path = srcPrefix ++ ct.modulePath - , content = ct.moduleContent - } - ) - customTypes - - let typesInitExports = - Prelude.List.map - CustomTypeGen.Output - TypesInit.Export - (\(ct : CustomTypeGen.Output) -> { moduleName = ct.moduleName, typeName = ct.typeName }) - customTypes - - let typesInitFiles = - if Prelude.List.null CustomTypeGen.Output customTypes - then [] : List Lude.File.Type - else [ { path = srcPrefix ++ "types/__init__.py" - , content = TypesInit.run { exports = typesInitExports } - } - ] - - let compositeNames = compositePgNames customTypes - - let enumNames = enumPgNames customTypes - - let hasCustomRegistration = - Prelude.Bool.not - ( Prelude.Bool.and - [ Prelude.List.null Text compositeNames - , Prelude.List.null Text enumNames - ] - ) - - let registerFiles = - if hasCustomRegistration - then [ { path = srcPrefix ++ "_register.py" - , content = - RegisterModule.run { compositeNames, enumNames, surface } - } - ] - else [] : List Lude.File.Type - - let staticFiles = - [ facade, topInit, coreModule, runtimeModule, statementsInit ] - - let allFiles = - staticFiles - # registerFiles - # rowsFiles - # typesInitFiles - # typeFiles - # statementFiles - - in Prelude.List.map Lude.File.Type Lude.File.Type withHeader allFiles - : Output -``` - -Note `topInit` (the `_generated/__init__.py` facade docstring file, defined earlier in the function and unchanged) is still referenced in `staticFiles` — leave its definition (around line 128-135 in the original) exactly as-is; only the block below it changes. - -- [ ] **Step 4: Drop `FacadeModule.dhall`'s dead surface-selection parameters** - -`generatedPrefix` and `statementsPath` exist only because the old design called `FacadeModule.run` twice with different values (once for the async facade at the package root, once for the sync facade one level down at `sync/__init__.py`). Now there is exactly one facade, always at the package root, so both become constants — hardcode them and drop them from `Params`. - -Replace the header comment and `Params` declaration (lines 13-24): - -```dhall --- A statement's public surface: the function and, when the query returns rows, --- its frozen Row dataclass. functionName doubles as the leaf module name. -let StatementExport = - { functionName : Text, rowClassName : Optional Text } - --- A custom type re-exported from types/: the leaf module name plus the class. -let TypeExport = { moduleName : Text, className : Text } - --- The facade always lives at the package root, importing from `_generated` --- (prefix `._generated`) and `_generated/statements` (statementsPath --- `statements`) — both constant now that exactly one surface is emitted per --- generate (Interpreters/Project.dhall picks it via config.sync). -let generatedPrefix = "._generated" - -let statementsPath = "statements" - -let Params = - { statements : List StatementExport - , types : List TypeExport - } -``` - -Replace every remaining `params.generatedPrefix` with `generatedPrefix` and every `params.statementsPath` with `statementsPath` in the rest of the file (the `runtimeBlock`, `typeBlock`, `rowBlock`, and `statementBlock` bindings inside `run`). - -Then, back in `Interpreters/Project.dhall`, simplify the `facade` binding from Step 3 above to drop the now-nonexistent fields: - -```dhall - let facade = - { path = packagePrefix ++ "__init__.py" - , content = - FacadeModule.run - { statements = facadeStatements, types = facadeTypes } - } -``` - -- [ ] **Step 5: Update the `syncContent` doc comment in `src/Templates/RuntimeModule.dhall`** - -Replace the comment at lines 84-88: - -```dhall --- The sync surface's body, selected in place of `content` at the same --- `_runtime.py` path when config.sync is True (Interpreters/Project.dhall). --- The five helpers are the same shape with `def`/`Connection`/`with`/no- --- `await`. JsonValue/NoRowError/require_array are re-exported from _core so --- both surfaces share one canonical identity rather than two equal-but- --- distinct definitions. -``` - -- [ ] **Step 6: Type-check the whole generator** - -Run: `cd python.gen && dhall type --file=src/package.dhall` -Expected: prints the generator's function type, no errors. - -- [ ] **Step 7: Delete the stale freeze file so the next `pgn generate` re-resolves the working tree** - -Run: `rm -f tests/fixture-project/freeze1.pgn.yaml` -Expected: file removed (pgn's freeze cache does not know the generator source changed underneath a stable relative path; a stale freeze would silently keep emitting the old two-tree output). - -- [ ] **Step 8: Commit** - -```bash -git add src/Structures/Surface.dhall src/Interpreters/Query.dhall src/Interpreters/Project.dhall \ - src/Templates/RuntimeModule.dhall src/Templates/FacadeModule.dhall -git commit -m "python.gen: make sync/async mutually exclusive at unified output paths" -``` - -(This task intentionally does not update `tests/fixture-project/project1.pgn.yaml` yet — the main `python:` artifact still has `sync: true` from before Task 1's rename, so at this point in the plan, generating against it produces sync-surface content at the paths async used to occupy, and `tests/golden/` has not been refreshed yet, so `test_generated_matches_golden` and the sync-specific round-trip tests are expected to fail until Task 3 and Task 4 land. This is fine mid-plan; the full suite is the gate at the end, not after every task.) - ---- - -### Task 3: Update the fixture project config - -**Files:** -- Modify: `tests/fixture-project/project1.pgn.yaml` - -**Interfaces:** -- Produces: a `python-sync` artifact (packageName `specimen-sync-client`) alongside the existing `python` artifact (now `sync` omitted, i.e. async by default), both ahead of the CI truncation marker. Task 4/5 consume `artifacts/python_sync` and the `specimen_sync_client` package name. - -- [ ] **Step 1: Rewrite `tests/fixture-project/project1.pgn.yaml`** - -```yaml -space: python-gen -name: fixture -version: 0.0.0 -postgres: 18 -artifacts: - python: - gen: ../../src/package.dhall - config: - packageName: specimen-client - python-sync: - gen: ../../src/package.dhall - config: - packageName: specimen-sync-client - sync: true - # The variants below pin pgn's actual YAML->Dhall decode semantics for the - # Optional Config knobs (undocumented by pgn itself); see - # tests/test_config_variants.py for the assertions. - python-name-only: - gen: ../../src/package.dhall - config: - packageName: name-only-client - python-sync-only: - gen: ../../src/package.dhall - config: - sync: true - python-empty: - gen: ../../src/package.dhall - config: {} - python-bare: - gen: ../../src/package.dhall - python-unknown-key: - gen: ../../src/package.dhall - config: - packageName: unknown-key-client - bogusField: 1 - python-null: - gen: ../../src/package.dhall - config: - packageName: null-client - sync: null -``` - -Note `python:` drops the `emitSync: true` key entirely (was: `packageName: specimen-client`, `emitSync: true`) — the primary golden fixture now represents the default (`sync` omitted → async), matching what most of `tests/test_generated.py` exercises. `python-sync` is new and sits *before* the `# The variants below` marker, so CI's truncation step (`.github/workflows/ci.yml`) keeps it alongside `python`. - -- [ ] **Step 2: Commit** - -```bash -git add tests/fixture-project/project1.pgn.yaml -git commit -m "python.gen: add python-sync fixture artifact, drop emitSync from main artifact" -``` - ---- - -### Task 4: Add the `tests/golden_sync/` shell and wire up golden regeneration for both artifacts - -**Files:** -- Create: `tests/golden_sync/pyproject.toml` -- Create: `tests/golden_sync/README.md` -- Create: `tests/golden_sync/src/specimen_sync_client/py.typed` -- Modify: `tests/_harness.py` -- Modify: `mise.toml` -- Modify: `tests/golden/pyproject.toml` (drop the now-absent `emitSync`-driven `sync/` package reference — see Step 5) - -**Interfaces:** -- Produces: `GOLDEN_DIR_SYNC` (a `Path` constant in `tests/_harness.py`), importable by `tests/conftest.py` and `tests/test_generated.py`. - -- [ ] **Step 1: Create `tests/golden_sync/pyproject.toml`** - -```toml -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[project] -name = "specimen-sync-client" -version = "0.0.0" -requires-python = ">=3.12" -dependencies = ["psycopg>=3.2"] - -[tool.hatch.build.targets.wheel] -packages = ["src/specimen_sync_client"] - -[tool.ruff] -exclude = ["src/specimen_sync_client/_generated"] -``` - -- [ ] **Step 2: Create `tests/golden_sync/src/specimen_sync_client/py.typed`** - -Empty file (PEP 561 marker, matching `tests/golden/src/specimen_client/py.typed`). - -```bash -mkdir -p tests/golden_sync/src/specimen_sync_client -touch tests/golden_sync/src/specimen_sync_client/py.typed -``` - -- [ ] **Step 3: Create `tests/golden_sync/README.md`** - -```markdown -# Golden files (sync surface) - -The sync-surface counterpart of `tests/golden/`: a committed full fixture -package generated with `config: { sync: true }`, package name -`specimen_sync_client`. Same structure and purpose as `tests/golden/README.md` -describes for the async (default) surface — the hand-written shell here -(`pyproject.toml`, `py.typed`) plus the generated `_generated/` subtree and -the generated package-root facade `src/specimen_sync_client/__init__.py`. - -`test_generated_sync_matches_golden` regenerates the `python-sync` fixture -artifact into a temp tree and asserts every file under the fresh -`_generated/` plus the facade equals its golden twin here, both ways (no -missing, no extra). `test_generated_sync_passes_basedpyright_strict` runs -basedpyright strict on this full golden package. - -## Updating the golden - -Run only when a generator change legitimately alters the sync surface's -output, and review the resulting diff before committing. - -```bash -mise run golden -``` - -`mise run golden` refreshes both `tests/golden/` (the `python` artifact) and -this directory (the `python-sync` artifact) in one pass — see `mise.toml`. -``` - -- [ ] **Step 4: Add `GOLDEN_DIR_SYNC` to `tests/_harness.py`** - -Line 28, immediately after `GOLDEN_DIR = HERE / "golden"`: - -```python -GOLDEN_DIR = HERE / "golden" -GOLDEN_DIR_SYNC = HERE / "golden_sync" -``` - -- [ ] **Step 5: Confirm `tests/golden/pyproject.toml` needs no change** - -`tests/golden/pyproject.toml`'s `[tool.ruff] exclude` and `packages` entries already reference only `src/specimen_client` (not a `sync/` sub-path), so this file needs no edit — the sync surface's shell lives entirely under the new `tests/golden_sync/`, a sibling directory, not nested inside the existing golden tree. (Listed in File Structure above out of caution; this step is a no-op confirmation, not an edit.) - -- [ ] **Step 6: Rewrite the `golden` task in `mise.toml`** - -Replace the `[tasks.golden]` block: - -```toml -# Regenerates the "python" and "python-sync" artifacts: a full 7-artifact -# generate peaks at ~31 GB RSS (memory goes to normalizing the generator -# closure per artifact, see ci.yml), so this keeps only the two artifacts -# golden actually needs instead of all 7. -[tasks.golden] -description = "Refresh tests/golden and tests/golden_sync from the working-tree src/" -run = ''' -#!/usr/bin/env bash -set -euo pipefail -root="$(pwd)" -tmp="$(mktemp -d)" -trap 'rm -rf "$tmp"' EXIT -cp -R "$root/tests/fixture-project/." "$tmp/fixture" -rm -f "$tmp/fixture/freeze1.pgn.yaml" -rm -rf "$tmp/fixture/artifacts" -python3 - "$tmp/fixture/project1.pgn.yaml" "$root/src/package.dhall" <<'EOF' -import sys -from pathlib import Path - -p = Path(sys.argv[1]) -head, sep, _ = p.read_text().partition(" # The variants below") -assert sep, "variants marker not found in project1.pgn.yaml" -_ = p.write_text(head.rstrip().replace("../../src/package.dhall", sys.argv[2]) + "\n") -EOF -cd "$tmp/fixture" -pgn --database-url "${PGN_TEST_DATABASE_URL:-postgresql://postgres:postgres@localhost:5432/postgres?sslmode=disable}" generate -rsync -a --delete artifacts/python/src/specimen_client/_generated/ "$root/tests/golden/src/specimen_client/_generated/" -cp artifacts/python/src/specimen_client/__init__.py "$root/tests/golden/src/specimen_client/__init__.py" -rsync -a --delete artifacts/python_sync/src/specimen_sync_client/_generated/ "$root/tests/golden_sync/src/specimen_sync_client/_generated/" -cp artifacts/python_sync/src/specimen_sync_client/__init__.py "$root/tests/golden_sync/src/specimen_sync_client/__init__.py" -echo "golden refreshed; review with: git diff tests/golden tests/golden_sync" -''' -``` - -Note the `cp .../sync/__init__.py` line from the original task is gone (no sync facade sub-path exists anymore); the sync surface's facade is now at the same top-level `__init__.py` position as async's, just in the sibling `python_sync` artifact directory. - -- [ ] **Step 7: Run the golden refresh** - -Run: `cd python.gen && mise run golden` -Expected: exits 0, prints `golden refreshed; review with: git diff tests/golden tests/golden_sync`. Needs a reachable Postgres (`PGN_TEST_DATABASE_URL`, default `postgresql://postgres:postgres@localhost:5432/postgres?sslmode=disable`). - -- [ ] **Step 8: Review the diff** - -Run: `cd python.gen && git status tests/golden tests/golden_sync && git diff tests/golden` -Expected: `tests/golden/src/specimen_client/_generated/sync/` and `tests/golden/src/specimen_client/sync/__init__.py` are deleted (14 files: `sync/__init__.py`, `sync/_register.py`, `sync/_runtime.py`, `sync/statements/__init__.py`, 10 `sync/statements/*.py`, package-root `sync/__init__.py`). `tests/golden_sync/src/specimen_sync_client/_generated/` is new and structurally mirrors what `tests/golden/src/specimen_client/_generated/` looked like *before* this plan minus the `sync/` nesting, with `def`/`Connection`/no-`await` content instead of `async def`/`AsyncConnection`/`await `. No other content in either tree should have changed (params, SQL, custom types, row shapes are untouched by this plan). - -- [ ] **Step 9: Commit** - -```bash -git add tests/golden tests/golden_sync tests/_harness.py mise.toml -git commit -m "python.gen: add tests/golden_sync, regenerate both golden trees for exclusive sync" -``` - ---- - -### Task 5: Update the pytest harness for two independent golden trees - -**Files:** -- Modify: `tests/conftest.py` -- Modify: `tests/test_generated.py` - -**Interfaces:** -- Consumes: `GOLDEN_DIR_SYNC` (Task 4), `artifacts/python_sync` (Task 3), package name `specimen_sync_client`. -- Produces: `full_package_sync` fixture (mirrors `full_package`), usable by any test needing an importable sync-surface package. - -- [ ] **Step 1: Add a `generated_tree_sync`-adjacent path helper and `full_package_sync` fixture to `tests/conftest.py`** - -Add after the existing `full_package` fixture (end of file): - -```python -@pytest.fixture(scope="session") -def full_package_sync(generated_tree: Path, tmp_path_factory: pytest.TempPathFactory) -> Path: - """The sync-surface counterpart of `full_package`. - - `generated_tree` already ran `pgn generate` against the full project - (all artifacts, including `python-sync`), so this reuses that one - subprocess call rather than invoking pgn again: it just points at the - sibling `python_sync` artifact directory instead of `python`. - """ - generated_tree_sync = generated_tree.parent.parent / "artifacts" / "python_sync" - root = tmp_path_factory.mktemp("pkg-sync") - shell_src = GOLDEN_DIR_SYNC / "src" - generated_src = generated_tree_sync / "src" - _ = shutil.copytree(shell_src, root / "src", ignore=shutil.ignore_patterns("_generated", "__init__.py")) - for pkg_dir in generated_src.iterdir(): - dest_pkg = root / "src" / pkg_dir.name - _ = shutil.copytree(pkg_dir / "_generated", dest_pkg / "_generated") - _ = shutil.copy2(pkg_dir / "__init__.py", dest_pkg / "__init__.py") - return root -``` - -Add `GOLDEN_DIR_SYNC` to the existing `from tests._harness import (...)` block at the top of the file: - -```python -from tests._harness import ( - FIXTURE_PROJECT, - GOLDEN_DIR, - GOLDEN_DIR_SYNC, - HERE, - SRC_DIR, - admin_database_url, - effective_database_name, - run_pgn, -) -``` - -- [ ] **Step 2: Update `tests/test_generated.py`'s module-level constants** - -Replace lines 33-39: - -```python -# The generator emits the /_generated subtree plus the package-root -# __init__.py facade; the rest of the shell (pyproject.toml, py.typed) is -# hand-written and lives in golden as a committed fixture, not produced by -# generate. -GENERATED_SUBTREE = Path("src/specimen_client/_generated") -FACADE_INIT = Path("src/specimen_client/__init__.py") -GENERATED_SUBTREE_SYNC = Path("src/specimen_sync_client/_generated") -FACADE_INIT_SYNC = Path("src/specimen_sync_client/__init__.py") -``` - -Update the import line to include `GOLDEN_DIR_SYNC`: - -```python -from tests._harness import FIXTURE_PROJECT, GOLDEN_DIR, GOLDEN_DIR_SYNC, HERE, ensure_droppable, run_pgn -``` - -- [ ] **Step 3: Simplify `test_generated_matches_golden`, add its sync counterpart** - -Replace the `for facade in (FACADE_INIT, SYNC_FACADE_INIT):` loop (lines 116-118) with a single check: - -```python - if (generated_tree / FACADE_INIT).read_text() != (GOLDEN_DIR / FACADE_INIT).read_text(): - mismatched.append(str(FACADE_INIT)) -``` - -Add a new test after `test_generated_matches_golden`: - -```python -def test_generated_sync_matches_golden(generated_tree: Path) -> None: - """Sync-surface counterpart of test_generated_matches_golden. - - generated_tree points at the "python" artifact; the sync surface's - output lives in the sibling "python_sync" artifact directory produced by - the same pgn generate call (see the generated_tree fixture). - """ - generated_tree_sync = generated_tree.parent.parent / "artifacts" / "python_sync" - produced_root = generated_tree_sync / GENERATED_SUBTREE_SYNC - golden_root = GOLDEN_DIR_SYNC / GENERATED_SUBTREE_SYNC - - produced = _relative_files(produced_root) - golden = _relative_files(golden_root) - - missing = sorted(str(p) for p in golden - produced) - extra = sorted(str(p) for p in produced - golden) - assert not missing, f"golden files not produced by the generator: {missing}" - assert not extra, f"generator emitted files absent from golden: {extra}" - - mismatched: list[str] = [] - for rel in sorted(produced, key=str): - if (produced_root / rel).read_text() != (golden_root / rel).read_text(): - mismatched.append(str(rel)) - - if (generated_tree_sync / FACADE_INIT_SYNC).read_text() != (GOLDEN_DIR_SYNC / FACADE_INIT_SYNC).read_text(): - mismatched.append(str(FACADE_INIT_SYNC)) - - assert not mismatched, ( - "generated sync output drifted from golden in: " - + ", ".join(mismatched) - + "\nupdate via: mise run golden (see tests/golden_sync/README.md)" - ) -``` - -- [ ] **Step 4: Add `test_generated_sync_passes_basedpyright_strict`** - -Add after `test_generated_passes_basedpyright_strict`: - -```python -def test_generated_sync_passes_basedpyright_strict(tmp_path: Path) -> None: - """basedpyright strict on the full sync-surface golden package.""" - config = tmp_path / "pyrightconfig.json" - _ = config.write_text( - json.dumps( - { - "pythonVersion": "3.12", - "typeCheckingMode": "strict", - "include": [str(GOLDEN_DIR_SYNC / "src")], - "venvPath": str(HARNESS_ROOT), - "venv": ".venv", - "reportMissingModuleSource": False, - } - ) - ) - result = subprocess.run( - ["basedpyright", "--project", str(config), "--outputjson"], - capture_output=True, - text=True, - ) - - if not result.stdout.strip(): - pytest.fail(f"basedpyright produced no JSON (exit {result.returncode}):\n{result.stderr}") - - summary = json.loads(result.stdout)["summary"] - assert summary["filesAnalyzed"] > 0, f"basedpyright analyzed no files; bad include path?\n{result.stdout}" - assert summary["errorCount"] == 0 and summary["warningCount"] == 0, ( - f"basedpyright strict reported issues: {summary}\n{result.stdout}" - ) -``` - -- [ ] **Step 5: Delete `test_roundtrip_sync_and_cross_surface_identity`, replace with `test_roundtrip_sync_surface`** - -Delete the whole function (lines 392-479 in the original — cross-surface identity is not a meaningful concept once each surface is its own independent generate with its own `_rows.py`/package). Replace with: - -```python -def test_roundtrip_sync_surface(full_package_sync: Path, roundtrip_db: str) -> None: - """The sync surface, generated independently with config.sync = true. - - Drives the generated sync functions (psycopg.Connection, no await) end - to end, at the same unified paths the async surface uses (no `.sync.` - sub-path) — this package was generated standalone, not alongside async. - """ - _apply_migrations(roundtrip_db) - import_module = _import_client_sync(full_package_sync) - - register = import_module("specimen_sync_client._generated._register") - mood_mod = import_module("specimen_sync_client._generated.types.mood") - point_mod = import_module("specimen_sync_client._generated.types.point_2_d") - insert = import_module("specimen_sync_client._generated.statements.insert_specimen") - get = import_module("specimen_sync_client._generated.statements.get_specimen") - by_moods = import_module("specimen_sync_client._generated.statements.list_specimens_by_moods") - bump = import_module("specimen_sync_client._generated.statements.bump_specimen_revision") - - Mood = mood_mod.Mood - Point2D = point_mod.Point2D - - conn = psycopg.connect(roundtrip_db, autocommit=True) - try: - register.register_types(conn) - - inserted = insert.insert_specimen( - conn, - doc_jsonb={"k": "v", "n": 1}, - feeling=Mood.HAPPY, - origin=Point2D(x=1.5, y=2.5), - flag=True, - small=1, - medium=2, - large=3, - ratio=0.5, - precise=0.25, - title="alpha", - code="C-1", - letter="x", - born_on=date(2020, 1, 2), - amount=Decimal("12.34"), - blob=b"\x00\x01", - doc_json=[1, 2, 3], - maybe_text=None, - maybe_int=None, - maybe_uuid=None, - maybe_ts=None, - maybe_num=None, - tags=["a", None, "b"], - related_ids=None, - grid=None, - moods=[Mood.HAPPY, None, Mood.SAD], - ) - assert isinstance(inserted.feeling, Mood) - assert inserted.feeling is Mood.HAPPY - assert isinstance(inserted.origin, Point2D) - assert inserted.doc_jsonb == {"k": "v", "n": 1} - assert inserted.moods == [Mood.HAPPY, None, Mood.SAD] - assert inserted.moods is not None - assert inserted.moods[0] is Mood.HAPPY - - specimen_id = inserted.id - hit = get.get_specimen(conn, id=specimen_id) - assert hit is not None - assert hit.id == specimen_id - assert get.get_specimen(conn, id=specimen_id + 10_000) is None - - mood_rows = by_moods.list_specimens_by_moods(conn, moods=[Mood.HAPPY]) - assert [r.id for r in mood_rows] == [specimen_id] - assert by_moods.list_specimens_by_moods(conn, moods=[Mood.SAD]) == [] - - affected = bump.bump_specimen_revision(conn, id=specimen_id) - assert affected == 1 - bumped = get.get_specimen(conn, id=specimen_id) - assert bumped is not None - assert bumped.rev == 2 - finally: - conn.close() -``` - -Add the `_import_client_sync` helper next to `_import_client` (same shape, different module-name prefix so `sys.modules` cache invalidation targets the right package): - -```python -def _import_client_sync(full_package_sync: Path): # noqa: ANN202 - dynamic module set - src = str(full_package_sync / "src") - if src not in sys.path: - sys.path.insert(0, src) - for name in list(sys.modules): - if name == "specimen_sync_client" or name.startswith("specimen_sync_client."): - del sys.modules[name] - return importlib.import_module -``` - -- [ ] **Step 6: Move `test_roundtrip_single_field_composite_sync` onto the sync-only package** - -Replace the function body: - -```python -def test_roundtrip_single_field_composite_sync(full_package_sync: Path, roundtrip_db: str) -> None: - """Sync-surface counterpart of test_roundtrip_single_field_composite.""" - _apply_migrations(roundtrip_db) - import_module = _import_client_sync(full_package_sync) - - register = import_module("specimen_sync_client._generated._register") - tag_mod = import_module("specimen_sync_client._generated.types.tag_value") - insert = import_module("specimen_sync_client._generated.statements.insert_tagged_item") - get = import_module("specimen_sync_client._generated.statements.get_tagged_item") - - TagValue = tag_mod.TagValue - - conn = psycopg.connect(roundtrip_db, autocommit=True) - try: - register.register_types(conn) - - inserted = insert.insert_tagged_item(conn, name="widget", tag=TagValue(value="blue")) - assert isinstance(inserted.tag, TagValue) - assert inserted.tag == TagValue(value="blue") - - hit = get.get_tagged_item(conn, id=inserted.id) - assert hit is not None - assert hit.tag == TagValue(value="blue") - finally: - conn.close() -``` - -- [ ] **Step 7: Run the generated-output test module** - -Run: `cd python.gen && mise run test -- tests/test_generated.py -v` -Expected: all tests pass, including the new `test_generated_sync_matches_golden`, `test_generated_sync_passes_basedpyright_strict`, `test_roundtrip_sync_surface`, and `test_roundtrip_single_field_composite_sync`. - -- [ ] **Step 8: Commit** - -```bash -git add tests/conftest.py tests/test_generated.py -git commit -m "python.gen: split sync-surface tests onto their own golden fixture" -``` - ---- - -### Task 6: Fix the config-variant tests' sync-detection signal - -**Files:** -- Modify: `tests/test_config_variants.py` - -**Interfaces:** -- Consumes: `_generated/_runtime.py` content (Task 2 made `def fetch_many`/`async def fetch_many` the sync/async tell, since there is no more `sync/` subdirectory to check for existence). - -- [ ] **Step 1: Rewrite `tests/test_config_variants.py`** - -```python -"""Pins pgn's actual YAML->Dhall decode semantics for the Optional Config knobs. - -pgn's decode behavior for record types is undocumented, so each artifact in -project1.pgn.yaml drives a different subset of `config` keys through the same -compile.dhall and the assertions below record what pgn 0.6.5 was observed to do, -not a documented contract. These variants are pinned by the directory/package -name and which surface (sync or async) they produce, not a full golden tree. -""" - -from __future__ import annotations - -import os -from pathlib import Path - -import pytest - -pytestmark = pytest.mark.skipif( - os.environ.get("HARNESS_CI_REDUCED") == "1", - reason=( - "variant decode semantics are pinned locally; CI runs the reduced " - "single-artifact project because pgn generate of all 7 artifacts " - "exhausts GitHub-hosted runner memory" - ), -) - - -def _artifact_src(generated_tree: Path, artifact_key: str) -> Path: - # generated_tree is /artifacts/python; artifact directories are - # named after the artifact key with "-" replaced by "_" (pgn's own doing, - # independent of the Dhall config below). - project_root = generated_tree.parent.parent - artifact_dir = artifact_key.replace("-", "_") - return project_root / "artifacts" / artifact_dir / "src" - - -def _package_dir(generated_tree: Path, artifact_key: str) -> Path: - src = _artifact_src(generated_tree, artifact_key) - packages = [p for p in src.iterdir() if p.is_dir()] - assert len(packages) == 1, f"expected exactly one package under {src}, found {packages}" - return packages[0] - - -def _is_sync(package: Path) -> bool: - """Whether the generated package's runtime module is the sync surface. - - There is no `sync/` subdirectory to check anymore (config.sync selects - which content is rendered at the *same* `_runtime.py` path); the async - body defines `async def fetch_many`, the sync body defines `def - fetch_many` with no `async`. - """ - runtime = (package / "_generated" / "_runtime.py").read_text() - return "async def fetch_many" not in runtime - - -def test_name_only_config_derives_package_and_defaults_sync_off(generated_tree: Path) -> None: - package = _package_dir(generated_tree, "python-name-only") - assert package.name == "name_only_client" - assert not _is_sync(package) - - -def test_sync_only_config_defaults_package_name_from_project(generated_tree: Path) -> None: - package = _package_dir(generated_tree, "python-sync-only") - assert package.name == "fixture" - assert _is_sync(package) - - -def test_empty_config_object_defaults_both_fields(generated_tree: Path) -> None: - package = _package_dir(generated_tree, "python-empty") - assert package.name == "fixture" - assert not _is_sync(package) - - -def test_absent_config_key_defaults_both_fields(generated_tree: Path) -> None: - """No `config:` key at all decodes the same as `config: {}` (None Config).""" - package = _package_dir(generated_tree, "python-bare") - assert package.name == "fixture" - assert not _is_sync(package) - - -def test_unknown_config_key_is_ignored_not_rejected(generated_tree: Path) -> None: - """An extra key not in the generator's Config type (bogusField) does not fail generation.""" - package = _package_dir(generated_tree, "python-unknown-key") - assert package.name == "unknown_key_client" - assert not _is_sync(package) - - -def test_null_value_decodes_as_absent_field(generated_tree: Path) -> None: - """`sync: null` decodes to None, same as omitting the key (default False).""" - package = _package_dir(generated_tree, "python-null") - assert package.name == "null_client" - assert not _is_sync(package) -``` - -- [ ] **Step 2: Run the config-variant tests** - -Run: `cd python.gen && mise run test -- tests/test_config_variants.py -v` -Expected: all six tests pass (unless `HARNESS_CI_REDUCED=1` is set locally, in which case they skip — matches existing behavior). - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_config_variants.py -git commit -m "python.gen: fix config-variant sync detection for unified output paths" -``` - ---- - -### Task 7: Update documentation - -**Files:** -- Modify: `README.md` -- Modify: `DESIGN.md` -- Modify: `CHANGELOG.md` - -- [ ] **Step 1: Update `README.md`'s config reference table** - -Replace the table at lines 76-80: - -```markdown -| key | type | default | -| --------------- | ------------------ | ------------------------------ | -| `packageName` | `Text` | the project name, kebab-cased | -| `sync` | `Bool` | `False` | -| `onUnsupported` | `"Fail" \| "Skip"` | `"Fail"` | -``` - -- [ ] **Step 2: Update the quickstart example (lines 36-42)** - -```yaml -artifacts: - python: - gen: https://raw.githubusercontent.com/slavashvets/python.gen/master/src/package.dhall - config: - packageName: my-db-client - sync: true -``` - -- [ ] **Step 3: Update "Key features" (line 21-23)** - -```markdown -- **Async or sync client over psycopg 3.** `config.sync` (default `False`) - picks exactly one surface — the async client (`psycopg.AsyncConnection`) or - the sync one (`psycopg.Connection`) — at the same output paths either way; - install `psycopg[binary]` for the fast C implementation. -``` - -- [ ] **Step 4: Update "Using the generated code" (lines 220-240)** - -Replace the closing paragraph and code block: - -```markdown -## Using the generated code - -Import from the flat facade, not from `_generated` directly: - -```python -from my_db_client import get_specimen, GetSpecimenRow, Mood -``` - -The facade re-exports every Row dataclass, every enum/composite, and every -statement function with `X as X` markers and a matching `__all__`. If the -project has composites or enums, call `register_types(conn)` once per -connection before relying on native composite decode or enum-array results; -scalar enums do not need it. - -With `sync: true`, the entire client is generated against -`psycopg.Connection` instead — same import path, same facade shape, only the -function signatures and I/O become synchronous: - -```python -from my_db_client import get_specimen, GetSpecimenRow, Mood - -def handler(conn): - row = get_specimen(conn, id=42) -``` - -A project that needs both an async backend and a sync consumer (e.g. a -Dagster pipeline) generates two artifacts pointed at this same `gen:`, one -with `sync: true` and one without, each with its own `packageName` — not one -artifact with both surfaces bundled together. -``` - -- [ ] **Step 5: Update `DESIGN.md` sections describing the surface mechanism** - -Rewrite the "What the generator emits" / "What the generator adds when `emitSync`" split (roughly lines 43-71) into a single section describing unconditional, surface-selected emission: - -```markdown -### What the generator emits - -``` -src//__init__.py # generated facade (re-exports everything) -src//_generated/__init__.py -src//_generated/_core.py # surface-agnostic: JsonValue, NoRowError, require_array -src//_generated/_runtime.py # I/O helpers for whichever surface config.sync picked -src//_generated/_register.py # only if composites or enums -src//_generated/_rows.py # shared Row dataclasses + decode functions -src//_generated/statements/__init__.py -src//_generated/statements/.py # one thin wrapper per query, def or async def -src//_generated/types/__init__.py # only if custom types exist -src//_generated/types/.py -``` - -Exactly one surface is emitted per generate, picked by `config.sync` -(`False`, the default, emits async; `True` emits sync) — the tree shape and -every import path are identical either way; only the statement modules' -`def`/`async def`, `Connection`/`AsyncConnection`, and `_runtime.py`'s body -change. A project that needs both surfaces generates two artifacts against -this same `gen:` with different `packageName`s, one with `sync: true` and -one without. -``` - -Rewrite the "Surface mechanism" section (roughly lines 192-217) to drop the "emits both, gated by config.emitSync" framing: - -```markdown -## 4. Surface mechanism (async / sync) - -```dhall -{ defKeyword : Text -- "async def" | "def" -, connType : Text -- "AsyncConnection" | "Connection" -, awaitKw : Text -- "await " | "" -, rowsImport : Text -- ".._rows" (same depth for both surfaces) -, corePrefix : Text -- ".._core" (same depth for both surfaces) -, typesPrefix : Text -- "..types" (same depth for both surfaces) -} -``` - -`Surface.async` and `Surface.sync` are the two values. `Interpreters/ -Project.dhall` picks one — `if config.sync then Surface.sync else -Surface.async` — and threads it through a single render pass: -`Interpreters/Query.dhall` calls the statement-module template once per -query (not twice), and `Interpreters/Project.dhall` picks the matching -`_runtime.py` body and, when custom types exist, the matching `_register.py` -body. `_rows.py` and `types/` are surface-agnostic and always render exactly -once, so switching `config.sync` costs only the thin I/O wrappers. -``` - -Update the facade section (roughly lines 400-409) to remove the two-facade description: - -```markdown -- every statement function from `_generated/statements/`. - -The facade lives at `/__init__.py` (prefix `._generated`, statements -under `statements`) regardless of which surface `config.sync` selected. -``` - -Update the Config-flow section's example (roughly lines 427-429, 503-509) to use `sync` in place of `emitSync`, matching Task 1/2's rename, e.g.: - -```dhall -let Config = { packageName : Optional Text, sync : Optional Bool, onUnsupported : Optional OnUnsupported.Mode } - -let Config/default = { packageName = None Text, sync = None Bool, onUnsupported = None OnUnsupported.Mode } -``` - -And the prose right below the second code block (roughly lines 500-514, "### Config flow"): replace every `emitSync` with `sync`, and `` `emitSync` to `False` `` with `` `sync` to `False` ``. - -Also update the "Generator decomposition" file-tree block (roughly lines 450-480): `Structures/Surface.dhall`'s comment changes from "async/sync token table (section 4)" to "async/sync token table, both surfaces at the same import depth (section 4)"; `Interpreters/Query.dhall`'s comment changes from "assemble one query: shared rowDef + async/sync statement modules" to "assemble one query: shared rowDef + one statement module for whichever surface config.sync picked". - -- [ ] **Step 6: Add a `CHANGELOG.md` "Upcoming" entry** - -Add to the top of the `# Upcoming` section: - -```markdown -- **Breaking:** `emitSync` is gone. In its place, `sync : Optional Bool` - (default `False`) picks exactly one surface per generate — async or sync — - emitted at the same unified paths either way (no more `sync/` subdirectory, - no more second package-root facade). Previously `emitSync: true` added a - second, nested sync tree alongside the always-emitted async one; a project - that needs both surfaces now generates two artifacts against this same - `gen:` with different `packageName`s, one with `sync: true` and one - without. See `docs/plans/2026-07-12-configurable-sync-output.md` for the - full rationale and migration shape. `tests/golden_sync/` is a new committed - golden fixture (`specimen_sync_client`) exercising the sync surface - end-to-end (basedpyright strict + round-trip), alongside the existing - `tests/golden/` (`specimen_client`, now async-only). -``` - -- [ ] **Step 7: Commit** - -```bash -git add README.md DESIGN.md CHANGELOG.md -git commit -m "python.gen: document the exclusive sync/async config change" -``` - ---- - -### Task 8: Full verification pass - -**Files:** none (verification only). - -- [ ] **Step 1: Run the full harness** - -Run: `cd python.gen && mise run test` -Expected: all tests pass (allow ~10 minutes; most of it is pgn's own type inference against Postgres). - -- [ ] **Step 2: Confirm no stray `emitSync` references anywhere in the repo** - -Run: `cd python.gen && grep -rn "emitSync" . --include="*.dhall" --include="*.md" --include="*.py" --include="*.yaml" --include="*.toml" | grep -v "^CHANGELOG.md:"` -Expected: no output (the `CHANGELOG.md` exclusion is because older, already-shipped entries legitimately still mention the old field name as history — only today's new entry from Task 7 should be free of it going forward, and that new entry doesn't mention `emitSync` at all so the grep with the exclusion is really just a safety margin, not an expected hit). - -- [ ] **Step 3: Confirm both golden trees are internally consistent with the new fixture config** - -Run: `cd python.gen && git status --short tests/golden tests/golden_sync` -Expected: clean (everything from Task 4's regeneration already committed). - -- [ ] **Step 4: Manually skim one generated statement file from each surface** - -Run: `head -30 tests/golden/src/specimen_client/_generated/statements/get_specimen.py tests/golden_sync/src/specimen_sync_client/_generated/statements/get_specimen.py` -Expected: the async file shows `from psycopg import AsyncConnection` and `async def get_specimen(...)`; the sync file shows `from psycopg import Connection` and `def get_specimen(...)` (no `async`), both importing from `.._rows`/`.._core` (not `..._rows`/`..._core`) — confirming Task 2's depth collapse took effect.