From 7fee358cf98c80b3f82e32f7e88f6d64f701b24a Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 09:19:23 +0300 Subject: [PATCH 01/11] feat(gen): migrate to gen-contract v4.0.1 / gen-sdk v2.0.0, adopt Sdk.Sigs Pins gen/Deps/Contract.dhall (new) and bumps gen/Deps/Sdk.dhall to gen-sdk v2.0.0, removing the gen/Deps/package.dhall barrel. Rewires Structures/CustomKind.dhall and all 11 Interpreters/*.dhall plus all 10 Templates/*.dhall to import Lude/Prelude/Contract/Sdk directly instead of through the barrel, replacing the local gen/Algebras/{Interpreter,Template} sig-constructors with gen-sdk's Sdk.Sigs.interpreter / Sdk.Sigs.template (now deleted). Matches the shape java.gen's current release already uses. Query.dhall, Result.dhall, and ResultColumns.dhall keep their pre-existing bare-export shape (like Member.dhall/ParamsMember.dhall) instead of adopting Sdk.Sigs.interpreter: their `run` functions take 3/4 curried arguments (config, lookup, [rowClassName], input), which doesn't fit Sigs.interpreter's fixed 2-argument Config -> Input -> Result shape. This extends the Member/ParamsMember exception already called out in the migration plan to these three files, confirmed against their actual call sites in Project.dhall/Query.dhall. gen/compile.dhall and gen/Gen.dhall still reference the removed barrel and are intentionally left broken until the next task rewrites the root entry point (Config.dhall/compile.dhall -> Interpret.dhall, Gen.dhall -> package.dhall). --- gen/Algebras/Interpreter.dhall | 21 ------------- gen/Algebras/Template.dhall | 5 --- gen/Algebras/package.dhall | 1 - gen/Deps/Contract.dhall | 2 ++ gen/Deps/Sdk.dhall | 4 +-- gen/Deps/package.dhall | 4 --- gen/Interpreters/CustomType.dhall | 25 +++++++++------ gen/Interpreters/Member.dhall | 27 ++++++++++------ gen/Interpreters/ParamsMember.dhall | 45 ++++++++++++++++----------- gen/Interpreters/Primitive.dhall | 23 +++++++++----- gen/Interpreters/Project.dhall | 25 +++++++++------ gen/Interpreters/Query.dhall | 23 +++++++++----- gen/Interpreters/QueryFragments.dhall | 23 ++++++++------ gen/Interpreters/Result.dhall | 26 ++++++++++------ gen/Interpreters/ResultColumns.dhall | 24 +++++++++----- gen/Interpreters/Scalar.dhall | 21 ++++++++----- gen/Interpreters/Value.dhall | 23 +++++++++----- gen/Structures/CustomKind.dhall | 4 +-- gen/Templates/CompositeModule.dhall | 12 +++---- gen/Templates/CoreModule.dhall | 4 +-- gen/Templates/EnumModule.dhall | 20 ++++++------ gen/Templates/FacadeModule.dhall | 8 ++--- gen/Templates/InitModule.dhall | 4 +-- gen/Templates/RegisterModule.dhall | 8 ++--- gen/Templates/RowsModule.dhall | 12 +++---- gen/Templates/RuntimeModule.dhall | 4 +-- gen/Templates/StatementModule.dhall | 12 +++---- gen/Templates/TypesInit.dhall | 8 ++--- 28 files changed, 230 insertions(+), 188 deletions(-) delete mode 100644 gen/Algebras/Interpreter.dhall delete mode 100644 gen/Algebras/Template.dhall delete mode 100644 gen/Algebras/package.dhall create mode 100644 gen/Deps/Contract.dhall delete mode 100644 gen/Deps/package.dhall diff --git a/gen/Algebras/Interpreter.dhall b/gen/Algebras/Interpreter.dhall deleted file mode 100644 index 7532153..0000000 --- a/gen/Algebras/Interpreter.dhall +++ /dev/null @@ -1,21 +0,0 @@ -let Deps = ../Deps/package.dhall - -let OnUnsupported = ../Structures/OnUnsupported.dhall - -let Config = - { packageName : Text - , importName : Text - , emitSync : Bool - , onUnsupported : OnUnsupported.Mode - } - -let module = - \(Input : Type) -> - \(Output : Type) -> - let Result = Deps.Lude.Compiled.Type Output - - let Run = Config -> Input -> Result - - in \(run : Run) -> { Input, Output, Result, Run, run } - -in { Config, module } diff --git a/gen/Algebras/Template.dhall b/gen/Algebras/Template.dhall deleted file mode 100644 index 15f2624..0000000 --- a/gen/Algebras/Template.dhall +++ /dev/null @@ -1,5 +0,0 @@ -let module = - \(Params : Type) -> - let Run = Params -> Text in \(run : Run) -> { Params, Run, run } - -in { module } diff --git a/gen/Algebras/package.dhall b/gen/Algebras/package.dhall deleted file mode 100644 index f0f64bd..0000000 --- a/gen/Algebras/package.dhall +++ /dev/null @@ -1 +0,0 @@ -{ Interpreter = ./Interpreter.dhall, Template = ./Template.dhall } diff --git a/gen/Deps/Contract.dhall b/gen/Deps/Contract.dhall new file mode 100644 index 0000000..ea9e805 --- /dev/null +++ b/gen/Deps/Contract.dhall @@ -0,0 +1,2 @@ +https://raw.githubusercontent.com/pgenie-io/gen-contract/v4.0.1/src/package.dhall + sha256:4a130ba7fbaa152a776babbb1bf2994a4833931ca76bde9bf6930d354225651e diff --git a/gen/Deps/Sdk.dhall b/gen/Deps/Sdk.dhall index 6cf6014..5602c43 100644 --- a/gen/Deps/Sdk.dhall +++ b/gen/Deps/Sdk.dhall @@ -1,3 +1,3 @@ -https://raw.githubusercontent.com/pgenie-io/gen-sdk/v0.11.0/dhall/package.dhall - sha256:8d43544ecb0e612406af3133bdbca51138c704a77a5a29ef62fe034d0e77a3a6 +https://raw.githubusercontent.com/pgenie-io/gen-sdk/v2.0.0/src/package.dhall + sha256:b9def6ab1179bc4aaae7fc6e91977f094f75934cd5755175c294a9e97ca71b15 as Source diff --git a/gen/Deps/package.dhall b/gen/Deps/package.dhall deleted file mode 100644 index b395cd9..0000000 --- a/gen/Deps/package.dhall +++ /dev/null @@ -1,4 +0,0 @@ -{ Sdk = ./Sdk.dhall -, Lude = ./Lude.dhall -, Prelude = ./Prelude.dhall -} diff --git a/gen/Interpreters/CustomType.dhall b/gen/Interpreters/CustomType.dhall index 7927f0a..676f9ab 100644 --- a/gen/Interpreters/CustomType.dhall +++ b/gen/Interpreters/CustomType.dhall @@ -1,16 +1,16 @@ -let Deps = ../Deps/package.dhall +let Lude = ../Deps/Lude.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Prelude = ../Deps/Prelude.dhall -let CustomKind = ../Structures/CustomKind.dhall +let Model = ../Deps/Contract.dhall -let Algebra = ../Algebras/Interpreter.dhall +let Sdk = ../Deps/Sdk.dhall -let Lude = Deps.Lude +let ImportSet = ../Structures/ImportSet.dhall -let Prelude = Deps.Prelude +let CustomKind = ../Structures/CustomKind.dhall -let Model = Deps.Sdk.Project +let OnUnsupported = ../Structures/OnUnsupported.dhall let MemberGen = ./Member.dhall @@ -18,6 +18,13 @@ let EnumModule = ../Templates/EnumModule.dhall let CompositeModule = ../Templates/CompositeModule.dhall +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } + let Input = Model.CustomType let TypeKind = < Enum | Composite > @@ -75,7 +82,7 @@ let renderExtraImports = ) let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(input : Input) -> let typeName = input.name.inPascalCase @@ -174,4 +181,4 @@ let run = } input.definition -in { Input, Output, TypeKind, run } +in Sdk.Sigs.interpreter Config Input Output run diff --git a/gen/Interpreters/Member.dhall b/gen/Interpreters/Member.dhall index 672dd18..8d2d27c 100644 --- a/gen/Interpreters/Member.dhall +++ b/gen/Interpreters/Member.dhall @@ -1,4 +1,8 @@ -let Deps = ../Deps/package.dhall +let Lude = ../Deps/Lude.dhall + +let Prelude = ../Deps/Prelude.dhall + +let Model = ../Deps/Contract.dhall let ImportSet = ../Structures/ImportSet.dhall @@ -6,14 +10,17 @@ let CustomKind = ../Structures/CustomKind.dhall let PyIdent = ../Structures/PyIdent.dhall -let Algebra = ../Algebras/Interpreter.dhall - -let Lude = Deps.Lude - -let Model = Deps.Sdk.Project +let OnUnsupported = ../Structures/OnUnsupported.dhall let Value = ./Value.dhall +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } + let Input = Model.Member -- decodeExpr is a Dhall function: given the source expression (e.g. row["x"] or @@ -29,7 +36,7 @@ let Output = } let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(lookup : CustomKind.Lookup) -> \(input : Input) -> -- Result-column / composite-field name becomes a dataclass field and decode @@ -95,7 +102,7 @@ let run = \(fields : List CustomKind.CompositeField) -> \(src : Text) -> let fieldTypes = - Deps.Prelude.Text.concatMapSep + Prelude.Text.concatMapSep ", " CustomKind.CompositeField (\(f : CustomKind.CompositeField) -> f.pyType) @@ -120,7 +127,7 @@ let run = , decodeExpr = passthroughDecode } , Custom = - Deps.Prelude.Optional.fold + Prelude.Optional.fold Model.Name value.scalar.customRef (Lude.Compiled.Type Output) @@ -235,6 +242,6 @@ let run = in Lude.Compiled.flatMap Value.Output Output buildOutput compiledValue -let Run = Algebra.Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output +let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output in { Input, Output, Run, run } diff --git a/gen/Interpreters/ParamsMember.dhall b/gen/Interpreters/ParamsMember.dhall index 47bef3f..2a4b469 100644 --- a/gen/Interpreters/ParamsMember.dhall +++ b/gen/Interpreters/ParamsMember.dhall @@ -1,17 +1,24 @@ -let Deps = ../Deps/package.dhall +let Lude = ../Deps/Lude.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Prelude = ../Deps/Prelude.dhall -let CustomKind = ../Structures/CustomKind.dhall +let Model = ../Deps/Contract.dhall -let Algebra = ../Algebras/Interpreter.dhall +let ImportSet = ../Structures/ImportSet.dhall -let Lude = Deps.Lude +let CustomKind = ../Structures/CustomKind.dhall -let Model = Deps.Sdk.Project +let OnUnsupported = ../Structures/OnUnsupported.dhall let Value = ./Value.dhall +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } + let Input = Model.Member let PyIdent = ../Structures/PyIdent.dhall @@ -192,7 +199,7 @@ let scalarIsJsonb = let valueIsArray = \(value : Model.Value) -> - Deps.Prelude.Optional.fold + Prelude.Optional.fold Model.ArraySettings value.arraySettings Bool @@ -204,15 +211,15 @@ let valueIsArray = -- the two must not be collapsed. let isJsonbScalar = \(value : Model.Value) -> - Deps.Prelude.Bool.and - [ scalarIsJsonb value, Deps.Prelude.Bool.not (valueIsArray value) ] + Prelude.Bool.and + [ scalarIsJsonb value, Prelude.Bool.not (valueIsArray value) ] let isJsonScalar = \(value : Model.Value) -> - Deps.Prelude.Bool.and + Prelude.Bool.and [ scalarIsJson value - , Deps.Prelude.Bool.not (scalarIsJsonb value) - , Deps.Prelude.Bool.not (valueIsArray value) + , Prelude.Bool.not (scalarIsJsonb value) + , Prelude.Bool.not (valueIsArray value) ] -- A json/jsonb ARRAY param has no faithful psycopg bind (Jsonb wraps a scalar, @@ -221,10 +228,10 @@ let isJsonScalar = -- in SQL is the supported route (the param then types as text[], not json[]). let isJsonArray = \(value : Model.Value) -> - Deps.Prelude.Bool.and [ scalarIsJson value, valueIsArray value ] + Prelude.Bool.and [ scalarIsJson value, valueIsArray value ] let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(lookup : CustomKind.Lookup) -> \(input : Input) -> let fieldName = pySafeName input.name.inSnakeCase @@ -251,7 +258,7 @@ let run = let compositeBind = \(fields : List CustomKind.CompositeField) -> let joinedFields = - Deps.Prelude.Text.concatMapSep + Prelude.Text.concatMapSep ", " CustomKind.CompositeField ( \(f : CustomKind.CompositeField) -> @@ -265,8 +272,8 @@ let run = -- as a 1-tuple, so force it for exactly one field; concatMapSep -- already inserts the internal comma for two or more. let trailingComma = - if Deps.Prelude.Natural.equal - ( Deps.Prelude.List.length + if Prelude.Natural.equal + ( Prelude.List.length CustomKind.CompositeField fields ) @@ -301,7 +308,7 @@ let run = , needsJsonbImport } - in Deps.Prelude.Optional.fold + in Prelude.Optional.fold Model.Name value.scalar.customRef (Lude.Compiled.Type Output) @@ -377,6 +384,6 @@ let run = in Lude.Compiled.flatMap Value.Output Output buildOutput compiledValue -let Run = Algebra.Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output +let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output in { Input, Output, Run, run } diff --git a/gen/Interpreters/Primitive.dhall b/gen/Interpreters/Primitive.dhall index 0e1cb70..052ef1b 100644 --- a/gen/Interpreters/Primitive.dhall +++ b/gen/Interpreters/Primitive.dhall @@ -1,10 +1,19 @@ -let Deps = ../Deps/package.dhall +let Lude = ../Deps/Lude.dhall + +let Model = ../Deps/Contract.dhall + +let Sdk = ../Deps/Sdk.dhall let ImportSet = ../Structures/ImportSet.dhall -let Algebra = ../Algebras/Interpreter.dhall +let OnUnsupported = ../Structures/OnUnsupported.dhall -let Model = Deps.Sdk.Project +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } let Input = Model.Primitive @@ -13,16 +22,16 @@ let Output = { pyType : Text, imports : ImportSet.Type } let supported = \(pyType : Text) -> \(imports : ImportSet.Type) -> - Deps.Lude.Compiled.ok Output { pyType, imports } + Lude.Compiled.ok Output { pyType, imports } let unsupported = \(pgType : Text) -> - Deps.Lude.Compiled.report Output [ pgType ] "Unsupported type" + Lude.Compiled.report Output [ pgType ] "Unsupported type" let plain = \(pyType : Text) -> supported pyType ImportSet.empty let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(input : Input) -> merge { Bit = unsupported "bit" @@ -89,4 +98,4 @@ let run = } input -in Algebra.module Input Output run +in Sdk.Sigs.interpreter Config Input Output run diff --git a/gen/Interpreters/Project.dhall b/gen/Interpreters/Project.dhall index d51438f..d4fa6d6 100644 --- a/gen/Interpreters/Project.dhall +++ b/gen/Interpreters/Project.dhall @@ -1,12 +1,10 @@ -let Deps = ../Deps/package.dhall +let Lude = ../Deps/Lude.dhall -let Algebra = ../Algebras/Interpreter.dhall +let Prelude = ../Deps/Prelude.dhall -let Lude = Deps.Lude +let Model = ../Deps/Contract.dhall -let Prelude = Deps.Prelude - -let Model = Deps.Sdk.Project +let Sdk = ../Deps/Sdk.dhall let CustomKind = ../Structures/CustomKind.dhall @@ -40,6 +38,13 @@ let OnUnsupported = ../Structures/OnUnsupported.dhall let Report = { path : List Text, message : Text } +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } + let Input = Model.Project let Output = Lude.Files.Type @@ -67,7 +72,7 @@ let withHeader = -- Internal interpreter config (importName etc.) is only needed to satisfy -- Value.run's signature; rendering a composite field's pyType does not read it. let lookupConfig - : Algebra.Config + : Config = { packageName = "" , importName = "" , emitSync = False @@ -175,7 +180,7 @@ let buildLookup = (\(_ : Model.Name) -> CustomKind.TypeKind.Absent) let combineOutputs = - \(config : Algebra.Config) -> + \(config : Config) -> \(input : Input) -> \(queries : List QueryGen.Output) -> -- Already the post-Skip-filter surviving set (see `run`); equal to @@ -430,7 +435,7 @@ let combineOutputs = let QueryCheck = { query : Model.Query, keep : Bool, warning : Optional Report } let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(input : Input) -> let skip = merge { Fail = False, Skip = True } config.onUnsupported @@ -542,4 +547,4 @@ let run = in Lude.Compiled.appendWarnings Output skipWarnings combined -in Algebra.module Input Output run +in Sdk.Sigs.interpreter Config Input Output run diff --git a/gen/Interpreters/Query.dhall b/gen/Interpreters/Query.dhall index b0e688d..daa1d47 100644 --- a/gen/Interpreters/Query.dhall +++ b/gen/Interpreters/Query.dhall @@ -1,6 +1,6 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Interpreter.dhall +let Lude = ../Deps/Lude.dhall let ImportSet = ../Structures/ImportSet.dhall @@ -10,6 +10,8 @@ let PyIdent = ../Structures/PyIdent.dhall let Surface = ../Structures/Surface.dhall +let OnUnsupported = ../Structures/OnUnsupported.dhall + let RowsModule = ../Templates/RowsModule.dhall let ResultModule = ./Result.dhall @@ -20,13 +22,16 @@ let ParamsMember = ./ParamsMember.dhall let StatementModule = ../Templates/StatementModule.dhall -let Prelude = Deps.Prelude - -let Lude = Deps.Lude +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } let Compiled = Lude.Compiled -let Model = Deps.Sdk.Project +let Model = ../Deps/Contract.dhall let Input = Model.Query @@ -46,7 +51,7 @@ let Output = } let render = - \(config : Algebra.Config) -> + \(config : Config) -> \(input : Input) -> \(result : ResultModule.Output) -> \(fragments : QueryFragmentsModule.Output) -> @@ -131,7 +136,7 @@ let render = } let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(lookup : CustomKind.Lookup) -> \(input : Input) -> let rowClassName = input.name.inPascalCase ++ "Row" @@ -172,4 +177,6 @@ let run = ) ) +let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output + in { Input, Output, run } diff --git a/gen/Interpreters/QueryFragments.dhall b/gen/Interpreters/QueryFragments.dhall index 066bc75..7813900 100644 --- a/gen/Interpreters/QueryFragments.dhall +++ b/gen/Interpreters/QueryFragments.dhall @@ -1,16 +1,21 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Interpreter.dhall +let Lude = ../Deps/Lude.dhall -let Prelude = Deps.Prelude +let Sdk = ../Deps/Sdk.dhall -let Sdk = Deps.Sdk +let Compiled = Lude.Compiled -let Lude = Deps.Lude +let Model = ../Deps/Contract.dhall -let Compiled = Lude.Compiled +let OnUnsupported = ../Structures/OnUnsupported.dhall -let Model = Deps.Sdk.Project +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } let Input = Model.QueryFragments @@ -47,8 +52,8 @@ let renderSql Prelude.Text.concatMap Model.QueryFragment renderFragment fragments let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(input : Input) -> Compiled.ok Output { sqlLiteral = renderSql input } -in Algebra.module Input Output run +in Sdk.Sigs.interpreter Config Input Output run diff --git a/gen/Interpreters/Result.dhall b/gen/Interpreters/Result.dhall index a6638e3..83f3ef1 100644 --- a/gen/Interpreters/Result.dhall +++ b/gen/Interpreters/Result.dhall @@ -1,20 +1,25 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Interpreter.dhall +let Lude = ../Deps/Lude.dhall + +let Model = ../Deps/Contract.dhall let ImportSet = ../Structures/ImportSet.dhall let CustomKind = ../Structures/CustomKind.dhall -let ResultColumns = ./ResultColumns.dhall - -let Prelude = Deps.Prelude +let OnUnsupported = ../Structures/OnUnsupported.dhall -let Lude = Deps.Lude +let ResultColumns = ./ResultColumns.dhall let Compiled = Lude.Compiled -let Model = Deps.Sdk.Project +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } let Input = Model.Result @@ -57,7 +62,7 @@ let cardinalityShape cardinality let rowsOutput = - \(config : Algebra.Config) -> + \(config : Config) -> \(lookup : CustomKind.Lookup) -> \(rowClassName : Text) -> \(rows : Model.ResultRows) -> @@ -84,7 +89,7 @@ let rowsOutput = (ResultColumns.run config lookup rowClassName columns) let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(lookup : CustomKind.Lookup) -> \(rowClassName : Text) -> \(input : Input) -> @@ -96,4 +101,7 @@ let run = } input +let Run = + Config -> CustomKind.Lookup -> Text -> Input -> Lude.Compiled.Type Output + in { Input, Output, RowClass, run } diff --git a/gen/Interpreters/ResultColumns.dhall b/gen/Interpreters/ResultColumns.dhall index ef550ec..b2fc737 100644 --- a/gen/Interpreters/ResultColumns.dhall +++ b/gen/Interpreters/ResultColumns.dhall @@ -1,20 +1,25 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Interpreter.dhall +let Lude = ../Deps/Lude.dhall + +let Model = ../Deps/Contract.dhall let ImportSet = ../Structures/ImportSet.dhall let CustomKind = ../Structures/CustomKind.dhall -let Member = ./Member.dhall - -let Prelude = Deps.Prelude +let OnUnsupported = ../Structures/OnUnsupported.dhall -let Lude = Deps.Lude +let Member = ./Member.dhall let Compiled = Lude.Compiled -let Model = Deps.Sdk.Project +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } let Input = List Model.Member @@ -53,7 +58,7 @@ let assemble } let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(lookup : CustomKind.Lookup) -> \(rowClassName : Text) -> \(input : Input) -> @@ -73,4 +78,7 @@ let run = input ) +let Run = + Config -> CustomKind.Lookup -> Text -> Input -> Lude.Compiled.Type Output + in { Input, Output, run } diff --git a/gen/Interpreters/Scalar.dhall b/gen/Interpreters/Scalar.dhall index b0eff70..63ec788 100644 --- a/gen/Interpreters/Scalar.dhall +++ b/gen/Interpreters/Scalar.dhall @@ -1,15 +1,22 @@ -let Deps = ../Deps/package.dhall +let Lude = ../Deps/Lude.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Model = ../Deps/Contract.dhall -let Algebra = ../Algebras/Interpreter.dhall +let Sdk = ../Deps/Sdk.dhall -let Lude = Deps.Lude +let ImportSet = ../Structures/ImportSet.dhall -let Model = Deps.Sdk.Project +let OnUnsupported = ../Structures/OnUnsupported.dhall let Primitive = ./Primitive.dhall +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } + let Input = Model.Scalar -- Passthrough for primitives; Custom is opaque here. The enum-vs-composite @@ -24,7 +31,7 @@ let Output = } let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(input : Input) -> merge { Primitive = @@ -52,4 +59,4 @@ let run = } input -in Algebra.module Input Output run /\ { ScalarDecode } +in Sdk.Sigs.interpreter Config Input Output run /\ { ScalarDecode } diff --git a/gen/Interpreters/Value.dhall b/gen/Interpreters/Value.dhall index de69fd0..e1c8d6a 100644 --- a/gen/Interpreters/Value.dhall +++ b/gen/Interpreters/Value.dhall @@ -1,17 +1,24 @@ -let Deps = ../Deps/package.dhall +let Lude = ../Deps/Lude.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Interpreter.dhall +let Model = ../Deps/Contract.dhall -let Lude = Deps.Lude +let Sdk = ../Deps/Sdk.dhall -let Prelude = Deps.Prelude +let ImportSet = ../Structures/ImportSet.dhall -let Model = Deps.Sdk.Project +let OnUnsupported = ../Structures/OnUnsupported.dhall let Scalar = ./Scalar.dhall +let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } + let Input = Model.Value let Output = @@ -23,7 +30,7 @@ let Output = } let run = - \(config : Algebra.Config) -> + \(config : Config) -> \(input : Input) -> Lude.Compiled.map Scalar.Output @@ -62,4 +69,4 @@ let run = ) (Scalar.run config input.scalar) -in Algebra.module Input Output run +in Sdk.Sigs.interpreter Config Input Output run diff --git a/gen/Structures/CustomKind.dhall b/gen/Structures/CustomKind.dhall index 22d3031..f2c4a1b 100644 --- a/gen/Structures/CustomKind.dhall +++ b/gen/Structures/CustomKind.dhall @@ -1,6 +1,4 @@ -let Deps = ../Deps/package.dhall - -let Model = Deps.Sdk.Project +let Model = ../Deps/Contract.dhall -- A composite field as the decode/encode sites need it: the Python attribute -- name and the rendered Python type (already nullability-applied). Threaded so diff --git a/gen/Templates/CompositeModule.dhall b/gen/Templates/CompositeModule.dhall index 198c77d..b1ab3e0 100644 --- a/gen/Templates/CompositeModule.dhall +++ b/gen/Templates/CompositeModule.dhall @@ -1,6 +1,6 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Template.dhall +let Sdk = ../Deps/Sdk.dhall let Field = { fieldName : Text, fieldType : Text } @@ -12,7 +12,7 @@ let Params = { typeName : Text, extraImports : List Text, fields : List Field } let run = \(params : Params) -> let fieldLines = - Deps.Prelude.Text.concatMapSep + Prelude.Text.concatMapSep "\n" Field ( \(field : Field) -> @@ -21,12 +21,12 @@ let run = params.fields let imports = - if Deps.Prelude.List.null Text params.extraImports + if Prelude.List.null Text params.extraImports then "from dataclasses import dataclass" else '' from dataclasses import dataclass - ${Deps.Prelude.Text.concatSep "\n" params.extraImports}'' + ${Prelude.Text.concatSep "\n" params.extraImports}'' in '' ${imports} @@ -43,4 +43,4 @@ let run = ${fieldLines} '' -in Algebra.module Params run /\ { Field } +in Sdk.Sigs.template Params run /\ { Field } diff --git a/gen/Templates/CoreModule.dhall b/gen/Templates/CoreModule.dhall index d3938c2..34a4943 100644 --- a/gen/Templates/CoreModule.dhall +++ b/gen/Templates/CoreModule.dhall @@ -1,4 +1,4 @@ -let Algebra = ../Algebras/Template.dhall +let Sdk = ../Deps/Sdk.dhall -- The surface-agnostic core of a generated package, emitted once at -- _generated/_core.py. It owns the names shared by every module and by both the @@ -43,4 +43,4 @@ let content = ) '' -in Algebra.module {} (\(_ : {}) -> content) +in Sdk.Sigs.template {} (\(_ : {}) -> content) diff --git a/gen/Templates/EnumModule.dhall b/gen/Templates/EnumModule.dhall index 7f82ebf..1ebf486 100644 --- a/gen/Templates/EnumModule.dhall +++ b/gen/Templates/EnumModule.dhall @@ -1,6 +1,6 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Template.dhall +let Sdk = ../Deps/Sdk.dhall let Variant = { memberName : Text, pgValue : Text } @@ -16,18 +16,18 @@ let run = let escapeLabel : Text -> Text = \(raw : Text) -> - Deps.Prelude.Function.composeList + Prelude.Function.composeList Text - [ Deps.Prelude.Text.replace "\\" "\\\\" - , Deps.Prelude.Text.replace "\r" "\\r" - , Deps.Prelude.Text.replace "\n" "\\n" - , Deps.Prelude.Text.replace "\t" "\\t" - , Deps.Prelude.Text.replace "\"" "\\\"" + [ Prelude.Text.replace "\\" "\\\\" + , Prelude.Text.replace "\r" "\\r" + , Prelude.Text.replace "\n" "\\n" + , Prelude.Text.replace "\t" "\\t" + , Prelude.Text.replace "\"" "\\\"" ] raw let memberLines = - Deps.Prelude.Text.concatMapSep + Prelude.Text.concatMapSep "\n" Variant ( \(variant : Variant) -> @@ -43,4 +43,4 @@ let run = ${memberLines} '' -in Algebra.module Params run /\ { Variant } +in Sdk.Sigs.template Params run /\ { Variant } diff --git a/gen/Templates/FacadeModule.dhall b/gen/Templates/FacadeModule.dhall index 2beb74c..18d6f2e 100644 --- a/gen/Templates/FacadeModule.dhall +++ b/gen/Templates/FacadeModule.dhall @@ -1,8 +1,6 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Template.dhall - -let Prelude = Deps.Prelude +let Sdk = ../Deps/Sdk.dhall -- A statement's public surface: the function and, when the query returns rows, -- its frozen Row dataclass. functionName doubles as the leaf module name. @@ -126,4 +124,4 @@ let run = ${allEntries}] '' -in Algebra.module Params run /\ { StatementExport, TypeExport } +in Sdk.Sigs.template Params run /\ { StatementExport, TypeExport } diff --git a/gen/Templates/InitModule.dhall b/gen/Templates/InitModule.dhall index f609ed6..cf5fc9f 100644 --- a/gen/Templates/InitModule.dhall +++ b/gen/Templates/InitModule.dhall @@ -1,4 +1,4 @@ -let Algebra = ../Algebras/Template.dhall +let Sdk = ../Deps/Sdk.dhall -- A minimal package __init__.py: a module docstring plus an empty __all__. -- Used for the _generated subpackage root and the statements subpackage. The @@ -15,4 +15,4 @@ let render = __all__: list[str] = [] '' -in Algebra.module Params render +in Sdk.Sigs.template Params render diff --git a/gen/Templates/RegisterModule.dhall b/gen/Templates/RegisterModule.dhall index ec6886b..cc94bfb 100644 --- a/gen/Templates/RegisterModule.dhall +++ b/gen/Templates/RegisterModule.dhall @@ -1,11 +1,9 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Template.dhall +let Sdk = ../Deps/Sdk.dhall let Surface = ../Structures/Surface.dhall -let Prelude = Deps.Prelude - -- Per-connection type registration, emitted once per surface. psycopg decodes an -- unregistered composite as a text string; registering its CompositeInfo makes -- it decode to a namedtuple, which the generated decode then splats into the @@ -96,4 +94,4 @@ let run = in Prelude.Text.concatSep "\n" allLines ++ "\n" -in Algebra.module Params run +in Sdk.Sigs.template Params run diff --git a/gen/Templates/RowsModule.dhall b/gen/Templates/RowsModule.dhall index 0107fa2..18da31b 100644 --- a/gen/Templates/RowsModule.dhall +++ b/gen/Templates/RowsModule.dhall @@ -1,10 +1,10 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Template.dhall +let Lude = ../Deps/Lude.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Sdk = ../Deps/Sdk.dhall -let Prelude = Deps.Prelude +let ImportSet = ../Structures/ImportSet.dhall -- The shared row module: every query's frozen Row dataclass and its decode -- function live here once, so the async and sync statement modules import the @@ -25,7 +25,7 @@ let indentAll \(text : Text) -> let pad = Prelude.Text.replicate n " " - in pad ++ Deps.Lude.Text.indentNonEmpty n text + in pad ++ Lude.Text.indentNonEmpty n text let importLineIf : Bool -> Text -> List Text @@ -118,4 +118,4 @@ let run = ++ Prelude.Text.concatMapSep "\n\n\n" RowDef renderRow params.rows ++ "\n" -in Algebra.module Params run /\ { RowDef } +in Sdk.Sigs.template Params run /\ { RowDef } diff --git a/gen/Templates/RuntimeModule.dhall b/gen/Templates/RuntimeModule.dhall index ddaf53a..baa56bc 100644 --- a/gen/Templates/RuntimeModule.dhall +++ b/gen/Templates/RuntimeModule.dhall @@ -1,4 +1,4 @@ -let Algebra = ../Algebras/Template.dhall +let Sdk = ../Deps/Sdk.dhall -- The fixed _runtime.py body, emitted once per generated package. No per-query -- customization. Mirrors DESIGN section 3 with two strict-clean adjustments the @@ -160,5 +160,5 @@ let syncContent = _ = cur.execute(sql, params) '' -in Algebra.module {} (\(_ : {}) -> content) +in Sdk.Sigs.template {} (\(_ : {}) -> content) /\ { runSync = \(_ : {}) -> syncContent } diff --git a/gen/Templates/StatementModule.dhall b/gen/Templates/StatementModule.dhall index f717aa7..f2a6f61 100644 --- a/gen/Templates/StatementModule.dhall +++ b/gen/Templates/StatementModule.dhall @@ -1,13 +1,13 @@ -let Algebra = ../Algebras/Template.dhall +let Prelude = ../Deps/Prelude.dhall -let Deps = ../Deps/package.dhall +let Lude = ../Deps/Lude.dhall + +let Sdk = ../Deps/Sdk.dhall let ImportSet = ../Structures/ImportSet.dhall let Surface = ../Structures/Surface.dhall -let Prelude = Deps.Prelude - -- Prefix every line (including the first) with `n` spaces, leaving blank lines -- untouched so trailing whitespace never appears. let indentAll @@ -16,7 +16,7 @@ let indentAll \(text : Text) -> let pad = Prelude.Text.replicate n " " - in pad ++ Deps.Lude.Text.indentNonEmpty n text + in pad ++ Lude.Text.indentNonEmpty n text -- A statement module is the thin per-surface I/O wrapper: it imports its Row -- dataclass and decode function from the shared `_rows` module and renders one @@ -187,7 +187,7 @@ let renderCall then "return ${await}${params.helperName}(conn, _SQL, params, ${params.decodeName})" else "return ${await}${params.helperName}(conn, _SQL, params)" -in Algebra.module +in Sdk.Sigs.template Params ( \(params : Params) -> renderImports params diff --git a/gen/Templates/TypesInit.dhall b/gen/Templates/TypesInit.dhall index 7583526..3182085 100644 --- a/gen/Templates/TypesInit.dhall +++ b/gen/Templates/TypesInit.dhall @@ -1,6 +1,6 @@ -let Deps = ../Deps/package.dhall +let Prelude = ../Deps/Prelude.dhall -let Algebra = ../Algebras/Template.dhall +let Sdk = ../Deps/Sdk.dhall let Export = { moduleName : Text, typeName : Text } @@ -9,7 +9,7 @@ let Params = { exports : List Export } let run = \(params : Params) -> let exportLines = - Deps.Prelude.Text.concatMapSep + Prelude.Text.concatMapSep "\n" Export ( \(export : Export) -> @@ -21,4 +21,4 @@ let run = ${exportLines} '' -in Algebra.module Params run /\ { Export } +in Sdk.Sigs.template Params run /\ { Export } From a5b4530f477c839541374ac485fa7892e76f52ae Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 09:31:19 +0300 Subject: [PATCH 02/11] feat(gen): rewrite compile.dhall/Gen.dhall as Interpret.dhall/package.dhall Task 4 of the Dhall dependency/layout migration: consumes Interpreters/Project.dhall's Sdk.Sigs.interpreter-shaped run and produces the Sdk.Sigs.generator-built entry point, now that the Deps barrel is gone (Tasks 1-3). - gen/compile.dhall -> gen/Interpret.dhall: drops the outer Optional Config unwrap (Sdk.Sigs.generator substitutes defaultConfig for the omitted-block case), keeps the per-field Prelude.Optional.fold defaults and importName derivation; imports Contract/Prelude directly instead of through the deleted barrel. - gen/Gen.dhall -> gen/package.dhall: builds Sdk.Sigs.generator Config Config/default interpret, with Config/default supplying an all-None Config. --- gen/Gen.dhall | 3 -- gen/Interpret.dhall | 54 ++++++++++++++++++++++++++++++++++++ gen/compile.dhall | 67 --------------------------------------------- gen/package.dhall | 16 +++++++++++ 4 files changed, 70 insertions(+), 70 deletions(-) delete mode 100644 gen/Gen.dhall create mode 100644 gen/Interpret.dhall delete mode 100644 gen/compile.dhall create mode 100644 gen/package.dhall diff --git a/gen/Gen.dhall b/gen/Gen.dhall deleted file mode 100644 index 9cb87fb..0000000 --- a/gen/Gen.dhall +++ /dev/null @@ -1,3 +0,0 @@ -let Sdk = ./Deps/Sdk.dhall - -in Sdk.module ./Config.dhall ./compile.dhall diff --git a/gen/Interpret.dhall b/gen/Interpret.dhall new file mode 100644 index 0000000..de4642f --- /dev/null +++ b/gen/Interpret.dhall @@ -0,0 +1,54 @@ +let Contract = ./Deps/Contract.dhall + +let Prelude = ./Deps/Prelude.dhall + +let Config = ./Config.dhall + +let OnUnsupported = ./Structures/OnUnsupported.dhall + +let ProjectInterpreter = ./Interpreters/Project.dhall + +-- Entry point handed to gen-sdk's Sdk.Sigs.generator as `interpret`. Each +-- field of Config is independently Optional, so a project may omit the +-- whole config block (Sdk.Sigs.generator substitutes an all-None +-- defaultConfig, see package.dhall) or any subset of its keys; `defaults` +-- collects every fallback in one place (packageName from the project name in +-- kebab case, emitSync off, onUnsupported Fail). The async surface is always +-- emitted; emitSync adds the sync mirror. +in \(config : Config) -> + \(project : Contract.Project) -> + let defaults = + { packageName = project.name.inKebabCase + , emitSync = False + , onUnsupported = OnUnsupported.Mode.Fail + } + + let packageName = + Prelude.Optional.fold + Text + config.packageName + Text + (\(t : Text) -> t) + defaults.packageName + + let emitSync = + Prelude.Optional.fold + Bool + config.emitSync + Bool + (\(b : Bool) -> b) + defaults.emitSync + + let onUnsupported = + Prelude.Optional.fold + OnUnsupported.Mode + config.onUnsupported + OnUnsupported.Mode + (\(m : OnUnsupported.Mode) -> m) + defaults.onUnsupported + + let importName = Prelude.Text.replace "-" "_" packageName + + let interpreterConfig = { packageName, importName, emitSync, onUnsupported } + + in ProjectInterpreter.run interpreterConfig project diff --git a/gen/compile.dhall b/gen/compile.dhall deleted file mode 100644 index cd12642..0000000 --- a/gen/compile.dhall +++ /dev/null @@ -1,67 +0,0 @@ -let Deps = ./Deps/package.dhall - -let Model = Deps.Sdk.Project - -let Prelude = Deps.Prelude - -let Config = ./Config.dhall - -let OnUnsupported = ./Structures/OnUnsupported.dhall - -let ProjectInterpreter = ./Interpreters/Project.dhall - --- Entry point handed to gen-sdk's module. `config` and each of its fields are --- Optional, so a project may omit the whole config block or any subset of its --- keys; `defaults` collects every fallback in one place (packageName from the --- project name in kebab case, emitSync off, onUnsupported Fail), so a future --- knob's default is added here alongside the others. The async surface is --- always emitted; emitSync adds the sync mirror. -in \(config : Optional Config) -> - \(project : Model.Project) -> - let defaults = - { packageName = project.name.inKebabCase - , emitSync = False - , onUnsupported = OnUnsupported.Mode.Fail - } - - let packageName = - Prelude.Optional.fold - Config - config - Text - (\(c : Config) -> - Prelude.Optional.fold Text c.packageName Text (\(t : Text) -> t) defaults.packageName - ) - defaults.packageName - - let emitSync = - Prelude.Optional.fold - Config - config - Bool - (\(c : Config) -> - Prelude.Optional.fold Bool c.emitSync Bool (\(b : Bool) -> b) defaults.emitSync - ) - defaults.emitSync - - let onUnsupported = - Prelude.Optional.fold - Config - config - OnUnsupported.Mode - ( \(c : Config) -> - Prelude.Optional.fold - OnUnsupported.Mode - c.onUnsupported - OnUnsupported.Mode - (\(m : OnUnsupported.Mode) -> m) - defaults.onUnsupported - ) - defaults.onUnsupported - - let importName = Prelude.Text.replace "-" "_" packageName - - let interpreterConfig = - { packageName, importName, emitSync, onUnsupported } - - in ProjectInterpreter.run interpreterConfig project diff --git a/gen/package.dhall b/gen/package.dhall new file mode 100644 index 0000000..3d07cf7 --- /dev/null +++ b/gen/package.dhall @@ -0,0 +1,16 @@ +let Sdk = ./Deps/Sdk.dhall + +let OnUnsupported = ./Structures/OnUnsupported.dhall + +let Config = ./Config.dhall + +let Config/default + : Config + = { packageName = None Text + , emitSync = None Bool + , onUnsupported = None OnUnsupported.Mode + } + +let interpret = ./Interpret.dhall + +in Sdk.Sigs.generator Config Config/default interpret From b13e42764765280e075d9390e9cea31dd5e3db8c Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 09:38:42 +0300 Subject: [PATCH 03/11] refactor(gen): rename gen/ to src/, move tests/Exhaustive.dhall to demos/ Pure directory move for gen/ -> src/ (relative imports inside are unaffected). tests/Exhaustive.dhall becomes demos/Exhaustive.dhall and is rewritten to use Sdk.Output.toFileMap, since Sdk.Sigs.generator-built modules no longer expose compileToFileMap. --- {tests => demos}/Exhaustive.dhall | 16 ++++++++-------- {gen => src}/Config.dhall | 0 {gen => src}/Deps/Contract.dhall | 0 {gen => src}/Deps/Lude.dhall | 0 {gen => src}/Deps/Prelude.dhall | 0 {gen => src}/Deps/Sdk.dhall | 0 {gen => src}/Interpret.dhall | 0 {gen => src}/Interpreters/CustomType.dhall | 0 {gen => src}/Interpreters/Member.dhall | 0 {gen => src}/Interpreters/ParamsMember.dhall | 0 {gen => src}/Interpreters/Primitive.dhall | 0 {gen => src}/Interpreters/Project.dhall | 0 {gen => src}/Interpreters/Query.dhall | 0 {gen => src}/Interpreters/QueryFragments.dhall | 0 {gen => src}/Interpreters/Result.dhall | 0 {gen => src}/Interpreters/ResultColumns.dhall | 0 {gen => src}/Interpreters/Scalar.dhall | 0 {gen => src}/Interpreters/Value.dhall | 0 {gen => src}/Structures/CustomKind.dhall | 0 {gen => src}/Structures/ImportSet.dhall | 0 {gen => src}/Structures/OnUnsupported.dhall | 0 {gen => src}/Structures/PyIdent.dhall | 0 {gen => src}/Structures/Surface.dhall | 0 {gen => src}/Templates/CompositeModule.dhall | 0 {gen => src}/Templates/CoreModule.dhall | 0 {gen => src}/Templates/EnumModule.dhall | 0 {gen => src}/Templates/FacadeModule.dhall | 0 {gen => src}/Templates/InitModule.dhall | 0 {gen => src}/Templates/RegisterModule.dhall | 0 {gen => src}/Templates/RowsModule.dhall | 0 {gen => src}/Templates/RuntimeModule.dhall | 0 {gen => src}/Templates/StatementModule.dhall | 0 {gen => src}/Templates/TypesInit.dhall | 0 {gen => src}/package.dhall | 0 34 files changed, 8 insertions(+), 8 deletions(-) rename {tests => demos}/Exhaustive.dhall (63%) rename {gen => src}/Config.dhall (100%) rename {gen => src}/Deps/Contract.dhall (100%) rename {gen => src}/Deps/Lude.dhall (100%) rename {gen => src}/Deps/Prelude.dhall (100%) rename {gen => src}/Deps/Sdk.dhall (100%) rename {gen => src}/Interpret.dhall (100%) rename {gen => src}/Interpreters/CustomType.dhall (100%) rename {gen => src}/Interpreters/Member.dhall (100%) rename {gen => src}/Interpreters/ParamsMember.dhall (100%) rename {gen => src}/Interpreters/Primitive.dhall (100%) rename {gen => src}/Interpreters/Project.dhall (100%) rename {gen => src}/Interpreters/Query.dhall (100%) rename {gen => src}/Interpreters/QueryFragments.dhall (100%) rename {gen => src}/Interpreters/Result.dhall (100%) rename {gen => src}/Interpreters/ResultColumns.dhall (100%) rename {gen => src}/Interpreters/Scalar.dhall (100%) rename {gen => src}/Interpreters/Value.dhall (100%) rename {gen => src}/Structures/CustomKind.dhall (100%) rename {gen => src}/Structures/ImportSet.dhall (100%) rename {gen => src}/Structures/OnUnsupported.dhall (100%) rename {gen => src}/Structures/PyIdent.dhall (100%) rename {gen => src}/Structures/Surface.dhall (100%) rename {gen => src}/Templates/CompositeModule.dhall (100%) rename {gen => src}/Templates/CoreModule.dhall (100%) rename {gen => src}/Templates/EnumModule.dhall (100%) rename {gen => src}/Templates/FacadeModule.dhall (100%) rename {gen => src}/Templates/InitModule.dhall (100%) rename {gen => src}/Templates/RegisterModule.dhall (100%) rename {gen => src}/Templates/RowsModule.dhall (100%) rename {gen => src}/Templates/RuntimeModule.dhall (100%) rename {gen => src}/Templates/StatementModule.dhall (100%) rename {gen => src}/Templates/TypesInit.dhall (100%) rename {gen => src}/package.dhall (100%) diff --git a/tests/Exhaustive.dhall b/demos/Exhaustive.dhall similarity index 63% rename from tests/Exhaustive.dhall rename to demos/Exhaustive.dhall index bfe8b84..35e8f28 100644 --- a/tests/Exhaustive.dhall +++ b/demos/Exhaustive.dhall @@ -1,9 +1,9 @@ -- Applies this generator to gen-sdk's shared cross-backend fixture project --- (the same "music_catalogue" project java.gen's own tests/Exhaustive.dhall +-- (the same "music_catalogue" project java.gen's own demos/Exhaustive.dhall -- exercises), so a Python client compiles from it and passes basedpyright -- strict. Pinned directly at gen-sdk's package.dhall, separately from --- gen/Deps/Sdk.dhall: that file only imports gen-sdk's `module.dhall` (the --- generator-construction function), which has no `Fixtures` field. +-- src/Deps/Sdk.dhall: that file only imports gen-sdk's `package.dhall` `as +-- Source` for RAM, and this fixture load doesn't need that mode. -- -- The fixture project deliberately covers PG types this generator does not -- support (box, inet, money, ranges, ...), so onUnsupported is set to Skip: @@ -13,13 +13,13 @@ -- Intended to be executed with: -- -- ```bash --- dhall to-directory-tree --file tests/Exhaustive.dhall --output --allow-path-separators +-- dhall to-directory-tree --file demos/Exhaustive.dhall --output --allow-path-separators -- ``` -let Sdk = ../gen/Deps/Sdk.dhall +let Sdk = ../src/Deps/Sdk.dhall -let Gen = ../gen/Gen.dhall +let Gen = ../src/package.dhall -let OnUnsupported = ../gen/Structures/OnUnsupported.dhall +let OnUnsupported = ../src/Structures/OnUnsupported.dhall let project = Sdk.Fixtures.Exhaustive @@ -30,4 +30,4 @@ let config = , onUnsupported = Some OnUnsupported.Mode.Skip } -in Gen.compileToFileMap config project +in Sdk.Output.toFileMap (Gen.compile config project) diff --git a/gen/Config.dhall b/src/Config.dhall similarity index 100% rename from gen/Config.dhall rename to src/Config.dhall diff --git a/gen/Deps/Contract.dhall b/src/Deps/Contract.dhall similarity index 100% rename from gen/Deps/Contract.dhall rename to src/Deps/Contract.dhall diff --git a/gen/Deps/Lude.dhall b/src/Deps/Lude.dhall similarity index 100% rename from gen/Deps/Lude.dhall rename to src/Deps/Lude.dhall diff --git a/gen/Deps/Prelude.dhall b/src/Deps/Prelude.dhall similarity index 100% rename from gen/Deps/Prelude.dhall rename to src/Deps/Prelude.dhall diff --git a/gen/Deps/Sdk.dhall b/src/Deps/Sdk.dhall similarity index 100% rename from gen/Deps/Sdk.dhall rename to src/Deps/Sdk.dhall diff --git a/gen/Interpret.dhall b/src/Interpret.dhall similarity index 100% rename from gen/Interpret.dhall rename to src/Interpret.dhall diff --git a/gen/Interpreters/CustomType.dhall b/src/Interpreters/CustomType.dhall similarity index 100% rename from gen/Interpreters/CustomType.dhall rename to src/Interpreters/CustomType.dhall diff --git a/gen/Interpreters/Member.dhall b/src/Interpreters/Member.dhall similarity index 100% rename from gen/Interpreters/Member.dhall rename to src/Interpreters/Member.dhall diff --git a/gen/Interpreters/ParamsMember.dhall b/src/Interpreters/ParamsMember.dhall similarity index 100% rename from gen/Interpreters/ParamsMember.dhall rename to src/Interpreters/ParamsMember.dhall diff --git a/gen/Interpreters/Primitive.dhall b/src/Interpreters/Primitive.dhall similarity index 100% rename from gen/Interpreters/Primitive.dhall rename to src/Interpreters/Primitive.dhall diff --git a/gen/Interpreters/Project.dhall b/src/Interpreters/Project.dhall similarity index 100% rename from gen/Interpreters/Project.dhall rename to src/Interpreters/Project.dhall diff --git a/gen/Interpreters/Query.dhall b/src/Interpreters/Query.dhall similarity index 100% rename from gen/Interpreters/Query.dhall rename to src/Interpreters/Query.dhall diff --git a/gen/Interpreters/QueryFragments.dhall b/src/Interpreters/QueryFragments.dhall similarity index 100% rename from gen/Interpreters/QueryFragments.dhall rename to src/Interpreters/QueryFragments.dhall diff --git a/gen/Interpreters/Result.dhall b/src/Interpreters/Result.dhall similarity index 100% rename from gen/Interpreters/Result.dhall rename to src/Interpreters/Result.dhall diff --git a/gen/Interpreters/ResultColumns.dhall b/src/Interpreters/ResultColumns.dhall similarity index 100% rename from gen/Interpreters/ResultColumns.dhall rename to src/Interpreters/ResultColumns.dhall diff --git a/gen/Interpreters/Scalar.dhall b/src/Interpreters/Scalar.dhall similarity index 100% rename from gen/Interpreters/Scalar.dhall rename to src/Interpreters/Scalar.dhall diff --git a/gen/Interpreters/Value.dhall b/src/Interpreters/Value.dhall similarity index 100% rename from gen/Interpreters/Value.dhall rename to src/Interpreters/Value.dhall diff --git a/gen/Structures/CustomKind.dhall b/src/Structures/CustomKind.dhall similarity index 100% rename from gen/Structures/CustomKind.dhall rename to src/Structures/CustomKind.dhall diff --git a/gen/Structures/ImportSet.dhall b/src/Structures/ImportSet.dhall similarity index 100% rename from gen/Structures/ImportSet.dhall rename to src/Structures/ImportSet.dhall diff --git a/gen/Structures/OnUnsupported.dhall b/src/Structures/OnUnsupported.dhall similarity index 100% rename from gen/Structures/OnUnsupported.dhall rename to src/Structures/OnUnsupported.dhall diff --git a/gen/Structures/PyIdent.dhall b/src/Structures/PyIdent.dhall similarity index 100% rename from gen/Structures/PyIdent.dhall rename to src/Structures/PyIdent.dhall diff --git a/gen/Structures/Surface.dhall b/src/Structures/Surface.dhall similarity index 100% rename from gen/Structures/Surface.dhall rename to src/Structures/Surface.dhall diff --git a/gen/Templates/CompositeModule.dhall b/src/Templates/CompositeModule.dhall similarity index 100% rename from gen/Templates/CompositeModule.dhall rename to src/Templates/CompositeModule.dhall diff --git a/gen/Templates/CoreModule.dhall b/src/Templates/CoreModule.dhall similarity index 100% rename from gen/Templates/CoreModule.dhall rename to src/Templates/CoreModule.dhall diff --git a/gen/Templates/EnumModule.dhall b/src/Templates/EnumModule.dhall similarity index 100% rename from gen/Templates/EnumModule.dhall rename to src/Templates/EnumModule.dhall diff --git a/gen/Templates/FacadeModule.dhall b/src/Templates/FacadeModule.dhall similarity index 100% rename from gen/Templates/FacadeModule.dhall rename to src/Templates/FacadeModule.dhall diff --git a/gen/Templates/InitModule.dhall b/src/Templates/InitModule.dhall similarity index 100% rename from gen/Templates/InitModule.dhall rename to src/Templates/InitModule.dhall diff --git a/gen/Templates/RegisterModule.dhall b/src/Templates/RegisterModule.dhall similarity index 100% rename from gen/Templates/RegisterModule.dhall rename to src/Templates/RegisterModule.dhall diff --git a/gen/Templates/RowsModule.dhall b/src/Templates/RowsModule.dhall similarity index 100% rename from gen/Templates/RowsModule.dhall rename to src/Templates/RowsModule.dhall diff --git a/gen/Templates/RuntimeModule.dhall b/src/Templates/RuntimeModule.dhall similarity index 100% rename from gen/Templates/RuntimeModule.dhall rename to src/Templates/RuntimeModule.dhall diff --git a/gen/Templates/StatementModule.dhall b/src/Templates/StatementModule.dhall similarity index 100% rename from gen/Templates/StatementModule.dhall rename to src/Templates/StatementModule.dhall diff --git a/gen/Templates/TypesInit.dhall b/src/Templates/TypesInit.dhall similarity index 100% rename from gen/Templates/TypesInit.dhall rename to src/Templates/TypesInit.dhall diff --git a/gen/package.dhall b/src/package.dhall similarity index 100% rename from gen/package.dhall rename to src/package.dhall From 2c9073fc3716a54536e77f2cebfe2878c2b21d0a Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 10:01:44 +0300 Subject: [PATCH 04/11] refactor(gen): update path references from gen/ to src/, tests/Exhaustive.dhall to demos/ Tasks 1-5 renamed gen/ to src/, gen/Gen.dhall to src/package.dhall, and tests/Exhaustive.dhall to demos/Exhaustive.dhall. This updates every external reference to the old paths across CI workflows, dev scripts, docs, and the bench harness. - .github/workflows/ci.yml, release.yml, build-contract-shell.sh: path refs. - README.md, AGENTS.md, DESIGN.md: doc path refs; DESIGN.md's section 10 tree diagram and entry-point description rewritten to match the actual post-move src/ layout (Interpret.dhall/package.dhall, no more Algebras/, Sdk.Sigs.generator instead of Sdk.module). - build.bash, mise.toml: dev-script path refs. - bench/generate.sh, bench/as-source.sh: gen/ -> src/ copies; the as-Source/plain-import sha256 substitution pair for gen-sdk is verified via `dhall hash` to be a no-op for v2.0.0 (both modes hash to the same value, unlike the old v0.11.0 pair), so only a comment remains; the lude pair is untouched since lude's pin is unchanged by this migration. - tests/fixture-project/project1.pgn.yaml: the fixture project's own `gen:` key still pointed at the now-deleted gen/Gen.dhall, which would break the harness and the `golden` mise task; updated to src/package.dhall. --- .github/scripts/build-contract-shell.sh | 2 +- .github/workflows/ci.yml | 8 ++--- .github/workflows/release.yml | 2 +- AGENTS.md | 2 +- DESIGN.md | 46 ++++++++++++++----------- README.md | 10 +++--- bench/as-source.sh | 24 ++++++++++--- bench/generate.sh | 26 ++++++++++---- mise.toml | 8 ++--- tests/fixture-project/project1.pgn.yaml | 14 ++++---- 10 files changed, 87 insertions(+), 55 deletions(-) diff --git a/.github/scripts/build-contract-shell.sh b/.github/scripts/build-contract-shell.sh index 3845a1d..9fbc79c 100755 --- a/.github/scripts/build-contract-shell.sh +++ b/.github/scripts/build-contract-shell.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Wraps the Dhall-generated package (from tests/Exhaustive.dhall) in a minimal +# Wraps the Dhall-generated package (from demos/Exhaustive.dhall) in a minimal # consumer shell, mirroring the hand-written tests/golden/ shell (pyproject.toml # + py.typed) so basedpyright strict runs against the same layout a real # consumer would import, per the full_package pattern in tests/conftest.py. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7664a04..b5d526f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,7 +121,7 @@ jobs: with: ref: ${{ inputs.ref || github.sha }} - # The `as Source` import mode on gen/Deps only changes how pgn loads the + # The `as Source` import mode on src/Deps only changes how pgn loads the # remote packages (unnormalized, to save RAM), not what they evaluate to. # The pinned action below bundles a dhall fork that predates the mode, so # strip it here; the evaluation is semantically identical either way. The @@ -132,16 +132,16 @@ jobs: shell: bash run: | set -euo pipefail - sed -i -e 's/^[[:space:]]*as Source$//' -e 's/^[[:space:]]*sha256:[0-9a-f]\{64\}$//' gen/Deps/*.dhall + sed -i -e 's/^[[:space:]]*as Source$//' -e 's/^[[:space:]]*sha256:[0-9a-f]\{64\}$//' src/Deps/*.dhall - # A plain, standard-Dhall evaluator cannot run this: gen/Interpreters/Project.dhall's + # A plain, standard-Dhall evaluator cannot run this: src/Interpreters/Project.dhall's # buildLookup and gen-sdk's own Fixtures.Exhaustive both use Text/equal, a builtin # from pgn's forked Dhall, absent from the upstream dhall-lang Prelude. This action # bundles that same fork. - name: Generate output from Dhall uses: nikita-volkov/dhall-directory-tree.github-action@60a18dc647d6daea805263ea0fed7bb8011f3bcd # v2 with: - dhall_file: tests/Exhaustive.dhall + dhall_file: demos/Exhaustive.dhall output_dir: contract-output - name: Assert the compile did not fail diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 25e92c7..91764bc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -86,7 +86,7 @@ jobs: id: resolve uses: nikita-volkov/dhall-resolve.github-action@7caaf1fdb40ac864bc02e37575f577ee084713a8 # v3 with: - file: gen/Gen.dhall + file: src/package.dhall minify: true - name: Prepare changelog for release diff --git a/AGENTS.md b/AGENTS.md index f6d2533..7da3b7b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,6 +33,6 @@ already says. ## Dhall -`gen/` pins its remote imports by sha256 (`gen/Deps/*.dhall`). Bump those +`src/` pins its remote imports by sha256 (`src/Deps/*.dhall`). Bump those deliberately, one at a time, and re-run the harness before committing a pin change. diff --git a/DESIGN.md b/DESIGN.md index bff5a2c..79e8a7c 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,7 +1,7 @@ # python.gen DESIGN Status: current. This describes the generator as shipped, not a plan. The code -is the source of truth: the Dhall generator under `gen/`, the fixture project +is the source of truth: the Dhall generator under `src/`, the fixture project and golden output under `tests/`. When this doc and the tree disagree, the tree wins. @@ -378,34 +378,37 @@ overwritten on every run. Do not hand-edit it. ## 10. Generator decomposition -`gen/Gen.dhall` is the entry point handed to gen-sdk: +`src/package.dhall` is the entry point handed to gen-sdk: ```dhall let Sdk = ./Deps/Sdk.dhall -in Sdk ./Config.dhall ./compile.dhall +let Config = ./Config.dhall + +let interpret = ./Interpret.dhall + +in Sdk.Sigs.generator Config Config/default interpret ``` -The Sdk `module` function has signature `\(Config : Type) -> \(compile) -> -{ contractVersion, Config, compile, compileToFileMap }`, and -`compile : Optional Config -> Project -> Lude.Compiled.Type Lude.Files.Type`, -where `Files.Type = List { path : Text, content : Text }`. `compile.dhall` -folds the optional user config into the internal interpreter config and -calls `Interpreters/Project.dhall`, which traverses queries and custom types -and assembles the file list. +`Sdk.Sigs.generator` has signature `\(Config : Type) -> \(defaultConfig : Config) -> +\(interpret : Config -> Contract.Project -> Contract.Output) -> ...`; it curries +`interpret` against `defaultConfig` whenever the user config is absent and hands +the result to gen-contract's `Contract.module`. `Interpret.dhall` folds the +optional user config into the internal interpreter config and calls +`Interpreters/Project.dhall`, which traverses queries and custom types and +assembles the file list (`Contract.Output`). -`gen/` mirrors a typical pgn gen-sdk generator, Python-flavored. The -algebra/interpreter/template split keeps assembly separate from rendering. +`src/` mirrors a typical pgn gen-sdk generator, Python-flavored: `Interpreters/` +assembles data, `Templates/` renders it to Python text. The interpreter/template +algebra signatures themselves live in gen-sdk's `Sdk.Sigs` (`interpreter.dhall`/ +`template.dhall`), not a local `Algebras/` dir. ```text -gen/ - Gen.dhall # Sdk Config compile (entry handed to gen-sdk) +src/ + package.dhall # Sdk.Sigs.generator Config Config/default interpret (entry handed to gen-sdk) Config.dhall # user config TYPE: { packageName, emitSync, onUnsupported } - compile.dhall # derive interpreter Config from user Config, call Project.run - Deps/ # pinned remote imports (gen-sdk module + Project, lude, Prelude) - Algebras/ - Interpreter.dhall # Config + `module Input Output run` ; Run = Config -> Input -> Compiled Output - Template.dhall # `module Params run` ; Run = Params -> Text + Interpret.dhall # derive interpreter Config from user Config, call Project.run + Deps/ # pinned remote imports: gen-sdk, gen-contract, lude, dhall Prelude Structures/ Surface.dhall # async/sync token table (section 4) CustomKind.dhall # Lookup : Name -> < Enum | Composite | Absent > + composite fields @@ -426,6 +429,7 @@ gen/ Project.dhall # traverse queries+customTypes, assemble all files + facade + header, # apply the Skip filter (section 11) Templates/ + CoreModule.dhall # shared _core.py: JsonValue, NoRowError/DecodeError, require_array RuntimeModule.dhall # async + sync _runtime.py bodies RowsModule.dhall # shared _rows.py (Row dataclasses + decode fns) StatementModule.dhall # one per-surface statement wrapper @@ -629,11 +633,11 @@ SQL rendering are largely driver-agnostic. CI runs two independent jobs (`.github/workflows/ci.yml`): `harness` (the pytest suite against a live Postgres) and `contract` (compiles gen-sdk's -`Fixtures.Exhaustive` via `tests/Exhaustive.dhall` and runs basedpyright +`Fixtures.Exhaustive` via `demos/Exhaustive.dhall` and runs basedpyright strict on the result). The `contract` job needs `nikita-volkov/dhall-directory-tree.github-action`, a Docker action bundling a forked Dhall evaluator; the local `dhall` CLI most people have installed is the standard dhall-lang build and does not -understand `Text/equal`, so it cannot run `tests/Exhaustive.dhall` directly. +understand `Text/equal`, so it cannot run `demos/Exhaustive.dhall` directly. Reproduce the `contract` job locally with [`act`](https://github.com/nektos/act) (not installed in this environment; `act -j contract` pulls the same pinned Docker action and runs the job as GitHub would). diff --git a/README.md b/README.md index db3226f..9e9a17f 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ name: ```yaml artifacts: python: - gen: https://raw.githubusercontent.com/slavashvets/python.gen/master/gen/Gen.dhall + gen: https://raw.githubusercontent.com/slavashvets/python.gen/master/src/package.dhall config: packageName: my-db-client emitSync: true @@ -61,10 +61,10 @@ database. | form | example | works? | | --------------- | --------------------------------------- | ------------------------------------------ | -| plain http(s) | `https://.../python.gen/gen/Gen.dhall` | yes; pgn fetches it and every relative import over HTTP | -| relative path | `../path/to/python.gen/gen/Gen.dhall` | yes, if you keep a local checkout next to your project | -| absolute path | `/abs/path/to/python.gen/gen/Gen.dhall` | yes, but the resulting freeze key is machine-specific | -| `file://` URL | `file:///abs/.../Gen.dhall` | rejected; pgn's project schema does not accept `file://` | +| plain http(s) | `https://.../python.gen/src/package.dhall` | yes; pgn fetches it and every relative import over HTTP | +| relative path | `../path/to/python.gen/src/package.dhall` | yes, if you keep a local checkout next to your project | +| absolute path | `/abs/path/to/python.gen/src/package.dhall` | yes, but the resulting freeze key is machine-specific | +| `file://` URL | `file:///abs/.../package.dhall` | rejected; pgn's project schema does not accept `file://` | Whichever form you use, the freeze file that caches the resolved generator (section "Freeze lifecycle" below) keys on the literal `gen:` value and diff --git a/bench/as-source.sh b/bench/as-source.sh index 203b973..c16945b 100755 --- a/bench/as-source.sh +++ b/bench/as-source.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Cold-cache benchmark of `pgn generate` on the fixture project, comparing the -# current gen (Deps imported `as Source`) against the same tree with the +# current src (Deps imported `as Source`) against the same tree with the # pre-as-Source Deps (plain pinned imports). Both variants run the same pgn # binary by default, so the measurement isolates the import mode itself. # @@ -24,7 +24,7 @@ trap 'rm -rf "$work"' EXIT prepare() { # $1 = variant root; mirrors the repo layout the fixture expects rm -rf "$1" && mkdir -p "$1/tests" - cp -R "$root/gen" "$1/gen" + cp -R "$root/src" "$1/src" cp -R "$root/tests/fixture-project" "$1/tests/fixture-project" rm -f "$1/tests/fixture-project/freeze1.pgn.yaml" rm -rf "$1/tests/fixture-project/artifacts" @@ -34,9 +34,23 @@ prepare() { # $1 = variant root; mirrors the repo layout the fixture expects # normalized-expression pins (an `as Source` pin hashes the import's source, # so the two modes need different sha256 values for the same version). strip_as_source() { # $1 = variant root - perl -i -ne 'print unless /^\s*as Source$/' "$1"/gen/Deps/*.dhall - perl -i -pe 's/8d43544ecb0e612406af3133bdbca51138c704a77a5a29ef62fe034d0e77a3a6/b9f7bb842345f3864c71e877fda4200306ba5c044a43e6f7713a23bc4769b91a/' "$1/gen/Deps/Sdk.dhall" - perl -i -pe 's/46b527b071eba96a17e76b4bc5774645714dd5b4355974d221e705aa7c126e77/14c43eec97972ae27afe3386ff937d04db66f84273d5551476361db12d2c4b50/' "$1/gen/Deps/Lude.dhall" + perl -i -ne 'print unless /^\s*as Source$/' "$1"/src/Deps/*.dhall + + # gen-sdk v2.0.0's src/package.dhall: `mise x -- dhall hash` against the + # live GitHub-hosted package (both `... as Source` and the plain import) + # returns the SAME sha256 (b9def6ab1179bc4aaae7fc6e91977f094f75934cd5755175c294a9e97ca71b15), + # matching the value already committed in src/Deps/Sdk.dhall -- so, unlike + # the old v0.11.0 pin this pair used to target (where the two genuinely + # differed: 8d43544e...->b9f7bb84...), no character swap is needed here + # after the strip above; the committed pin already equals the plain-import + # target. See docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md + # Task 6 for how this was verified (dhall's local import cache made the + # lookup instant; a cold, uncached fetch of a *different* URL hung in this + # sandbox, so treat network reachability here as best-effort, not given). + + # lude v5.1.0 is unchanged by this migration (see src/Deps/Lude.dhall), so + # its existing as-Source -> plain-import hash swap below is still correct. + perl -i -pe 's/46b527b071eba96a17e76b4bc5774645714dd5b4355974d221e705aa7c126e77/14c43eec97972ae27afe3386ff937d04db66f84273d5551476361db12d2c4b50/' "$1/src/Deps/Lude.dhall" } measure() { # $1 = label, $2 = variant root, $3 = pgn binary diff --git a/bench/generate.sh b/bench/generate.sh index a2acd50..2769621 100755 --- a/bench/generate.sh +++ b/bench/generate.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Build the fixture client from the working-tree gen, in one of two variants: -# with - gen/Deps as committed (imports `as Source`) +# Build the fixture client from the working-tree src, in one of two variants: +# with - src/Deps as committed (imports `as Source`) # without - the pre-as-Source Deps (mode stripped, normalized-expression # pins restored; byte-identical to commit fb78869's Deps) # Output lands in ./demo-with-as-source or ./demo-without-as-source. @@ -15,15 +15,29 @@ url="${PGN_TEST_DATABASE_URL:-postgresql://postgres:postgres@localhost:5432/post out="$root/demo-$variant-as-source" rm -rf "$out" && mkdir -p "$out/tests" -cp -R "$root/gen" "$out/gen" +cp -R "$root/src" "$out/src" cp -R "$root/tests/fixture-project" "$out/tests/fixture-project" rm -f "$out/tests/fixture-project/freeze1.pgn.yaml" rm -rf "$out/tests/fixture-project/artifacts" if [ "$variant" = without ]; then - perl -i -ne 'print unless /^\s*as Source$/' "$out"/gen/Deps/*.dhall - perl -i -pe 's/8d43544ecb0e612406af3133bdbca51138c704a77a5a29ef62fe034d0e77a3a6/b9f7bb842345f3864c71e877fda4200306ba5c044a43e6f7713a23bc4769b91a/' "$out/gen/Deps/Sdk.dhall" - perl -i -pe 's/46b527b071eba96a17e76b4bc5774645714dd5b4355974d221e705aa7c126e77/14c43eec97972ae27afe3386ff937d04db66f84273d5551476361db12d2c4b50/' "$out/gen/Deps/Lude.dhall" + perl -i -ne 'print unless /^\s*as Source$/' "$out"/src/Deps/*.dhall + + # gen-sdk v2.0.0's src/package.dhall: `mise x -- dhall hash` against the + # live GitHub-hosted package (both `... as Source` and the plain import) + # returns the SAME sha256 (b9def6ab1179bc4aaae7fc6e91977f094f75934cd5755175c294a9e97ca71b15), + # matching the value already committed in src/Deps/Sdk.dhall -- so, unlike + # the old v0.11.0 pin this pair used to target (where the two genuinely + # differed: 8d43544e...->b9f7bb84...), no character swap is needed here + # after the strip above; the committed pin already equals the plain-import + # target. See docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md + # Task 6 for how this was verified (dhall's local import cache made the + # lookup instant; a cold, uncached fetch of a *different* URL hung in this + # sandbox, so treat network reachability here as best-effort, not given). + + # lude v5.1.0 is unchanged by this migration (see src/Deps/Lude.dhall), so + # its existing as-Source -> plain-import hash swap below is still correct. + perl -i -pe 's/46b527b071eba96a17e76b4bc5774645714dd5b4355974d221e705aa7c126e77/14c43eec97972ae27afe3386ff937d04db66f84273d5551476361db12d2c4b50/' "$out/src/Deps/Lude.dhall" fi cd "$out/tests/fixture-project" diff --git a/mise.toml b/mise.toml index 040920b..68903bf 100644 --- a/mise.toml +++ b/mise.toml @@ -35,10 +35,10 @@ run = "bench/generate.sh without" # Regenerates only the "python" artifact: a full 7-artifact generate peaks at # ~31 GB RSS (memory goes to normalizing the generator closure per artifact, # see ci.yml), a single-artifact one at ~10 GB. The temp copy guarantees a -# fresh resolve of the working-tree gen/ (no stale freeze) and keeps pgn's +# fresh resolve of the working-tree src/ (no stale freeze) and keeps pgn's # scratch files out of the repo. [tasks.golden] -description = "Refresh tests/golden from the working-tree gen/ (single-artifact run)" +description = "Refresh tests/golden from the working-tree src/ (single-artifact run)" run = ''' #!/usr/bin/env bash set -euo pipefail @@ -48,14 +48,14 @@ trap 'rm -rf "$tmp"' EXIT cp -R "$root/tests/fixture-project/." "$tmp/fixture" rm -f "$tmp/fixture/freeze1.pgn.yaml" rm -rf "$tmp/fixture/artifacts" -python3 - "$tmp/fixture/project1.pgn.yaml" "$root/gen/Gen.dhall" <<'EOF' +python3 - "$tmp/fixture/project1.pgn.yaml" "$root/src/package.dhall" <<'EOF' import sys from pathlib import Path p = Path(sys.argv[1]) head, sep, _ = p.read_text().partition(" # The variants below") assert sep, "variants marker not found in project1.pgn.yaml" -_ = p.write_text(head.rstrip().replace("../../gen/Gen.dhall", sys.argv[2]) + "\n") +_ = p.write_text(head.rstrip().replace("../../src/package.dhall", sys.argv[2]) + "\n") EOF cd "$tmp/fixture" pgn --database-url "${PGN_TEST_DATABASE_URL:-postgresql://postgres:postgres@localhost:5432/postgres?sslmode=disable}" generate diff --git a/tests/fixture-project/project1.pgn.yaml b/tests/fixture-project/project1.pgn.yaml index f6bec82..ff75999 100644 --- a/tests/fixture-project/project1.pgn.yaml +++ b/tests/fixture-project/project1.pgn.yaml @@ -4,7 +4,7 @@ version: 0.0.0 postgres: 18 artifacts: python: - gen: ../../gen/Gen.dhall + gen: ../../src/package.dhall config: packageName: specimen-client emitSync: true @@ -12,25 +12,25 @@ artifacts: # Optional Config knobs (undocumented by pgn itself); see # tests/test_config_variants.py for the assertions. python-name-only: - gen: ../../gen/Gen.dhall + gen: ../../src/package.dhall config: packageName: name-only-client python-sync-only: - gen: ../../gen/Gen.dhall + gen: ../../src/package.dhall config: emitSync: true python-empty: - gen: ../../gen/Gen.dhall + gen: ../../src/package.dhall config: {} python-bare: - gen: ../../gen/Gen.dhall + gen: ../../src/package.dhall python-unknown-key: - gen: ../../gen/Gen.dhall + gen: ../../src/package.dhall config: packageName: unknown-key-client bogusField: 1 python-null: - gen: ../../gen/Gen.dhall + gen: ../../src/package.dhall config: packageName: null-client emitSync: null From a30ac3209dfc7556fe82bacb2b7ca7ac52d5f359 Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 10:05:34 +0300 Subject: [PATCH 05/11] fix(gen): update gen/ path references in the pytest harness to src/ tests/_harness.py's GEN_DIR (and its two consumers) still pointed the harness at the repo-root gen/ directory, which Task 5 deleted when it renamed gen/ to src/. Every test using the session-scoped generated_tree fixture, or building its own copytree in test_unsupported_types.py, would have hit FileNotFoundError on the very first pgn run. Renamed the constant to SRC_DIR (was misleading otherwise), pointed it at src/, matched the copied-tree destination directory name to what project1.pgn.yaml's `gen: ../../src/package.dhall` (fixed in the prior commit) actually resolves against, and fixed the two hardcoded yaml literals in test_unsupported_types.py plus conftest.py's docstring. Confirmed via `python3 -m py_compile` and a repo-wide grep for any remaining gen/-as-directory or GEN_DIR references in .py files (none). --- tests/_harness.py | 2 +- tests/conftest.py | 10 +++++----- tests/test_unsupported_types.py | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/_harness.py b/tests/_harness.py index 037ac41..9e5ffbd 100644 --- a/tests/_harness.py +++ b/tests/_harness.py @@ -23,7 +23,7 @@ DEFAULT_MAX_RSS_GB = 40.0 HERE = Path(__file__).resolve().parent -GEN_DIR = HERE.parent / "gen" +SRC_DIR = HERE.parent / "src" FIXTURE_PROJECT = HERE / "fixture-project" GOLDEN_DIR = HERE / "golden" diff --git a/tests/conftest.py b/tests/conftest.py index d0ccc23..5bac571 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,9 +15,9 @@ from tests._harness import ( FIXTURE_PROJECT, - GEN_DIR, GOLDEN_DIR, HERE, + SRC_DIR, admin_database_url, effective_database_name, run_pgn, @@ -67,15 +67,15 @@ def fixture_copy(tmp_path: Path) -> Path: def generated_tree(pgn_bin: str, pgn_admin_url: str, tmp_path_factory: pytest.TempPathFactory) -> Path: """Generate the fixture client once and return its artifacts/python dir. - The generator path in project1.pgn.yaml is `../../gen/Gen.dhall`, + The generator path in project1.pgn.yaml is `../../src/package.dhall`, relative to the fixture project. To keep it resolvable the copy mirrors the - real layout: `/gen` and `/tests/fixture-project`. The freeze + real layout: `/src` and `/tests/fixture-project`. The freeze file is dropped so pgn re-resolves the working-tree generator instead of a - cached hash (a stale freeze makes pgn ignore gen/ edits and silently + cached hash (a stale freeze makes pgn ignore src/ edits and silently emit the old output). """ root = tmp_path_factory.mktemp("pygen") - _ = shutil.copytree(GEN_DIR, root / "gen") + _ = shutil.copytree(SRC_DIR, root / "src") project = shutil.copytree(FIXTURE_PROJECT, root / "tests" / "fixture-project") (project / "freeze1.pgn.yaml").unlink(missing_ok=True) shutil.rmtree(project / "artifacts", ignore_errors=True) diff --git a/tests/test_unsupported_types.py b/tests/test_unsupported_types.py index 9d99eab..642db87 100644 --- a/tests/test_unsupported_types.py +++ b/tests/test_unsupported_types.py @@ -24,14 +24,14 @@ import pytest -from tests._harness import FIXTURE_PROJECT, GEN_DIR, HERE, run_pgn +from tests._harness import FIXTURE_PROJECT, HERE, SRC_DIR, run_pgn HARNESS_ROOT = HERE.parent def test_unsupported_pg_type_fails_loudly(pgn_bin: str, pgn_admin_url: str, tmp_path: Path) -> None: root = tmp_path / "pygen" - _ = shutil.copytree(GEN_DIR, root / "gen") + _ = shutil.copytree(SRC_DIR, root / "src") project = shutil.copytree(FIXTURE_PROJECT, root / "tests" / "fixture-project") (project / "freeze1.pgn.yaml").unlink(missing_ok=True) shutil.rmtree(project / "artifacts", ignore_errors=True) @@ -51,7 +51,7 @@ def test_unsupported_pg_type_fails_loudly(pgn_bin: str, pgn_admin_url: str, tmp_ def test_json_array_param_fails_loudly(pgn_bin: str, pgn_admin_url: str, tmp_path: Path) -> None: root = tmp_path / "pygen" - _ = shutil.copytree(GEN_DIR, root / "gen") + _ = shutil.copytree(SRC_DIR, root / "src") project = shutil.copytree(FIXTURE_PROJECT, root / "tests" / "fixture-project") (project / "freeze1.pgn.yaml").unlink(missing_ok=True) shutil.rmtree(project / "artifacts", ignore_errors=True) @@ -66,7 +66,7 @@ def test_json_array_param_fails_loudly(pgn_bin: str, pgn_admin_url: str, tmp_pat "postgres: 18\n" "artifacts:\n" " python:\n" - " gen: ../../gen/Gen.dhall\n" + " gen: ../../src/package.dhall\n" " config:\n" " onUnsupported: Fail\n" ) @@ -104,7 +104,7 @@ def test_skip_unsupported_drops_offending_units_and_cascades( golden package is held to. """ root = tmp_path / "pygen" - _ = shutil.copytree(GEN_DIR, root / "gen") + _ = shutil.copytree(SRC_DIR, root / "src") project = shutil.copytree(FIXTURE_PROJECT, root / "tests" / "fixture-project") (project / "freeze1.pgn.yaml").unlink(missing_ok=True) shutil.rmtree(project / "artifacts", ignore_errors=True) @@ -129,7 +129,7 @@ def test_skip_unsupported_drops_offending_units_and_cascades( "postgres: 18\n" "artifacts:\n" " python:\n" - " gen: ../../gen/Gen.dhall\n" + " gen: ../../src/package.dhall\n" " config:\n" " onUnsupported: Skip\n" ) From ffea69fe9ccccfafaef73b343fc9514cd6b21840 Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 14:19:59 +0300 Subject: [PATCH 06/11] docs(gen): add Task 7 CHANGELOG entry, fix stale Config.dhall comment Documents the gen-contract v4.0.1 / gen-sdk v2.0.0 migration and gen/->src/ layout restructure under # Upcoming, and updates Config.dhall's doc comment to reference Interpret.dhall instead of the old compile.dhall name. --- CHANGELOG.md | 12 ++++++++++++ src/Config.dhall | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cc7b2c..f98fe04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Upcoming +- Migrated the generator's internal dependencies to `gen-contract` v4.0.1 + and `gen-sdk` v2.0.0, adopting `Sdk.Sigs` in place of the local + `Algebras/` module, and restructured the repository layout to match the + pGenie generator architecture: implementation moved from `gen/` to + `src/`, the public entry point renamed from `gen/Gen.dhall` to + `src/package.dhall`, and the fixture driver moved from + `tests/Exhaustive.dhall` to `demos/Exhaustive.dhall`. No change to + generated output or the public Dhall interface (`artifacts..gen` + URLs pointing at a previously-released `resolved.dhall` are unaffected; + only the next release's URL path changes, from `.../gen/Gen.dhall` — the + unresolved source path some projects may reference directly instead of a + frozen release — to `.../src/package.dhall`). - The test harness now runs every pgn subprocess in its own process group under an RSS watchdog: a thread polls `ps -o rss=` every 2 s and, on breach of `PGN_MAX_RSS_GB` (default 40 GB), kills the whole group and fails the test with diff --git a/src/Config.dhall b/src/Config.dhall index 7ec1fe6..1377127 100644 --- a/src/Config.dhall +++ b/src/Config.dhall @@ -5,7 +5,7 @@ -- unsupported statement/type and its dependents, with a warning) when a query -- or custom type hits a PG shape the generator cannot render; see -- Structures/OnUnsupported.dhall. All fields are Optional so a project may omit --- the whole config block or any subset of its keys; compile.dhall supplies the +-- the whole config block or any subset of its keys; Interpret.dhall supplies the -- defaults. let OnUnsupported = ./Structures/OnUnsupported.dhall From 9c33a23dbbec070c530430900ea7f770054a4fcd Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 14:29:51 +0300 Subject: [PATCH 07/11] refactor(gen): drop dead Run type-alias bindings in Query/Result/ResultColumns These were declared but never exported (unlike Member.dhall/ParamsMember.dhall, which do export Run), left over from documenting each file's actual multi-arg call signature during the gen-sdk v2 migration. Flagged by final review as harmless but pointless; removing rather than exporting keeps these three files' tails exactly as they were pre-migration, matching the migration's "keep bare exports verbatim" instruction for this exception group. --- src/Interpreters/Query.dhall | 2 -- src/Interpreters/Result.dhall | 3 --- src/Interpreters/ResultColumns.dhall | 3 --- 3 files changed, 8 deletions(-) diff --git a/src/Interpreters/Query.dhall b/src/Interpreters/Query.dhall index daa1d47..cce3fcd 100644 --- a/src/Interpreters/Query.dhall +++ b/src/Interpreters/Query.dhall @@ -177,6 +177,4 @@ let run = ) ) -let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output - in { Input, Output, run } diff --git a/src/Interpreters/Result.dhall b/src/Interpreters/Result.dhall index 83f3ef1..5d652a0 100644 --- a/src/Interpreters/Result.dhall +++ b/src/Interpreters/Result.dhall @@ -101,7 +101,4 @@ let run = } input -let Run = - Config -> CustomKind.Lookup -> Text -> Input -> Lude.Compiled.Type Output - in { Input, Output, RowClass, run } diff --git a/src/Interpreters/ResultColumns.dhall b/src/Interpreters/ResultColumns.dhall index b2fc737..e0bbfaa 100644 --- a/src/Interpreters/ResultColumns.dhall +++ b/src/Interpreters/ResultColumns.dhall @@ -78,7 +78,4 @@ let run = input ) -let Run = - Config -> CustomKind.Lookup -> Text -> Input -> Lude.Compiled.Type Output - in { Input, Output, run } From 15c28361de3192aa2d0e896fa32076c5f841ece3 Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 16:30:39 +0300 Subject: [PATCH 08/11] The plan --- .../plans/2026-07-11-gen-sdk-v2-migration.md | 666 ++++++++++++++++++ 1 file changed, 666 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md diff --git a/docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md b/docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md new file mode 100644 index 0000000..5e0499c --- /dev/null +++ b/docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md @@ -0,0 +1,666 @@ +# python.gen: migrate to gen-contract v4.0.1 / gen-sdk v2.0.0, adopt architecture layout + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Bring `python.gen` up to the same structural shape as `java.gen`'s +last release (`v1.1.0`): pin `gen-contract`/`gen-sdk` directly (no bundled +SDK), adopt `Sdk.Sigs` in place of the local `Algebras/` folder, and move to +the `src/`-rooted repo layout the normative architecture doc describes — with +**zero change to generated output**. + +**Architecture:** Reference is +`gen-sdk/docs/generator-architecture.md` (already read in full) and +`java.gen`'s current `master` (tag `v1.1.0`, commit `658508c`). `python.gen` +is currently on pre-split `gen-sdk v0.11.0` (no `Sigs`, bundles `Project` +itself) with a hand-rolled `gen/Algebras/{Interpreter,Template}.dhall`, a +`gen/Deps/package.dhall` barrel, and the old `gen/` + `tests/Exhaustive.dhall` +layout — i.e. it has never done *any* of the steps `java.gen` went through +(`gen-migration-plan.md` at the pgenie repo root covers `rust.gen`/`haskell.gen`, +which already had `Sigs`; it does not cover `python.gen` at all). This plan +folds all of `java.gen`'s historical steps into one destination state, since +there is no reason to recreate `java.gen`'s intermediate commits. + +**Tech Stack:** Dhall (fork `pgn`'s dhall, package name `dhll`, needed for the +`Text/equal` builtin and `as Source` import mode — see Global Constraints), +Python 3.12 / `uv` / `pytest` for the harness, `mise` for tool pinning. + +## Global Constraints + +- **No behavior change.** This is a dependency/layout migration, not a + feature change. If `demos/Exhaustive.dhall` (see Task 5) produces different + file paths or content than `tests/Exhaustive.dhall` did before the move, + that is a bug in the migration, not an expected diff. +- **Do not touch interpreter algorithms.** `Interpreters/Project.dhall`'s + `Skip`/`Fail` logic (`typeSucceeds`/`queryChecks`/`effectiveQueries`) is + deliberately hand-rolled instead of using `Typeclasses.Classes.Alternative` + the way `java.gen` does, per an explicit in-file comment: multiple + `QueryGen.run`/`CustomTypeGen.run` call sites for the same query measurably + multiplied Dhall normalization time (seconds → minutes), confirmed by wall-time + bisection. This is orthogonal to the Deps/Sigs migration — leave it as is. + Do **not** add `src/Deps/Typeclasses.dhall` since nothing will use it. +- **`Interpreters/Member.dhall` and `Interpreters/ParamsMember.dhall` keep + their 3-argument `Run` type** (`Config -> CustomKind.Lookup -> Input -> + Compiled Output`) instead of conforming to `Sdk.Sigs.interpreter`'s fixed + 2-argument shape (`Config -> Input -> Result`). `java.gen`'s own + `Member.dhall` doesn't need a lookup table; `python.gen`'s does (custom-type + name resolution — see the `buildLookup`/`IndexedCustomType` comments in + `Interpreters/Project.dhall`). Bundling `Lookup` into `Config` or `Input` + to force-fit the Sig is a bigger, separate refactor — out of scope here. + These two files get the Deps-import and `Algebra.Config` → local `Config` + changes (Task 2) but keep their existing bare `{ Input, Output, Run, run }` + export, not `Sdk.Sigs.interpreter Config Input Output run`. +- **Config narrowing is deferred.** The architecture doc's ideal is each + interpreter declaring only the `Config` fields it (and its children) needs. + `java.gen` does this trivially (`{ useOptional : Bool }` everywhere, since + that's its whole config). `python.gen`'s config has 4 fields + (`packageName`, `importName`, `emitSync`, `onUnsupported`); auditing exactly + which fields each of the 11 interpreters actually reads and narrowing each + is real, separate work with real risk of missing a field some deeply nested + path needs. This plan keeps the **same 4-field `Config` record, declared + locally and verbatim in each interpreter module** (no more shared + `Algebra.Config` alias) — this satisfies "`Config` is a parameter each + module declares itself," just not narrowed. Note it as a follow-up, don't + do it now. +- **Local verification gap.** This sandbox has a global `dhall`/`pgn` (cabal + build, `dhll-1.42.3`) that is **not** the `mise`-pinned `pgn v0.9.1` this + repo's CI/tests actually use — a plain `dhall type --file tests/Exhaustive.dhall` + here fails on the *pre-migration* tree already, with an `as Source` + hash-integrity mismatch on `gen/Deps/Sdk.dhall` (confirmed during planning: + expected `8d43544e...`, actual `573b4655...`). This is a toolchain mismatch, + not evidence of a real problem. **Whoever executes this plan must run + verification through `mise x -- dhall ...` / `mise x -- pgn ...` / `mise x + -- uv run pytest`**, matching `mise.toml`'s pin, not a bare global `dhall`. + If `mise` isn't available in the execution environment either, at minimum + run `dhall format --transitive` (syntax-only, tool-version-agnostic) and + flag that deeper verification (`dhall type`, fixture diff, pytest, + basedpyright) still needs to happen on a properly provisioned machine/CI + before this is considered done. + +--- + +## File Structure + +``` +src/ (was gen/) + package.dhall (was Gen.dhall — now built via Sdk.Sigs.generator) + Config.dhall (unchanged content, moved) + Interpret.dhall (was compile.dhall — Config no longer Optional at top) + Deps/ + Contract.dhall (NEW — gen-contract v4.0.1 pin) + Sdk.dhall (bumped gen-sdk v0.11.0 → v2.0.0) + Lude.dhall (unchanged content, moved) + Prelude.dhall (unchanged content, moved) + (package.dhall barrel REMOVED) + Interpreters/ (11 files: Deps.Sdk.Project → Deps.Contract, Algebra → Sdk.Sigs) + Templates/ (10 files: Algebra → Sdk.Sigs; 7 also de-barrel Deps) + Structures/ (CustomKind.dhall: Deps.Sdk.Project → Deps.Contract; others untouched) + (Algebras/ REMOVED) +demos/ + Exhaustive.dhall (was tests/Exhaustive.dhall — rewritten for Sdk.Output.toFileMap) +tests/ (Python pytest harness — unchanged except any gen/-path references) +.github/workflows/{ci,release}.yml, .github/scripts/build-contract-shell.sh, +README.md, AGENTS.md, DESIGN.md, build.bash, bench/*.sh, mise.toml + (path references updated: gen/ → src/, tests/Exhaustive.dhall → demos/Exhaustive.dhall) +``` + +--- + +### Task 1: Add the new Deps pins and remove the `Deps/package.dhall` barrel + +**Files:** +- Create: `gen/Deps/Contract.dhall` +- Modify: `gen/Deps/Sdk.dhall` +- Delete: `gen/Deps/package.dhall` +- Modify (de-barrel): every file that currently has `let Deps = ../Deps/package.dhall` (see the full list in Tasks 2–3 — do this as part of those tasks, not twice) + +This task only stages the new pins; Task 2 is where the fallout (broken +`Sdk.Project`/`Sdk.Fixtures` references, `Algebras/` removal) gets fixed. Do +not try to get `dhall type` green after this task alone — it won't be, and +that's expected (same as `gen-migration-plan.md`'s phase-0 commit 2 for +`java.gen`). Fold Task 1 and Task 2 into one commit if you'd rather not carry +a known-broken intermediate state. + +- [ ] **Step 1: Create `gen/Deps/Contract.dhall`** — the exact pin `java.gen` + and `gen-sdk` itself use: + +```dhall +https://raw.githubusercontent.com/pgenie-io/gen-contract/v4.0.1/src/package.dhall + sha256:4a130ba7fbaa152a776babbb1bf2994a4833931ca76bde9bf6930d354225651e +``` + +- [ ] **Step 2: Bump `gen/Deps/Sdk.dhall`** to `gen-sdk v2.0.0` (same pin + `java.gen`'s `src/Deps/Sdk.dhall` uses). Preserve the existing `as Source` + import mode (see `AGENTS.md`/CI comments on why `python.gen` uses it — + `java.gen` doesn't, but that's an intentional `python.gen`-specific RAM + optimization, not something this migration should undo): + +```dhall +https://raw.githubusercontent.com/pgenie-io/gen-sdk/v2.0.0/src/package.dhall + sha256:b9def6ab1179bc4aaae7fc6e91977f094f75934cd5755175c294a9e97ca71b15 + as Source +``` + + If keeping `as Source` here, its hash is a *source*-text hash, not the + semantic hash above copied from `java.gen` (which imports plainly). Verify + with `dhall hash` against the raw URL using the repo's pinned toolchain + (`mise x -- dhall hash <<< 'https://raw.githubusercontent.com/pgenie-io/gen-sdk/v2.0.0/src/package.dhall as Source'` + or equivalent) before trusting the semantic hash verbatim in `as Source` + mode — don't guess. + +- [ ] **Step 3: Delete `gen/Deps/package.dhall`.** + +- [ ] **Step 4: Commit** (or fold into Task 2's commit). + +--- + +### Task 2: Rewire `Structures/CustomKind.dhall` and all 11 `Interpreters/*.dhall` + +**Files:** +- Modify: `gen/Structures/CustomKind.dhall` +- Modify: `gen/Interpreters/{CustomType,Member,ParamsMember,Primitive,Project,Query,QueryFragments,Result,ResultColumns,Scalar,Value}.dhall` +- Delete: `gen/Algebras/` (all three files) + +**The mechanical recipe, applied to every file above:** + +1. Replace `let Deps = ../Deps/package.dhall` with direct imports of exactly + what the file uses. Every one of these files uses `Deps.Sdk.Project` + (→ becomes a direct `Deps/Contract.dhall` import) and `Deps.Lude`/`Deps.Prelude` + (→ direct imports). Concretely, replace: + ```dhall + let Deps = ../Deps/package.dhall + ``` + with (only the lines this particular file actually needs — check with + `grep -n 'Deps\.' ` first): + ```dhall + let Lude = ../Deps/Lude.dhall + + let Prelude = ../Deps/Prelude.dhall + + let Model = ../Deps/Contract.dhall + ``` + and change every remaining `Deps.Lude` → `Lude`, `Deps.Prelude` → `Prelude` + in the body. `QueryFragments.dhall` additionally has an unused + `let Sdk = Deps.Sdk` line (line 7) — drop it, nothing in the file + references the `Sdk` binding. + +2. Replace the line `let Model = Deps.Sdk.Project` (now redundant with step + 1's `Model` binding) — don't duplicate it, step 1 already introduces + `Model` pointed at `Deps/Contract.dhall`. + +3. Delete `let Algebra = ../Algebras/Interpreter.dhall`. + +4. Add a local `Config` type declaration (same 4 fields everywhere per the + Global Constraints note on deferred narrowing): + ```dhall + let Config = + { packageName : Text + , importName : Text + , emitSync : Bool + , onUnsupported : OnUnsupported.Mode + } + ``` + This needs `OnUnsupported = ../Structures/OnUnsupported.dhall` imported in + any file that doesn't already import it (check first — `Project.dhall` + already does). + +5. Change every `\(config : Algebra.Config) ->` to `\(config : Config) ->`. + +6. Change the tail: + - **10 of the 11 files** (`CustomType`, `Primitive`, `Project`, `Query`, + `QueryFragments`, `Result`, `ResultColumns`, `Scalar`, `Value` — 9 files, + not 10; `Member`/`ParamsMember` are the exception below) end with + `Algebra.module Input Output run` (or, for `Scalar.dhall`, + `Algebra.module Input Output run /\ { ScalarDecode }`). Change to: + ```dhall + Sdk.Sigs.interpreter Config Input Output run + ``` + (`Scalar.dhall`: `Sdk.Sigs.interpreter Config Input Output run /\ { ScalarDecode }`), + which needs `let Sdk = ../Deps/Sdk.dhall` imported (it isn't currently, + since `Deps.Sdk.Project` used to come through the barrel — add it). + - **`Member.dhall` and `ParamsMember.dhall`** keep their existing tail + verbatim: `in { Input, Output, Run, run }`, just with `Algebra.Config` → + `Config` in the `Run` type alias line + (`let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output`). + No `Sdk.Sigs.interpreter` here — see Global Constraints. + +7. `Interpreters/Project.dhall` specifically: its two `\(config : Algebra.Config) ->` + occurrences (the `combineOutputs` and `run` functions) both become + `\(config : Config) ->`; its `lookupConfig : Algebra.Config` type + annotation (used to type-check `Value.run` calls for composite-field + rendering) becomes `lookupConfig : Config`. No other logic in this file + changes — the `Skip`/`Fail` machinery is untouched per Global Constraints. + +- [ ] **Step 1: Apply the recipe to all 11 `Interpreters/*.dhall` files and `Structures/CustomKind.dhall`.** +- [ ] **Step 2: Delete `gen/Algebras/`.** +- [ ] **Step 3: Verify** (with the `mise`-pinned toolchain, not the bare local `dhall` — see Global Constraints): + ```bash + mise x -- dhall type --file gen/Interpreters/Project.dhall + ``` + Expected: prints the interpreter's type (a record with `Input`, `Output`, + `Result`, `Run`, `run` fields) with no error. This alone pulls in every + other `Interpreters/*.dhall` transitively, so it's a full check of this task. +- [ ] **Step 4: Commit.** + +--- + +### Task 3: Rewire the 10 `Templates/*.dhall` files + +**Files:** +- Modify: `gen/Templates/{CompositeModule,CoreModule,EnumModule,FacadeModule,InitModule,RegisterModule,RowsModule,RuntimeModule,StatementModule,TypesInit}.dhall` + +**Recipe:** + +1. `CoreModule.dhall`, `InitModule.dhall`, `RuntimeModule.dhall` don't import + `Deps` at all (no `Prelude`/`Lude` need) — only change: drop + `let Algebra = ../Algebras/Template.dhall`, add `let Sdk = ../Deps/Sdk.dhall`, + and change the tail. `CoreModule`/`RuntimeModule` end with + `Algebra.module {} (\(_ : {}) -> content)` → `Sdk.Sigs.template {} (\(_ : {}) -> content)`. + `InitModule` ends with `Algebra.module Params render` → + `Sdk.Sigs.template Params render`. + +2. The other 7 (`CompositeModule`, `EnumModule`, `FacadeModule`, + `RegisterModule`, `RowsModule`, `StatementModule`, `TypesInit`) currently + have `let Deps = ../Deps/package.dhall`, and only ever use + `Deps.Prelude.*` (all seven) and, additionally, `Deps.Lude.Text.indentNonEmpty` + (`RowsModule`, `StatementModule` only — confirmed by + `grep -n 'Deps\.' gen/Templates/*.dhall` during planning). Replace the + barrel import with: + ```dhall + let Prelude = ../Deps/Prelude.dhall + ``` + adding `let Lude = ../Deps/Lude.dhall` only in `RowsModule.dhall` and + `StatementModule.dhall`. Then replace `Deps.Prelude.` → `Prelude.` and + `Deps.Lude.` → `Lude.` throughout each file's body. Drop + `let Algebra = ../Algebras/Template.dhall`, add `let Sdk = ../Deps/Sdk.dhall`. + +3. Tails for these 7: `Algebra.module Params run` → + `Sdk.Sigs.template Params run` (`FacadeModule`, `RegisterModule`, + `StatementModule`); with a combined record for the other 4: + `Algebra.module Params run /\ { Field }` (`CompositeModule`) → + `Sdk.Sigs.template Params run /\ { Field }`; `/\ { Variant }` (`EnumModule`); + `/\ { StatementExport, TypeExport }` (`FacadeModule` — check which of + `FacadeModule`/others actually has this combinator vs a plain + `Algebra.module Params run`, per the earlier grep output, before editing — + don't assume, re-`grep -n 'Algebra.module' gen/Templates/*.dhall` and + match each file's exact current tail); `/\ { RowDef }` (`RowsModule`); + `/\ { Export }` (`TypesInit`). + +- [ ] **Step 1: Apply the recipe to all 10 files.** +- [ ] **Step 2: Verify:** + ```bash + mise x -- dhall type --file gen/Interpreters/Project.dhall + ``` + (Templates are only reachable transitively through `Interpreters/Project.dhall` + and its children, same as Task 2 — this single check covers both tasks once + both are done. If running Task 2 and 3 as separate commits, this step will + fail after Task 2 alone if any interpreter references a not-yet-updated + template's old shape; if so, do Tasks 2 and 3 as one commit instead.) +- [ ] **Step 3: Commit.** + +--- + +### Task 4: Rewrite the root entry point (`Config.dhall`, `compile.dhall` → `Interpret.dhall`, `Gen.dhall` → `package.dhall`) + +**Files:** +- Modify (move, content unchanged): `gen/Config.dhall` +- Modify (move + rewrite): `gen/compile.dhall` → `gen/Interpret.dhall` +- Modify (move + rewrite): `gen/Gen.dhall` → `gen/package.dhall` + +(Paths shown as `gen/...` here since Task 5 does the `gen/` → `src/` directory +move; do this task first, in place, then Task 5 is a pure `git mv` sweep with +no further content changes.) + +**Interfaces:** +- Consumes: `Interpreters/Project.dhall`'s `run` (produced by Task 2, + now `Sdk.Sigs.interpreter`-shaped: `.run : Config -> Contract.Project -> Compiled Output`). +- Produces: `package.dhall`'s `Sdk.Sigs.generator`-built value + (`{ contractVersion, Config, compile }`), consumed by Task 5's + `demos/Exhaustive.dhall` and by any pGenie project's `artifacts..gen` URL. + +`Sdk.Sigs.generator`'s shape (from the architecture doc): +```dhall +\(Config : Type) -> +\(defaultConfig : Config) -> +\(interpret : Config -> Contract.Project -> Contract.Output) -> + let compile = \(config : Optional Config) -> + merge { None = interpret defaultConfig, Some = interpret } config + in Contract.module Config compile +``` +Note `interpret` takes a **bare** `Config`, not `Optional Config` — the outer +"config block omitted entirely" case is handled once, by substituting +`defaultConfig`, not by `interpret` itself. `python.gen`'s current +`compile.dhall` handles *two* levels of optionality (the whole block, and +each field within it) with a doubled `Prelude.Optional.fold`. Only the outer +level goes away; each field inside `Config` stays individually `Optional` (so +a project can supply `emitSync: true` alone and still get default +`packageName`/`onUnsupported`) — that per-field defaulting is +`python.gen`-specific richness `java.gen` doesn't have (its `Config` has one +non-Optional `Bool` field), and this migration must not lose it. + +- [ ] **Step 1: `gen/Config.dhall`** — content unchanged, just confirm it + still reads (no edits needed here; listed for completeness since Task 5 + moves the file). + +- [ ] **Step 2: Rewrite `gen/compile.dhall` as `gen/Interpret.dhall`** — + drop the outer `Optional Config` unwrap (the two outermost + `Prelude.Optional.fold Config config Text (\(c : Config) -> ...)` / + `... Bool ...` / `... OnUnsupported.Mode ...` wrappers), keep everything + else (the per-field defaults, `importName` derivation) as is: + +```dhall +let Deps = ./Deps/package.dhall + +-- NOTE: Task 5 changes this to ./Deps/Contract.dhall / ./Deps/Prelude.dhall +-- directly once the Deps barrel is gone (Task 1) — write it that way now, +-- don't reintroduce the barrel: +let Contract = ./Deps/Contract.dhall + +let Prelude = ./Deps/Prelude.dhall + +let Config = ./Config.dhall + +let OnUnsupported = ./Structures/OnUnsupported.dhall + +let ProjectInterpreter = ./Interpreters/Project.dhall + +-- Entry point handed to gen-sdk's Sdk.Sigs.generator as `interpret`. Each +-- field of Config is independently Optional, so a project may omit the +-- whole config block (Sdk.Sigs.generator substitutes an all-None +-- defaultConfig, see package.dhall) or any subset of its keys; `defaults` +-- collects every fallback in one place (packageName from the project name in +-- kebab case, emitSync off, onUnsupported Fail). The async surface is always +-- emitted; emitSync adds the sync mirror. +in \(config : Config) -> + \(project : Contract.Project) -> + let defaults = + { packageName = project.name.inKebabCase + , emitSync = False + , onUnsupported = OnUnsupported.Mode.Fail + } + + let packageName = + Prelude.Optional.fold + Text + config.packageName + Text + (\(t : Text) -> t) + defaults.packageName + + let emitSync = + Prelude.Optional.fold + Bool + config.emitSync + Bool + (\(b : Bool) -> b) + defaults.emitSync + + let onUnsupported = + Prelude.Optional.fold + OnUnsupported.Mode + config.onUnsupported + OnUnsupported.Mode + (\(m : OnUnsupported.Mode) -> m) + defaults.onUnsupported + + let importName = Prelude.Text.replace "-" "_" packageName + + let interpreterConfig = { packageName, importName, emitSync, onUnsupported } + + in ProjectInterpreter.run interpreterConfig project +``` + +- [ ] **Step 3: Rewrite `gen/Gen.dhall` as `gen/package.dhall`:** + +```dhall +let Sdk = ./Deps/Sdk.dhall + +let OnUnsupported = ./Structures/OnUnsupported.dhall + +let Config = ./Config.dhall + +let Config/default + : Config + = { packageName = None Text + , emitSync = None Bool + , onUnsupported = None OnUnsupported.Mode + } + +let interpret = ./Interpret.dhall + +in Sdk.Sigs.generator Config Config/default interpret +``` + +- [ ] **Step 4: Verify:** + ```bash + mise x -- dhall type --file gen/package.dhall + ``` + Expected type: a record with `contractVersion`, `Config`, `compile` fields + (`compile : Optional Config -> Contract.Project -> Contract.Output`). +- [ ] **Step 5: Commit.** + +--- + +### Task 5: Move `gen/` → `src/`, `tests/Exhaustive.dhall` → `demos/Exhaustive.dhall` + +**Files:** +- Move: `gen/` → `src/` (whole tree, `git mv`) +- Move + rewrite: `tests/Exhaustive.dhall` → `demos/Exhaustive.dhall` + +- [ ] **Step 1:** + ```bash + git mv gen src + mkdir -p demos + git mv tests/Exhaustive.dhall demos/Exhaustive.dhall + ``` + All the `../Deps/...`, `./Interpreters/...`, `../Templates/...` style + relative imports inside `src/` are untouched by this move (they're relative + to their own file, not to the repo root), so no content changes are needed + inside `src/` itself from the move alone. + +- [ ] **Step 2: Rewrite `demos/Exhaustive.dhall`.** Its old body called + `Gen.compileToFileMap config project` — `Sdk.Sigs.generator`-built modules + don't have a `compileToFileMap` field (per the architecture doc: "there is + no `compileToFileMap` on the module — turning an `Output` into files is the + caller's job, via `Sdk.Output.toFileMap`"). New content: + +```dhall +-- Applies this generator to gen-sdk's shared cross-backend fixture project +-- (the same "music_catalogue" project java.gen's own demos/Exhaustive.dhall +-- exercises), so a Python client compiles from it and passes basedpyright +-- strict. Pinned directly at gen-sdk's package.dhall, separately from +-- src/Deps/Sdk.dhall: that file only imports gen-sdk's `package.dhall` `as +-- Source` for RAM, and this fixture load doesn't need that mode. +-- +-- The fixture project deliberately covers PG types this generator does not +-- support (box, inet, money, ranges, ...), so onUnsupported is set to Skip: +-- those statements/types are dropped with a warning instead of aborting the +-- whole compile. +-- +-- Intended to be executed with: +-- +-- ```bash +-- dhall to-directory-tree --file demos/Exhaustive.dhall --output --allow-path-separators +-- ``` +let Sdk = ../src/Deps/Sdk.dhall + +let Gen = ../src/package.dhall + +let OnUnsupported = ../src/Structures/OnUnsupported.dhall + +let project = Sdk.Fixtures.Exhaustive + +let config = + Some + { packageName = None Text + , emitSync = Some True + , onUnsupported = Some OnUnsupported.Mode.Skip + } + +in Sdk.Output.toFileMap (Gen.compile config project) +``` + +- [ ] **Step 3: Verify:** + ```bash + mise x -- dhall type --file demos/Exhaustive.dhall + ``` + Expected: `List { mapKey : Text, mapValue : Text }` (or however this + fork/version of Dhall renders `Prelude.Map.Type Text Text`), no error. +- [ ] **Step 4: Commit.** + +--- + +### Task 6: Update every external reference to the old paths + +**Files:** +- Modify: `.github/workflows/ci.yml` +- Modify: `.github/workflows/release.yml` +- Modify: `.github/scripts/build-contract-shell.sh` +- Modify: `README.md` +- Modify: `AGENTS.md` +- Modify: `DESIGN.md` +- Modify: `build.bash` +- Modify: `bench/generate.sh`, `bench/as-source.sh` +- Modify: `mise.toml` (the `golden` task) + +**Path substitutions to apply everywhere they occur** (verify each hit with +`grep -rn` first — don't blind-sed across the whole repo, `tests/golden/` +contains generated Python that must NOT be touched): + +| Old | New | +|---|---| +| `gen/Gen.dhall` | `src/package.dhall` | +| `gen/Deps/*.dhall` | `src/Deps/*.dhall` | +| `tests/Exhaustive.dhall` | `demos/Exhaustive.dhall` | +| `gen/` (prose/dir references) | `src/` | + +Specific known hits (from `grep -rn "gen/Gen\.dhall\|gen/Deps\|tests/Exhaustive"` +run during planning): + +- `.github/workflows/ci.yml`: the `contract` job's "Strip `as Source`..." + step does `sed -i ... gen/Deps/*.dhall` → `src/Deps/*.dhall`; the + "Generate output from Dhall" step's `dhall_file: tests/Exhaustive.dhall` → + `demos/Exhaustive.dhall`. +- `.github/workflows/release.yml`: the "Resolve Dhall" step's + `file: gen/Gen.dhall` → `file: src/package.dhall`. +- `.github/scripts/build-contract-shell.sh`: comment references + `tests/Exhaustive.dhall` → `demos/Exhaustive.dhall` (comment only, verify + no functional path argument needs changing — it's invoked with + `contract-output` as a positional arg per `ci.yml`, not a hardcoded path). +- `README.md`: line ~38 `gen: https://raw.githubusercontent.com/slavashvets/python.gen/master/gen/Gen.dhall` + → `.../src/package.dhall`; lines ~64-66, the three example URLs + (`.../gen/Gen.dhall`) → `.../src/package.dhall`. +- `AGENTS.md`: line ~36 "`gen/` pins its remote imports by sha256 + (`gen/Deps/*.dhall`)" → "`src/` pins its remote imports by sha256 + (`src/Deps/*.dhall`)". +- `DESIGN.md`: line 4 (`gen/`), line 381 (`gen/Gen.dhall` "is the entry point + handed to gen-sdk"), line 397 (`gen/` mirrors...), line 401 (the `gen/` + tree diagram — replace with the new `src/` tree, matching Task 5's actual + post-move layout), lines 632/636 (`tests/Exhaustive.dhall` → `demos/Exhaustive.dhall`). +- `build.bash`: this is a scratch/dev script (mostly commented-out lines) — + update the live lines: `target=tests/Exhaustive.dhall` → + `target=demos/Exhaustive.dhall`; the commented `# target=gen/Gen.dhall` and + `# dhall freeze gen/Deps/*.dhall` lines → `src/` equivalents (keep them + commented, just fix the paths so they're not stale if uncommented later). +- `bench/generate.sh`, `bench/as-source.sh`: both `cp -R "$root/gen" "$out/gen"` / + `"$1/gen"` → `"$root/src" "$out/src"` (and update the `gen/Deps/*.dhall` + perl substitutions to `src/Deps/*.dhall`). **The `as Source` → plain-import + sha256 substitutions in both scripts' `strip_as_source` are pinned to the + *old* `gen-sdk v0.11.0`/`lude v5.1.0` source-vs-normalized hash pairs** + (`8d43544e...`→`b9f7bb84...` for Sdk, `46b527b0...`→`14c43eec...` for Lude). + Since `Task 1` bumps `gen-sdk` to `v2.0.0`, these substitution pairs are now + wrong and must be recomputed for the new pin using the repo's actual pinned + toolchain (`mise x -- dhall hash` on the plain, non-`as-Source` import) — + don't guess these; if the recompute can't happen in this pass, leave a + `# TODO` in the script rather than shipping a silently-wrong benchmark. +- `mise.toml`'s `golden` task: the `python3 - ... "$root/gen/Gen.dhall"` arg + and the fixture-project string replace target `"../../gen/Gen.dhall"` → + `"../../src/package.dhall"`. + +- [ ] **Step 1: Apply all substitutions above.** +- [ ] **Step 2: Confirm no stragglers:** + ```bash + grep -rn "gen/Gen\.dhall\|gen/Deps\|gen/Interpreters\|gen/Templates\|gen/Structures\|gen/Config\.dhall\|gen/compile\.dhall\|tests/Exhaustive" \ + --include="*.md" --include="*.yml" --include="*.yaml" --include="*.toml" --include="*.sh" --include="*.bash" . + ``` + Expected: no output (everything left under `tests/golden/` or `tests/fixture-project/` + that isn't a generator-path reference is fine and out of scope — check any + hit manually rather than assuming). +- [ ] **Step 3: Commit.** + +--- + +### Task 7: Format, verify end-to-end, update CHANGELOG + +- [ ] **Step 1: Format everything:** + ```bash + mise x -- dhall format --transitive src/package.dhall + mise x -- dhall format --transitive demos/Exhaustive.dhall + ``` + +- [ ] **Step 2: Full type-check:** + ```bash + mise x -- dhall type --file src/package.dhall + mise x -- dhall type --file demos/Exhaustive.dhall + ``` + +- [ ] **Step 3: Regenerate the Exhaustive fixture and confirm no diff in + output** (this is the load-bearing check — everything above only proves + the Dhall type-checks, not that it still produces the same files): + ```bash + mise x -- dhall to-directory-tree --allow-path-separators --file demos/Exhaustive.dhall --output /tmp/pygen-after + ``` + Compare against a snapshot taken from the pre-migration tree the same way + (`git stash`, regenerate to `/tmp/pygen-before`, `git stash pop`, `diff -rq + /tmp/pygen-before /tmp/pygen-after`). Expected: **no diff**. Any diff here + is a migration bug, not an intentional update — per Global Constraints, go + fix it rather than accepting the new output. + +- [ ] **Step 4: Run the Python harness:** + ```bash + mise x -- uv sync + mise x -- uv run pytest tests -v + ``` + (Needs a reachable Postgres — `PGN_TEST_DATABASE_URL`, see `ci.yml` for the + Docker Compose equivalent — and the `mise`-pinned `pgn 0.9.1`, since the + harness shells out to it.) Expected: all green, no new failures relative to + a pre-migration run. + +- [ ] **Step 5: Add a CHANGELOG.md entry** under `# Upcoming` (the file + already starts with that heading), non-breaking, modeled on `java.gen`'s + own `v1.1.0` entry: + ```markdown + - Migrated the generator's internal dependencies to `gen-contract` v4.0.1 + and `gen-sdk` v2.0.0, adopting `Sdk.Sigs` in place of the local + `Algebras/` module, and restructured the repository layout to match the + pGenie generator architecture: implementation moved from `gen/` to + `src/`, the public entry point renamed from `gen/Gen.dhall` to + `src/package.dhall`, and the fixture driver moved from + `tests/Exhaustive.dhall` to `demos/Exhaustive.dhall`. No change to + generated output or the public Dhall interface (`artifacts..gen` + URLs pointing at a previously-released `resolved.dhall` are unaffected; + only the next release's URL path changes, from `.../gen/Gen.dhall` — the + unresolved source path some projects may reference directly instead of a + frozen release — to `.../src/package.dhall`). + ``` + Adjust the last parenthetical if no project in practice points at the + unresolved source path (check `README.md`'s own recommended usage — if it + only ever recommends the frozen `resolved.dhall` release asset, simplify + this to "no change to generated output or the public Dhall interface"). + +- [ ] **Step 6: Commit.** + +## Deferred / explicitly out of scope (record as follow-ups, don't do now) + +- Narrowing each interpreter's `Config` to only the fields it needs (see + Global Constraints). +- Giving `Member.dhall`/`ParamsMember.dhall` a real `Sdk.Sigs.interpreter` + shape by folding `CustomKind.Lookup` into `Config` or `Input` (see Global + Constraints). +- Adding a `Name` interpreter (`java.gen` has `Interpreters/Name.dhall` + centralizing identifier casing/escaping; `python.gen` calls + `Structures/PyIdent.dhall` ad hoc from several interpreters instead). This + is a real architecture-doc deviation but not something `java.gen`'s last + release changed — separate task if wanted. +- Recomputing the `as Source` bench-script hash pairs for `gen-sdk v2.0.0` / + `lude v5.1.0` (flagged inline in Task 6 — needs the real toolchain, not + guessable). From 81ea82c81dee169431839b06bfa530175aebf92f Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sat, 11 Jul 2026 17:31:09 +0300 Subject: [PATCH 09/11] Revert "The plan" This reverts commit 15c28361de3192aa2d0e896fa32076c5f841ece3. --- .../plans/2026-07-11-gen-sdk-v2-migration.md | 666 ------------------ 1 file changed, 666 deletions(-) delete mode 100644 docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md diff --git a/docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md b/docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md deleted file mode 100644 index 5e0499c..0000000 --- a/docs/superpowers/plans/2026-07-11-gen-sdk-v2-migration.md +++ /dev/null @@ -1,666 +0,0 @@ -# python.gen: migrate to gen-contract v4.0.1 / gen-sdk v2.0.0, adopt architecture layout - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Bring `python.gen` up to the same structural shape as `java.gen`'s -last release (`v1.1.0`): pin `gen-contract`/`gen-sdk` directly (no bundled -SDK), adopt `Sdk.Sigs` in place of the local `Algebras/` folder, and move to -the `src/`-rooted repo layout the normative architecture doc describes — with -**zero change to generated output**. - -**Architecture:** Reference is -`gen-sdk/docs/generator-architecture.md` (already read in full) and -`java.gen`'s current `master` (tag `v1.1.0`, commit `658508c`). `python.gen` -is currently on pre-split `gen-sdk v0.11.0` (no `Sigs`, bundles `Project` -itself) with a hand-rolled `gen/Algebras/{Interpreter,Template}.dhall`, a -`gen/Deps/package.dhall` barrel, and the old `gen/` + `tests/Exhaustive.dhall` -layout — i.e. it has never done *any* of the steps `java.gen` went through -(`gen-migration-plan.md` at the pgenie repo root covers `rust.gen`/`haskell.gen`, -which already had `Sigs`; it does not cover `python.gen` at all). This plan -folds all of `java.gen`'s historical steps into one destination state, since -there is no reason to recreate `java.gen`'s intermediate commits. - -**Tech Stack:** Dhall (fork `pgn`'s dhall, package name `dhll`, needed for the -`Text/equal` builtin and `as Source` import mode — see Global Constraints), -Python 3.12 / `uv` / `pytest` for the harness, `mise` for tool pinning. - -## Global Constraints - -- **No behavior change.** This is a dependency/layout migration, not a - feature change. If `demos/Exhaustive.dhall` (see Task 5) produces different - file paths or content than `tests/Exhaustive.dhall` did before the move, - that is a bug in the migration, not an expected diff. -- **Do not touch interpreter algorithms.** `Interpreters/Project.dhall`'s - `Skip`/`Fail` logic (`typeSucceeds`/`queryChecks`/`effectiveQueries`) is - deliberately hand-rolled instead of using `Typeclasses.Classes.Alternative` - the way `java.gen` does, per an explicit in-file comment: multiple - `QueryGen.run`/`CustomTypeGen.run` call sites for the same query measurably - multiplied Dhall normalization time (seconds → minutes), confirmed by wall-time - bisection. This is orthogonal to the Deps/Sigs migration — leave it as is. - Do **not** add `src/Deps/Typeclasses.dhall` since nothing will use it. -- **`Interpreters/Member.dhall` and `Interpreters/ParamsMember.dhall` keep - their 3-argument `Run` type** (`Config -> CustomKind.Lookup -> Input -> - Compiled Output`) instead of conforming to `Sdk.Sigs.interpreter`'s fixed - 2-argument shape (`Config -> Input -> Result`). `java.gen`'s own - `Member.dhall` doesn't need a lookup table; `python.gen`'s does (custom-type - name resolution — see the `buildLookup`/`IndexedCustomType` comments in - `Interpreters/Project.dhall`). Bundling `Lookup` into `Config` or `Input` - to force-fit the Sig is a bigger, separate refactor — out of scope here. - These two files get the Deps-import and `Algebra.Config` → local `Config` - changes (Task 2) but keep their existing bare `{ Input, Output, Run, run }` - export, not `Sdk.Sigs.interpreter Config Input Output run`. -- **Config narrowing is deferred.** The architecture doc's ideal is each - interpreter declaring only the `Config` fields it (and its children) needs. - `java.gen` does this trivially (`{ useOptional : Bool }` everywhere, since - that's its whole config). `python.gen`'s config has 4 fields - (`packageName`, `importName`, `emitSync`, `onUnsupported`); auditing exactly - which fields each of the 11 interpreters actually reads and narrowing each - is real, separate work with real risk of missing a field some deeply nested - path needs. This plan keeps the **same 4-field `Config` record, declared - locally and verbatim in each interpreter module** (no more shared - `Algebra.Config` alias) — this satisfies "`Config` is a parameter each - module declares itself," just not narrowed. Note it as a follow-up, don't - do it now. -- **Local verification gap.** This sandbox has a global `dhall`/`pgn` (cabal - build, `dhll-1.42.3`) that is **not** the `mise`-pinned `pgn v0.9.1` this - repo's CI/tests actually use — a plain `dhall type --file tests/Exhaustive.dhall` - here fails on the *pre-migration* tree already, with an `as Source` - hash-integrity mismatch on `gen/Deps/Sdk.dhall` (confirmed during planning: - expected `8d43544e...`, actual `573b4655...`). This is a toolchain mismatch, - not evidence of a real problem. **Whoever executes this plan must run - verification through `mise x -- dhall ...` / `mise x -- pgn ...` / `mise x - -- uv run pytest`**, matching `mise.toml`'s pin, not a bare global `dhall`. - If `mise` isn't available in the execution environment either, at minimum - run `dhall format --transitive` (syntax-only, tool-version-agnostic) and - flag that deeper verification (`dhall type`, fixture diff, pytest, - basedpyright) still needs to happen on a properly provisioned machine/CI - before this is considered done. - ---- - -## File Structure - -``` -src/ (was gen/) - package.dhall (was Gen.dhall — now built via Sdk.Sigs.generator) - Config.dhall (unchanged content, moved) - Interpret.dhall (was compile.dhall — Config no longer Optional at top) - Deps/ - Contract.dhall (NEW — gen-contract v4.0.1 pin) - Sdk.dhall (bumped gen-sdk v0.11.0 → v2.0.0) - Lude.dhall (unchanged content, moved) - Prelude.dhall (unchanged content, moved) - (package.dhall barrel REMOVED) - Interpreters/ (11 files: Deps.Sdk.Project → Deps.Contract, Algebra → Sdk.Sigs) - Templates/ (10 files: Algebra → Sdk.Sigs; 7 also de-barrel Deps) - Structures/ (CustomKind.dhall: Deps.Sdk.Project → Deps.Contract; others untouched) - (Algebras/ REMOVED) -demos/ - Exhaustive.dhall (was tests/Exhaustive.dhall — rewritten for Sdk.Output.toFileMap) -tests/ (Python pytest harness — unchanged except any gen/-path references) -.github/workflows/{ci,release}.yml, .github/scripts/build-contract-shell.sh, -README.md, AGENTS.md, DESIGN.md, build.bash, bench/*.sh, mise.toml - (path references updated: gen/ → src/, tests/Exhaustive.dhall → demos/Exhaustive.dhall) -``` - ---- - -### Task 1: Add the new Deps pins and remove the `Deps/package.dhall` barrel - -**Files:** -- Create: `gen/Deps/Contract.dhall` -- Modify: `gen/Deps/Sdk.dhall` -- Delete: `gen/Deps/package.dhall` -- Modify (de-barrel): every file that currently has `let Deps = ../Deps/package.dhall` (see the full list in Tasks 2–3 — do this as part of those tasks, not twice) - -This task only stages the new pins; Task 2 is where the fallout (broken -`Sdk.Project`/`Sdk.Fixtures` references, `Algebras/` removal) gets fixed. Do -not try to get `dhall type` green after this task alone — it won't be, and -that's expected (same as `gen-migration-plan.md`'s phase-0 commit 2 for -`java.gen`). Fold Task 1 and Task 2 into one commit if you'd rather not carry -a known-broken intermediate state. - -- [ ] **Step 1: Create `gen/Deps/Contract.dhall`** — the exact pin `java.gen` - and `gen-sdk` itself use: - -```dhall -https://raw.githubusercontent.com/pgenie-io/gen-contract/v4.0.1/src/package.dhall - sha256:4a130ba7fbaa152a776babbb1bf2994a4833931ca76bde9bf6930d354225651e -``` - -- [ ] **Step 2: Bump `gen/Deps/Sdk.dhall`** to `gen-sdk v2.0.0` (same pin - `java.gen`'s `src/Deps/Sdk.dhall` uses). Preserve the existing `as Source` - import mode (see `AGENTS.md`/CI comments on why `python.gen` uses it — - `java.gen` doesn't, but that's an intentional `python.gen`-specific RAM - optimization, not something this migration should undo): - -```dhall -https://raw.githubusercontent.com/pgenie-io/gen-sdk/v2.0.0/src/package.dhall - sha256:b9def6ab1179bc4aaae7fc6e91977f094f75934cd5755175c294a9e97ca71b15 - as Source -``` - - If keeping `as Source` here, its hash is a *source*-text hash, not the - semantic hash above copied from `java.gen` (which imports plainly). Verify - with `dhall hash` against the raw URL using the repo's pinned toolchain - (`mise x -- dhall hash <<< 'https://raw.githubusercontent.com/pgenie-io/gen-sdk/v2.0.0/src/package.dhall as Source'` - or equivalent) before trusting the semantic hash verbatim in `as Source` - mode — don't guess. - -- [ ] **Step 3: Delete `gen/Deps/package.dhall`.** - -- [ ] **Step 4: Commit** (or fold into Task 2's commit). - ---- - -### Task 2: Rewire `Structures/CustomKind.dhall` and all 11 `Interpreters/*.dhall` - -**Files:** -- Modify: `gen/Structures/CustomKind.dhall` -- Modify: `gen/Interpreters/{CustomType,Member,ParamsMember,Primitive,Project,Query,QueryFragments,Result,ResultColumns,Scalar,Value}.dhall` -- Delete: `gen/Algebras/` (all three files) - -**The mechanical recipe, applied to every file above:** - -1. Replace `let Deps = ../Deps/package.dhall` with direct imports of exactly - what the file uses. Every one of these files uses `Deps.Sdk.Project` - (→ becomes a direct `Deps/Contract.dhall` import) and `Deps.Lude`/`Deps.Prelude` - (→ direct imports). Concretely, replace: - ```dhall - let Deps = ../Deps/package.dhall - ``` - with (only the lines this particular file actually needs — check with - `grep -n 'Deps\.' ` first): - ```dhall - let Lude = ../Deps/Lude.dhall - - let Prelude = ../Deps/Prelude.dhall - - let Model = ../Deps/Contract.dhall - ``` - and change every remaining `Deps.Lude` → `Lude`, `Deps.Prelude` → `Prelude` - in the body. `QueryFragments.dhall` additionally has an unused - `let Sdk = Deps.Sdk` line (line 7) — drop it, nothing in the file - references the `Sdk` binding. - -2. Replace the line `let Model = Deps.Sdk.Project` (now redundant with step - 1's `Model` binding) — don't duplicate it, step 1 already introduces - `Model` pointed at `Deps/Contract.dhall`. - -3. Delete `let Algebra = ../Algebras/Interpreter.dhall`. - -4. Add a local `Config` type declaration (same 4 fields everywhere per the - Global Constraints note on deferred narrowing): - ```dhall - let Config = - { packageName : Text - , importName : Text - , emitSync : Bool - , onUnsupported : OnUnsupported.Mode - } - ``` - This needs `OnUnsupported = ../Structures/OnUnsupported.dhall` imported in - any file that doesn't already import it (check first — `Project.dhall` - already does). - -5. Change every `\(config : Algebra.Config) ->` to `\(config : Config) ->`. - -6. Change the tail: - - **10 of the 11 files** (`CustomType`, `Primitive`, `Project`, `Query`, - `QueryFragments`, `Result`, `ResultColumns`, `Scalar`, `Value` — 9 files, - not 10; `Member`/`ParamsMember` are the exception below) end with - `Algebra.module Input Output run` (or, for `Scalar.dhall`, - `Algebra.module Input Output run /\ { ScalarDecode }`). Change to: - ```dhall - Sdk.Sigs.interpreter Config Input Output run - ``` - (`Scalar.dhall`: `Sdk.Sigs.interpreter Config Input Output run /\ { ScalarDecode }`), - which needs `let Sdk = ../Deps/Sdk.dhall` imported (it isn't currently, - since `Deps.Sdk.Project` used to come through the barrel — add it). - - **`Member.dhall` and `ParamsMember.dhall`** keep their existing tail - verbatim: `in { Input, Output, Run, run }`, just with `Algebra.Config` → - `Config` in the `Run` type alias line - (`let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output`). - No `Sdk.Sigs.interpreter` here — see Global Constraints. - -7. `Interpreters/Project.dhall` specifically: its two `\(config : Algebra.Config) ->` - occurrences (the `combineOutputs` and `run` functions) both become - `\(config : Config) ->`; its `lookupConfig : Algebra.Config` type - annotation (used to type-check `Value.run` calls for composite-field - rendering) becomes `lookupConfig : Config`. No other logic in this file - changes — the `Skip`/`Fail` machinery is untouched per Global Constraints. - -- [ ] **Step 1: Apply the recipe to all 11 `Interpreters/*.dhall` files and `Structures/CustomKind.dhall`.** -- [ ] **Step 2: Delete `gen/Algebras/`.** -- [ ] **Step 3: Verify** (with the `mise`-pinned toolchain, not the bare local `dhall` — see Global Constraints): - ```bash - mise x -- dhall type --file gen/Interpreters/Project.dhall - ``` - Expected: prints the interpreter's type (a record with `Input`, `Output`, - `Result`, `Run`, `run` fields) with no error. This alone pulls in every - other `Interpreters/*.dhall` transitively, so it's a full check of this task. -- [ ] **Step 4: Commit.** - ---- - -### Task 3: Rewire the 10 `Templates/*.dhall` files - -**Files:** -- Modify: `gen/Templates/{CompositeModule,CoreModule,EnumModule,FacadeModule,InitModule,RegisterModule,RowsModule,RuntimeModule,StatementModule,TypesInit}.dhall` - -**Recipe:** - -1. `CoreModule.dhall`, `InitModule.dhall`, `RuntimeModule.dhall` don't import - `Deps` at all (no `Prelude`/`Lude` need) — only change: drop - `let Algebra = ../Algebras/Template.dhall`, add `let Sdk = ../Deps/Sdk.dhall`, - and change the tail. `CoreModule`/`RuntimeModule` end with - `Algebra.module {} (\(_ : {}) -> content)` → `Sdk.Sigs.template {} (\(_ : {}) -> content)`. - `InitModule` ends with `Algebra.module Params render` → - `Sdk.Sigs.template Params render`. - -2. The other 7 (`CompositeModule`, `EnumModule`, `FacadeModule`, - `RegisterModule`, `RowsModule`, `StatementModule`, `TypesInit`) currently - have `let Deps = ../Deps/package.dhall`, and only ever use - `Deps.Prelude.*` (all seven) and, additionally, `Deps.Lude.Text.indentNonEmpty` - (`RowsModule`, `StatementModule` only — confirmed by - `grep -n 'Deps\.' gen/Templates/*.dhall` during planning). Replace the - barrel import with: - ```dhall - let Prelude = ../Deps/Prelude.dhall - ``` - adding `let Lude = ../Deps/Lude.dhall` only in `RowsModule.dhall` and - `StatementModule.dhall`. Then replace `Deps.Prelude.` → `Prelude.` and - `Deps.Lude.` → `Lude.` throughout each file's body. Drop - `let Algebra = ../Algebras/Template.dhall`, add `let Sdk = ../Deps/Sdk.dhall`. - -3. Tails for these 7: `Algebra.module Params run` → - `Sdk.Sigs.template Params run` (`FacadeModule`, `RegisterModule`, - `StatementModule`); with a combined record for the other 4: - `Algebra.module Params run /\ { Field }` (`CompositeModule`) → - `Sdk.Sigs.template Params run /\ { Field }`; `/\ { Variant }` (`EnumModule`); - `/\ { StatementExport, TypeExport }` (`FacadeModule` — check which of - `FacadeModule`/others actually has this combinator vs a plain - `Algebra.module Params run`, per the earlier grep output, before editing — - don't assume, re-`grep -n 'Algebra.module' gen/Templates/*.dhall` and - match each file's exact current tail); `/\ { RowDef }` (`RowsModule`); - `/\ { Export }` (`TypesInit`). - -- [ ] **Step 1: Apply the recipe to all 10 files.** -- [ ] **Step 2: Verify:** - ```bash - mise x -- dhall type --file gen/Interpreters/Project.dhall - ``` - (Templates are only reachable transitively through `Interpreters/Project.dhall` - and its children, same as Task 2 — this single check covers both tasks once - both are done. If running Task 2 and 3 as separate commits, this step will - fail after Task 2 alone if any interpreter references a not-yet-updated - template's old shape; if so, do Tasks 2 and 3 as one commit instead.) -- [ ] **Step 3: Commit.** - ---- - -### Task 4: Rewrite the root entry point (`Config.dhall`, `compile.dhall` → `Interpret.dhall`, `Gen.dhall` → `package.dhall`) - -**Files:** -- Modify (move, content unchanged): `gen/Config.dhall` -- Modify (move + rewrite): `gen/compile.dhall` → `gen/Interpret.dhall` -- Modify (move + rewrite): `gen/Gen.dhall` → `gen/package.dhall` - -(Paths shown as `gen/...` here since Task 5 does the `gen/` → `src/` directory -move; do this task first, in place, then Task 5 is a pure `git mv` sweep with -no further content changes.) - -**Interfaces:** -- Consumes: `Interpreters/Project.dhall`'s `run` (produced by Task 2, - now `Sdk.Sigs.interpreter`-shaped: `.run : Config -> Contract.Project -> Compiled Output`). -- Produces: `package.dhall`'s `Sdk.Sigs.generator`-built value - (`{ contractVersion, Config, compile }`), consumed by Task 5's - `demos/Exhaustive.dhall` and by any pGenie project's `artifacts..gen` URL. - -`Sdk.Sigs.generator`'s shape (from the architecture doc): -```dhall -\(Config : Type) -> -\(defaultConfig : Config) -> -\(interpret : Config -> Contract.Project -> Contract.Output) -> - let compile = \(config : Optional Config) -> - merge { None = interpret defaultConfig, Some = interpret } config - in Contract.module Config compile -``` -Note `interpret` takes a **bare** `Config`, not `Optional Config` — the outer -"config block omitted entirely" case is handled once, by substituting -`defaultConfig`, not by `interpret` itself. `python.gen`'s current -`compile.dhall` handles *two* levels of optionality (the whole block, and -each field within it) with a doubled `Prelude.Optional.fold`. Only the outer -level goes away; each field inside `Config` stays individually `Optional` (so -a project can supply `emitSync: true` alone and still get default -`packageName`/`onUnsupported`) — that per-field defaulting is -`python.gen`-specific richness `java.gen` doesn't have (its `Config` has one -non-Optional `Bool` field), and this migration must not lose it. - -- [ ] **Step 1: `gen/Config.dhall`** — content unchanged, just confirm it - still reads (no edits needed here; listed for completeness since Task 5 - moves the file). - -- [ ] **Step 2: Rewrite `gen/compile.dhall` as `gen/Interpret.dhall`** — - drop the outer `Optional Config` unwrap (the two outermost - `Prelude.Optional.fold Config config Text (\(c : Config) -> ...)` / - `... Bool ...` / `... OnUnsupported.Mode ...` wrappers), keep everything - else (the per-field defaults, `importName` derivation) as is: - -```dhall -let Deps = ./Deps/package.dhall - --- NOTE: Task 5 changes this to ./Deps/Contract.dhall / ./Deps/Prelude.dhall --- directly once the Deps barrel is gone (Task 1) — write it that way now, --- don't reintroduce the barrel: -let Contract = ./Deps/Contract.dhall - -let Prelude = ./Deps/Prelude.dhall - -let Config = ./Config.dhall - -let OnUnsupported = ./Structures/OnUnsupported.dhall - -let ProjectInterpreter = ./Interpreters/Project.dhall - --- Entry point handed to gen-sdk's Sdk.Sigs.generator as `interpret`. Each --- field of Config is independently Optional, so a project may omit the --- whole config block (Sdk.Sigs.generator substitutes an all-None --- defaultConfig, see package.dhall) or any subset of its keys; `defaults` --- collects every fallback in one place (packageName from the project name in --- kebab case, emitSync off, onUnsupported Fail). The async surface is always --- emitted; emitSync adds the sync mirror. -in \(config : Config) -> - \(project : Contract.Project) -> - let defaults = - { packageName = project.name.inKebabCase - , emitSync = False - , onUnsupported = OnUnsupported.Mode.Fail - } - - let packageName = - Prelude.Optional.fold - Text - config.packageName - Text - (\(t : Text) -> t) - defaults.packageName - - let emitSync = - Prelude.Optional.fold - Bool - config.emitSync - Bool - (\(b : Bool) -> b) - defaults.emitSync - - let onUnsupported = - Prelude.Optional.fold - OnUnsupported.Mode - config.onUnsupported - OnUnsupported.Mode - (\(m : OnUnsupported.Mode) -> m) - defaults.onUnsupported - - let importName = Prelude.Text.replace "-" "_" packageName - - let interpreterConfig = { packageName, importName, emitSync, onUnsupported } - - in ProjectInterpreter.run interpreterConfig project -``` - -- [ ] **Step 3: Rewrite `gen/Gen.dhall` as `gen/package.dhall`:** - -```dhall -let Sdk = ./Deps/Sdk.dhall - -let OnUnsupported = ./Structures/OnUnsupported.dhall - -let Config = ./Config.dhall - -let Config/default - : Config - = { packageName = None Text - , emitSync = None Bool - , onUnsupported = None OnUnsupported.Mode - } - -let interpret = ./Interpret.dhall - -in Sdk.Sigs.generator Config Config/default interpret -``` - -- [ ] **Step 4: Verify:** - ```bash - mise x -- dhall type --file gen/package.dhall - ``` - Expected type: a record with `contractVersion`, `Config`, `compile` fields - (`compile : Optional Config -> Contract.Project -> Contract.Output`). -- [ ] **Step 5: Commit.** - ---- - -### Task 5: Move `gen/` → `src/`, `tests/Exhaustive.dhall` → `demos/Exhaustive.dhall` - -**Files:** -- Move: `gen/` → `src/` (whole tree, `git mv`) -- Move + rewrite: `tests/Exhaustive.dhall` → `demos/Exhaustive.dhall` - -- [ ] **Step 1:** - ```bash - git mv gen src - mkdir -p demos - git mv tests/Exhaustive.dhall demos/Exhaustive.dhall - ``` - All the `../Deps/...`, `./Interpreters/...`, `../Templates/...` style - relative imports inside `src/` are untouched by this move (they're relative - to their own file, not to the repo root), so no content changes are needed - inside `src/` itself from the move alone. - -- [ ] **Step 2: Rewrite `demos/Exhaustive.dhall`.** Its old body called - `Gen.compileToFileMap config project` — `Sdk.Sigs.generator`-built modules - don't have a `compileToFileMap` field (per the architecture doc: "there is - no `compileToFileMap` on the module — turning an `Output` into files is the - caller's job, via `Sdk.Output.toFileMap`"). New content: - -```dhall --- Applies this generator to gen-sdk's shared cross-backend fixture project --- (the same "music_catalogue" project java.gen's own demos/Exhaustive.dhall --- exercises), so a Python client compiles from it and passes basedpyright --- strict. Pinned directly at gen-sdk's package.dhall, separately from --- src/Deps/Sdk.dhall: that file only imports gen-sdk's `package.dhall` `as --- Source` for RAM, and this fixture load doesn't need that mode. --- --- The fixture project deliberately covers PG types this generator does not --- support (box, inet, money, ranges, ...), so onUnsupported is set to Skip: --- those statements/types are dropped with a warning instead of aborting the --- whole compile. --- --- Intended to be executed with: --- --- ```bash --- dhall to-directory-tree --file demos/Exhaustive.dhall --output --allow-path-separators --- ``` -let Sdk = ../src/Deps/Sdk.dhall - -let Gen = ../src/package.dhall - -let OnUnsupported = ../src/Structures/OnUnsupported.dhall - -let project = Sdk.Fixtures.Exhaustive - -let config = - Some - { packageName = None Text - , emitSync = Some True - , onUnsupported = Some OnUnsupported.Mode.Skip - } - -in Sdk.Output.toFileMap (Gen.compile config project) -``` - -- [ ] **Step 3: Verify:** - ```bash - mise x -- dhall type --file demos/Exhaustive.dhall - ``` - Expected: `List { mapKey : Text, mapValue : Text }` (or however this - fork/version of Dhall renders `Prelude.Map.Type Text Text`), no error. -- [ ] **Step 4: Commit.** - ---- - -### Task 6: Update every external reference to the old paths - -**Files:** -- Modify: `.github/workflows/ci.yml` -- Modify: `.github/workflows/release.yml` -- Modify: `.github/scripts/build-contract-shell.sh` -- Modify: `README.md` -- Modify: `AGENTS.md` -- Modify: `DESIGN.md` -- Modify: `build.bash` -- Modify: `bench/generate.sh`, `bench/as-source.sh` -- Modify: `mise.toml` (the `golden` task) - -**Path substitutions to apply everywhere they occur** (verify each hit with -`grep -rn` first — don't blind-sed across the whole repo, `tests/golden/` -contains generated Python that must NOT be touched): - -| Old | New | -|---|---| -| `gen/Gen.dhall` | `src/package.dhall` | -| `gen/Deps/*.dhall` | `src/Deps/*.dhall` | -| `tests/Exhaustive.dhall` | `demos/Exhaustive.dhall` | -| `gen/` (prose/dir references) | `src/` | - -Specific known hits (from `grep -rn "gen/Gen\.dhall\|gen/Deps\|tests/Exhaustive"` -run during planning): - -- `.github/workflows/ci.yml`: the `contract` job's "Strip `as Source`..." - step does `sed -i ... gen/Deps/*.dhall` → `src/Deps/*.dhall`; the - "Generate output from Dhall" step's `dhall_file: tests/Exhaustive.dhall` → - `demos/Exhaustive.dhall`. -- `.github/workflows/release.yml`: the "Resolve Dhall" step's - `file: gen/Gen.dhall` → `file: src/package.dhall`. -- `.github/scripts/build-contract-shell.sh`: comment references - `tests/Exhaustive.dhall` → `demos/Exhaustive.dhall` (comment only, verify - no functional path argument needs changing — it's invoked with - `contract-output` as a positional arg per `ci.yml`, not a hardcoded path). -- `README.md`: line ~38 `gen: https://raw.githubusercontent.com/slavashvets/python.gen/master/gen/Gen.dhall` - → `.../src/package.dhall`; lines ~64-66, the three example URLs - (`.../gen/Gen.dhall`) → `.../src/package.dhall`. -- `AGENTS.md`: line ~36 "`gen/` pins its remote imports by sha256 - (`gen/Deps/*.dhall`)" → "`src/` pins its remote imports by sha256 - (`src/Deps/*.dhall`)". -- `DESIGN.md`: line 4 (`gen/`), line 381 (`gen/Gen.dhall` "is the entry point - handed to gen-sdk"), line 397 (`gen/` mirrors...), line 401 (the `gen/` - tree diagram — replace with the new `src/` tree, matching Task 5's actual - post-move layout), lines 632/636 (`tests/Exhaustive.dhall` → `demos/Exhaustive.dhall`). -- `build.bash`: this is a scratch/dev script (mostly commented-out lines) — - update the live lines: `target=tests/Exhaustive.dhall` → - `target=demos/Exhaustive.dhall`; the commented `# target=gen/Gen.dhall` and - `# dhall freeze gen/Deps/*.dhall` lines → `src/` equivalents (keep them - commented, just fix the paths so they're not stale if uncommented later). -- `bench/generate.sh`, `bench/as-source.sh`: both `cp -R "$root/gen" "$out/gen"` / - `"$1/gen"` → `"$root/src" "$out/src"` (and update the `gen/Deps/*.dhall` - perl substitutions to `src/Deps/*.dhall`). **The `as Source` → plain-import - sha256 substitutions in both scripts' `strip_as_source` are pinned to the - *old* `gen-sdk v0.11.0`/`lude v5.1.0` source-vs-normalized hash pairs** - (`8d43544e...`→`b9f7bb84...` for Sdk, `46b527b0...`→`14c43eec...` for Lude). - Since `Task 1` bumps `gen-sdk` to `v2.0.0`, these substitution pairs are now - wrong and must be recomputed for the new pin using the repo's actual pinned - toolchain (`mise x -- dhall hash` on the plain, non-`as-Source` import) — - don't guess these; if the recompute can't happen in this pass, leave a - `# TODO` in the script rather than shipping a silently-wrong benchmark. -- `mise.toml`'s `golden` task: the `python3 - ... "$root/gen/Gen.dhall"` arg - and the fixture-project string replace target `"../../gen/Gen.dhall"` → - `"../../src/package.dhall"`. - -- [ ] **Step 1: Apply all substitutions above.** -- [ ] **Step 2: Confirm no stragglers:** - ```bash - grep -rn "gen/Gen\.dhall\|gen/Deps\|gen/Interpreters\|gen/Templates\|gen/Structures\|gen/Config\.dhall\|gen/compile\.dhall\|tests/Exhaustive" \ - --include="*.md" --include="*.yml" --include="*.yaml" --include="*.toml" --include="*.sh" --include="*.bash" . - ``` - Expected: no output (everything left under `tests/golden/` or `tests/fixture-project/` - that isn't a generator-path reference is fine and out of scope — check any - hit manually rather than assuming). -- [ ] **Step 3: Commit.** - ---- - -### Task 7: Format, verify end-to-end, update CHANGELOG - -- [ ] **Step 1: Format everything:** - ```bash - mise x -- dhall format --transitive src/package.dhall - mise x -- dhall format --transitive demos/Exhaustive.dhall - ``` - -- [ ] **Step 2: Full type-check:** - ```bash - mise x -- dhall type --file src/package.dhall - mise x -- dhall type --file demos/Exhaustive.dhall - ``` - -- [ ] **Step 3: Regenerate the Exhaustive fixture and confirm no diff in - output** (this is the load-bearing check — everything above only proves - the Dhall type-checks, not that it still produces the same files): - ```bash - mise x -- dhall to-directory-tree --allow-path-separators --file demos/Exhaustive.dhall --output /tmp/pygen-after - ``` - Compare against a snapshot taken from the pre-migration tree the same way - (`git stash`, regenerate to `/tmp/pygen-before`, `git stash pop`, `diff -rq - /tmp/pygen-before /tmp/pygen-after`). Expected: **no diff**. Any diff here - is a migration bug, not an intentional update — per Global Constraints, go - fix it rather than accepting the new output. - -- [ ] **Step 4: Run the Python harness:** - ```bash - mise x -- uv sync - mise x -- uv run pytest tests -v - ``` - (Needs a reachable Postgres — `PGN_TEST_DATABASE_URL`, see `ci.yml` for the - Docker Compose equivalent — and the `mise`-pinned `pgn 0.9.1`, since the - harness shells out to it.) Expected: all green, no new failures relative to - a pre-migration run. - -- [ ] **Step 5: Add a CHANGELOG.md entry** under `# Upcoming` (the file - already starts with that heading), non-breaking, modeled on `java.gen`'s - own `v1.1.0` entry: - ```markdown - - Migrated the generator's internal dependencies to `gen-contract` v4.0.1 - and `gen-sdk` v2.0.0, adopting `Sdk.Sigs` in place of the local - `Algebras/` module, and restructured the repository layout to match the - pGenie generator architecture: implementation moved from `gen/` to - `src/`, the public entry point renamed from `gen/Gen.dhall` to - `src/package.dhall`, and the fixture driver moved from - `tests/Exhaustive.dhall` to `demos/Exhaustive.dhall`. No change to - generated output or the public Dhall interface (`artifacts..gen` - URLs pointing at a previously-released `resolved.dhall` are unaffected; - only the next release's URL path changes, from `.../gen/Gen.dhall` — the - unresolved source path some projects may reference directly instead of a - frozen release — to `.../src/package.dhall`). - ``` - Adjust the last parenthetical if no project in practice points at the - unresolved source path (check `README.md`'s own recommended usage — if it - only ever recommends the frozen `resolved.dhall` release asset, simplify - this to "no change to generated output or the public Dhall interface"). - -- [ ] **Step 6: Commit.** - -## Deferred / explicitly out of scope (record as follow-ups, don't do now) - -- Narrowing each interpreter's `Config` to only the fields it needs (see - Global Constraints). -- Giving `Member.dhall`/`ParamsMember.dhall` a real `Sdk.Sigs.interpreter` - shape by folding `CustomKind.Lookup` into `Config` or `Input` (see Global - Constraints). -- Adding a `Name` interpreter (`java.gen` has `Interpreters/Name.dhall` - centralizing identifier casing/escaping; `python.gen` calls - `Structures/PyIdent.dhall` ad hoc from several interpreters instead). This - is a real architecture-doc deviation but not something `java.gen`'s last - release changed — separate task if wanted. -- Recomputing the `as Source` bench-script hash pairs for `gen-sdk v2.0.0` / - `lude v5.1.0` (flagged inline in Task 6 — needs the real toolchain, not - guessable). From 7b3ef57f68105b5c5dbfee2a1e41117fea135e96 Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sun, 12 Jul 2026 06:16:12 +0300 Subject: [PATCH 10/11] Fix --- CHANGELOG.md | 40 ++ DESIGN.md | 294 +++++--- ...26-07-11-encounter-order-custom-imports.md | 327 +++++++++ .../2026-07-11-reusable-custom-type-codecs.md | 633 ++++++++++++++++++ docs/upstream-asks.md | 12 + src/Config.dhall | 15 - src/Deps/Lude.dhall | 5 +- src/Deps/Prelude.dhall | 1 - src/Deps/Sdk.dhall | 1 - src/Interpret.dhall | 54 -- src/Interpreters/CustomType.dhall | 14 +- src/Interpreters/Member.dhall | 180 ++--- src/Interpreters/ParamsMember.dhall | 121 +--- src/Interpreters/Project.dhall | 180 ++--- src/Interpreters/Query.dhall | 11 +- src/Interpreters/Result.dhall | 27 +- src/Interpreters/ResultColumns.dhall | 10 +- src/Structures/CustomKind.dhall | 28 - src/Structures/ImportSet.dhall | 115 +--- src/Templates/CompositeModule.dhall | 42 +- src/Templates/EnumModule.dhall | 11 + src/Templates/RowsModule.dhall | 2 +- src/Templates/StatementModule.dhall | 2 +- src/package.dhall | 21 +- tests/test_config_variants.py | 2 +- 25 files changed, 1483 insertions(+), 665 deletions(-) create mode 100644 docs/plans/2026-07-11-encounter-order-custom-imports.md create mode 100644 docs/plans/2026-07-11-reusable-custom-type-codecs.md delete mode 100644 src/Config.dhall delete mode 100644 src/Interpret.dhall delete mode 100644 src/Structures/CustomKind.dhall diff --git a/CHANGELOG.md b/CHANGELOG.md index f98fe04..b55b2bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,45 @@ # Upcoming +- `buildLookup` (`Interpreters/Project.dhall`) and, with it, this generator's + last dependency on pgn's fork-only `Text/equal` builtin are removed from + `src/`: custom-type decode/encode now dispatches through named + `_decode`/`_encode` methods generated onto each custom type's own Python + class (`CompositeModule.dhall`/`EnumModule.dhall`), called by name from + every reference site, instead of resolving classification and fields via a + project-wide structural search (`grep -rn "Text/equal" src` now returns + only two explanatory comments, zero invocations). Array (dims > 0) + decode/encode is built at the call site (`Member.dhall`/ + `ParamsMember.dhall`) instead of a third per-type method, delegating only + the per-element transform to `_decode`/`_encode`: an earlier draft this + session added a per-type `_decode_array` to `EnumModule.dhall`, but it + could not express `elementIsNullable` (a per-column fact, not a per-type + one) and silently broke nullable-element enum-array decode and + enum-array param encode — both working, corpus-exercised paths — caught + by the final whole-branch review and fixed before merge. Behavior change: + because the call site is now kind-uniform, a 1-D composite-array column + or param is no longer rejected at Dhall-generation time the way it used + to be, and — unlike the `_decode_array` design it replaces — no longer + depends on `basedpyright strict` catching a missing method either, since + `_decode`/`_encode` genuinely exist on a composite class too. **This path + has not been exercised against real Postgres, and `tests/golden/` has NOT + been regenerated for this change this session** — the composite-array + fixture addition, its golden regeneration, and confirming actual Postgres + round-trip behavior are a known, deliberate gap in this commit, deferred + to a follow-up pass on a properly provisioned machine (see + `docs/plans/2026-07-11-reusable-custom-type-codecs.md`). + Separately, a composite field nesting another custom type is *also* no + longer rejected at generation time: the `nestedLookup = Absent` stub that + used to force it down the same loud-fail path is gone (it only existed + to satisfy `Member.run`'s old signature). This is not the same kind of + change as the composite-array case above, though — `CompositeModule.dhall`'s + `_decode`/`_encode` still do a blind flat `cast(tuple[...], src)`/splat, + unchanged by this refactor, and never recurse into the nested type's own + codec, so the field silently decodes/encodes wrong rather than being + caught by a type checker. Because the failure mode is `cast()`, which + suppresses type-checking on its argument by design, this is **not** + expected to be caught by `basedpyright strict`. It is a real, silent + architecture gap, flagged here as an open follow-up design question, not + a shipped or backstopped behavior change. - Migrated the generator's internal dependencies to `gen-contract` v4.0.1 and `gen-sdk` v2.0.0, adopting `Sdk.Sigs` in place of the local `Algebras/` module, and restructured the repository layout to match the diff --git a/DESIGN.md b/DESIGN.md index 79e8a7c..952e4cc 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -125,18 +125,28 @@ those the decode is a `cast(, row[""])` to satisfy strict - enum column: `Mood(cast(str, row["feeling"]))`; nullable guards `None`. - enum array column: element-wise rebuild, - `[Mood(v) for v in cast(list[str], row["..."])]`, with per-element and - outer `None` guards driven by `elementIsNullable` / column nullability. - Requires the enum's TypeInfo registered (section 6). + `[Mood._decode(v) for v in cast(list[str], row["..."])]`, with per-element + and outer `None` guards driven by `elementIsNullable` / column nullability, + built at the reference site in `Member.dhall` rather than a per-type array + method (see below). Requires the enum's TypeInfo registered (section 6). - composite column: psycopg returns a namedtuple once the composite is registered; decode is `TypeName(*cast(tuple[...], row["..."]))` with the exact field types. `Member.run` (`Interpreters/Member.dhall`) builds `decodeExpr` as a `Text -> Text` function next to the type info, so `_rows.py` and `types/` -compose the same logic. A composite array, or an enum array with -`dims > 1`, is unimplemented and fails loudly (`Compiled.report`) rather -than emitting wrong Python. +compose the same logic. Any custom-type array with `dims > 1` is +unimplemented and fails loudly (`Compiled.report`), regardless of kind. For +`dims == 1`, `Member.dhall` builds the list comprehension itself +(`[${typeName}._decode(v) for v in cast(...)]`, with per-element/outer +`None` guards driven by `value.elementIsNullable`/`isNullable`) instead of +calling a per-type array method: a per-type, zero-argument method has no way +to see `elementIsNullable`, a per-*column* fact, so it cannot express it. +This makes the branch kind-uniform: a 1-D composite-array column now +type-checks and attempts a real decode the same way an enum-array column +does, instead of being rejected at Dhall-generation time the way it used +to be. See section 12 for why that changed and for the currently-unverified +state of that path. --- @@ -226,10 +236,15 @@ no partial output for the affected query under `Fail`. Add a mapping to The param side (`Interpreters/ParamsMember.dhall`) carries the same loud-fail contract for bind shapes psycopg cannot adapt faithfully. It -rejects, rather than silently mis-binds: - -- a `json`/`jsonb` ARRAY param (`Jsonb` wraps a scalar, not element-wise). -- a composite ARRAY param (psycopg cannot adapt the dataclass array). +still rejects a `json`/`jsonb` ARRAY param (`Jsonb` wraps a scalar, not +element-wise). A composite ARRAY param is no longer rejected here the way +it used to be (see section 12): encode branches on `Natural/isZero +value.dims`, calling `._encode()` for a scalar custom-type param and +building `[x._encode() for x in ]` (with the same +`elementIsNullable`/outer-nullable guards as decode) for an array one, so a +composite-array param now type-checks and attempts a genuine per-element +encode rather than depending on `basedpyright strict` to catch a missing +method. ### Arrays and nullability @@ -276,12 +291,34 @@ composite (a single `value: str | None` field) exists specifically to cover this, exercised as both a param and a `RETURNING` result column in the same statement, with a round-trip test. -Composite fields cannot themselves reference another custom type. -`CustomType.dhall` hardcodes the nested-member lookup to `Absent` -(`nestedLookup`), so a composite nesting a composite (or an enum) fails the -same loud-fail path as any other unresolvable reference rather than -guessing a decode. Widening this is possible but has not been needed by any -project this generator has shipped against yet. +Composite fields nesting another custom type used to be explicitly +rejected: `CustomType.dhall` called `Member.run` with a `nestedLookup` stub +hardcoded to `Absent`, forcing the same loud-fail path as any other +unresolvable reference rather than guessing a decode. That stub is gone — +it existed only to satisfy `Member.run`'s old signature, and was deleted +along with `buildLookup` (section 12) — but its removal does not make this +case work; it only removed the one thing that used to reject it at +generation time. `Member.run` does compute a named-codec `decodeExpr` +(`${typeName}._decode(...)`) for a `Custom`-typed field, but +`CustomType.dhall`'s Composite branch never threads it anywhere: it maps +each member down to a flat `{fieldName, fieldType}` pair (the `Field` shape +`Templates/CompositeModule.dhall` takes) and discards `decodeExpr` +entirely. `CompositeModule.dhall`'s `_decode`/`_encode` — unchanged by this +refactor — render a single blind `${typeName}(*cast(tuple[...], src))` +splat and a flat `(self.field1, ...)` tuple; neither ever calls a nested +field's own `_decode`/`_encode`. So a composite field whose own type is +another custom type still does not decode/encode correctly at +runtime — it is just no longer *rejected* at generation time the way it +used to be. Unlike the composite-array case (section 12), this is **not** +expected to be caught by `basedpyright strict`: `cast()` exists +specifically to suppress type-checking on its argument, so the checker +sees exactly the annotated field type and raises nothing. This is a real, +silent architecture gap introduced by this refactor — flagged here as an +open follow-up design question (should `CustomType.dhall` thread a +member's own `decodeExpr` through to `CompositeModule.dhall`, or does +`_decode` need to become field-aware instead of a blind tuple cast?), not +something fixed in this commit and not on the same footing as the +composite-array case's deferred-but-backstopped behavior change. --- @@ -383,20 +420,27 @@ overwritten on every run. Do not hand-edit it. ```dhall let Sdk = ./Deps/Sdk.dhall -let Config = ./Config.dhall +let OnUnsupported = ./Structures/OnUnsupported.dhall + +let ProjectInterpreter = ./Interpreters/Project.dhall + +let Config = { packageName : Optional Text, emitSync : Optional Bool, onUnsupported : Optional OnUnsupported.Mode } -let interpret = ./Interpret.dhall +let Config/default = { packageName = None Text, emitSync = None Bool, onUnsupported = None OnUnsupported.Mode } -in Sdk.Sigs.generator Config Config/default interpret +in Sdk.Sigs.generator Config Config/default ProjectInterpreter.run ``` `Sdk.Sigs.generator` has signature `\(Config : Type) -> \(defaultConfig : Config) -> \(interpret : Config -> Contract.Project -> Contract.Output) -> ...`; it curries `interpret` against `defaultConfig` whenever the user config is absent and hands -the result to gen-contract's `Contract.module`. `Interpret.dhall` folds the -optional user config into the internal interpreter config and calls -`Interpreters/Project.dhall`, which traverses queries and custom types and -assembles the file list (`Contract.Output`). +the result to gen-contract's `Contract.module`. `Config` is passed straight +through to `Interpreters/Project.dhall` as that interpreter's own `Config` -- +there is no separate config type or resolve step in between, matching every +other gen (java.gen, rust.gen, haskell.gen). `Project.run` folds the optional +user config into the fully-resolved internal config itself (see "Config flow" +below), traverses queries and custom types, and assembles the file list +(`Contract.Output`). `src/` mirrors a typical pgn gen-sdk generator, Python-flavored: `Interpreters/` assembles data, `Templates/` renders it to Python text. The interpreter/template @@ -405,13 +449,10 @@ algebra signatures themselves live in gen-sdk's `Sdk.Sigs` (`interpreter.dhall`/ ```text src/ - package.dhall # Sdk.Sigs.generator Config Config/default interpret (entry handed to gen-sdk) - Config.dhall # user config TYPE: { packageName, emitSync, onUnsupported } - Interpret.dhall # derive interpreter Config from user Config, call Project.run + package.dhall # Config, Config/default, Sdk.Sigs.generator Config Config/default ProjectInterpreter.run Deps/ # pinned remote imports: gen-sdk, gen-contract, lude, dhall Prelude Structures/ Surface.dhall # async/sync token table (section 4) - CustomKind.dhall # Lookup : Name -> < Enum | Composite | Absent > + composite fields ImportSet.dhall # per-module import flags + combine PyIdent.dhall # sanitize names that become Python identifiers (keyword -> name_) OnUnsupported.dhall # < Fail | Skip > (section 11) @@ -438,29 +479,39 @@ src/ EnumModule.dhall / CompositeModule.dhall / TypesInit.dhall / InitModule.dhall ``` -The cross-cutting dependency is the custom-type lookup: `Scalar`/`Value`/ -`Primitive` stop at "Custom + Name"; `Project.run` builds a -`CustomKind.Lookup : Name -> < Enum | Composite | Absent >` from the -(post-Skip-filter) custom types and threads it to `Query.run` -> -`Result`/`ResultColumns`/`ParamsMember`/`Member`. The lookup compares by -`name.inSnakeCase` (the column carries a distinct `Name` occurrence, so -record identity cannot be relied on) and carries the type's alphabetical -index so per-module custom-type import blocks stay sorted. Section 12 -covers why this one comparison still needs a Dhall fork builtin. +The cross-cutting dependency used to be a project-wide custom-type lookup: +`Project.run` built a `CustomKind.Lookup : Name -> < Enum | Composite | +Absent >` from the (post-Skip-filter) custom types and threaded it to +`Query.run` -> `Result`/`ResultColumns`/`ParamsMember`/`Member`. That +lookup, and `Structures/CustomKind.dhall` itself, are deleted. `Scalar`/ +`Value`/`Primitive` still stop at "Custom + Name", but `Member.dhall`/ +`ParamsMember.dhall` now resolve a `Custom` reference by calling the +generated class's `_decode`/`_encode` method directly, keyed off +`name.inPascalCase` — no project-wide search, no classification step +threaded through the query pipeline. Array (dims > 0) decode/encode stays +local to the call site rather than becoming a third per-type method, +because `elementIsNullable` is a per-column fact a per-type method cannot +see. Section 12 covers the removal and the behavior change it introduced. +The old lookup's alphabetical index, used to keep per-module custom-type +import blocks sorted and deduped, is also gone: `ImportSet.dhall` now +renders custom-type imports in encounter order, unsorted and undeduped +(see the comment at its top). ### Config flow -`Config.dhall` is the user-facing type `{ packageName : Optional Text, -emitSync : Optional Bool, onUnsupported : Optional OnUnsupported.Mode }`. -`compile.dhall` folds the optional user config, and each of its optional -fields, into the internal interpreter Config `{ packageName, importName, -emitAsync = True, emitSync, onUnsupported }`, with `packageName` falling -back to the project name in kebab case, `emitSync` to `False`, and -`onUnsupported` to `Fail`. `importName` = `packageName` with `-` -> `_`. -`emitAsync` is always `True`. A project's artifact config can therefore omit -`config:` entirely, supply `config: {}`, or set any subset of the three -keys; see the README's Config reference for the decode semantics pgn itself -applies before this fold ever runs. +`package.dhall`'s `Config` is the user-facing type `{ packageName : Optional +Text, emitSync : Optional Bool, onUnsupported : Optional OnUnsupported.Mode +}`, passed straight through to `Interpreters/Project.dhall` as its own +`Config` -- there is no separate config type or resolve step in between. +`Project.run` folds the optional config, and each of its optional fields, +into the fully-resolved `ResolvedConfig` it passes to every interpreter +below it: `{ packageName, importName, emitSync, onUnsupported }`, with +`packageName` falling back to the project name in kebab case, `emitSync` to +`False`, and `onUnsupported` to `Fail`. `importName` = `packageName` with +`-` -> `_`. A project's artifact config can therefore omit `config:` +entirely, supply `config: {}`, or set any subset of the three keys; see the +README's Config reference for the decode semantics pgn itself applies +before `Project.run` ever sees the value. --- @@ -474,13 +525,18 @@ aborts the whole `Project.run` non-zero via `Lude.Compiled.report`. drop the smallest self-consistent unit and keep the rest of the project generating. `Project.dhall`'s `run` computes, once, whether each custom type and each query would have compiled cleanly (`typeSucceeds` / the `keep` -field of a per-query `QueryCheck`), filters the failing ones out before -building the custom-type lookup and the final query list, and separately -collects a `Report` per dropped unit into `combined.warnings`. A query that -references a skipped custom type resolves that reference to `Absent` -through the (already-filtered) lookup and fails its own compile the same -way any other unsupported shape would, which is what makes the drop cascade -without any special-cased "type X depends on type Y" bookkeeping. +field of a per-query `QueryCheck`), filters the failing ones out to produce +the final surviving custom-type and query lists, and separately collects a +`Report` per dropped unit into `combined.warnings`. + +Whether a query referencing a *specific* skipped custom type still cascades +into its own compile failure is worth re-checking rather than assumed: the +project-wide custom-type lookup this cascade used to route through is gone +(section 12), and neither `Member.dhall` nor `ParamsMember.dhall` currently +take the surviving custom-type list as an input to cross-check a +`customRef` name against. This is unrelated to the `Text/equal` removal and +out of scope for this pass; flagged here only so the Skip-mode cascade +isn't assumed unchanged without someone verifying it. The precedent for this shape is java.gen, an earlier gen-sdk generator for a different target language: it skips unconditionally and silently (an @@ -496,34 +552,108 @@ generator is already ready for that; it isn't waiting on it. --- -## 12. Forked-Dhall (`Text/equal`) dependency risk - -ACCEPTED RISK, recorded so nobody is surprised. pgn is a prebuilt binary -that embeds a FORKED Dhall providing a `Text/equal` builtin, which is not -part of the upstream Dhall standard Prelude. This generator uses it in -exactly one place left: `Interpreters/Project.dhall`'s `buildLookup`, to -match a custom type by its snake-case name while building the -`CustomKind.Lookup`. Everywhere else that used to need name equality -(keyword sanitizing in `PyIdent.dhall`) has since been rewritten against a -`Text/replace`-based trick that needs no fork builtin at all; section 13 -covers how. - -`buildLookup`'s use is harder to remove the same way: it returns a -structural `TypeKind` value, not `Text`, so the replace-based marker trick -(built for sanitizing a `Text -> Text` name) doesn't carry over as-is. -gen-sdk's own `Fixtures` module relies on the same builtin, so a Dhall -evaluator without it can't even typecheck gen-sdk's full package entry -point, only the narrower `module.dhall`/`Project.dhall` imports this -generator pins directly. The clean fix is upstream: a `kind` tag or a -`Natural` index carried directly on `Scalar.Custom`, so the lookup becomes -an equality-free structural match. That is a planned ask against gen-sdk, -not something this generator can do unilaterally. - -Consequence: end-to-end regeneration requires the pgn binary (for its -embedded fork Dhall), a live Postgres, and currently-live remote Dhall -imports (`Deps/*` resolve gen-sdk, lude, and the Prelude over the network, -pinned by sha256). There is no pure-upstream-dhall path to reproduce the -output today. +## 12. Forked-Dhall (`Text/equal`) dependency: removed from this generator + +RESOLVED, not an accepted risk anymore. `Interpreters/Project.dhall`'s +`buildLookup` was this generator's last consumer of pgn's FORKED-Dhall-only +`Text/equal` builtin (not part of the upstream Dhall standard Prelude): it +matched a custom type by its snake-case name while building a +`CustomKind.Lookup : Name -> < Enum | Composite | Absent >`, threaded +through `Query.run` so `Result`/`ResultColumns`/`ParamsMember`/`Member` +could classify a `Custom` reference and pull its fields. `buildLookup` and +`CustomKind.Lookup` usage are removed in commit `ffdd9bd`; the now-empty +`Structures/CustomKind.dhall` file itself is deleted separately in commit +`ab8f4df`. + +In their place, `CompositeModule.dhall`/`EnumModule.dhall` now emit a +`_decode`/`_encode` method directly onto each generated custom type's +Python class, covering the scalar (`dims == 0`) case. `Member.dhall` and +`ParamsMember.dhall` call it by name off `name.inPascalCase` at the +reference site (e.g. `${typeName}._decode(src)`) instead of resolving +classification/fields via a project-wide name search. No name-equality +comparison is needed at all anymore, so there's nothing left for +`Text/equal` to do here. `grep -rn "Text/equal" src` confirms this: it +returns exactly two hits, both explanatory comments about why a mechanism +does *not* use the builtin (`Structures/ImportSet.dhall:7`, +`Structures/PyIdent.dhall:48`), zero actual invocations anywhere in `src/`. + +Array (dims > 0) decode/encode is deliberately NOT a third per-type method. +An earlier draft this session gave `EnumModule.dhall` a `_decode_array` +staticmethod, but the final whole-branch review caught that a per-type, +zero-argument array codec cannot express `elementIsNullable` — an +`Optional`-array-settings field that varies per *column*, not per type (a +`moods: list[Mood | None] | None` column needs a different per-element guard +than a `list[Mood]` column of the same enum). That method hardcoded the +non-nullable-element shape unconditionally, silently breaking +nullable-element enum-array decode (a runtime crash on any `NULL` array +element) and, symmetrically, `ParamsMember.dhall`'s unconditional +`${field}._encode()` broke enum-array param encode (calling `._encode()` on +a `list`). Both were working, corpus-exercised paths before this refactor. +The fix moves array handling back to the call site, exactly where it lived +before this refactor: `Member.dhall`'s dims==1 branch and +`ParamsMember.dhall`'s `Natural/isZero value.dims` branch build the list +comprehension locally, reading `value.elementIsNullable`/`value.dims` off +the column- or param-local `Value.Output`, and delegate only the +per-element transform to `${typeName}._decode(v)` / `x._encode()`. This +restores exact parity with the pre-refactor `enumArrayDecode`/array-param +behavior for enums (verified against +`tests/golden/src/specimen_client/_generated/_rows.py`'s `moods` column and +`statements/insert_specimen.py`'s `moods` param — same shape, just calling +`._decode`/`._encode` per element instead of the enum constructor/bare +pass-through). + +A behavior change worth flagging, now unavoidable rather than accidental: +because the call site's array branch is kind-uniform (the same +`${typeName}._decode(v)`/`x._encode()` call per element regardless of +whether `typeName` is an enum or a composite), a 1-D composite-array column +or param is no longer rejected at Dhall-generation time the way it used to +be (the old "Array of a composite type is not supported" reports are +gone), and — unlike the `_decode_array` design it replaces — no longer +depends on `basedpyright strict` catching a missing method either, since +`CompositeModule.dhall`'s `_decode`/`_encode` genuinely exist. A +composite-array column/param now type-checks and attempts a real +per-element decode/encode. **This path remains unverified against real +Postgres either way** — composite arrays were never tested before this +refactor (they were rejected outright at generation time) and aren't +tested now (same deferred gap, just no longer expected to fail statically). +Adding that fixture, regenerating golden, and confirming actual Postgres +round-trip behavior for a composite-array column/param are deferred to a +follow-up pass on a properly provisioned machine; see +`docs/plans/2026-07-11-reusable-custom-type-codecs.md` for the original +design decision (Option A, chosen deliberately) and the deferred task list. + +A second, related change here: `CustomType.dhall`'s Composite branch used +to call `Member.run` with a `nestedLookup` stub hardcoded to `Absent`, +which forced a composite field nesting another custom type down the same +loud-fail path as any unresolvable reference (see section 5). That stub is +gone — it existed only to satisfy `Member.run`'s old signature, and was +deleted along with `buildLookup` — but, unlike the composite-array case +just above, this is not "the same behavior change, just unexercised." +`CompositeModule.dhall`'s `_decode`/`_encode` do a blind flat +`cast(tuple[...], src)`/splat and never recurse into a nested field's own +codec, unchanged by this refactor; removing the stub only removed the +thing that used to reject a nested custom-type composite field at +generation time, it did not make that field decode/encode correctly. And +because the failure mode runs through `cast()` — which suppresses +type-checking on its argument by design — this is **not** expected to be +caught by `basedpyright strict` the way the composite-array case is. This +is a real, silent architecture gap, not a deferred-but-backstopped +behavior change; see section 5 for detail, and treat it as an open +follow-up design question rather than something covered by the +composite-array follow-up plan. + +This removal is scoped to this generator's own Dhall source. It does not +unblock end-to-end regeneration: `demos/Exhaustive.dhall`/`mise run golden` +still needs the pinned pgn binary (for its embedded fork Dhall) regardless, +because gen-sdk's own `Fixtures` module independently relies on the same +`Text/equal` builtin, and this change doesn't touch gen-sdk. A live +Postgres and currently-live remote Dhall imports (`Deps/*` resolve gen-sdk, +lude, and the Prelude over the network, pinned by sha256) are also still +required for that path. + +Section 13 covers a different, still-in-place mechanism (`PyIdent.dhall`'s +keyword sanitizing), which never depended on `buildLookup` and was already +rewritten off `Text/equal` before this change; it is unaffected either way. --- diff --git a/docs/plans/2026-07-11-encounter-order-custom-imports.md b/docs/plans/2026-07-11-encounter-order-custom-imports.md new file mode 100644 index 0000000..7d1c663 --- /dev/null +++ b/docs/plans/2026-07-11-encounter-order-custom-imports.md @@ -0,0 +1,327 @@ +# Encounter-Order Custom-Type Imports Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Stop requiring a project-wide `order : Natural` to sort and dedupe custom-type import lines in `ImportSet.dhall`. Emit them in encounter order (the order the referencing columns/params are declared) instead, and accept that de-duplicating two references to the *same* custom type within one file is not achievable in vanilla Dhall — verify it isn't actually needed by the current corpus, and document the tradeoff rather than reintroduing a lookup to avoid it. + +**Architecture:** `ImportSet.dhall` today dedupes and alphabetizes custom-type imports by carrying a `dedupKey : Natural` — the type's alphabetical index in `project.customTypes` — through every `CustomImport` value, specifically because Dhall has no `Text` comparison to sort or dedupe on `moduleName`/`className` directly (see the file's own header comment). That `order` value's only source was `buildLookup` (`Interpreters/Project.dhall`), which the companion plan (`2026-07-11-reusable-custom-type-codecs.md`) deletes. Once it's gone, `ImportSet.dhall` has nothing to key on. Rather than re-deriving a Natural surrogate some other way, this plan removes the sort/dedup step and lets `ImportSet.combine`'s existing `List/fold` order (already the query's declared column/param order — a real, already-computed, non-Text-comparison ordering) stand as the output order. + +**Tech Stack:** Dhall (dhall-lang 1.42), Python 3.12 generated output, pytest golden-file harness (`mise run test`, `mise run golden`). + +## Global Constraints + +- **Depends on** `docs/plans/2026-07-11-reusable-custom-type-codecs.md` — that plan's Task 3/4 call `ImportSet.custom customImport` (no `order` argument). Land this plan's Task 1 first, or in the same PR; `ImportSet.customEnum`/`customComposite` (which this plan deletes) are exactly what those tasks stop calling. +- No behavior change to the four already-Natural-keyed stdlib import flags (`uuid`, `datetime`, `date`, `time`, `timedelta`, `decimal`, `jsonb`, `json`, `jsonValue`, `enumArray`) — those are plain `Bool` OR's today and are untouched by this plan. +- Golden fixture output must be regenerated and diffed (`tests/golden/`), not hand-edited. +- Every Dhall file touched must independently type-check: `dhall type --file=`. + +--- + +## Why dedup can't be preserved without reintroducing a lookup + +Worth writing down since it's not obvious and the temptation to "just find a clever `Text/replace` trick" is real — this was checked directly against `PyIdent.dhall`'s working pattern (DESIGN.md section 13) before concluding it doesn't generalize: + +`PyIdent.dhall`'s `sanitizeAgainst` tests a runtime `Text` against a small, **fixed, compile-time-literal** candidate list (the 35 Python keywords) — one known literal at a time, folded. That's why the two-`Text/replace` trick works: one side of every comparison is always a literal. + +Import dedup needs the opposite shape: is this **runtime** `moduleName` (derived from a `Name` that varies per project) equal to any of the **other runtime** `moduleName`s already collected? Both sides are dynamic. No sequence of `Text/replace` calls can decide that, because deciding it requires producing a `Bool` from two non-literal `Text` values, which is exactly the operation Dhall doesn't have (and which pgn's forked `Text/equal` exists to provide). This isn't a missing trick — dedup of dynamically-computed `Text` is unconditionally impossible in vanilla Dhall. The only ways to get it back are: (a) a fork builtin (what we're removing), (b) a pre-assigned `Natural` id per distinct value (what `order` was — sourced from a project-wide search, i.e. `buildLookup`, also being removed), or (c) don't need it. + +This plan takes (c), having checked how much it costs to: + +```bash +grep -rl "^from \.\.types\." tests/golden/src 2>/dev/null | while read f; do + n=$(grep -c "^from \.\.types\." "$f") + [ "$n" -gt 1 ] && echo "$f: $n" +done +``` + +No output — verified during design (see conversation record / re-run before Task 2 to confirm it's still true after Doc 1's regeneration). No file in the current fixture corpus imports the same custom type twice. The risk is real but currently unexercised: if a future query selects the same composite/enum type via two different columns, the generated file will contain two identical `from ..types.X import Y` lines — syntactically valid, harmless to `basedpyright strict` and to Python's import system, just visually redundant. Task 2 adds a corpus case that exercises this on purpose so the tradeoff is documented against real output, not just asserted. + +--- + +## File Structure + +| File | Change | +|---|---| +| `src/Structures/ImportSet.dhall` | Drop `dedupKey` from `CustomImport`; delete `dedupCustoms`, `sortCustoms`, `eqNat`, `leNat`, `sortedCustoms`; collapse `customEnum`/`customComposite` into the existing `custom`; `combine` plain-concatenates `customTypes` instead of deduping. | +| `src/Templates/RowsModule.dhall`, `src/Templates/StatementModule.dhall` | Read `imports.customTypes` directly instead of `ImportSet.sortedCustoms imports`. | +| `tests/golden/` | Regenerate via `mise run golden`; review the (likely negligible) reordering of custom-type import lines from alphabetical to declaration order. | +| `python.gen/DESIGN.md` | Note the ordering change where section 12/13 currently describe the alphabetical-by-`order` scheme. | + +--- + +### Task 1: Simplify `ImportSet.dhall` + +**Files:** +- Modify: `src/Structures/ImportSet.dhall` + +**Interfaces:** +- `CustomImport` loses `dedupKey : Natural` → becomes `{ className : Text, moduleName : Text }`. +- `custom : CustomImport -> Self` — unchanged signature, now the only constructor (no more `customEnum`/`customComposite`). +- `combine : Self -> Self -> Self` — `customTypes` field is now a plain list concatenation. +- `sortedCustoms` is deleted. Callers read `.customTypes` directly. + +- [ ] **Step 1: Replace the file** + +```dhall +let Prelude = ../Deps/Prelude.dhall + +-- A custom-type import line: "from ..types. import ". +-- Emitted in encounter order (the order the referencing columns/params were +-- declared), not sorted. Dhall (upstream) has no Text comparison, so there is +-- no way to alphabetize or dedupe by moduleName/className without either the +-- pgn fork's Text/equal or a project-wide Natural id (previously `order`, +-- sourced from Project.dhall's buildLookup — see +-- docs/plans/2026-07-11-encounter-order-custom-imports.md for why that's +-- gone and why dedup isn't reintroduced some other way). Two references to +-- the same type currently produce two identical lines; harmless to Python +-- and to basedpyright, just not deduped. +let CustomImport = { className : Text, moduleName : Text } + +let Self = + { uuid : Bool + , datetime : Bool + , date : Bool + , time : Bool + , timedelta : Bool + , decimal : Bool + , jsonb : Bool + , json : Bool + , jsonValue : Bool + , enumArray : Bool + , customTypes : List CustomImport + } + +let base = + { uuid = False + , datetime = False + , date = False + , time = False + , timedelta = False + , decimal = False + , jsonb = False + , json = False + , jsonValue = False + , enumArray = False + , customTypes = [] : List CustomImport + } + +let empty + : Self + = base + +let uuid + : Self + = base // { uuid = True } + +let datetime + : Self + = base // { datetime = True } + +let date + : Self + = base // { date = True } + +let time + : Self + = base // { time = True } + +let timedelta + : Self + = base // { timedelta = True } + +let decimal + : Self + = base // { decimal = True } + +let jsonb + : Self + = base // { jsonb = True } + +let json + : Self + = base // { json = True } + +let jsonValue + : Self + = base // { jsonValue = True } + +let enumArray + : Self + = base // { enumArray = True } + +let custom + : CustomImport -> Self + = \(c : CustomImport) -> base // { customTypes = [ c ] } + +let combine = + \(left : Self) -> + \(right : Self) -> + { uuid = left.uuid || right.uuid + , datetime = left.datetime || right.datetime + , date = left.date || right.date + , time = left.time || right.time + , timedelta = left.timedelta || right.timedelta + , decimal = left.decimal || right.decimal + , jsonb = left.jsonb || right.jsonb + , json = left.json || right.json + , jsonValue = left.jsonValue || right.jsonValue + , enumArray = left.enumArray || right.enumArray + , customTypes = left.customTypes # right.customTypes + } + +let combineAll + : List Self -> Self + = \(sets : List Self) -> List/fold Self sets Self combine empty + +in { Type = Self + , CustomImport + , empty + , uuid + , datetime + , date + , time + , timedelta + , decimal + , jsonb + , json + , jsonValue + , enumArray + , custom + , combine + , combineAll + } +``` + +- [ ] **Step 2: Type-check** + +Run: `dhall type --file=src/Structures/ImportSet.dhall` +Expected: prints the record-of-functions signature, no error. + +- [ ] **Step 3: Commit** + +```bash +git add src/Structures/ImportSet.dhall +git commit -m "python.gen: drop order-based sort/dedup from ImportSet, use encounter order" +``` + +--- + +### Task 2: Update the two render call sites + +**Files:** +- Modify: `src/Templates/RowsModule.dhall`, `src/Templates/StatementModule.dhall` + +- [ ] **Step 1: `RowsModule.dhall:58`** + +Change: +```dhall + (ImportSet.sortedCustoms imports) +``` +to: +```dhall + imports.customTypes +``` + +- [ ] **Step 2: `StatementModule.dhall:82`** + +Same change: +```dhall + imports.customTypes +``` + +- [ ] **Step 3: Type-check both** + +Run: `dhall type --file=src/Templates/RowsModule.dhall && dhall type --file=src/Templates/StatementModule.dhall` +Expected: both print their signatures, no error. + +- [ ] **Step 4: Commit** + +```bash +git add src/Templates/RowsModule.dhall src/Templates/StatementModule.dhall +git commit -m "python.gen: render custom-type imports in encounter order" +``` + +--- + +### Task 3: Regenerate golden fixtures and verify the dedup gap directly + +**Files:** +- Modify: `tests/fixture-project/` (temporary, to exercise the dedup gap — see Step 1) +- Regenerate: `tests/golden/` + +- [ ] **Step 1: Confirm today's corpus has no same-type-twice case** + +Run: +```bash +grep -rl "^from \.\.types\." tests/golden/src 2>/dev/null | while read f; do + n=$(grep -c "^from \.\.types\." "$f") + [ "$n" -gt 1 ] && echo "$f: $n" +done +``` +Expected: no output (re-confirms the design-time check above, against the *current* golden tree before this plan's regeneration). + +- [ ] **Step 2: Regenerate golden** + +Run: `mise run golden` +Expected: succeeds. `git diff tests/golden` shows custom-type import lines reordered from alphabetical to declaration order in files with 2+ distinct custom-type imports (e.g. wherever `Mood` and `Point2D` are both imported today — check whether the query's own column order already happens to be alphabetical for that file; if so the diff is empty there and this is confirmed low-risk for the current corpus). + +- [ ] **Step 3: Deliberately add a same-type-twice column to the fixture project** + +Add a query (or extend an existing one) whose result row or param list references the *same* composite or enum type through two different columns/params — e.g. two `mood`-typed columns in one query. This is new fixture surface, not present today; add it under `tests/fixture-project/queries/`. + +Run: `mise run golden` +Expected: succeeds; the regenerated file for that query contains **two** identical `from ..types.mood import Mood` lines (confirm with `grep -c` on the specific file). This is the one visible, accepted consequence of this plan — capture it in the diff review, don't silently let it slip into `tests/golden` unremarked. + +- [ ] **Step 4: Decide whether to keep the same-type-twice fixture case** + +Keeping it in the committed corpus makes the tradeoff a permanent, visible regression test (future readers see the duplicate import and the comment in `ImportSet.dhall` explaining it, instead of being surprised by it later). Removing it keeps the golden diff minimal for this change. Either is fine — this plan recommends **keeping it**, since an accepted-but-invisible tradeoff tends to resurface as a bug report; make the call and note it in the commit message either way. + +- [ ] **Step 5: Run the full test suite** + +Run: `mise run test` +Expected: all pass. Duplicate import lines don't fail `basedpyright strict` (Python tolerates redundant imports) or the golden byte-comparison (it compares against the freshly-committed golden, not some independent expectation). + +- [ ] **Step 6: Commit** + +```bash +git add tests/golden tests/fixture-project +git commit -m "python.gen: regenerate golden fixtures for encounter-order imports" +``` + +--- + +### Task 4: Update `DESIGN.md` + +**Files:** +- Modify: `python.gen/DESIGN.md` + +- [ ] **Step 1: Update section 12/13 cross-references** + +Wherever DESIGN.md currently describes `order`/alphabetical import sorting (it's referenced in passing around sections 12-13 and in code comments already updated by `docs/plans/2026-07-11-reusable-custom-type-codecs.md`'s Task 8), add a short note: custom-type imports are emitted in encounter (declaration) order, not sorted, since the `order` Natural no longer exists once `buildLookup` is gone; same-type-twice references are not deduped (a Dhall limitation, not an oversight — see `src/Structures/ImportSet.dhall`'s header comment for the full reasoning). Point at the fixture case from Task 3 if kept. + +- [ ] **Step 2: Commit** + +```bash +git add python.gen/DESIGN.md +git commit -m "python.gen: document encounter-order custom-type imports in DESIGN.md" +``` + +--- + +## Self-Review + +**Spec coverage:** Task 1 is the actual mechanism change (drop the Natural key, drop sort/dedup). Task 2 fixes the two render call sites that would otherwise reference a deleted `sortedCustoms`. Task 3 regenerates and — importantly — deliberately exercises the one behavior change (duplicate imports for a same-type-twice reference) instead of letting it go unverified. Task 4 keeps DESIGN.md truthful. + +**Placeholder scan:** no TBDs; every step names an exact file, an exact diff, or an exact command with expected output. + +**Dependency on the companion plan:** called out at the top (Global Constraints) and repeated in the companion plan's own self-review — `docs/plans/2026-07-11-reusable-custom-type-codecs.md` Task 3/4 call `ImportSet.custom` with no `order` argument, which only type-checks after this plan's Task 1. Sequence: this plan's Task 1 → companion plan's Tasks 3-6 → this plan's Tasks 2-4 (Task 2 touches templates the companion plan doesn't touch, so it can land anytime after Task 1, but golden regeneration in either plan's Task 8/3 should happen once, after both are code-complete, not twice). + +## Execution Handoff + +Plan complete and saved to `python.gen/docs/plans/2026-07-11-encounter-order-custom-imports.md`. Two execution options: + +**1. Subagent-Driven (recommended)** - dispatch a fresh subagent per task, review between tasks, fast iteration + +**2. Inline Execution** - execute tasks in this session using executing-plans, batch execution with checkpoints + +Which approach? diff --git a/docs/plans/2026-07-11-reusable-custom-type-codecs.md b/docs/plans/2026-07-11-reusable-custom-type-codecs.md new file mode 100644 index 0000000..b2f880c --- /dev/null +++ b/docs/plans/2026-07-11-reusable-custom-type-codecs.md @@ -0,0 +1,633 @@ +# Reusable Custom-Type Codecs Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Delete `buildLookup`/`CustomKind.Lookup` — and with it python.gen's last dependency on pgn's forked `Text/equal` builtin (DESIGN.md section 12) — by generating one `_decode`/`_encode` codec per custom type in `types/.py`, called by name from every reference site, instead of re-deriving decode/encode logic (field types, Composite-vs-Enum classification) at each call site. + +**Architecture:** `CustomType.dhall` already visits every custom type once with its full definition in hand (composite fields or enum variants) and emits `types/.py`. Today it stops at the dataclass/enum class. This plan makes it also emit a `_decode` staticmethod (and, for composites, an `_encode` instance method) on that same class. Every column/param reference site (`Member.dhall`, `ParamsMember.dhall`) currently has to search `project.customTypes` by name to find out whether it's looking at a composite or an enum, because it re-derives the decode/encode expression inline. Once decode/encode is a named method reachable from the `Name` the reference site already carries (`name.inPascalCase`), the reference site just calls it — no search, no classification, no `Text/equal`. + +**Tech Stack:** Dhall (dhall-lang 1.42 vendored via `dhall` CLI; pgn's forked interpreter for the parts still using it), Python 3.12 generated output (psycopg3), basedpyright strict, pytest golden-file harness (`mise run test`, `mise run golden`). + +## Global Constraints + +- No behavior change to non-custom-type (primitive) decode/encode paths. +- Golden fixture output (`tests/golden/`) must be regenerated and diffed, not hand-edited (per `tests/golden/README.md`). +- `mise run test` (pytest, includes `test_generated_passes_basedpyright_strict`) must pass after regeneration. +- Every Dhall file touched must independently type-check: `dhall type --file=`. +- Don't touch `Primitive.dhall`, `Scalar.dhall`, `Value.dhall`, or the `Model`/`Contract` dependency — this plan is scoped to what python.gen can do unilaterally, with the current, unmodified `Scalar = < Primitive : Primitive | Custom : Name >` contract. + +--- + +## Design decision needed before Task 3: array-of-custom-type behavior + +Today, `Member.dhall`'s Custom branch treats arrays differently by kind, because it already knows the kind from `lookup name`: +- **Enum arrays (1-D):** supported — `enumArrayDecode` emits an element-wise list comprehension. +- **Composite arrays:** rejected at generation time — `"Array of a composite type is not supported (element-wise decode is unimplemented)"` (Member.dhall:219) and, for params, `"Array of a composite type as a parameter is not supported"` (ParamsMember.dhall:359). + +Once decode/encode become named methods called unconditionally (no classification at the call site), there's no Dhall-level way to keep rejecting *only* composite arrays without reintroducing some form of lookup. Two ways forward: + +- **A — Recommended: uniform array codec, kind-specific availability.** Every custom type's template *may* define `_decode_array` (a staticmethod that turns the raw array value into a `list[...]`); `EnumModule.dhall` defines it, `CompositeModule.dhall` does not. `Member.dhall` always emits `f"{typeName}._decode_array({src})"` for a 1-D custom-type array, regardless of kind. If that's ever generated against a composite, `basedpyright strict` (already gating `mise run test`, see `tests/golden/README.md`) fails on the missing attribute — a real check, just moved from `pgn generate` time to CI time. Composite-array *params* work the same way: encode always calls `f"{fieldName}._encode()"`; if `fieldName`'s inferred type is `list[Point2D]`, basedpyright flags the missing `._encode` on `list`. This is a genuine, if later, safety net — not a silent hole. The `"Absent"` case (a `customRef` name matching no generated module) degrades the same way: the emitted `from ..types. import ` fails as an unresolved import, also caught by basedpyright strict. +- **B — Preserve today's exact rejection.** Keep a minimal kind signal alive somewhere reachable without a name search — no such source was found during design (see the exploration notes below); the only ones available (project-wide `customTypes` list, or a `Scalar.Custom` contract change) reintroduce either the search or the upstream ask this plan is explicitly trying to avoid. Pick this only if lifting the composite-array restriction is unacceptable without a live Postgres verification first. + +This plan is written for **Option A**. Composite-array support was never tested (DESIGN.md and the fixture corpus have no composite-array column — verified via `grep -rn "^from \.\.types\." tests/golden/`, no file references the same custom type twice, and none of the fixture's composite columns are arrays), so Task 8 includes adding one to the fixture project specifically to exercise this path before it ships. If that Postgres test reveals composite-array decode genuinely doesn't work end-to-end (not just an assumption), fall back to Option B and keep the `"not supported"` report, sourced from a small dedicated check rather than full `buildLookup` (open a follow-up plan; don't block this one on it). + +--- + +## File Structure + +| File | Change | +|---|---| +| `src/Templates/CompositeModule.dhall` | Add `_decode` (staticmethod) and `_encode` (instance method) to the generated dataclass. | +| `src/Templates/EnumModule.dhall` | Add `_decode` and `_decode_array` (staticmethods) and `_encode` (instance method, identity) to the generated `StrEnum`. | +| `src/Interpreters/Member.dhall` | Custom branch calls `{typeName}._decode(...)` / `{typeName}._decode_array(...)` unconditionally; drop the `lookup` parameter and the `Enum`/`Composite`/`Absent` merge. | +| `src/Interpreters/ParamsMember.dhall` | Custom branch calls `{fieldName}._encode()` unconditionally; drop `lookup` and the merge. | +| `src/Interpreters/ResultColumns.dhall`, `src/Interpreters/Result.dhall`, `src/Interpreters/Query.dhall` | Drop the `lookup : CustomKind.Lookup` parameter they only thread through. | +| `src/Interpreters/CustomType.dhall` | Drop `nestedLookup` (nothing left to pass it to). | +| `src/Interpreters/Project.dhall` | Delete `buildLookup`, `IndexedCustomType`, `compositeFields`, `memberPyType`, `lookupConfig` — all exist only to feed `buildLookup`. | +| `src/Structures/CustomKind.dhall` | Delete the file; nothing imports it after the above. | +| `python.gen/DESIGN.md` | Rewrite section 12 (no longer an accepted risk — resolved) and section 13 (unchanged content, but cross-reference updates). | +| `python.gen/docs/upstream-asks.md` | Remove ask 3 (resolved without the upstream change) or mark it withdrawn. | +| `tests/golden/` | Regenerate via `mise run golden`. | + +--- + +### Task 1: Add `_decode`/`_encode` to `CompositeModule.dhall` + +**Files:** +- Modify: `src/Templates/CompositeModule.dhall` + +**Interfaces:** +- Consumes: same `Params = { typeName : Text, extraImports : List Text, fields : List Field }` as today, `Field = { fieldName : Text, fieldType : Text }` — no signature change. +- Produces: the rendered module text now defines `_decode(src: object) -> ""` (staticmethod) and `_encode(self) -> tuple[...]` (instance method) on the dataclass, callable by any generator downstream as `TypeName._decode(x)` / `value._encode()`. + +This was prototyped and verified against `dhall type` and `dhall text` (rendered output checked with `python3 -c "compile(...)"` for both a two-field and a one-field composite — the one-field case needs the trailing-comma tuple, same edge case `compositeBind` already handles today). + +- [ ] **Step 1: Replace the template body** + +```dhall +let Prelude = ../Deps/Prelude.dhall + +let Sdk = ../Deps/Sdk.dhall + +let Field = { fieldName : Text, fieldType : Text } + +-- extraImports are the extra import lines a field type needs (e.g. +-- "from uuid import UUID"). They sit between the dataclass import and the class, +-- separated by one blank line; when empty only the dataclass import is emitted. +let Params = { typeName : Text, extraImports : List Text, fields : List Field } + +let run = + \(params : Params) -> + let fieldLines = + Prelude.Text.concatMapSep + "\n" + Field + ( \(field : Field) -> + " ${field.fieldName}: ${field.fieldType}" + ) + params.fields + + let fieldCount = Prelude.List.length Field params.fields + + let fieldTypesJoined = + Prelude.Text.concatMapSep + ", " + Field + (\(field : Field) -> field.fieldType) + params.fields + + let selfFieldsJoined = + Prelude.Text.concatMapSep + ", " + Field + (\(field : Field) -> "self.${field.fieldName}") + params.fields + + -- Python only treats trailing-comma parens as a 1-tuple; concatMapSep + -- never emits an internal comma for a single-element list, so force one + -- here. Mirrors ParamsMember.dhall's existing compositeBind trick. + let encodeTupleExpr = + if Prelude.Natural.equal fieldCount 1 + then "(${selfFieldsJoined},)" + else "(${selfFieldsJoined})" + + -- _decode/_encode are emitted as literal lines (not a nested multi-line + -- ''...'' block) because Dhall dedents a multi-line literal against its + -- OWN source indentation before splicing it into the outer literal; a + -- nested block loses its intended 4/8-space class-body indentation. + -- Verified against `dhall text` during design. + let codecMethods = + "\n" + ++ " @staticmethod\n" + ++ " def _decode(src: object) -> \"${params.typeName}\":\n" + ++ " return ${params.typeName}(*cast(tuple[${fieldTypesJoined}], src))\n" + ++ "\n" + ++ " def _encode(self) -> tuple[${fieldTypesJoined}]:\n" + ++ " return ${encodeTupleExpr}" + + let imports = + if Prelude.List.null Text params.extraImports + then "from dataclasses import dataclass\nfrom typing import cast" + else '' + from dataclasses import dataclass + from typing import cast + + ${Prelude.Text.concatSep "\n" params.extraImports}'' + + in '' + ${imports} + + + @dataclass(frozen=True, slots=True) + class ${params.typeName}: + """Decoding/encoding this composite requires register_types(conn) first. + + Without per-connection registration psycopg returns the value as a + raw string, which the generated _decode cannot splat into the dataclass. + """ + + ${fieldLines} + ${codecMethods} + '' + +in Sdk.Sigs.template Params run /\ { Field } +``` + +- [ ] **Step 2: Type-check** + +Run: `dhall type --file=src/Templates/CompositeModule.dhall` +Expected: prints the `{ Field : Type, Params : Type, Run : Type, run : ... }` signature, no error. + +- [ ] **Step 3: Spot-render and validate as Python** + +```bash +cat > /tmp/render_composite.dhall <<'EOF' +let CompositeModule = ./src/Templates/CompositeModule.dhall +in CompositeModule.run + { typeName = "Point2D" + , extraImports = [] : List Text + , fields = + [ { fieldName = "x", fieldType = "int" } + , { fieldName = "y", fieldType = "int" } + ] + } +EOF +dhall text --file=/tmp/render_composite.dhall | python3 -c "import sys; compile(sys.stdin.read(), 'point2d.py', 'exec')" && echo OK +``` +Expected: `OK`, and eyeballing the output shows `_decode`/`_encode` indented as class members (4 spaces), not module-level. + +- [ ] **Step 4: Commit** + +```bash +git add src/Templates/CompositeModule.dhall +git commit -m "python.gen: emit _decode/_encode on generated composite dataclasses" +``` + +--- + +### Task 2: Add `_decode`/`_decode_array`/`_encode` to `EnumModule.dhall` + +**Files:** +- Modify: `src/Templates/EnumModule.dhall` + +**Interfaces:** +- Consumes: same `Params = { typeName : Text, variants : List Variant }` — no signature change. +- Produces: `_decode(src: object) -> ""`, `_decode_array(src: object) -> list[""]`, `_encode(self) -> ""` (identity — psycopg binds the `StrEnum` instance directly, matching today's `defaultBind`). + +The scalar/array decode bodies are copied verbatim from `Member.dhall`'s current `enumDecode`/`enumArrayDecode` (Member.dhall:61-82), just moved from "Dhall builds inline Python text at every call site" to "Dhall builds it once, into the class." + +- [ ] **Step 1: Replace the template body** + +```dhall +let Prelude = ../Deps/Prelude.dhall + +let Sdk = ../Deps/Sdk.dhall + +let Variant = { memberName : Text, pgValue : Text } + +let Params = { typeName : Text, variants : List Variant } + +let run = + \(params : Params) -> + -- pgValue is interpolated into a single-line double-quoted Python literal; a + -- label may legally contain a backslash, quote, or control character, so + -- escape them to keep the literal valid and value-equal to the DB label. + -- Order is load-bearing: backslash first (so the escapes added below are not + -- re-escaped), then the control chars, then the closing quote. + let escapeLabel + : Text -> Text + = \(raw : Text) -> + Prelude.Function.composeList + Text + [ Prelude.Text.replace "\\" "\\\\" + , Prelude.Text.replace "\r" "\\r" + , Prelude.Text.replace "\n" "\\n" + , Prelude.Text.replace "\t" "\\t" + , Prelude.Text.replace "\"" "\\\"" + ] + raw + + let memberLines = + Prelude.Text.concatMapSep + "\n" + Variant + ( \(variant : Variant) -> + " ${variant.memberName} = \"${escapeLabel variant.pgValue}\"" + ) + params.variants + + let codecMethods = + "\n" + ++ " @staticmethod\n" + ++ " def _decode(src: object) -> \"${params.typeName}\":\n" + ++ " return ${params.typeName}(cast(str, src))\n" + ++ "\n" + ++ " @staticmethod\n" + ++ " def _decode_array(src: object) -> list[\"${params.typeName}\"]:\n" + ++ " return [\n" + ++ " ${params.typeName}(v)\n" + ++ " for v in cast(list[str], require_array(src))\n" + ++ " ]\n" + ++ "\n" + ++ " def _encode(self) -> \"${params.typeName}\":\n" + ++ " return self" + + in '' + from enum import StrEnum + from typing import cast + + from .._runtime import require_array + + + class ${params.typeName}(StrEnum): + ${memberLines} + ${codecMethods} + '' + +in Sdk.Sigs.template Params run /\ { Variant } +``` + +> `require_array` must already be importable from `.._runtime` relative to `types/.py` — confirm the relative import depth matches `types/`'s actual nesting (one level under the package root per `CustomType.dhall`'s `modulePath = "types/${moduleName}.py"`) before running Step 2; adjust to `.._runtime` vs `..._runtime` to match what `Member.dhall`'s current `enumArrayDecode` assumes at its own call sites (check `ImportSet.dhall`'s handling of `require_array` imports today for the exact existing relative path convention, since this moves an existing import from call sites into the shared module). + +- [ ] **Step 2: Type-check** + +Run: `dhall type --file=src/Templates/EnumModule.dhall` +Expected: signature prints, no error. + +- [ ] **Step 3: Spot-render and validate as Python** + +```bash +cat > /tmp/render_enum.dhall <<'EOF' +let EnumModule = ./src/Templates/EnumModule.dhall +in EnumModule.run + { typeName = "Mood" + , variants = + [ { memberName = "HAPPY", pgValue = "happy" } + , { memberName = "SAD", pgValue = "sad" } + ] + } +EOF +dhall text --file=/tmp/render_enum.dhall | python3 -c "import sys; compile(sys.stdin.read(), 'mood.py', 'exec')" && echo OK +``` +Expected: `OK`. + +- [ ] **Step 4: Commit** + +```bash +git add src/Templates/EnumModule.dhall +git commit -m "python.gen: emit _decode/_decode_array/_encode on generated enum classes" +``` + +--- + +### Task 3: Simplify `Member.dhall`'s Custom branch + +**Files:** +- Modify: `src/Interpreters/Member.dhall` + +**Interfaces:** +- Consumes: `value.scalar.customRef : Optional Model.Name` (unchanged — still comes straight off `Scalar.run`, see `src/Interpreters/Scalar.dhall:44-52`), `value.dims : Natural` (unchanged). +- Produces: `Run = Config -> Input -> Lude.Compiled.Type Output` — **drops the `CustomKind.Lookup` parameter**. Every caller (`ResultColumns.dhall`, `CustomType.dhall`) updates in Task 5/6. + +- [ ] **Step 1: Replace the Custom branch (Member.dhall:129-232) and the `Run` alias (Member.dhall:245)** + +Delete the `merge { Enum = ...; Composite = ...; Absent = ... } (lookup name)` block and the `CustomKind.CompositeField`-typed `compositeDecode` helper (lines 100-116, now dead — the same logic lives in `CompositeModule.dhall`'s `_decode` now). Replace with: + +```dhall + , Custom = + Prelude.Optional.fold + Model.Name + value.scalar.customRef + (Lude.Compiled.Type Output) + ( \(name : Model.Name) -> + let typeName = name.inPascalCase + + let customImport = + { className = typeName, moduleName = name.inSnakeCase } + + let mkOutput = + \(customImports : ImportSet.Type) -> + \(decodeExpr : Text -> Text) -> + { fieldName + , pgName = input.pgName + , pyType + , isNullable = input.isNullable + , imports = + ImportSet.combine baseImports customImports + , decodeExpr + } + + let wrapNullable = + \(call : Text -> Text) -> + \(src : Text) -> + if input.isNullable + then "None if ${src} is None else ${call src}" + else call src + + let dimsIsOne = + Natural/isZero (Natural/subtract 1 value.dims) + + in if Natural/isZero value.dims + then Lude.Compiled.ok + Output + ( mkOutput + (ImportSet.custom customImport) + ( wrapNullable + (\(src : Text) -> "${typeName}._decode(${src})") + ) + ) + else if dimsIsOne + then Lude.Compiled.ok + Output + ( mkOutput + (ImportSet.custom customImport) + ( wrapNullable + ( \(src : Text) -> + "${typeName}._decode_array(${src})" + ) + ) + ) + else Lude.Compiled.report + Output + [ input.pgName, name.inSnakeCase ] + "Array of dimensionality > 1 is not supported" + ) + ( Lude.Compiled.report + Output + [ input.pgName ] + "Custom scalar without a customRef name" + ) +``` + +Note this drops the `Enum`/`Composite` import-set split (`ImportSet.customEnum`/`ImportSet.customComposite`) in favor of one `ImportSet.custom` call — see Doc 2 (`2026-07-11-encounter-order-custom-imports.md`) for why `ImportSet.customEnum`/`customComposite` collapse into a single `ImportSet.custom` once `order` no longer exists. **Land Doc 2 in the same branch as this task, or `dhall type` will fail here** — Task 3 depends on `ImportSet.CustomImport` no longer requiring a `dedupKey`/`order` field. + +- [ ] **Step 2: Drop the `lookup` parameter from `Run`** + +Change: +```dhall +let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output +``` +to: +```dhall +let Run = Config -> Input -> Lude.Compiled.Type Output +``` +and the `run` definition's `\(lookup : CustomKind.Lookup) ->` (Member.dhall:40) — delete that line entirely, since `run` no longer takes it. + +- [ ] **Step 3: Delete the now-dead `CustomKind` import (Member.dhall:9) and the dead `compositeDecode` helper (Member.dhall:100-116)** + +- [ ] **Step 4: Type-check** + +Run: `dhall type --file=src/Interpreters/Member.dhall` +Expected: fails until Task 5 updates `ResultColumns.dhall` (Member's only caller) to stop passing `lookup` — that's fine, type-check `Member.dhall` standalone by temporarily checking `Sdk.Sigs.interpreter Config Input Output run` in isolation, or proceed straight to Task 5 and type-check the pair together. Don't commit Task 3 alone; commit Tasks 3+5+6 together (they're one type-checking unit — Dhall won't let you land a signature change without updating every call site in the same change). + +--- + +### Task 4: Simplify `ParamsMember.dhall`'s Custom branch + +**Files:** +- Modify: `src/Interpreters/ParamsMember.dhall` + +**Interfaces:** +- Consumes: same as Task 3. +- Produces: `Run = Config -> Input -> Lude.Compiled.Type Output` — drops `CustomKind.Lookup`. + +- [ ] **Step 1: Replace the Custom branch (ParamsMember.dhall:311-367)** + +Delete `compositeBind` (lines 258-288, now dead — logic moved into `CompositeModule.dhall`'s `_encode`) and the `merge { Enum = ...; Composite = ...; Absent = ... } (lookup name)` block. Replace with: + +```dhall + in Prelude.Optional.fold + Model.Name + value.scalar.customRef + (Lude.Compiled.Type Output) + ( \(name : Model.Name) -> + let customImport = + { className = name.inPascalCase + , moduleName = name.inSnakeCase + } + + let encodeExpr = + if input.isNullable + then "None if ${fieldName} is None else ${fieldName}._encode()" + else "${fieldName}._encode()" + + in Lude.Compiled.ok + Output + ( mkOutput + (ImportSet.combine value.imports (ImportSet.custom customImport)) + encodeExpr + ) + ) + ( if isJsonArrayParam + then Lude.Compiled.report + Output + [ input.pgName ] + "json/jsonb array as a parameter is not supported" + else Lude.Compiled.ok + Output + (mkOutput value.imports defaultBind) + ) +``` + +This drops the `Natural/isZero value.dims` guard that used to reject composite-array params — per the Design decision above (Option A), an array-of-composite param now generates `{fieldName}._encode()` where `fieldName`'s type is `list[Point2D]`; `list` has no `._encode`, so `basedpyright strict` catches it. Confirm this in Task 8's basedpyright run specifically, not just eyeball it. + +- [ ] **Step 2: Drop the `lookup` parameter from `Run` (ParamsMember.dhall:387) and `run`'s `\(lookup : CustomKind.Lookup) ->` (ParamsMember.dhall:235)** + +- [ ] **Step 3: Delete the dead `CustomKind` import (ParamsMember.dhall:9)** + +- [ ] **Step 4: Type-check together with Task 3** + +Run: `dhall type --file=src/Interpreters/ParamsMember.dhall` +Expected: same caveat as Task 3 Step 4 — its caller (`Query.dhall`) still passes `lookup` until Task 5. + +--- + +### Task 5: Drop `lookup` threading from `ResultColumns.dhall`, `Result.dhall`, `Query.dhall` + +**Files:** +- Modify: `src/Interpreters/ResultColumns.dhall`, `src/Interpreters/Result.dhall`, `src/Interpreters/Query.dhall` + +**Interfaces:** +- Each of these only forwards `lookup` to a callee; none inspect it. Removing it is mechanical. + +- [ ] **Step 1: `ResultColumns.dhall`** — delete `\(lookup : CustomKind.Lookup) ->` (line 62) and change `Member.run config lookup member` (line 76) to `Member.run config member`. Delete the `CustomKind` import (line 9) if nothing else in the file uses it — verify with `grep -n CustomKind src/Interpreters/ResultColumns.dhall` after editing. + +- [ ] **Step 2: `Result.dhall`** — delete both `\(lookup : CustomKind.Lookup) ->` occurrences (lines 66, 93), change `ResultColumns.run config lookup rowClassName columns` (line 89) to `ResultColumns.run config rowClassName columns`, and `rowsOutput config lookup rowClassName` (line 100) to `rowsOutput config rowClassName` — check `rowsOutput`'s own definition for a `lookup` parameter to drop too (it wasn't in the earlier grep excerpt; read the file before editing to confirm). Delete the `CustomKind` import (line 9) if unused after. + +- [ ] **Step 3: `Query.dhall`** — delete `\(lookup : CustomKind.Lookup) ->` (line 140), change `ResultModule.run config lookup rowClassName input.result` (line 156) to drop `lookup`, and `ParamsMember.run config lookup member` (line 173) to `ParamsMember.run config member`. Delete the `CustomKind` import (line 7) if unused after. + +- [ ] **Step 4: Type-check each file standalone** + +Run: `dhall type --file=src/Interpreters/ResultColumns.dhall && dhall type --file=src/Interpreters/Result.dhall && dhall type --file=src/Interpreters/Query.dhall` +Expected: all three print their signatures. `Query.dhall` will still fail until `Project.dhall` (Task 6) stops passing `lookup` to `QueryGen.run` — that's the last link. + +--- + +### Task 6: Delete `buildLookup` and its support code from `Project.dhall` + +**Files:** +- Modify: `src/Interpreters/Project.dhall` + +- [ ] **Step 1: Delete dead helpers** + +Delete, in order (they only exist to feed `buildLookup`, confirmed by re-reading the file top to bottom — nothing else calls `lookupConfig`, `memberPyType`, `compositeFields`, `IndexedCustomType`, or `buildLookup` itself): +- `lookupConfig` (lines 74-80) +- `memberPyType` (lines 85-96) +- `compositeFields` (lines 98-108) +- The local `CompositeField = { fieldName : Text, pyType : Text }` (line 7) — dead once `compositeFields`/`memberPyType` are gone +- `IndexedCustomType` (line 142) +- `buildLookup` (lines 147-181) + +- [ ] **Step 2: Update the call site** + +Find `let lookup = buildLookup effectiveCustomTypes` (line 479) — delete it, and change both call sites that pass `lookup`: +- `QueryGen.run config lookup query` (line 503) → `QueryGen.run config query` +- `(\(query : Model.Query) -> QueryGen.run config lookup query)` (line 524) → `(\(query : Model.Query) -> QueryGen.run config query)` + +- [ ] **Step 3: Delete the dead `CustomKind` import (line 9)** + +- [ ] **Step 4: Type-check the whole chain** + +Run: `dhall type --file=src/Interpreters/Project.dhall` +Expected: prints the module signature with no error — this is the point where Tasks 3-6 all click together (Project → Query → Result/ParamsMember → ResultColumns/Member all now agree on the lookup-free signatures). + +- [ ] **Step 5: Commit Tasks 3-6 together** + +```bash +git add src/Interpreters/Member.dhall src/Interpreters/ParamsMember.dhall \ + src/Interpreters/ResultColumns.dhall src/Interpreters/Result.dhall \ + src/Interpreters/Query.dhall src/Interpreters/Project.dhall +git commit -m "python.gen: call custom-type codecs by name instead of resolving via buildLookup" +``` + +--- + +### Task 7: Drop `nestedLookup` from `CustomType.dhall`, delete `CustomKind.dhall` + +**Files:** +- Modify: `src/Interpreters/CustomType.dhall` +- Delete: `src/Structures/CustomKind.dhall` + +- [ ] **Step 1: `CustomType.dhall`** — delete `nestedLookup` (lines 51-53) and its comment, and change `MemberGen.run config nestedLookup m` (line 127) to `MemberGen.run config m`. Delete the `CustomKind` import (line 11). + +- [ ] **Step 2: Confirm nothing else references `CustomKind`** + +Run: `grep -rln "CustomKind" src` +Expected: no output. + +- [ ] **Step 3: Delete the file** + +```bash +git rm src/Structures/CustomKind.dhall +``` + +- [ ] **Step 4: Type-check the full package entry point** + +Run: `dhall type --file=src/package.dhall` +Expected: prints the top-level module signature, no error. (This is the first point a full-package check is meaningful — earlier steps only checked individual interpreter files.) + +- [ ] **Step 5: Commit** + +```bash +git add src/Interpreters/CustomType.dhall +git commit -m "python.gen: delete CustomKind.dhall, the last remnant of buildLookup" +``` + +--- + +### Task 8: Regenerate golden fixtures, verify, update docs + +**Files:** +- Modify: `tests/fixture-project/` (add a composite-array column per the Design decision above) +- Regenerate: `tests/golden/` +- Modify: `python.gen/DESIGN.md`, `python.gen/docs/upstream-asks.md`, `python.gen/CHANGELOG.md` + +- [ ] **Step 1: Add a composite-array test column to the fixture project** + +Find the fixture's composite-type column definitions under `tests/fixture-project/` (query/table SQL referencing a composite type, e.g. the `Point2D`-typed column used in `insert_specimen`/`get_specimen` per the golden output). Add one query or column that selects/inserts an *array* of that composite type — this is the first real exercise of the Option A fallback path (`basedpyright strict` should catch it if `_decode_array`/`_encode` aren't defined on `Point2D`, since Task 1 deliberately didn't add them to `CompositeModule.dhall`). + +Confirm the expected failure mode first: + +Run: `mise run golden` (needs `PGN_TEST_DATABASE_URL` pointing at a live Postgres — see `tests/golden/README.md`) +Expected: generation succeeds (no Dhall-level rejection — that's the point of Option A), but the regenerated file calls `Point2D._decode_array(...)` or `list[Point2D]._encode()`-shaped code that doesn't exist. + +Run: `mise run test` +Expected: `test_generated_passes_basedpyright_strict` FAILS, citing the missing attribute. This confirms the safety net from the Design decision actually fires. Once confirmed, either: + - revert the fixture addition (if you don't want composite arrays in the committed golden corpus yet), or + - implement `_decode_array`/an array-aware `_encode` on `CompositeModule.dhall` for real and keep the fixture (a follow-up, out of this plan's scope — flag it, don't scope-creep this task). + +- [ ] **Step 2: Remove the composite-array addition (unless implementing it for real per Step 1)** + +- [ ] **Step 3: Regenerate golden for real** + +Run: `mise run golden` +Expected: succeeds, rewrites `tests/golden/src/specimen_client/_generated/**` and both facades. + +- [ ] **Step 4: Review the diff** + +Run: `git diff tests/golden` +Expected: every composite/enum decode/encode call site now reads `TypeName._decode(...)`/`TypeName._decode_array(...)`/`value._encode()` instead of the old inlined `cast(tuple[...], ...)`/`(x.a, x.b)` expressions; `types/point_2_d.py`, `types/mood.py`, `types/tag_value.py` each gain the new methods. No unrelated files change. + +- [ ] **Step 5: Run the full test suite** + +Run: `mise run test` +Expected: all pass, including `test_generated_passes_basedpyright_strict`. + +- [ ] **Step 6: Update `DESIGN.md`** + +Rewrite section 12 (`## 12. Forked-Dhall (Text/equal) dependency risk`) — it's no longer an accepted risk for python.gen; state plainly that `buildLookup` is gone and `Text/equal` is no longer used anywhere in this generator's own Dhall source (grep to confirm: `grep -rn "Text/equal" src` returns nothing). Keep a short note that `demos/Exhaustive.dhall`/`mise run golden` still needs the pinned pgn binary regardless, because `gen-sdk`'s own `Fixtures` module independently uses the fork builtin — this plan doesn't touch that, and it isn't blocked on it. Cross-reference section 13 (unchanged — `PyIdent.dhall`'s trick is unrelated and still in place). + +- [ ] **Step 7: Update `docs/upstream-asks.md`** + +Remove ask 3 (`## 3. gen-sdk: kind tag or Natural index on Scalar.Custom`) or mark it explicitly withdrawn with one line explaining why (`buildLookup`'s only consumer was resolved locally by generating named codecs instead of resolving structural type info by search — see `docs/plans/2026-07-11-reusable-custom-type-codecs.md`). Don't delete the file's other two asks (pragma parsing, warnings printing) — they're unrelated and still open. + +- [ ] **Step 8: Update `CHANGELOG.md`** + +Add an entry under the appropriate section describing the behavior change from the Design decision (composite-array columns/params no longer rejected at generation time; verify at basedpyright-strict time instead) if Step 1's fixture addition was kept, or note it as a documented-but-untested path if reverted. + +- [ ] **Step 9: Commit** + +```bash +git add tests/golden tests/fixture-project python.gen/DESIGN.md python.gen/docs/upstream-asks.md python.gen/CHANGELOG.md +git commit -m "python.gen: regenerate golden fixtures for reusable custom-type codecs" +``` + +--- + +## Self-Review + +**Spec coverage:** Task 1-2 build the reusable codecs (the actual "fix the root cause"). Tasks 3-4 make the two reference sites (decode, encode) call them. Tasks 5-7 remove the now-dead plumbing (`lookup` threading, `buildLookup`, `CustomKind.dhall`) so nothing is left half-migrated. Task 8 proves it against the real toolchain and updates the two docs (`DESIGN.md`, `upstream-asks.md`) that currently assert this is unfixable — both would otherwise go stale and mislead the next reader. + +**Open question carried forward, not silently resolved:** the array-of-composite behavior change (Design decision, Option A vs B) is a real product decision, flagged explicitly rather than picked unilaterally in the diff. Task 8 Step 1 is designed to surface the actual runtime behavior (does `basedpyright strict` really catch it, does composite-array decode actually work against real Postgres) before committing to either branch. + +**Dependency on the companion plan:** Task 3 explicitly calls out that it needs `docs/plans/2026-07-11-encounter-order-custom-imports.md` (`ImportSet.dhall`'s `order`/`dedupKey` removal) landed first or alongside — `ImportSet.customEnum`/`customComposite` currently *require* an `order : Natural` that only `buildLookup` produced. Implement that plan first, or fold both into one PR; don't land this plan's Task 3 against the unmodified `ImportSet.dhall`. + +## Execution Handoff + +Plan complete and saved to `python.gen/docs/plans/2026-07-11-reusable-custom-type-codecs.md`. Two execution options: + +**1. Subagent-Driven (recommended)** - dispatch a fresh subagent per task, review between tasks, fast iteration + +**2. Inline Execution** - execute tasks in this session using executing-plans, batch execution with checkpoints + +Which approach? diff --git a/docs/upstream-asks.md b/docs/upstream-asks.md index 44c3417..8d10b37 100644 --- a/docs/upstream-asks.md +++ b/docs/upstream-asks.md @@ -102,6 +102,18 @@ and the new stderr output only appears when warnings are non-empty. ## 3. gen-sdk: `kind` tag or `Natural` index on `Scalar.Custom` +**WITHDRAWN.** `buildLookup`'s only consumer of the fork-only `Text/equal` +builtin was resolved locally, with no gen-sdk contract change needed: +custom-type decode/encode now dispatches through named +`_decode`/`_decode_array`/`_encode` methods generated onto each custom +type's own Python class (`CompositeModule.dhall`/`EnumModule.dhall`), +called by name from every reference site, instead of resolving +classification/fields via a project-wide structural search. `buildLookup` +and `Structures/CustomKind.dhall` are deleted; see +`docs/plans/2026-07-11-reusable-custom-type-codecs.md` and DESIGN.md +section 12. This section is kept for the historical record of why the ask +existed, not as an open request. + ### Motivation This is the ask already planned in DESIGN.md, section 12. The generator's diff --git a/src/Config.dhall b/src/Config.dhall deleted file mode 100644 index 1377127..0000000 --- a/src/Config.dhall +++ /dev/null @@ -1,15 +0,0 @@ --- User-facing config for this generator. `emitSync` adds a parallel sync surface --- (psycopg.Connection) alongside the default async one, so one project can serve --- both an async backend and a sync (Dagster) consumer from shared Row types. --- `onUnsupported` picks Fail (default, abort loudly) or Skip (drop the --- unsupported statement/type and its dependents, with a warning) when a query --- or custom type hits a PG shape the generator cannot render; see --- Structures/OnUnsupported.dhall. All fields are Optional so a project may omit --- the whole config block or any subset of its keys; Interpret.dhall supplies the --- defaults. -let OnUnsupported = ./Structures/OnUnsupported.dhall - -in { packageName : Optional Text - , emitSync : Optional Bool - , onUnsupported : Optional OnUnsupported.Mode - } : Type diff --git a/src/Deps/Lude.dhall b/src/Deps/Lude.dhall index 30f1f5a..a4b5116 100644 --- a/src/Deps/Lude.dhall +++ b/src/Deps/Lude.dhall @@ -1,3 +1,2 @@ -https://raw.githubusercontent.com/codemine-io/lude.dhall/v5.1.0/src/package.dhall - sha256:46b527b071eba96a17e76b4bc5774645714dd5b4355974d221e705aa7c126e77 - as Source +https://raw.githubusercontent.com/codemine-io/lude.dhall/v5.2.0/src/package.dhall + sha256:b04ff9a38c8be087dfacfe76d057fc5272afec60cac28c9e6c1b78cade7ff6ef diff --git a/src/Deps/Prelude.dhall b/src/Deps/Prelude.dhall index 54107fc..76e2a7a 100644 --- a/src/Deps/Prelude.dhall +++ b/src/Deps/Prelude.dhall @@ -1,3 +1,2 @@ https://raw.githubusercontent.com/dhall-lang/dhall-lang/v23.1.0/Prelude/package.dhall sha256:931cbfae9d746c4611b07633ab1e547637ab4ba138b16bf65ef1b9ad66a60b7f - as Source diff --git a/src/Deps/Sdk.dhall b/src/Deps/Sdk.dhall index 5602c43..18f9fce 100644 --- a/src/Deps/Sdk.dhall +++ b/src/Deps/Sdk.dhall @@ -1,3 +1,2 @@ https://raw.githubusercontent.com/pgenie-io/gen-sdk/v2.0.0/src/package.dhall sha256:b9def6ab1179bc4aaae7fc6e91977f094f75934cd5755175c294a9e97ca71b15 - as Source diff --git a/src/Interpret.dhall b/src/Interpret.dhall deleted file mode 100644 index de4642f..0000000 --- a/src/Interpret.dhall +++ /dev/null @@ -1,54 +0,0 @@ -let Contract = ./Deps/Contract.dhall - -let Prelude = ./Deps/Prelude.dhall - -let Config = ./Config.dhall - -let OnUnsupported = ./Structures/OnUnsupported.dhall - -let ProjectInterpreter = ./Interpreters/Project.dhall - --- Entry point handed to gen-sdk's Sdk.Sigs.generator as `interpret`. Each --- field of Config is independently Optional, so a project may omit the --- whole config block (Sdk.Sigs.generator substitutes an all-None --- defaultConfig, see package.dhall) or any subset of its keys; `defaults` --- collects every fallback in one place (packageName from the project name in --- kebab case, emitSync off, onUnsupported Fail). The async surface is always --- emitted; emitSync adds the sync mirror. -in \(config : Config) -> - \(project : Contract.Project) -> - let defaults = - { packageName = project.name.inKebabCase - , emitSync = False - , onUnsupported = OnUnsupported.Mode.Fail - } - - let packageName = - Prelude.Optional.fold - Text - config.packageName - Text - (\(t : Text) -> t) - defaults.packageName - - let emitSync = - Prelude.Optional.fold - Bool - config.emitSync - Bool - (\(b : Bool) -> b) - defaults.emitSync - - let onUnsupported = - Prelude.Optional.fold - OnUnsupported.Mode - config.onUnsupported - OnUnsupported.Mode - (\(m : OnUnsupported.Mode) -> m) - defaults.onUnsupported - - let importName = Prelude.Text.replace "-" "_" packageName - - let interpreterConfig = { packageName, importName, emitSync, onUnsupported } - - in ProjectInterpreter.run interpreterConfig project diff --git a/src/Interpreters/CustomType.dhall b/src/Interpreters/CustomType.dhall index 676f9ab..abdf94f 100644 --- a/src/Interpreters/CustomType.dhall +++ b/src/Interpreters/CustomType.dhall @@ -8,8 +8,6 @@ let Sdk = ../Deps/Sdk.dhall let ImportSet = ../Structures/ImportSet.dhall -let CustomKind = ../Structures/CustomKind.dhall - let OnUnsupported = ../Structures/OnUnsupported.dhall let MemberGen = ./Member.dhall @@ -44,14 +42,6 @@ let Output = , kind : TypeKind } --- Composite fields could in principle reference other custom types, but pgn never --- nests customs in our corpus and CustomType.run does not receive the project --- lookup (Project.run threads it only to queries). Resolving any nested custom to --- Absent makes Member.run fail loudly instead of guessing a type. -let nestedLookup - : CustomKind.Lookup - = \(_ : Model.Name) -> CustomKind.TypeKind.Absent - -- Render the stdlib/runtime imports a composite field type needs, in a fixed -- order so output stays byte-stable. Reads only the standard flags; nested custom -- types are out of Wave 2 scope (no composite in the corpus references one). @@ -123,9 +113,7 @@ let run = = Lude.Compiled.traverseList Model.Member MemberGen.Output - ( \(m : Model.Member) -> - MemberGen.run config nestedLookup m - ) + ( \(m : Model.Member) -> MemberGen.run config m ) members let assemble = diff --git a/src/Interpreters/Member.dhall b/src/Interpreters/Member.dhall index 8d2d27c..c4e8273 100644 --- a/src/Interpreters/Member.dhall +++ b/src/Interpreters/Member.dhall @@ -4,9 +4,9 @@ let Prelude = ../Deps/Prelude.dhall let Model = ../Deps/Contract.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Sdk = ../Deps/Sdk.dhall -let CustomKind = ../Structures/CustomKind.dhall +let ImportSet = ../Structures/ImportSet.dhall let PyIdent = ../Structures/PyIdent.dhall @@ -37,7 +37,6 @@ let Output = let run = \(config : Config) -> - \(lookup : CustomKind.Lookup) -> \(input : Input) -> -- Result-column / composite-field name becomes a dataclass field and decode -- kwarg, so a keyword-named column must be sanitized; row["..."] keeps the @@ -58,63 +57,6 @@ let run = let passthroughDecode = \(src : Text) -> "cast(${castTarget}, ${src})" - let enumDecode = - \(enumName : Text) -> - \(src : Text) -> - let call = "${enumName}(cast(str, ${src}))" - - in if input.isNullable - then "None if ${src} is None else ${call}" - else call - - -- psycopg returns an enum array as a list of text, so each element - -- is rebuilt into the StrEnum. The cast pins the iterable's element - -- type; the per-element None guard mirrors elementIsNullable and the - -- outer None guard mirrors a nullable column. - let enumArrayDecode = - \(enumName : Text) -> - \(src : Text) -> - let elemCast = - if value.elementIsNullable - then "list[str | None]" - else "list[str]" - - let elemDecode = - if value.elementIsNullable - then "None if v is None else ${enumName}(v)" - else "${enumName}(v)" - - -- require_array fails loudly if the array came back as - -- text (the connection did not register the enum type) - -- instead of iterating a string into bogus members. - let elements = - "[${elemDecode} for v in cast(${elemCast}, require_array(${src}))]" - - in if input.isNullable - then "None if ${src} is None else ${elements}" - else elements - - -- Composites are registered per connection (see register_types), - -- so psycopg returns a namedtuple. The fixed-length tuple cast - -- with each field's exact pyType lets the splat satisfy strict. - let compositeDecode = - \(typeName : Text) -> - \(fields : List CustomKind.CompositeField) -> - \(src : Text) -> - let fieldTypes = - Prelude.Text.concatMapSep - ", " - CustomKind.CompositeField - (\(f : CustomKind.CompositeField) -> f.pyType) - fields - - let call = - "${typeName}(*cast(tuple[${fieldTypes}], ${src}))" - - in if input.isNullable - then "None if ${src} is None else ${call}" - else call - in merge { Passthrough = Lude.Compiled.ok @@ -135,11 +77,7 @@ let run = let typeName = name.inPascalCase let customImport = - \(order : Natural) -> - { className = typeName - , moduleName = name.inSnakeCase - , order - } + { className = typeName, moduleName = name.inSnakeCase } let mkOutput = \(customImports : ImportSet.Type) -> @@ -149,81 +87,57 @@ let run = , pyType , isNullable = input.isNullable , imports = - ImportSet.combine - baseImports - customImports + ImportSet.combine baseImports customImports , decodeExpr } - -- A 1-D enum array decodes element-wise; a scalar - -- custom (dims == 0) keeps the single-value decode. - -- Composite arrays and dims > 1 are unimplemented, - -- so fail loudly rather than emit wrong Python. + let wrapNullable = + \(call : Text -> Text) -> + \(src : Text) -> + if input.isNullable + then "None if ${src} is None else ${call src}" + else call src + let dimsIsOne = Natural/isZero (Natural/subtract 1 value.dims) - in merge - { Enum = - \(order : Natural) -> - let enumImport = - ImportSet.customEnum - (customImport order) - - in if Natural/isZero value.dims - then Lude.Compiled.ok - Output - ( mkOutput - enumImport - (enumDecode typeName) - ) - else if dimsIsOne - then Lude.Compiled.ok - Output - ( mkOutput - ( ImportSet.combine - enumImport - ImportSet.enumArray - ) - (enumArrayDecode typeName) - ) - else Lude.Compiled.report - Output - [ input.pgName - , name.inSnakeCase - ] - "Array of an enum with dimensionality > 1 is not supported" - , Composite = - \ ( composite - : { fields : - List CustomKind.CompositeField - , order : Natural - } - ) -> - if Natural/isZero value.dims - then Lude.Compiled.ok - Output - ( mkOutput - ( ImportSet.customComposite - (customImport composite.order) - ) - ( compositeDecode - typeName - composite.fields + in if Natural/isZero value.dims + then Lude.Compiled.ok + Output + ( mkOutput + (ImportSet.custom customImport) + ( wrapNullable + (\(src : Text) -> "${typeName}._decode(${src})") + ) + ) + else if dimsIsOne + then let elemCast = + if value.elementIsNullable + then "list[str | None]" + else "list[str]" + + let elemDecode = + if value.elementIsNullable + then "None if v is None else ${typeName}._decode(v)" + else "${typeName}._decode(v)" + + in Lude.Compiled.ok + Output + ( mkOutput + ( ImportSet.combine + (ImportSet.custom customImport) + ImportSet.enumArray + ) + ( wrapNullable + ( \(src : Text) -> + "[${elemDecode} for v in cast(${elemCast}, require_array(${src}))]" ) ) - else Lude.Compiled.report - Output - [ input.pgName - , name.inSnakeCase - ] - "Array of a composite type is not supported (element-wise decode is unimplemented)" - , Absent = - Lude.Compiled.report + ) + else Lude.Compiled.report Output - [ name.inSnakeCase ] - "Custom type not found in project customTypes" - } - (lookup name) + [ input.pgName, name.inSnakeCase ] + "Array of dimensionality > 1 is not supported" ) ( Lude.Compiled.report Output @@ -242,6 +156,4 @@ let run = in Lude.Compiled.flatMap Value.Output Output buildOutput compiledValue -let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output - -in { Input, Output, Run, run } +in Sdk.Sigs.interpreter Config Input Output run diff --git a/src/Interpreters/ParamsMember.dhall b/src/Interpreters/ParamsMember.dhall index 2a4b469..457631d 100644 --- a/src/Interpreters/ParamsMember.dhall +++ b/src/Interpreters/ParamsMember.dhall @@ -4,9 +4,9 @@ let Prelude = ../Deps/Prelude.dhall let Model = ../Deps/Contract.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Sdk = ../Deps/Sdk.dhall -let CustomKind = ../Structures/CustomKind.dhall +let ImportSet = ../Structures/ImportSet.dhall let OnUnsupported = ../Structures/OnUnsupported.dhall @@ -232,7 +232,6 @@ let isJsonArray = let run = \(config : Config) -> - \(lookup : CustomKind.Lookup) -> \(input : Input) -> let fieldName = pySafeName input.name.inSnakeCase @@ -253,40 +252,6 @@ let run = then wrapJson "Jsonb" else if needsJsonImport then wrapJson "Json" else fieldName - -- psycopg binds a bare tuple to an anonymous composite, but cannot adapt - -- a dataclass, so a composite param is converted to its field tuple. - let compositeBind = - \(fields : List CustomKind.CompositeField) -> - let joinedFields = - Prelude.Text.concatMapSep - ", " - CustomKind.CompositeField - ( \(f : CustomKind.CompositeField) -> - "${fieldName}.${f.fieldName}" - ) - fields - - -- concatMapSep never emits a separator for a single-element list, so - -- a one-field composite would render "(x.f)": parens around a bare - -- expression, not a tuple. Python only treats trailing-comma parens - -- as a 1-tuple, so force it for exactly one field; concatMapSep - -- already inserts the internal comma for two or more. - let trailingComma = - if Prelude.Natural.equal - ( Prelude.List.length - CustomKind.CompositeField - fields - ) - 1 - then "," - else "" - - let tupleExpr = "(" ++ joinedFields ++ trailingComma ++ ")" - - in if input.isNullable - then "None if ${fieldName} is None else ${tupleExpr}" - else tupleExpr - let buildOutput = \(value : Value.Output) -> let pyType = @@ -314,56 +279,36 @@ let run = (Lude.Compiled.Type Output) ( \(name : Model.Name) -> let customImport = - \(order : Natural) -> - { className = name.inPascalCase - , moduleName = name.inSnakeCase - , order - } - - in merge - { Enum = - \(order : Natural) -> - Lude.Compiled.ok - Output - ( mkOutput - ( ImportSet.combine - value.imports - ( ImportSet.customEnum - (customImport order) - ) - ) - defaultBind - ) - , Composite = - \ ( composite - : { fields : - List CustomKind.CompositeField - , order : Natural - } - ) -> - if Natural/isZero value.dims - then Lude.Compiled.ok - Output - ( mkOutput - ( ImportSet.combine - value.imports - ( ImportSet.customComposite - (customImport composite.order) - ) - ) - (compositeBind composite.fields) - ) - else Lude.Compiled.report - Output - [ input.pgName, name.inSnakeCase ] - "Array of a composite type as a parameter is not supported" - , Absent = - Lude.Compiled.report - Output - [ name.inSnakeCase ] - "Custom type not found in project customTypes" + { className = name.inPascalCase + , moduleName = name.inSnakeCase } - (lookup name) + + let scalarEncode = + if input.isNullable + then "None if ${fieldName} is None else ${fieldName}._encode()" + else "${fieldName}._encode()" + + let arrayElemEncode = + if value.elementIsNullable + then "None if x is None else x._encode()" + else "x._encode()" + + let arrayEncode = + let base = "[${arrayElemEncode} for x in ${fieldName}]" + + in if input.isNullable + then "None if ${fieldName} is None else ${base}" + else base + + let encodeExpr = + if Natural/isZero value.dims then scalarEncode else arrayEncode + + in Lude.Compiled.ok + Output + ( mkOutput + (ImportSet.combine value.imports (ImportSet.custom customImport)) + encodeExpr + ) ) ( if isJsonArrayParam then Lude.Compiled.report @@ -384,6 +329,4 @@ let run = in Lude.Compiled.flatMap Value.Output Output buildOutput compiledValue -let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output - -in { Input, Output, Run, run } +in Sdk.Sigs.interpreter Config Input Output run diff --git a/src/Interpreters/Project.dhall b/src/Interpreters/Project.dhall index d4fa6d6..f142a99 100644 --- a/src/Interpreters/Project.dhall +++ b/src/Interpreters/Project.dhall @@ -6,12 +6,6 @@ let Model = ../Deps/Contract.dhall let Sdk = ../Deps/Sdk.dhall -let CustomKind = ../Structures/CustomKind.dhall - -let PyIdent = ../Structures/PyIdent.dhall - -let Value = ./Value.dhall - let QueryGen = ./Query.dhall let CustomTypeGen = ./CustomType.dhall @@ -38,7 +32,20 @@ let OnUnsupported = ../Structures/OnUnsupported.dhall let Report = { path : List Text, message : Text } +-- The generator's public Config: every field is independently Optional, so a +-- project may omit the whole `config:` block or any subset of its keys. +-- `run` below resolves the fallbacks itself (packageName from the project +-- name, emitSync off, onUnsupported Fail); there is no separate config type +-- or resolve step between package.dhall and here. let Config = + { packageName : Optional Text + , emitSync : Optional Bool + , onUnsupported : Optional OnUnsupported.Mode + } + +-- The fully-resolved shape every downstream interpreter (Query, CustomType, +-- and everything below them) actually declares as its own `Config`. +let ResolvedConfig = { packageName : Text , importName : Text , emitSync : Bool @@ -69,44 +76,6 @@ let withHeader = \(file : Lude.File.Type) -> { path = file.path, content = generatedHeader ++ file.content } --- Internal interpreter config (importName etc.) is only needed to satisfy --- Value.run's signature; rendering a composite field's pyType does not read it. -let lookupConfig - : Config - = { packageName = "" - , importName = "" - , emitSync = False - , onUnsupported = OnUnsupported.Mode.Fail - } - --- Render a composite member's Python type purely. The Err branch is unreachable --- for a valid composite (CustomType.run fails the whole generation first on any --- unsupported field type), so the fallback string is dead. -let memberPyType = - \(member : Model.Member) -> - let valuePyType = - merge - { Ok = - \(wrapped : { value : Value.Output, warnings : List Report }) -> - wrapped.value.pyType - , Err = \(_ : Report) -> "object" - } - (Value.run lookupConfig member.value) - - in valuePyType ++ (if member.isNullable then " | None" else "") - -let compositeFields = - \(members : List Model.Member) -> - Prelude.List.map - Model.Member - CustomKind.CompositeField - ( \(member : Model.Member) -> - { fieldName = PyIdent.pySafeName member.name.inSnakeCase - , pyType = memberPyType member - } - ) - members - -- Schema-qualified type names for psycopg CompositeInfo.fetch (search-path -- safe). Reads CustomTypeGen.Output (already a combineOutputs parameter, and -- already the post-Skip-filter surviving set), not Model.CustomType, so @@ -136,58 +105,18 @@ let enumPgNames = customTypes ) --- A custom type referenced by a Scalar.Custom name resolves by matching its --- snake-case key. The query/column carries a distinct Name occurrence, so the --- lookup compares by inSnakeCase rather than relying on record identity. -let IndexedCustomType = { index : Natural, value : Model.CustomType } - --- pgn emits customTypes alphabetically by name, so the list index is the type's --- alphabetical order. The import renderer sorts on it to keep the per-module --- `from ..types.X import Y` block alphabetical independent of column order. -let buildLookup = - \(customTypes : List Model.CustomType) -> - Prelude.List.fold - IndexedCustomType - (Prelude.List.indexed Model.CustomType customTypes) - CustomKind.Lookup - ( \(entry : IndexedCustomType) -> - \(rest : CustomKind.Lookup) -> - \(name : Model.Name) -> - let ct = entry.value - - -- Text/equal is a pgn embedded-Dhall builtin, absent from the Dhall - -- standard 23.1 Prelude. PyIdent.dhall's replace-trick cannot stand in - -- here: this branch returns a structural TypeKind, not Text. gen-sdk's - -- Fixtures module relies on the same builtin; a kind tag or a Natural - -- index on Scalar.Custom is a planned upstream ask. - in if Text/equal name.inSnakeCase ct.name.inSnakeCase - then merge - { Composite = - \(members : List Model.Member) -> - CustomKind.TypeKind.Composite - { fields = compositeFields members - , order = entry.index - } - , Enum = - \(_ : List Model.EnumVariant) -> - CustomKind.TypeKind.Enum entry.index - , Domain = - \(_ : Model.Value) -> CustomKind.TypeKind.Absent - } - ct.definition - else rest name - ) - (\(_ : Model.Name) -> CustomKind.TypeKind.Absent) - let combineOutputs = - \(config : Config) -> + \(config : ResolvedConfig) -> \(input : Input) -> - \(queries : List QueryGen.Output) -> -- Already the post-Skip-filter surviving set (see `run`); equal to - -- input.customTypes' compiled outputs verbatim when nothing was skipped + -- input.queries' compiled outputs verbatim when nothing was skipped -- (including every Fail-mode run, since Fail never drops anything). - -- Facade/typesInit/register entries are built from this, not - -- input.customTypes, so a skipped type leaves no dangling export. + -- Facade/statement/row entries are built from this, not input.queries, + -- so a skipped query leaves no dangling export. + \(queries : List QueryGen.Output) -> + -- Same as above, but for customTypes. Facade/typesInit/register + -- entries are built from this, not input.customTypes, so a skipped + -- type leaves no dangling export. \(customTypes : List CustomTypeGen.Output) -> -- The generator emits the _generated subtree plus the package-root -- __init__.py facade. The rest of the shell (pyproject.toml, py.typed) is @@ -423,21 +352,54 @@ let combineOutputs = -- warning list. Custom types use a pair of plain functions instead of an -- equivalent record (see `typeSucceeds`/`typeWarning` below) purely because -- that was the faster shape empirically for the type side, and the query --- side re-uses `queryChecks` because `lookup` (built from the surviving --- custom types) threads into every query's Member/ParamsMember resolution: --- calling QueryGen.run config lookup query from more than one place in this --- function (once to decide keep/drop, again to render, again for a --- warning -- each a fresh, separate call site in the source) measurably --- multiplies Dhall's normalization cost per extra call site, confirmed by --- bisection against `pgn generate` wall time (a few seconds regressed to --- minutes with three call sites; this file keeps it to two: one to build --- `queryChecks`, one for the final render). +-- side re-uses `queryChecks` because calling QueryGen.run config query from +-- more than one place in this function (once to decide keep/drop, again to +-- render, again for a warning -- each a fresh, separate call site in the +-- source) measurably multiplies Dhall's normalization cost per extra call +-- site, confirmed by bisection against `pgn generate` wall time (a few +-- seconds regressed to minutes with three call sites; this file keeps it to +-- two: one to build `queryChecks`, one for the final render). let QueryCheck = { query : Model.Query, keep : Bool, warning : Optional Report } let run = \(config : Config) -> \(input : Input) -> - let skip = merge { Fail = False, Skip = True } config.onUnsupported + -- config's fields are independently Optional (a project may omit the + -- whole config: block or any subset of its keys); the fallbacks are + -- resolved here rather than in a separate config type or resolve + -- step, since this is the root interpreter and Config above is + -- exactly the generator's public Config. + let packageName = + Prelude.Optional.fold + Text + config.packageName + Text + (\(t : Text) -> t) + input.name.inKebabCase + + let emitSync = + Prelude.Optional.fold + Bool + config.emitSync + Bool + (\(b : Bool) -> b) + False + + let onUnsupported = + Prelude.Optional.fold + OnUnsupported.Mode + config.onUnsupported + OnUnsupported.Mode + (\(m : OnUnsupported.Mode) -> m) + OnUnsupported.Mode.Fail + + let importName = Prelude.Text.replace "-" "_" packageName + + let resolvedConfig + : ResolvedConfig + = { packageName, importName, emitSync, onUnsupported } + + let skip = merge { Fail = False, Skip = True } resolvedConfig.onUnsupported let typeSucceeds : Model.CustomType -> Bool @@ -448,7 +410,7 @@ let run = True , Err = \(_ : Report) -> False } - (CustomTypeGen.run config ct) + (CustomTypeGen.run resolvedConfig ct) -- Nested under the type's own name so the warning names the type -- that failed, not just the inner member/column that triggered it @@ -463,7 +425,7 @@ let run = , Err = \(err : Report) -> Some { path = [ ct.name.inSnakeCase ] # err.path, message = err.message } } - (CustomTypeGen.run config ct) + (CustomTypeGen.run resolvedConfig ct) -- A skipped custom type resolves to Absent for any query that -- references it, and that query's own Member/ParamsMember @@ -476,8 +438,6 @@ let run = then Prelude.List.filter Model.CustomType typeSucceeds input.customTypes else input.customTypes - let lookup = buildLookup effectiveCustomTypes - -- Fail mode: identical to the pre-Skip code (traverseList straight -- over input.customTypes), so its error message/path is unchanged. let typesForCombine @@ -485,7 +445,7 @@ let run = = Lude.Compiled.traverseList Model.CustomType CustomTypeGen.Output - (\(ct : Model.CustomType) -> CustomTypeGen.run config ct) + (\(ct : Model.CustomType) -> CustomTypeGen.run resolvedConfig ct) effectiveCustomTypes let queryChecks @@ -500,7 +460,7 @@ let run = { query, keep = True, warning = None Report } , Err = \(err : Report) -> { query, keep = False, warning = Some err } } - (QueryGen.run config lookup query) + (QueryGen.run resolvedConfig query) ) input.queries @@ -521,7 +481,7 @@ let run = = Lude.Compiled.traverseList Model.Query QueryGen.Output - (\(query : Model.Query) -> QueryGen.run config lookup query) + (\(query : Model.Query) -> QueryGen.run resolvedConfig query) effectiveQueries let skipWarnings @@ -541,7 +501,7 @@ let run = (List QueryGen.Output) (List CustomTypeGen.Output) Output - (combineOutputs config input) + (combineOutputs resolvedConfig input) queriesForCombine typesForCombine diff --git a/src/Interpreters/Query.dhall b/src/Interpreters/Query.dhall index cce3fcd..b3e86cd 100644 --- a/src/Interpreters/Query.dhall +++ b/src/Interpreters/Query.dhall @@ -2,9 +2,9 @@ let Prelude = ../Deps/Prelude.dhall let Lude = ../Deps/Lude.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Sdk = ../Deps/Sdk.dhall -let CustomKind = ../Structures/CustomKind.dhall +let ImportSet = ../Structures/ImportSet.dhall let PyIdent = ../Structures/PyIdent.dhall @@ -137,7 +137,6 @@ let render = let run = \(config : Config) -> - \(lookup : CustomKind.Lookup) -> \(input : Input) -> let rowClassName = input.name.inPascalCase ++ "Row" @@ -153,7 +152,7 @@ let run = ( Compiled.nest ResultModule.Output "result" - (ResultModule.run config lookup rowClassName input.result) + (ResultModule.run (config /\ { rowClassName }) input.result) ) ( Compiled.nest QueryFragmentsModule.Output @@ -170,11 +169,11 @@ let run = Compiled.nest ParamsMember.Output member.pgName - (ParamsMember.run config lookup member) + (ParamsMember.run config member) ) input.params ) ) ) -in { Input, Output, run } +in Sdk.Sigs.interpreter Config Input Output run diff --git a/src/Interpreters/Result.dhall b/src/Interpreters/Result.dhall index 5d652a0..b20190d 100644 --- a/src/Interpreters/Result.dhall +++ b/src/Interpreters/Result.dhall @@ -4,9 +4,9 @@ let Lude = ../Deps/Lude.dhall let Model = ../Deps/Contract.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Sdk = ../Deps/Sdk.dhall -let CustomKind = ../Structures/CustomKind.dhall +let ImportSet = ../Structures/ImportSet.dhall let OnUnsupported = ../Structures/OnUnsupported.dhall @@ -14,11 +14,17 @@ let ResultColumns = ./ResultColumns.dhall let Compiled = Lude.Compiled +-- rowClassName is supplied by the caller (Query.dhall derives it from the +-- query's own name) rather than living on Model.Result, so it rides on this +-- interpreter's own local Config instead of widening Input away from +-- Model.Result. ResultColumns below does not need it, so it is projected +-- back down to the narrower shared shape at that call site. let Config = { packageName : Text , importName : Text , emitSync : Bool , onUnsupported : OnUnsupported.Mode + , rowClassName : Text } let Input = Model.Result @@ -63,10 +69,8 @@ let cardinalityShape let rowsOutput = \(config : Config) -> - \(lookup : CustomKind.Lookup) -> - \(rowClassName : Text) -> \(rows : Model.ResultRows) -> - let shape = cardinalityShape rows.cardinality rowClassName + let shape = cardinalityShape rows.cardinality config.rowClassName let columns = Prelude.NonEmpty.toList Model.Member rows.columns @@ -78,7 +82,7 @@ let rowsOutput = { returnType = shape.returnType , helperName = shape.helperName , rowClass = Some - { name = rowClassName + { name = config.rowClassName , fieldsBlock = cols.fieldsBlock , decodeBlock = cols.decodeBlock } @@ -86,19 +90,20 @@ let rowsOutput = , callsDecode = True } ) - (ResultColumns.run config lookup rowClassName columns) + ( ResultColumns.run + config.{ packageName, importName, emitSync, onUnsupported } + columns + ) let run = \(config : Config) -> - \(lookup : CustomKind.Lookup) -> - \(rowClassName : Text) -> \(input : Input) -> merge { Void = Compiled.ok Output (noResult "None" "execute_void") , RowsAffected = Compiled.ok Output (noResult "int" "execute_rows_affected") - , Rows = rowsOutput config lookup rowClassName + , Rows = rowsOutput config } input -in { Input, Output, RowClass, run } +in Sdk.Sigs.interpreter Config Input Output run /\ { RowClass } diff --git a/src/Interpreters/ResultColumns.dhall b/src/Interpreters/ResultColumns.dhall index e0bbfaa..6a589a2 100644 --- a/src/Interpreters/ResultColumns.dhall +++ b/src/Interpreters/ResultColumns.dhall @@ -4,9 +4,9 @@ let Lude = ../Deps/Lude.dhall let Model = ../Deps/Contract.dhall -let ImportSet = ../Structures/ImportSet.dhall +let Sdk = ../Deps/Sdk.dhall -let CustomKind = ../Structures/CustomKind.dhall +let ImportSet = ../Structures/ImportSet.dhall let OnUnsupported = ../Structures/OnUnsupported.dhall @@ -59,8 +59,6 @@ let assemble let run = \(config : Config) -> - \(lookup : CustomKind.Lookup) -> - \(rowClassName : Text) -> \(input : Input) -> Compiled.map (List Member.Output) @@ -73,9 +71,9 @@ let run = Compiled.nest Member.Output member.pgName - (Member.run config lookup member) + (Member.run config member) ) input ) -in { Input, Output, run } +in Sdk.Sigs.interpreter Config Input Output run diff --git a/src/Structures/CustomKind.dhall b/src/Structures/CustomKind.dhall deleted file mode 100644 index f2c4a1b..0000000 --- a/src/Structures/CustomKind.dhall +++ /dev/null @@ -1,28 +0,0 @@ -let Model = ../Deps/Contract.dhall - --- A composite field as the decode/encode sites need it: the Python attribute --- name and the rendered Python type (already nullability-applied). Threaded so --- a column decode can build `Point2D(*cast(tuple[x, y], src))` and a param --- encode can build `(value.x, value.y)` without re-reading the customType. -let CompositeField = { fieldName : Text, pyType : Text } - --- Classification of a custom type referenced by a Scalar.Custom name. Composite --- carries its field list so callers render per-field decode/encode. Absent means --- the name did not resolve against project.customTypes, which is a model --- inconsistency the caller turns into a Compiled error. --- Enum and Composite both carry `order`, the type's alphabetical position in --- project.customTypes; the import renderer sorts on it so the per-module --- `from ..types.X import Y` block stays alphabetical regardless of column order. --- NOTE: union is named TypeKind because `Kind` is a reserved Dhall keyword. -let TypeKind = - < Enum : Natural - | Composite : { fields : List CompositeField, order : Natural } - | Absent - > - --- Resolves a custom-type Name to its TypeKind. Project.run builds this from --- project.customTypes and threads it into Member/ParamsMember (and onward to --- Result/ResultColumns) so those interpreters can pick the right decode/encode. -let Lookup = Model.Name -> TypeKind - -in { TypeKind, Lookup, CompositeField } diff --git a/src/Structures/ImportSet.dhall b/src/Structures/ImportSet.dhall index 04fc239..b5c7b81 100644 --- a/src/Structures/ImportSet.dhall +++ b/src/Structures/ImportSet.dhall @@ -1,11 +1,16 @@ let Prelude = ../Deps/Prelude.dhall -- A custom-type import line: "from ..types. import ". --- dedupKey identifies the type for deduplication and orders the rendered import --- block. Dhall (upstream) has no Text comparison, so the caller assigns each --- distinct custom type its alphabetical position in project.customTypes as the --- key; dedup keys on it and the render step sorts ascending to stay alphabetical. -let CustomImport = { className : Text, moduleName : Text, dedupKey : Natural } +-- Emitted in encounter order (the order the referencing columns/params were +-- declared), not sorted. Dhall (upstream) has no Text comparison, so there is +-- no way to alphabetize or dedupe by moduleName/className without either the +-- pgn fork's Text/equal or a project-wide Natural id (previously `order`, +-- sourced from Project.dhall's buildLookup — see +-- docs/plans/2026-07-11-encounter-order-custom-imports.md for why that's +-- gone and why dedup isn't reintroduced some other way). Two references to +-- the same type currently produce two identical lines; harmless to Python +-- and to basedpyright, just not deduped. +let CustomImport = { className : Text, moduleName : Text } let Self = { uuid : Bool @@ -83,97 +88,6 @@ let custom : CustomImport -> Self = \(c : CustomImport) -> base // { customTypes = [ c ] } --- Named-type imports dedup and sort on `order`, the type's alphabetical index in --- project.customTypes. Two references to the same type carry the same order and --- collapse; distinct types keep distinct keys so the render step can sort them. -let customEnum - : { className : Text, moduleName : Text, order : Natural } -> Self - = \(c : { className : Text, moduleName : Text, order : Natural }) -> - custom - { className = c.className, moduleName = c.moduleName, dedupKey = c.order } - -let customComposite - : { className : Text, moduleName : Text, order : Natural } -> Self - = \(c : { className : Text, moduleName : Text, order : Natural }) -> - custom - { className = c.className, moduleName = c.moduleName, dedupKey = c.order } - -let eqNat = - \(a : Natural) -> - \(b : Natural) -> - Natural/isZero (Natural/subtract a b) - && Natural/isZero (Natural/subtract b a) - -let dedupCustoms - : List CustomImport -> List CustomImport - = \(items : List CustomImport) -> - let State = { seen : List Natural, acc : List CustomImport } - - let step = - \(item : CustomImport) -> - \(state : State) -> - let known = - Prelude.List.any - Natural - (\(k : Natural) -> eqNat k item.dedupKey) - state.seen - - in if known - then state - else { seen = state.seen # [ item.dedupKey ] - , acc = state.acc # [ item ] - } - - in ( List/fold - CustomImport - items - State - step - { seen = [] : List Natural, acc = [] : List CustomImport } - ).acc - -let leNat = \(a : Natural) -> \(b : Natural) -> Natural/isZero (Natural/subtract b a) - --- Insertion sort over custom imports by ascending dedupKey (the alphabetical --- order). The list is tiny (one or two types per module), so the cost is moot --- and the result is a stable alphabetical import block. -let sortCustoms - : List CustomImport -> List CustomImport - = \(items : List CustomImport) -> - let insert = - \(item : CustomImport) -> - \(acc : List CustomImport) -> - let State = { placed : Bool, out : List CustomImport } - - -- foldLeft so the scan runs left-to-right and `item` lands before - -- the FIRST element that is >= it (a right List/fold visits the - -- list back-to-front and would insert before the last such - -- element, mis-ordering blocks of 3+ types). - let step = - \(state : State) -> - \(cur : CustomImport) -> - if state.placed - then { placed = True, out = state.out # [ cur ] } - else if leNat item.dedupKey cur.dedupKey - then { placed = True - , out = state.out # [ item, cur ] - } - else { placed = False, out = state.out # [ cur ] } - - let folded = - Prelude.List.foldLeft - CustomImport - acc - State - step - { placed = False, out = [] : List CustomImport } - - in if folded.placed - then folded.out - else folded.out # [ item ] - - in List/fold CustomImport items (List CustomImport) insert ([] : List CustomImport) - let combine = \(left : Self) -> \(right : Self) -> @@ -187,17 +101,13 @@ let combine = , json = left.json || right.json , jsonValue = left.jsonValue || right.jsonValue , enumArray = left.enumArray || right.enumArray - , customTypes = dedupCustoms (left.customTypes # right.customTypes) + , customTypes = left.customTypes # right.customTypes } let combineAll : List Self -> Self = \(sets : List Self) -> List/fold Self sets Self combine empty -let sortedCustoms - : Self -> List CustomImport - = \(self : Self) -> sortCustoms self.customTypes - in { Type = Self , CustomImport , empty @@ -212,9 +122,6 @@ in { Type = Self , jsonValue , enumArray , custom - , customEnum - , customComposite , combine , combineAll - , sortedCustoms } diff --git a/src/Templates/CompositeModule.dhall b/src/Templates/CompositeModule.dhall index b1ab3e0..32c4ee5 100644 --- a/src/Templates/CompositeModule.dhall +++ b/src/Templates/CompositeModule.dhall @@ -20,11 +20,50 @@ let run = ) params.fields + let fieldCount = Prelude.List.length Field params.fields + + let fieldTypesJoined = + Prelude.Text.concatMapSep + ", " + Field + (\(field : Field) -> field.fieldType) + params.fields + + let selfFieldsJoined = + Prelude.Text.concatMapSep + ", " + Field + (\(field : Field) -> "self.${field.fieldName}") + params.fields + + -- Python only treats trailing-comma parens as a 1-tuple; concatMapSep + -- never emits an internal comma for a single-element list, so force one + -- here. Mirrors ParamsMember.dhall's existing compositeBind trick. + let encodeTupleExpr = + if Prelude.Natural.equal fieldCount 1 + then "(${selfFieldsJoined},)" + else "(${selfFieldsJoined})" + + -- _decode/_encode are emitted as literal lines (not a nested multi-line + -- ''...'' block) because Dhall dedents a multi-line literal against its + -- OWN source indentation before splicing it into the outer literal; a + -- nested block loses its intended 4/8-space class-body indentation. + -- Verified against `dhall text` during design. + let codecMethods = + "\n" + ++ " @staticmethod\n" + ++ " def _decode(src: object) -> \"${params.typeName}\":\n" + ++ " return ${params.typeName}(*cast(tuple[${fieldTypesJoined}], src))\n" + ++ "\n" + ++ " def _encode(self) -> tuple[${fieldTypesJoined}]:\n" + ++ " return ${encodeTupleExpr}" + let imports = if Prelude.List.null Text params.extraImports - then "from dataclasses import dataclass" + then "from dataclasses import dataclass\nfrom typing import cast" else '' from dataclasses import dataclass + from typing import cast ${Prelude.Text.concatSep "\n" params.extraImports}'' @@ -41,6 +80,7 @@ let run = """ ${fieldLines} + ${codecMethods} '' in Sdk.Sigs.template Params run /\ { Field } diff --git a/src/Templates/EnumModule.dhall b/src/Templates/EnumModule.dhall index 1ebf486..04c4bfe 100644 --- a/src/Templates/EnumModule.dhall +++ b/src/Templates/EnumModule.dhall @@ -35,12 +35,23 @@ let run = ) params.variants + let codecMethods = + "\n" + ++ " @staticmethod\n" + ++ " def _decode(src: object) -> \"${params.typeName}\":\n" + ++ " return ${params.typeName}(cast(str, src))\n" + ++ "\n" + ++ " def _encode(self) -> \"${params.typeName}\":\n" + ++ " return self" + in '' from enum import StrEnum + from typing import cast class ${params.typeName}(StrEnum): ${memberLines} + ${codecMethods} '' in Sdk.Sigs.template Params run /\ { Variant } diff --git a/src/Templates/RowsModule.dhall b/src/Templates/RowsModule.dhall index 18da31b..b1d5b13 100644 --- a/src/Templates/RowsModule.dhall +++ b/src/Templates/RowsModule.dhall @@ -55,7 +55,7 @@ let customImportLines ( \(c : ImportSet.CustomImport) -> "from .types." ++ c.moduleName ++ " import " ++ c.className ) - (ImportSet.sortedCustoms imports) + imports.customTypes let renderImports : ImportSet.Type -> Text diff --git a/src/Templates/StatementModule.dhall b/src/Templates/StatementModule.dhall index f2a6f61..325e21f 100644 --- a/src/Templates/StatementModule.dhall +++ b/src/Templates/StatementModule.dhall @@ -79,7 +79,7 @@ let customImportLines ( \(c : ImportSet.CustomImport) -> "from ${typesPrefix}.${c.moduleName} import ${c.className}" ) - (ImportSet.sortedCustoms imports) + imports.customTypes let rowsImportLine : Params -> List Text diff --git a/src/package.dhall b/src/package.dhall index 3d07cf7..5161367 100644 --- a/src/package.dhall +++ b/src/package.dhall @@ -2,7 +2,22 @@ let Sdk = ./Deps/Sdk.dhall let OnUnsupported = ./Structures/OnUnsupported.dhall -let Config = ./Config.dhall +let ProjectInterpreter = ./Interpreters/Project.dhall + +-- User-facing config for this generator. `emitSync` adds a parallel sync +-- surface (psycopg.Connection) alongside the default async one, so one +-- project can serve both an async backend and a sync (Dagster) consumer from +-- shared Row types. `onUnsupported` picks Fail (default, abort loudly) or +-- Skip (drop the unsupported statement/type and its dependents, with a +-- warning) when a query or custom type hits a PG shape the generator cannot +-- render; see Structures/OnUnsupported.dhall. All fields are Optional so a +-- project may omit the whole config block or any subset of its keys; +-- Interpreters/Project.dhall's `run` supplies the defaults. +let Config = + { packageName : Optional Text + , emitSync : Optional Bool + , onUnsupported : Optional OnUnsupported.Mode + } : Type let Config/default : Config @@ -11,6 +26,4 @@ let Config/default , onUnsupported = None OnUnsupported.Mode } -let interpret = ./Interpret.dhall - -in Sdk.Sigs.generator Config Config/default interpret +in Sdk.Sigs.generator Config Config/default ProjectInterpreter.run diff --git a/tests/test_config_variants.py b/tests/test_config_variants.py index 9b23d18..e0d3ecf 100644 --- a/tests/test_config_variants.py +++ b/tests/test_config_variants.py @@ -66,7 +66,7 @@ def test_absent_config_key_defaults_both_fields(generated_tree: Path) -> None: def test_unknown_config_key_is_ignored_not_rejected(generated_tree: Path) -> None: - """An extra key not in Config.dhall (bogusField) does not fail generation.""" + """An extra key not in the generator's Config type (bogusField) does not fail generation.""" package = _package_dir(generated_tree, "python-unknown-key") assert package.name == "unknown_key_client" assert not (package / "sync").exists() From 76255508f57f17230e1a8f153810ca234db3582b Mon Sep 17 00:00:00 2001 From: Nikita Volkov Date: Sun, 12 Jul 2026 06:29:44 +0300 Subject: [PATCH 11/11] Drop the plans --- ...26-07-11-encounter-order-custom-imports.md | 327 --------- .../2026-07-11-reusable-custom-type-codecs.md | 633 ------------------ 2 files changed, 960 deletions(-) delete mode 100644 docs/plans/2026-07-11-encounter-order-custom-imports.md delete mode 100644 docs/plans/2026-07-11-reusable-custom-type-codecs.md diff --git a/docs/plans/2026-07-11-encounter-order-custom-imports.md b/docs/plans/2026-07-11-encounter-order-custom-imports.md deleted file mode 100644 index 7d1c663..0000000 --- a/docs/plans/2026-07-11-encounter-order-custom-imports.md +++ /dev/null @@ -1,327 +0,0 @@ -# Encounter-Order Custom-Type Imports Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Stop requiring a project-wide `order : Natural` to sort and dedupe custom-type import lines in `ImportSet.dhall`. Emit them in encounter order (the order the referencing columns/params are declared) instead, and accept that de-duplicating two references to the *same* custom type within one file is not achievable in vanilla Dhall — verify it isn't actually needed by the current corpus, and document the tradeoff rather than reintroduing a lookup to avoid it. - -**Architecture:** `ImportSet.dhall` today dedupes and alphabetizes custom-type imports by carrying a `dedupKey : Natural` — the type's alphabetical index in `project.customTypes` — through every `CustomImport` value, specifically because Dhall has no `Text` comparison to sort or dedupe on `moduleName`/`className` directly (see the file's own header comment). That `order` value's only source was `buildLookup` (`Interpreters/Project.dhall`), which the companion plan (`2026-07-11-reusable-custom-type-codecs.md`) deletes. Once it's gone, `ImportSet.dhall` has nothing to key on. Rather than re-deriving a Natural surrogate some other way, this plan removes the sort/dedup step and lets `ImportSet.combine`'s existing `List/fold` order (already the query's declared column/param order — a real, already-computed, non-Text-comparison ordering) stand as the output order. - -**Tech Stack:** Dhall (dhall-lang 1.42), Python 3.12 generated output, pytest golden-file harness (`mise run test`, `mise run golden`). - -## Global Constraints - -- **Depends on** `docs/plans/2026-07-11-reusable-custom-type-codecs.md` — that plan's Task 3/4 call `ImportSet.custom customImport` (no `order` argument). Land this plan's Task 1 first, or in the same PR; `ImportSet.customEnum`/`customComposite` (which this plan deletes) are exactly what those tasks stop calling. -- No behavior change to the four already-Natural-keyed stdlib import flags (`uuid`, `datetime`, `date`, `time`, `timedelta`, `decimal`, `jsonb`, `json`, `jsonValue`, `enumArray`) — those are plain `Bool` OR's today and are untouched by this plan. -- Golden fixture output must be regenerated and diffed (`tests/golden/`), not hand-edited. -- Every Dhall file touched must independently type-check: `dhall type --file=`. - ---- - -## Why dedup can't be preserved without reintroducing a lookup - -Worth writing down since it's not obvious and the temptation to "just find a clever `Text/replace` trick" is real — this was checked directly against `PyIdent.dhall`'s working pattern (DESIGN.md section 13) before concluding it doesn't generalize: - -`PyIdent.dhall`'s `sanitizeAgainst` tests a runtime `Text` against a small, **fixed, compile-time-literal** candidate list (the 35 Python keywords) — one known literal at a time, folded. That's why the two-`Text/replace` trick works: one side of every comparison is always a literal. - -Import dedup needs the opposite shape: is this **runtime** `moduleName` (derived from a `Name` that varies per project) equal to any of the **other runtime** `moduleName`s already collected? Both sides are dynamic. No sequence of `Text/replace` calls can decide that, because deciding it requires producing a `Bool` from two non-literal `Text` values, which is exactly the operation Dhall doesn't have (and which pgn's forked `Text/equal` exists to provide). This isn't a missing trick — dedup of dynamically-computed `Text` is unconditionally impossible in vanilla Dhall. The only ways to get it back are: (a) a fork builtin (what we're removing), (b) a pre-assigned `Natural` id per distinct value (what `order` was — sourced from a project-wide search, i.e. `buildLookup`, also being removed), or (c) don't need it. - -This plan takes (c), having checked how much it costs to: - -```bash -grep -rl "^from \.\.types\." tests/golden/src 2>/dev/null | while read f; do - n=$(grep -c "^from \.\.types\." "$f") - [ "$n" -gt 1 ] && echo "$f: $n" -done -``` - -No output — verified during design (see conversation record / re-run before Task 2 to confirm it's still true after Doc 1's regeneration). No file in the current fixture corpus imports the same custom type twice. The risk is real but currently unexercised: if a future query selects the same composite/enum type via two different columns, the generated file will contain two identical `from ..types.X import Y` lines — syntactically valid, harmless to `basedpyright strict` and to Python's import system, just visually redundant. Task 2 adds a corpus case that exercises this on purpose so the tradeoff is documented against real output, not just asserted. - ---- - -## File Structure - -| File | Change | -|---|---| -| `src/Structures/ImportSet.dhall` | Drop `dedupKey` from `CustomImport`; delete `dedupCustoms`, `sortCustoms`, `eqNat`, `leNat`, `sortedCustoms`; collapse `customEnum`/`customComposite` into the existing `custom`; `combine` plain-concatenates `customTypes` instead of deduping. | -| `src/Templates/RowsModule.dhall`, `src/Templates/StatementModule.dhall` | Read `imports.customTypes` directly instead of `ImportSet.sortedCustoms imports`. | -| `tests/golden/` | Regenerate via `mise run golden`; review the (likely negligible) reordering of custom-type import lines from alphabetical to declaration order. | -| `python.gen/DESIGN.md` | Note the ordering change where section 12/13 currently describe the alphabetical-by-`order` scheme. | - ---- - -### Task 1: Simplify `ImportSet.dhall` - -**Files:** -- Modify: `src/Structures/ImportSet.dhall` - -**Interfaces:** -- `CustomImport` loses `dedupKey : Natural` → becomes `{ className : Text, moduleName : Text }`. -- `custom : CustomImport -> Self` — unchanged signature, now the only constructor (no more `customEnum`/`customComposite`). -- `combine : Self -> Self -> Self` — `customTypes` field is now a plain list concatenation. -- `sortedCustoms` is deleted. Callers read `.customTypes` directly. - -- [ ] **Step 1: Replace the file** - -```dhall -let Prelude = ../Deps/Prelude.dhall - --- A custom-type import line: "from ..types. import ". --- Emitted in encounter order (the order the referencing columns/params were --- declared), not sorted. Dhall (upstream) has no Text comparison, so there is --- no way to alphabetize or dedupe by moduleName/className without either the --- pgn fork's Text/equal or a project-wide Natural id (previously `order`, --- sourced from Project.dhall's buildLookup — see --- docs/plans/2026-07-11-encounter-order-custom-imports.md for why that's --- gone and why dedup isn't reintroduced some other way). Two references to --- the same type currently produce two identical lines; harmless to Python --- and to basedpyright, just not deduped. -let CustomImport = { className : Text, moduleName : Text } - -let Self = - { uuid : Bool - , datetime : Bool - , date : Bool - , time : Bool - , timedelta : Bool - , decimal : Bool - , jsonb : Bool - , json : Bool - , jsonValue : Bool - , enumArray : Bool - , customTypes : List CustomImport - } - -let base = - { uuid = False - , datetime = False - , date = False - , time = False - , timedelta = False - , decimal = False - , jsonb = False - , json = False - , jsonValue = False - , enumArray = False - , customTypes = [] : List CustomImport - } - -let empty - : Self - = base - -let uuid - : Self - = base // { uuid = True } - -let datetime - : Self - = base // { datetime = True } - -let date - : Self - = base // { date = True } - -let time - : Self - = base // { time = True } - -let timedelta - : Self - = base // { timedelta = True } - -let decimal - : Self - = base // { decimal = True } - -let jsonb - : Self - = base // { jsonb = True } - -let json - : Self - = base // { json = True } - -let jsonValue - : Self - = base // { jsonValue = True } - -let enumArray - : Self - = base // { enumArray = True } - -let custom - : CustomImport -> Self - = \(c : CustomImport) -> base // { customTypes = [ c ] } - -let combine = - \(left : Self) -> - \(right : Self) -> - { uuid = left.uuid || right.uuid - , datetime = left.datetime || right.datetime - , date = left.date || right.date - , time = left.time || right.time - , timedelta = left.timedelta || right.timedelta - , decimal = left.decimal || right.decimal - , jsonb = left.jsonb || right.jsonb - , json = left.json || right.json - , jsonValue = left.jsonValue || right.jsonValue - , enumArray = left.enumArray || right.enumArray - , customTypes = left.customTypes # right.customTypes - } - -let combineAll - : List Self -> Self - = \(sets : List Self) -> List/fold Self sets Self combine empty - -in { Type = Self - , CustomImport - , empty - , uuid - , datetime - , date - , time - , timedelta - , decimal - , jsonb - , json - , jsonValue - , enumArray - , custom - , combine - , combineAll - } -``` - -- [ ] **Step 2: Type-check** - -Run: `dhall type --file=src/Structures/ImportSet.dhall` -Expected: prints the record-of-functions signature, no error. - -- [ ] **Step 3: Commit** - -```bash -git add src/Structures/ImportSet.dhall -git commit -m "python.gen: drop order-based sort/dedup from ImportSet, use encounter order" -``` - ---- - -### Task 2: Update the two render call sites - -**Files:** -- Modify: `src/Templates/RowsModule.dhall`, `src/Templates/StatementModule.dhall` - -- [ ] **Step 1: `RowsModule.dhall:58`** - -Change: -```dhall - (ImportSet.sortedCustoms imports) -``` -to: -```dhall - imports.customTypes -``` - -- [ ] **Step 2: `StatementModule.dhall:82`** - -Same change: -```dhall - imports.customTypes -``` - -- [ ] **Step 3: Type-check both** - -Run: `dhall type --file=src/Templates/RowsModule.dhall && dhall type --file=src/Templates/StatementModule.dhall` -Expected: both print their signatures, no error. - -- [ ] **Step 4: Commit** - -```bash -git add src/Templates/RowsModule.dhall src/Templates/StatementModule.dhall -git commit -m "python.gen: render custom-type imports in encounter order" -``` - ---- - -### Task 3: Regenerate golden fixtures and verify the dedup gap directly - -**Files:** -- Modify: `tests/fixture-project/` (temporary, to exercise the dedup gap — see Step 1) -- Regenerate: `tests/golden/` - -- [ ] **Step 1: Confirm today's corpus has no same-type-twice case** - -Run: -```bash -grep -rl "^from \.\.types\." tests/golden/src 2>/dev/null | while read f; do - n=$(grep -c "^from \.\.types\." "$f") - [ "$n" -gt 1 ] && echo "$f: $n" -done -``` -Expected: no output (re-confirms the design-time check above, against the *current* golden tree before this plan's regeneration). - -- [ ] **Step 2: Regenerate golden** - -Run: `mise run golden` -Expected: succeeds. `git diff tests/golden` shows custom-type import lines reordered from alphabetical to declaration order in files with 2+ distinct custom-type imports (e.g. wherever `Mood` and `Point2D` are both imported today — check whether the query's own column order already happens to be alphabetical for that file; if so the diff is empty there and this is confirmed low-risk for the current corpus). - -- [ ] **Step 3: Deliberately add a same-type-twice column to the fixture project** - -Add a query (or extend an existing one) whose result row or param list references the *same* composite or enum type through two different columns/params — e.g. two `mood`-typed columns in one query. This is new fixture surface, not present today; add it under `tests/fixture-project/queries/`. - -Run: `mise run golden` -Expected: succeeds; the regenerated file for that query contains **two** identical `from ..types.mood import Mood` lines (confirm with `grep -c` on the specific file). This is the one visible, accepted consequence of this plan — capture it in the diff review, don't silently let it slip into `tests/golden` unremarked. - -- [ ] **Step 4: Decide whether to keep the same-type-twice fixture case** - -Keeping it in the committed corpus makes the tradeoff a permanent, visible regression test (future readers see the duplicate import and the comment in `ImportSet.dhall` explaining it, instead of being surprised by it later). Removing it keeps the golden diff minimal for this change. Either is fine — this plan recommends **keeping it**, since an accepted-but-invisible tradeoff tends to resurface as a bug report; make the call and note it in the commit message either way. - -- [ ] **Step 5: Run the full test suite** - -Run: `mise run test` -Expected: all pass. Duplicate import lines don't fail `basedpyright strict` (Python tolerates redundant imports) or the golden byte-comparison (it compares against the freshly-committed golden, not some independent expectation). - -- [ ] **Step 6: Commit** - -```bash -git add tests/golden tests/fixture-project -git commit -m "python.gen: regenerate golden fixtures for encounter-order imports" -``` - ---- - -### Task 4: Update `DESIGN.md` - -**Files:** -- Modify: `python.gen/DESIGN.md` - -- [ ] **Step 1: Update section 12/13 cross-references** - -Wherever DESIGN.md currently describes `order`/alphabetical import sorting (it's referenced in passing around sections 12-13 and in code comments already updated by `docs/plans/2026-07-11-reusable-custom-type-codecs.md`'s Task 8), add a short note: custom-type imports are emitted in encounter (declaration) order, not sorted, since the `order` Natural no longer exists once `buildLookup` is gone; same-type-twice references are not deduped (a Dhall limitation, not an oversight — see `src/Structures/ImportSet.dhall`'s header comment for the full reasoning). Point at the fixture case from Task 3 if kept. - -- [ ] **Step 2: Commit** - -```bash -git add python.gen/DESIGN.md -git commit -m "python.gen: document encounter-order custom-type imports in DESIGN.md" -``` - ---- - -## Self-Review - -**Spec coverage:** Task 1 is the actual mechanism change (drop the Natural key, drop sort/dedup). Task 2 fixes the two render call sites that would otherwise reference a deleted `sortedCustoms`. Task 3 regenerates and — importantly — deliberately exercises the one behavior change (duplicate imports for a same-type-twice reference) instead of letting it go unverified. Task 4 keeps DESIGN.md truthful. - -**Placeholder scan:** no TBDs; every step names an exact file, an exact diff, or an exact command with expected output. - -**Dependency on the companion plan:** called out at the top (Global Constraints) and repeated in the companion plan's own self-review — `docs/plans/2026-07-11-reusable-custom-type-codecs.md` Task 3/4 call `ImportSet.custom` with no `order` argument, which only type-checks after this plan's Task 1. Sequence: this plan's Task 1 → companion plan's Tasks 3-6 → this plan's Tasks 2-4 (Task 2 touches templates the companion plan doesn't touch, so it can land anytime after Task 1, but golden regeneration in either plan's Task 8/3 should happen once, after both are code-complete, not twice). - -## Execution Handoff - -Plan complete and saved to `python.gen/docs/plans/2026-07-11-encounter-order-custom-imports.md`. Two execution options: - -**1. Subagent-Driven (recommended)** - dispatch a fresh subagent per task, review between tasks, fast iteration - -**2. Inline Execution** - execute tasks in this session using executing-plans, batch execution with checkpoints - -Which approach? diff --git a/docs/plans/2026-07-11-reusable-custom-type-codecs.md b/docs/plans/2026-07-11-reusable-custom-type-codecs.md deleted file mode 100644 index b2f880c..0000000 --- a/docs/plans/2026-07-11-reusable-custom-type-codecs.md +++ /dev/null @@ -1,633 +0,0 @@ -# Reusable Custom-Type Codecs Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Delete `buildLookup`/`CustomKind.Lookup` — and with it python.gen's last dependency on pgn's forked `Text/equal` builtin (DESIGN.md section 12) — by generating one `_decode`/`_encode` codec per custom type in `types/.py`, called by name from every reference site, instead of re-deriving decode/encode logic (field types, Composite-vs-Enum classification) at each call site. - -**Architecture:** `CustomType.dhall` already visits every custom type once with its full definition in hand (composite fields or enum variants) and emits `types/.py`. Today it stops at the dataclass/enum class. This plan makes it also emit a `_decode` staticmethod (and, for composites, an `_encode` instance method) on that same class. Every column/param reference site (`Member.dhall`, `ParamsMember.dhall`) currently has to search `project.customTypes` by name to find out whether it's looking at a composite or an enum, because it re-derives the decode/encode expression inline. Once decode/encode is a named method reachable from the `Name` the reference site already carries (`name.inPascalCase`), the reference site just calls it — no search, no classification, no `Text/equal`. - -**Tech Stack:** Dhall (dhall-lang 1.42 vendored via `dhall` CLI; pgn's forked interpreter for the parts still using it), Python 3.12 generated output (psycopg3), basedpyright strict, pytest golden-file harness (`mise run test`, `mise run golden`). - -## Global Constraints - -- No behavior change to non-custom-type (primitive) decode/encode paths. -- Golden fixture output (`tests/golden/`) must be regenerated and diffed, not hand-edited (per `tests/golden/README.md`). -- `mise run test` (pytest, includes `test_generated_passes_basedpyright_strict`) must pass after regeneration. -- Every Dhall file touched must independently type-check: `dhall type --file=`. -- Don't touch `Primitive.dhall`, `Scalar.dhall`, `Value.dhall`, or the `Model`/`Contract` dependency — this plan is scoped to what python.gen can do unilaterally, with the current, unmodified `Scalar = < Primitive : Primitive | Custom : Name >` contract. - ---- - -## Design decision needed before Task 3: array-of-custom-type behavior - -Today, `Member.dhall`'s Custom branch treats arrays differently by kind, because it already knows the kind from `lookup name`: -- **Enum arrays (1-D):** supported — `enumArrayDecode` emits an element-wise list comprehension. -- **Composite arrays:** rejected at generation time — `"Array of a composite type is not supported (element-wise decode is unimplemented)"` (Member.dhall:219) and, for params, `"Array of a composite type as a parameter is not supported"` (ParamsMember.dhall:359). - -Once decode/encode become named methods called unconditionally (no classification at the call site), there's no Dhall-level way to keep rejecting *only* composite arrays without reintroducing some form of lookup. Two ways forward: - -- **A — Recommended: uniform array codec, kind-specific availability.** Every custom type's template *may* define `_decode_array` (a staticmethod that turns the raw array value into a `list[...]`); `EnumModule.dhall` defines it, `CompositeModule.dhall` does not. `Member.dhall` always emits `f"{typeName}._decode_array({src})"` for a 1-D custom-type array, regardless of kind. If that's ever generated against a composite, `basedpyright strict` (already gating `mise run test`, see `tests/golden/README.md`) fails on the missing attribute — a real check, just moved from `pgn generate` time to CI time. Composite-array *params* work the same way: encode always calls `f"{fieldName}._encode()"`; if `fieldName`'s inferred type is `list[Point2D]`, basedpyright flags the missing `._encode` on `list`. This is a genuine, if later, safety net — not a silent hole. The `"Absent"` case (a `customRef` name matching no generated module) degrades the same way: the emitted `from ..types. import ` fails as an unresolved import, also caught by basedpyright strict. -- **B — Preserve today's exact rejection.** Keep a minimal kind signal alive somewhere reachable without a name search — no such source was found during design (see the exploration notes below); the only ones available (project-wide `customTypes` list, or a `Scalar.Custom` contract change) reintroduce either the search or the upstream ask this plan is explicitly trying to avoid. Pick this only if lifting the composite-array restriction is unacceptable without a live Postgres verification first. - -This plan is written for **Option A**. Composite-array support was never tested (DESIGN.md and the fixture corpus have no composite-array column — verified via `grep -rn "^from \.\.types\." tests/golden/`, no file references the same custom type twice, and none of the fixture's composite columns are arrays), so Task 8 includes adding one to the fixture project specifically to exercise this path before it ships. If that Postgres test reveals composite-array decode genuinely doesn't work end-to-end (not just an assumption), fall back to Option B and keep the `"not supported"` report, sourced from a small dedicated check rather than full `buildLookup` (open a follow-up plan; don't block this one on it). - ---- - -## File Structure - -| File | Change | -|---|---| -| `src/Templates/CompositeModule.dhall` | Add `_decode` (staticmethod) and `_encode` (instance method) to the generated dataclass. | -| `src/Templates/EnumModule.dhall` | Add `_decode` and `_decode_array` (staticmethods) and `_encode` (instance method, identity) to the generated `StrEnum`. | -| `src/Interpreters/Member.dhall` | Custom branch calls `{typeName}._decode(...)` / `{typeName}._decode_array(...)` unconditionally; drop the `lookup` parameter and the `Enum`/`Composite`/`Absent` merge. | -| `src/Interpreters/ParamsMember.dhall` | Custom branch calls `{fieldName}._encode()` unconditionally; drop `lookup` and the merge. | -| `src/Interpreters/ResultColumns.dhall`, `src/Interpreters/Result.dhall`, `src/Interpreters/Query.dhall` | Drop the `lookup : CustomKind.Lookup` parameter they only thread through. | -| `src/Interpreters/CustomType.dhall` | Drop `nestedLookup` (nothing left to pass it to). | -| `src/Interpreters/Project.dhall` | Delete `buildLookup`, `IndexedCustomType`, `compositeFields`, `memberPyType`, `lookupConfig` — all exist only to feed `buildLookup`. | -| `src/Structures/CustomKind.dhall` | Delete the file; nothing imports it after the above. | -| `python.gen/DESIGN.md` | Rewrite section 12 (no longer an accepted risk — resolved) and section 13 (unchanged content, but cross-reference updates). | -| `python.gen/docs/upstream-asks.md` | Remove ask 3 (resolved without the upstream change) or mark it withdrawn. | -| `tests/golden/` | Regenerate via `mise run golden`. | - ---- - -### Task 1: Add `_decode`/`_encode` to `CompositeModule.dhall` - -**Files:** -- Modify: `src/Templates/CompositeModule.dhall` - -**Interfaces:** -- Consumes: same `Params = { typeName : Text, extraImports : List Text, fields : List Field }` as today, `Field = { fieldName : Text, fieldType : Text }` — no signature change. -- Produces: the rendered module text now defines `_decode(src: object) -> ""` (staticmethod) and `_encode(self) -> tuple[...]` (instance method) on the dataclass, callable by any generator downstream as `TypeName._decode(x)` / `value._encode()`. - -This was prototyped and verified against `dhall type` and `dhall text` (rendered output checked with `python3 -c "compile(...)"` for both a two-field and a one-field composite — the one-field case needs the trailing-comma tuple, same edge case `compositeBind` already handles today). - -- [ ] **Step 1: Replace the template body** - -```dhall -let Prelude = ../Deps/Prelude.dhall - -let Sdk = ../Deps/Sdk.dhall - -let Field = { fieldName : Text, fieldType : Text } - --- extraImports are the extra import lines a field type needs (e.g. --- "from uuid import UUID"). They sit between the dataclass import and the class, --- separated by one blank line; when empty only the dataclass import is emitted. -let Params = { typeName : Text, extraImports : List Text, fields : List Field } - -let run = - \(params : Params) -> - let fieldLines = - Prelude.Text.concatMapSep - "\n" - Field - ( \(field : Field) -> - " ${field.fieldName}: ${field.fieldType}" - ) - params.fields - - let fieldCount = Prelude.List.length Field params.fields - - let fieldTypesJoined = - Prelude.Text.concatMapSep - ", " - Field - (\(field : Field) -> field.fieldType) - params.fields - - let selfFieldsJoined = - Prelude.Text.concatMapSep - ", " - Field - (\(field : Field) -> "self.${field.fieldName}") - params.fields - - -- Python only treats trailing-comma parens as a 1-tuple; concatMapSep - -- never emits an internal comma for a single-element list, so force one - -- here. Mirrors ParamsMember.dhall's existing compositeBind trick. - let encodeTupleExpr = - if Prelude.Natural.equal fieldCount 1 - then "(${selfFieldsJoined},)" - else "(${selfFieldsJoined})" - - -- _decode/_encode are emitted as literal lines (not a nested multi-line - -- ''...'' block) because Dhall dedents a multi-line literal against its - -- OWN source indentation before splicing it into the outer literal; a - -- nested block loses its intended 4/8-space class-body indentation. - -- Verified against `dhall text` during design. - let codecMethods = - "\n" - ++ " @staticmethod\n" - ++ " def _decode(src: object) -> \"${params.typeName}\":\n" - ++ " return ${params.typeName}(*cast(tuple[${fieldTypesJoined}], src))\n" - ++ "\n" - ++ " def _encode(self) -> tuple[${fieldTypesJoined}]:\n" - ++ " return ${encodeTupleExpr}" - - let imports = - if Prelude.List.null Text params.extraImports - then "from dataclasses import dataclass\nfrom typing import cast" - else '' - from dataclasses import dataclass - from typing import cast - - ${Prelude.Text.concatSep "\n" params.extraImports}'' - - in '' - ${imports} - - - @dataclass(frozen=True, slots=True) - class ${params.typeName}: - """Decoding/encoding this composite requires register_types(conn) first. - - Without per-connection registration psycopg returns the value as a - raw string, which the generated _decode cannot splat into the dataclass. - """ - - ${fieldLines} - ${codecMethods} - '' - -in Sdk.Sigs.template Params run /\ { Field } -``` - -- [ ] **Step 2: Type-check** - -Run: `dhall type --file=src/Templates/CompositeModule.dhall` -Expected: prints the `{ Field : Type, Params : Type, Run : Type, run : ... }` signature, no error. - -- [ ] **Step 3: Spot-render and validate as Python** - -```bash -cat > /tmp/render_composite.dhall <<'EOF' -let CompositeModule = ./src/Templates/CompositeModule.dhall -in CompositeModule.run - { typeName = "Point2D" - , extraImports = [] : List Text - , fields = - [ { fieldName = "x", fieldType = "int" } - , { fieldName = "y", fieldType = "int" } - ] - } -EOF -dhall text --file=/tmp/render_composite.dhall | python3 -c "import sys; compile(sys.stdin.read(), 'point2d.py', 'exec')" && echo OK -``` -Expected: `OK`, and eyeballing the output shows `_decode`/`_encode` indented as class members (4 spaces), not module-level. - -- [ ] **Step 4: Commit** - -```bash -git add src/Templates/CompositeModule.dhall -git commit -m "python.gen: emit _decode/_encode on generated composite dataclasses" -``` - ---- - -### Task 2: Add `_decode`/`_decode_array`/`_encode` to `EnumModule.dhall` - -**Files:** -- Modify: `src/Templates/EnumModule.dhall` - -**Interfaces:** -- Consumes: same `Params = { typeName : Text, variants : List Variant }` — no signature change. -- Produces: `_decode(src: object) -> ""`, `_decode_array(src: object) -> list[""]`, `_encode(self) -> ""` (identity — psycopg binds the `StrEnum` instance directly, matching today's `defaultBind`). - -The scalar/array decode bodies are copied verbatim from `Member.dhall`'s current `enumDecode`/`enumArrayDecode` (Member.dhall:61-82), just moved from "Dhall builds inline Python text at every call site" to "Dhall builds it once, into the class." - -- [ ] **Step 1: Replace the template body** - -```dhall -let Prelude = ../Deps/Prelude.dhall - -let Sdk = ../Deps/Sdk.dhall - -let Variant = { memberName : Text, pgValue : Text } - -let Params = { typeName : Text, variants : List Variant } - -let run = - \(params : Params) -> - -- pgValue is interpolated into a single-line double-quoted Python literal; a - -- label may legally contain a backslash, quote, or control character, so - -- escape them to keep the literal valid and value-equal to the DB label. - -- Order is load-bearing: backslash first (so the escapes added below are not - -- re-escaped), then the control chars, then the closing quote. - let escapeLabel - : Text -> Text - = \(raw : Text) -> - Prelude.Function.composeList - Text - [ Prelude.Text.replace "\\" "\\\\" - , Prelude.Text.replace "\r" "\\r" - , Prelude.Text.replace "\n" "\\n" - , Prelude.Text.replace "\t" "\\t" - , Prelude.Text.replace "\"" "\\\"" - ] - raw - - let memberLines = - Prelude.Text.concatMapSep - "\n" - Variant - ( \(variant : Variant) -> - " ${variant.memberName} = \"${escapeLabel variant.pgValue}\"" - ) - params.variants - - let codecMethods = - "\n" - ++ " @staticmethod\n" - ++ " def _decode(src: object) -> \"${params.typeName}\":\n" - ++ " return ${params.typeName}(cast(str, src))\n" - ++ "\n" - ++ " @staticmethod\n" - ++ " def _decode_array(src: object) -> list[\"${params.typeName}\"]:\n" - ++ " return [\n" - ++ " ${params.typeName}(v)\n" - ++ " for v in cast(list[str], require_array(src))\n" - ++ " ]\n" - ++ "\n" - ++ " def _encode(self) -> \"${params.typeName}\":\n" - ++ " return self" - - in '' - from enum import StrEnum - from typing import cast - - from .._runtime import require_array - - - class ${params.typeName}(StrEnum): - ${memberLines} - ${codecMethods} - '' - -in Sdk.Sigs.template Params run /\ { Variant } -``` - -> `require_array` must already be importable from `.._runtime` relative to `types/.py` — confirm the relative import depth matches `types/`'s actual nesting (one level under the package root per `CustomType.dhall`'s `modulePath = "types/${moduleName}.py"`) before running Step 2; adjust to `.._runtime` vs `..._runtime` to match what `Member.dhall`'s current `enumArrayDecode` assumes at its own call sites (check `ImportSet.dhall`'s handling of `require_array` imports today for the exact existing relative path convention, since this moves an existing import from call sites into the shared module). - -- [ ] **Step 2: Type-check** - -Run: `dhall type --file=src/Templates/EnumModule.dhall` -Expected: signature prints, no error. - -- [ ] **Step 3: Spot-render and validate as Python** - -```bash -cat > /tmp/render_enum.dhall <<'EOF' -let EnumModule = ./src/Templates/EnumModule.dhall -in EnumModule.run - { typeName = "Mood" - , variants = - [ { memberName = "HAPPY", pgValue = "happy" } - , { memberName = "SAD", pgValue = "sad" } - ] - } -EOF -dhall text --file=/tmp/render_enum.dhall | python3 -c "import sys; compile(sys.stdin.read(), 'mood.py', 'exec')" && echo OK -``` -Expected: `OK`. - -- [ ] **Step 4: Commit** - -```bash -git add src/Templates/EnumModule.dhall -git commit -m "python.gen: emit _decode/_decode_array/_encode on generated enum classes" -``` - ---- - -### Task 3: Simplify `Member.dhall`'s Custom branch - -**Files:** -- Modify: `src/Interpreters/Member.dhall` - -**Interfaces:** -- Consumes: `value.scalar.customRef : Optional Model.Name` (unchanged — still comes straight off `Scalar.run`, see `src/Interpreters/Scalar.dhall:44-52`), `value.dims : Natural` (unchanged). -- Produces: `Run = Config -> Input -> Lude.Compiled.Type Output` — **drops the `CustomKind.Lookup` parameter**. Every caller (`ResultColumns.dhall`, `CustomType.dhall`) updates in Task 5/6. - -- [ ] **Step 1: Replace the Custom branch (Member.dhall:129-232) and the `Run` alias (Member.dhall:245)** - -Delete the `merge { Enum = ...; Composite = ...; Absent = ... } (lookup name)` block and the `CustomKind.CompositeField`-typed `compositeDecode` helper (lines 100-116, now dead — the same logic lives in `CompositeModule.dhall`'s `_decode` now). Replace with: - -```dhall - , Custom = - Prelude.Optional.fold - Model.Name - value.scalar.customRef - (Lude.Compiled.Type Output) - ( \(name : Model.Name) -> - let typeName = name.inPascalCase - - let customImport = - { className = typeName, moduleName = name.inSnakeCase } - - let mkOutput = - \(customImports : ImportSet.Type) -> - \(decodeExpr : Text -> Text) -> - { fieldName - , pgName = input.pgName - , pyType - , isNullable = input.isNullable - , imports = - ImportSet.combine baseImports customImports - , decodeExpr - } - - let wrapNullable = - \(call : Text -> Text) -> - \(src : Text) -> - if input.isNullable - then "None if ${src} is None else ${call src}" - else call src - - let dimsIsOne = - Natural/isZero (Natural/subtract 1 value.dims) - - in if Natural/isZero value.dims - then Lude.Compiled.ok - Output - ( mkOutput - (ImportSet.custom customImport) - ( wrapNullable - (\(src : Text) -> "${typeName}._decode(${src})") - ) - ) - else if dimsIsOne - then Lude.Compiled.ok - Output - ( mkOutput - (ImportSet.custom customImport) - ( wrapNullable - ( \(src : Text) -> - "${typeName}._decode_array(${src})" - ) - ) - ) - else Lude.Compiled.report - Output - [ input.pgName, name.inSnakeCase ] - "Array of dimensionality > 1 is not supported" - ) - ( Lude.Compiled.report - Output - [ input.pgName ] - "Custom scalar without a customRef name" - ) -``` - -Note this drops the `Enum`/`Composite` import-set split (`ImportSet.customEnum`/`ImportSet.customComposite`) in favor of one `ImportSet.custom` call — see Doc 2 (`2026-07-11-encounter-order-custom-imports.md`) for why `ImportSet.customEnum`/`customComposite` collapse into a single `ImportSet.custom` once `order` no longer exists. **Land Doc 2 in the same branch as this task, or `dhall type` will fail here** — Task 3 depends on `ImportSet.CustomImport` no longer requiring a `dedupKey`/`order` field. - -- [ ] **Step 2: Drop the `lookup` parameter from `Run`** - -Change: -```dhall -let Run = Config -> CustomKind.Lookup -> Input -> Lude.Compiled.Type Output -``` -to: -```dhall -let Run = Config -> Input -> Lude.Compiled.Type Output -``` -and the `run` definition's `\(lookup : CustomKind.Lookup) ->` (Member.dhall:40) — delete that line entirely, since `run` no longer takes it. - -- [ ] **Step 3: Delete the now-dead `CustomKind` import (Member.dhall:9) and the dead `compositeDecode` helper (Member.dhall:100-116)** - -- [ ] **Step 4: Type-check** - -Run: `dhall type --file=src/Interpreters/Member.dhall` -Expected: fails until Task 5 updates `ResultColumns.dhall` (Member's only caller) to stop passing `lookup` — that's fine, type-check `Member.dhall` standalone by temporarily checking `Sdk.Sigs.interpreter Config Input Output run` in isolation, or proceed straight to Task 5 and type-check the pair together. Don't commit Task 3 alone; commit Tasks 3+5+6 together (they're one type-checking unit — Dhall won't let you land a signature change without updating every call site in the same change). - ---- - -### Task 4: Simplify `ParamsMember.dhall`'s Custom branch - -**Files:** -- Modify: `src/Interpreters/ParamsMember.dhall` - -**Interfaces:** -- Consumes: same as Task 3. -- Produces: `Run = Config -> Input -> Lude.Compiled.Type Output` — drops `CustomKind.Lookup`. - -- [ ] **Step 1: Replace the Custom branch (ParamsMember.dhall:311-367)** - -Delete `compositeBind` (lines 258-288, now dead — logic moved into `CompositeModule.dhall`'s `_encode`) and the `merge { Enum = ...; Composite = ...; Absent = ... } (lookup name)` block. Replace with: - -```dhall - in Prelude.Optional.fold - Model.Name - value.scalar.customRef - (Lude.Compiled.Type Output) - ( \(name : Model.Name) -> - let customImport = - { className = name.inPascalCase - , moduleName = name.inSnakeCase - } - - let encodeExpr = - if input.isNullable - then "None if ${fieldName} is None else ${fieldName}._encode()" - else "${fieldName}._encode()" - - in Lude.Compiled.ok - Output - ( mkOutput - (ImportSet.combine value.imports (ImportSet.custom customImport)) - encodeExpr - ) - ) - ( if isJsonArrayParam - then Lude.Compiled.report - Output - [ input.pgName ] - "json/jsonb array as a parameter is not supported" - else Lude.Compiled.ok - Output - (mkOutput value.imports defaultBind) - ) -``` - -This drops the `Natural/isZero value.dims` guard that used to reject composite-array params — per the Design decision above (Option A), an array-of-composite param now generates `{fieldName}._encode()` where `fieldName`'s type is `list[Point2D]`; `list` has no `._encode`, so `basedpyright strict` catches it. Confirm this in Task 8's basedpyright run specifically, not just eyeball it. - -- [ ] **Step 2: Drop the `lookup` parameter from `Run` (ParamsMember.dhall:387) and `run`'s `\(lookup : CustomKind.Lookup) ->` (ParamsMember.dhall:235)** - -- [ ] **Step 3: Delete the dead `CustomKind` import (ParamsMember.dhall:9)** - -- [ ] **Step 4: Type-check together with Task 3** - -Run: `dhall type --file=src/Interpreters/ParamsMember.dhall` -Expected: same caveat as Task 3 Step 4 — its caller (`Query.dhall`) still passes `lookup` until Task 5. - ---- - -### Task 5: Drop `lookup` threading from `ResultColumns.dhall`, `Result.dhall`, `Query.dhall` - -**Files:** -- Modify: `src/Interpreters/ResultColumns.dhall`, `src/Interpreters/Result.dhall`, `src/Interpreters/Query.dhall` - -**Interfaces:** -- Each of these only forwards `lookup` to a callee; none inspect it. Removing it is mechanical. - -- [ ] **Step 1: `ResultColumns.dhall`** — delete `\(lookup : CustomKind.Lookup) ->` (line 62) and change `Member.run config lookup member` (line 76) to `Member.run config member`. Delete the `CustomKind` import (line 9) if nothing else in the file uses it — verify with `grep -n CustomKind src/Interpreters/ResultColumns.dhall` after editing. - -- [ ] **Step 2: `Result.dhall`** — delete both `\(lookup : CustomKind.Lookup) ->` occurrences (lines 66, 93), change `ResultColumns.run config lookup rowClassName columns` (line 89) to `ResultColumns.run config rowClassName columns`, and `rowsOutput config lookup rowClassName` (line 100) to `rowsOutput config rowClassName` — check `rowsOutput`'s own definition for a `lookup` parameter to drop too (it wasn't in the earlier grep excerpt; read the file before editing to confirm). Delete the `CustomKind` import (line 9) if unused after. - -- [ ] **Step 3: `Query.dhall`** — delete `\(lookup : CustomKind.Lookup) ->` (line 140), change `ResultModule.run config lookup rowClassName input.result` (line 156) to drop `lookup`, and `ParamsMember.run config lookup member` (line 173) to `ParamsMember.run config member`. Delete the `CustomKind` import (line 7) if unused after. - -- [ ] **Step 4: Type-check each file standalone** - -Run: `dhall type --file=src/Interpreters/ResultColumns.dhall && dhall type --file=src/Interpreters/Result.dhall && dhall type --file=src/Interpreters/Query.dhall` -Expected: all three print their signatures. `Query.dhall` will still fail until `Project.dhall` (Task 6) stops passing `lookup` to `QueryGen.run` — that's the last link. - ---- - -### Task 6: Delete `buildLookup` and its support code from `Project.dhall` - -**Files:** -- Modify: `src/Interpreters/Project.dhall` - -- [ ] **Step 1: Delete dead helpers** - -Delete, in order (they only exist to feed `buildLookup`, confirmed by re-reading the file top to bottom — nothing else calls `lookupConfig`, `memberPyType`, `compositeFields`, `IndexedCustomType`, or `buildLookup` itself): -- `lookupConfig` (lines 74-80) -- `memberPyType` (lines 85-96) -- `compositeFields` (lines 98-108) -- The local `CompositeField = { fieldName : Text, pyType : Text }` (line 7) — dead once `compositeFields`/`memberPyType` are gone -- `IndexedCustomType` (line 142) -- `buildLookup` (lines 147-181) - -- [ ] **Step 2: Update the call site** - -Find `let lookup = buildLookup effectiveCustomTypes` (line 479) — delete it, and change both call sites that pass `lookup`: -- `QueryGen.run config lookup query` (line 503) → `QueryGen.run config query` -- `(\(query : Model.Query) -> QueryGen.run config lookup query)` (line 524) → `(\(query : Model.Query) -> QueryGen.run config query)` - -- [ ] **Step 3: Delete the dead `CustomKind` import (line 9)** - -- [ ] **Step 4: Type-check the whole chain** - -Run: `dhall type --file=src/Interpreters/Project.dhall` -Expected: prints the module signature with no error — this is the point where Tasks 3-6 all click together (Project → Query → Result/ParamsMember → ResultColumns/Member all now agree on the lookup-free signatures). - -- [ ] **Step 5: Commit Tasks 3-6 together** - -```bash -git add src/Interpreters/Member.dhall src/Interpreters/ParamsMember.dhall \ - src/Interpreters/ResultColumns.dhall src/Interpreters/Result.dhall \ - src/Interpreters/Query.dhall src/Interpreters/Project.dhall -git commit -m "python.gen: call custom-type codecs by name instead of resolving via buildLookup" -``` - ---- - -### Task 7: Drop `nestedLookup` from `CustomType.dhall`, delete `CustomKind.dhall` - -**Files:** -- Modify: `src/Interpreters/CustomType.dhall` -- Delete: `src/Structures/CustomKind.dhall` - -- [ ] **Step 1: `CustomType.dhall`** — delete `nestedLookup` (lines 51-53) and its comment, and change `MemberGen.run config nestedLookup m` (line 127) to `MemberGen.run config m`. Delete the `CustomKind` import (line 11). - -- [ ] **Step 2: Confirm nothing else references `CustomKind`** - -Run: `grep -rln "CustomKind" src` -Expected: no output. - -- [ ] **Step 3: Delete the file** - -```bash -git rm src/Structures/CustomKind.dhall -``` - -- [ ] **Step 4: Type-check the full package entry point** - -Run: `dhall type --file=src/package.dhall` -Expected: prints the top-level module signature, no error. (This is the first point a full-package check is meaningful — earlier steps only checked individual interpreter files.) - -- [ ] **Step 5: Commit** - -```bash -git add src/Interpreters/CustomType.dhall -git commit -m "python.gen: delete CustomKind.dhall, the last remnant of buildLookup" -``` - ---- - -### Task 8: Regenerate golden fixtures, verify, update docs - -**Files:** -- Modify: `tests/fixture-project/` (add a composite-array column per the Design decision above) -- Regenerate: `tests/golden/` -- Modify: `python.gen/DESIGN.md`, `python.gen/docs/upstream-asks.md`, `python.gen/CHANGELOG.md` - -- [ ] **Step 1: Add a composite-array test column to the fixture project** - -Find the fixture's composite-type column definitions under `tests/fixture-project/` (query/table SQL referencing a composite type, e.g. the `Point2D`-typed column used in `insert_specimen`/`get_specimen` per the golden output). Add one query or column that selects/inserts an *array* of that composite type — this is the first real exercise of the Option A fallback path (`basedpyright strict` should catch it if `_decode_array`/`_encode` aren't defined on `Point2D`, since Task 1 deliberately didn't add them to `CompositeModule.dhall`). - -Confirm the expected failure mode first: - -Run: `mise run golden` (needs `PGN_TEST_DATABASE_URL` pointing at a live Postgres — see `tests/golden/README.md`) -Expected: generation succeeds (no Dhall-level rejection — that's the point of Option A), but the regenerated file calls `Point2D._decode_array(...)` or `list[Point2D]._encode()`-shaped code that doesn't exist. - -Run: `mise run test` -Expected: `test_generated_passes_basedpyright_strict` FAILS, citing the missing attribute. This confirms the safety net from the Design decision actually fires. Once confirmed, either: - - revert the fixture addition (if you don't want composite arrays in the committed golden corpus yet), or - - implement `_decode_array`/an array-aware `_encode` on `CompositeModule.dhall` for real and keep the fixture (a follow-up, out of this plan's scope — flag it, don't scope-creep this task). - -- [ ] **Step 2: Remove the composite-array addition (unless implementing it for real per Step 1)** - -- [ ] **Step 3: Regenerate golden for real** - -Run: `mise run golden` -Expected: succeeds, rewrites `tests/golden/src/specimen_client/_generated/**` and both facades. - -- [ ] **Step 4: Review the diff** - -Run: `git diff tests/golden` -Expected: every composite/enum decode/encode call site now reads `TypeName._decode(...)`/`TypeName._decode_array(...)`/`value._encode()` instead of the old inlined `cast(tuple[...], ...)`/`(x.a, x.b)` expressions; `types/point_2_d.py`, `types/mood.py`, `types/tag_value.py` each gain the new methods. No unrelated files change. - -- [ ] **Step 5: Run the full test suite** - -Run: `mise run test` -Expected: all pass, including `test_generated_passes_basedpyright_strict`. - -- [ ] **Step 6: Update `DESIGN.md`** - -Rewrite section 12 (`## 12. Forked-Dhall (Text/equal) dependency risk`) — it's no longer an accepted risk for python.gen; state plainly that `buildLookup` is gone and `Text/equal` is no longer used anywhere in this generator's own Dhall source (grep to confirm: `grep -rn "Text/equal" src` returns nothing). Keep a short note that `demos/Exhaustive.dhall`/`mise run golden` still needs the pinned pgn binary regardless, because `gen-sdk`'s own `Fixtures` module independently uses the fork builtin — this plan doesn't touch that, and it isn't blocked on it. Cross-reference section 13 (unchanged — `PyIdent.dhall`'s trick is unrelated and still in place). - -- [ ] **Step 7: Update `docs/upstream-asks.md`** - -Remove ask 3 (`## 3. gen-sdk: kind tag or Natural index on Scalar.Custom`) or mark it explicitly withdrawn with one line explaining why (`buildLookup`'s only consumer was resolved locally by generating named codecs instead of resolving structural type info by search — see `docs/plans/2026-07-11-reusable-custom-type-codecs.md`). Don't delete the file's other two asks (pragma parsing, warnings printing) — they're unrelated and still open. - -- [ ] **Step 8: Update `CHANGELOG.md`** - -Add an entry under the appropriate section describing the behavior change from the Design decision (composite-array columns/params no longer rejected at generation time; verify at basedpyright-strict time instead) if Step 1's fixture addition was kept, or note it as a documented-but-untested path if reverted. - -- [ ] **Step 9: Commit** - -```bash -git add tests/golden tests/fixture-project python.gen/DESIGN.md python.gen/docs/upstream-asks.md python.gen/CHANGELOG.md -git commit -m "python.gen: regenerate golden fixtures for reusable custom-type codecs" -``` - ---- - -## Self-Review - -**Spec coverage:** Task 1-2 build the reusable codecs (the actual "fix the root cause"). Tasks 3-4 make the two reference sites (decode, encode) call them. Tasks 5-7 remove the now-dead plumbing (`lookup` threading, `buildLookup`, `CustomKind.dhall`) so nothing is left half-migrated. Task 8 proves it against the real toolchain and updates the two docs (`DESIGN.md`, `upstream-asks.md`) that currently assert this is unfixable — both would otherwise go stale and mislead the next reader. - -**Open question carried forward, not silently resolved:** the array-of-composite behavior change (Design decision, Option A vs B) is a real product decision, flagged explicitly rather than picked unilaterally in the diff. Task 8 Step 1 is designed to surface the actual runtime behavior (does `basedpyright strict` really catch it, does composite-array decode actually work against real Postgres) before committing to either branch. - -**Dependency on the companion plan:** Task 3 explicitly calls out that it needs `docs/plans/2026-07-11-encounter-order-custom-imports.md` (`ImportSet.dhall`'s `order`/`dedupKey` removal) landed first or alongside — `ImportSet.customEnum`/`customComposite` currently *require* an `order : Natural` that only `buildLookup` produced. Implement that plan first, or fold both into one PR; don't land this plan's Task 3 against the unmodified `ImportSet.dhall`. - -## Execution Handoff - -Plan complete and saved to `python.gen/docs/plans/2026-07-11-reusable-custom-type-codecs.md`. Two execution options: - -**1. Subagent-Driven (recommended)** - dispatch a fresh subagent per task, review between tasks, fast iteration - -**2. Inline Execution** - execute tasks in this session using executing-plans, batch execution with checkpoints - -Which approach?