From 616c68ada9e2101834ab59de540d849305fb3cdd Mon Sep 17 00:00:00 2001 From: Micro Liu Date: Sun, 28 Sep 2025 10:53:35 +0800 Subject: [PATCH 1/4] feat: support global variables --- _xtool/pydump/pydump.go | 21 +++++++++++++++- symbol/symbol.go | 3 ++- tool/pygen/genvar.go | 38 ++++++++++++++++++++++++++++ tool/pygen/pygen.go | 4 ++- tool/pygen/pygen_test.go | 15 +++++++++++ tool/pygen/testdata/var/demo.py | 15 +++++++++++ tool/pygen/testdata/var/expect.go | 41 +++++++++++++++++++++++++++++++ 7 files changed, 134 insertions(+), 3 deletions(-) create mode 100644 tool/pygen/genvar.go create mode 100644 tool/pygen/testdata/var/demo.py create mode 100644 tool/pygen/testdata/var/expect.go diff --git a/_xtool/pydump/pydump.go b/_xtool/pydump/pydump.go index 7b8be43..aaaa7c8 100644 --- a/_xtool/pydump/pydump.go +++ b/_xtool/pydump/pydump.go @@ -20,6 +20,19 @@ var pyFuncTypes = map[string]bool{ "_ArrayFunctionDispatcher": true, } +var pyVarTypes = map[string]struct{}{ + "int": {}, + "float": {}, + "complex": {}, + "bool": {}, + "str": {}, + "list": {}, + "tuple": {}, + "set": {}, + "dict": {}, + "NoneType": {}, +} + func extractSignatureFromDoc(doc, funcName string) string { lines := strings.SplitN(doc, "\n\n", 2) if len(lines) == 0 { @@ -98,8 +111,14 @@ func pydump(moduleName string) (*symbol.Module, error) { if pyFuncTypes[sym.Type] { sym.Sig = getSignature(val, sym) modInstance.Functions = append(modInstance.Functions, sym) + continue + } + // variables + if _, ok := pyVarTypes[sym.Type]; ok { + modInstance.Variables = append(modInstance.Variables, sym) + continue } - // TODO: variables, classes, etc. + // TODO: classes, etc. } return modInstance, nil } diff --git a/symbol/symbol.go b/symbol/symbol.go index e0e5d2b..a6caebc 100644 --- a/symbol/symbol.go +++ b/symbol/symbol.go @@ -10,5 +10,6 @@ type Symbol struct { type Module struct { Name string `json:"name"` // python module name Functions []*Symbol `json:"functions"` // package functions - // TODO: variables, classes, etc. + Variables []*Symbol `json:"variables"` // package variables + // TODO: classes, etc. } diff --git a/tool/pygen/genvar.go b/tool/pygen/genvar.go new file mode 100644 index 0000000..6d2898a --- /dev/null +++ b/tool/pygen/genvar.go @@ -0,0 +1,38 @@ +package pygen + +import ( + "github.com/goplus/gogen" + "go/ast" + "go/token" + "github.com/goplus/llpyg/symbol" +) + + +func (ctx *context) genVars(pkg *gogen.Package, syms []*symbol.Symbol) { + names := make(map[string]struct{}) + for _, sym := range syms { + if sym.Name == "" || sym.Name[0] == '_' { + continue + } + name := ctx.genName(sym.Name, -1) + _, exist := names[name] + // avoid name conflict + for exist { + name = name + "_" + _, exist = names[name] + } + names[name] = struct{}{} + ctx.genVar(pkg, sym, name) + } +} + +func (ctx *context) genVar(pkg *gogen.Package, sym *symbol.Symbol, goName string) { + def := pkg.NewVarDefs(pkg.Types.Scope()) + // linkname + docList := make([]*ast.Comment, 0, 2) + goLinkname := "//go:linkname " + goName + " py." + sym.Name + docList = append(docList, &ast.Comment{Text: goLinkname}) + def.SetComments(&ast.CommentGroup{List: docList}) + // new + def.New(token.NoPos, ctx.objPtr, goName) +} diff --git a/tool/pygen/pygen.go b/tool/pygen/pygen.go index 905398d..a63c61b 100644 --- a/tool/pygen/pygen.go +++ b/tool/pygen/pygen.go @@ -103,7 +103,9 @@ func (ctx *context) genMod(pkg *gogen.Package, mod *symbol.Module) { funcMap[sym.Name] = true ctx.genFunc(pkg, sym) } - // TODO: class, variable, etc. + // variables + ctx.genVars(pkg, mod.Variables) + // TODO: class, etc. } diff --git a/tool/pygen/pygen_test.go b/tool/pygen/pygen_test.go index 2a4d8ec..f38b2b2 100644 --- a/tool/pygen/pygen_test.go +++ b/tool/pygen/pygen_test.go @@ -41,6 +41,21 @@ func TestGenFunc(t *testing.T) { t.Logf("test gen func pass") } +func TestGenVar(t *testing.T) { + prepareEnv("./testdata/var") + mod, err := pydump("demo") + if err != nil { + t.Fatal(err) + } + ctx := createGoPackage(mod) + ctx.genVars(ctx.pkg, mod.Variables) + err = compareWithExpected(t, ctx, "testdata/var/expect.go") + if err != nil { + t.Fatalf("test gen var failed: %v", err) + } + t.Logf("test gen var pass") +} + func compareWithExpected(t *testing.T, ctx *context, expectedPath string) error { outFilePath := "./temp/actual_git.go" dir := filepath.Dir(outFilePath) diff --git a/tool/pygen/testdata/var/demo.py b/tool/pygen/testdata/var/demo.py new file mode 100644 index 0000000..3166cb2 --- /dev/null +++ b/tool/pygen/testdata/var/demo.py @@ -0,0 +1,15 @@ +int_var = 30 +Int_var = 25 +float_var = 175.5 +complex_var = 3 + 4j +bool_var = True + +str_var = "Hello, World!" +list_var = [1, 2, 3] +tuple_var = (1, 2, 3) + +set_var = {1, 2, 3} + +dict_var = {1: "one", 2: "two", 3: "three"} + +none_var = None diff --git a/tool/pygen/testdata/var/expect.go b/tool/pygen/testdata/var/expect.go new file mode 100644 index 0000000..a4f978b --- /dev/null +++ b/tool/pygen/testdata/var/expect.go @@ -0,0 +1,41 @@ +package demo + +import ( + "github.com/goplus/lib/py" + _ "unsafe" +) + +const LLGoPackage = "py.demo" + +//go:linkname IntVar py.int_var +var IntVar *py.Object + +//go:linkname IntVar_ py.Int_var +var IntVar_ *py.Object + +//go:linkname FloatVar py.float_var +var FloatVar *py.Object + +//go:linkname ComplexVar py.complex_var +var ComplexVar *py.Object + +//go:linkname BoolVar py.bool_var +var BoolVar *py.Object + +//go:linkname StrVar py.str_var +var StrVar *py.Object + +//go:linkname ListVar py.list_var +var ListVar *py.Object + +//go:linkname TupleVar py.tuple_var +var TupleVar *py.Object + +//go:linkname SetVar py.set_var +var SetVar *py.Object + +//go:linkname DictVar py.dict_var +var DictVar *py.Object + +//go:linkname NoneVar py.none_var +var NoneVar *py.Object From 3fac898f47fcec69ecc2e3f8536d8bbf625226df Mon Sep 17 00:00:00 2001 From: Micro Liu Date: Fri, 10 Oct 2025 11:18:16 +0800 Subject: [PATCH 2/4] fix vartype to datatype --- _xtool/pydump/pydump.go | 4 ++-- tool/pygen/genvar.go | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/_xtool/pydump/pydump.go b/_xtool/pydump/pydump.go index aaaa7c8..f20ab03 100644 --- a/_xtool/pydump/pydump.go +++ b/_xtool/pydump/pydump.go @@ -20,7 +20,7 @@ var pyFuncTypes = map[string]bool{ "_ArrayFunctionDispatcher": true, } -var pyVarTypes = map[string]struct{}{ +var pyDataTypes = map[string]struct{}{ "int": {}, "float": {}, "complex": {}, @@ -114,7 +114,7 @@ func pydump(moduleName string) (*symbol.Module, error) { continue } // variables - if _, ok := pyVarTypes[sym.Type]; ok { + if _, ok := pyDataTypes[sym.Type]; ok { modInstance.Variables = append(modInstance.Variables, sym) continue } diff --git a/tool/pygen/genvar.go b/tool/pygen/genvar.go index 6d2898a..96bcd1c 100644 --- a/tool/pygen/genvar.go +++ b/tool/pygen/genvar.go @@ -1,13 +1,12 @@ package pygen import ( - "github.com/goplus/gogen" "go/ast" "go/token" + "github.com/goplus/gogen" "github.com/goplus/llpyg/symbol" ) - func (ctx *context) genVars(pkg *gogen.Package, syms []*symbol.Symbol) { names := make(map[string]struct{}) for _, sym := range syms { From 915c37229ff4f63c71a2ce27e0f89f85a5dc8d61 Mon Sep 17 00:00:00 2001 From: Micro Liu Date: Fri, 10 Oct 2025 12:45:41 +0800 Subject: [PATCH 3/4] add variable comment issue link --- tool/pygen/genvar.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tool/pygen/genvar.go b/tool/pygen/genvar.go index 96bcd1c..5bda54e 100644 --- a/tool/pygen/genvar.go +++ b/tool/pygen/genvar.go @@ -25,6 +25,7 @@ func (ctx *context) genVars(pkg *gogen.Package, syms []*symbol.Symbol) { } } +// current can not get variable comment, see https://github.com/goplus/llpyg/issues/43 func (ctx *context) genVar(pkg *gogen.Package, sym *symbol.Symbol, goName string) { def := pkg.NewVarDefs(pkg.Types.Scope()) // linkname @@ -32,6 +33,6 @@ func (ctx *context) genVar(pkg *gogen.Package, sym *symbol.Symbol, goName string goLinkname := "//go:linkname " + goName + " py." + sym.Name docList = append(docList, &ast.Comment{Text: goLinkname}) def.SetComments(&ast.CommentGroup{List: docList}) - // new + def.New(token.NoPos, ctx.objPtr, goName) } From ab413619f25956bba157b2df1752cb53f471f0e8 Mon Sep 17 00:00:00 2001 From: Micro Liu Date: Fri, 10 Oct 2025 15:58:59 +0800 Subject: [PATCH 4/4] use py basic data type to judge global variable --- _xtool/pydump/pydump.go | 29 +++++++++++++++++------------ tool/pygen/testdata/var/demo.py | 22 +++++++++++++++++++--- tool/pygen/testdata/var/expect.go | 19 +++++++++++++++++-- 3 files changed, 53 insertions(+), 17 deletions(-) diff --git a/_xtool/pydump/pydump.go b/_xtool/pydump/pydump.go index f20ab03..01c5113 100644 --- a/_xtool/pydump/pydump.go +++ b/_xtool/pydump/pydump.go @@ -20,17 +20,22 @@ var pyFuncTypes = map[string]bool{ "_ArrayFunctionDispatcher": true, } -var pyDataTypes = map[string]struct{}{ - "int": {}, - "float": {}, - "complex": {}, - "bool": {}, - "str": {}, - "list": {}, - "tuple": {}, - "set": {}, - "dict": {}, - "NoneType": {}, +var pyBasicDataTypes = map[string]struct{}{ + "int": {}, + "float": {}, + "complex": {}, + "bool": {}, + "str": {}, + "list": {}, + "tuple": {}, + "range": {}, + "set": {}, + "frozenset": {}, + "dict": {}, + "bytes": {}, + "bytearray": {}, + "memoryview": {}, + "NoneType": {}, } func extractSignatureFromDoc(doc, funcName string) string { @@ -114,7 +119,7 @@ func pydump(moduleName string) (*symbol.Module, error) { continue } // variables - if _, ok := pyDataTypes[sym.Type]; ok { + if _, ok := pyBasicDataTypes[sym.Type]; ok { modInstance.Variables = append(modInstance.Variables, sym) continue } diff --git a/tool/pygen/testdata/var/demo.py b/tool/pygen/testdata/var/demo.py index 3166cb2..9a050b2 100644 --- a/tool/pygen/testdata/var/demo.py +++ b/tool/pygen/testdata/var/demo.py @@ -1,15 +1,31 @@ -int_var = 30 +# Numeric types +int_var = 42 Int_var = 25 -float_var = 175.5 +float_var = 3.14 complex_var = 3 + 4j + +# Boolean type bool_var = True +# String type str_var = "Hello, World!" + +# Sequence types list_var = [1, 2, 3] tuple_var = (1, 2, 3) +range_var = range(10) +# Mapping type +dict_var = {1: "one", 2: "two", 3: "three"} + +# Set types set_var = {1, 2, 3} +frozenset_var = frozenset({1, 2, 3}) -dict_var = {1: "one", 2: "two", 3: "three"} +# Binary types +bytes_var = b"hello" +bytearray_var = bytearray(b"hello") +mv_var = memoryview(b"hello") +# Special type none_var = None diff --git a/tool/pygen/testdata/var/expect.go b/tool/pygen/testdata/var/expect.go index a4f978b..2661a56 100644 --- a/tool/pygen/testdata/var/expect.go +++ b/tool/pygen/testdata/var/expect.go @@ -31,11 +31,26 @@ var ListVar *py.Object //go:linkname TupleVar py.tuple_var var TupleVar *py.Object -//go:linkname SetVar py.set_var -var SetVar *py.Object +//go:linkname RangeVar py.range_var +var RangeVar *py.Object //go:linkname DictVar py.dict_var var DictVar *py.Object +//go:linkname SetVar py.set_var +var SetVar *py.Object + +//go:linkname FrozensetVar py.frozenset_var +var FrozensetVar *py.Object + +//go:linkname BytesVar py.bytes_var +var BytesVar *py.Object + +//go:linkname BytearrayVar py.bytearray_var +var BytearrayVar *py.Object + +//go:linkname MvVar py.mv_var +var MvVar *py.Object + //go:linkname NoneVar py.none_var var NoneVar *py.Object