From 3e1dfa8e34e205b4f28c2f27b3958a292e7896b7 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 17 Dec 2019 12:16:50 -0800 Subject: [PATCH 01/23] first pass of tree-shaker --- compiler/stz-el.stanza | 146 +++++++++++++++++++++++++++++++++++++++ examples/calculus.stanza | 1 - 2 files changed, 146 insertions(+), 1 deletion(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index 2017111e3..4c3968944 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -33,7 +33,151 @@ public defn lower-unoptimized (epackage:EPackage) -> EPackage : ;========================= Lowering ========================= ;============================================================ +;; how to print out id table? + +defn roots (epackage:EPackage) -> Seqable : + for e in exps(epackage) filter : + ;; e is EInit|EDefGlobal|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject + e is EInit|EDefGlobal|EDefStruct|EDefType|EDefObject|EDefTypeObject + +defn methods (epackage:EPackage) -> IntTable> : + val tab = IntTable>(List()) + for e in exps(epackage) do : + match(e) : + (e:EDefmethod) : tab[multi(e)] = cons(n(e), tab[multi(e)]) + (e) : false + tab + +defn defs (epackage:EPackage) -> IntTable : + val tab = IntTable() + for e in exps(epackage) do : + match(e) : + (e:EDefGlobal|EDefn|EDefClosure|EDefmulti|EDefmethod) : tab[n(e)] = e + (e:EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : tab[n(e)] = e + (e) : false + tab + +defn walk-reachable-defs (e:ETExp, q:Queue, defs:IntTable, methods:IntTable>, visited?:IntSet, reachables:IntSet, defined-objects:IntSet) : + match(e) : + (e:EDefGlobal|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : add(reachables, n(e)) + (e) : false + let walk (e:ELItem = e) : + defn maybe-walk (n:Int) : + if add(visited?, n) : + if key?(defs, n) : + add(reachables, n) + add(q, defs[n]) + match(e) : + (e:EVar|EVarLoc) : + maybe-walk(n(e)) + if key?(methods, n(e)) : + for m in methods[n(e)] do : maybe-walk(m) + (e:EDefObject|ELocalObj) : + add(defined-objects, n(e)) + do(walk, e) + (e:ELItem) : + do(walk, e) + +defn find-defined-objects (epackage:EPackage) -> IntSet : + val defined-objects = IntSet() + defn* walk (e:ELItem) : + match(e) : + (e:EDefObject|ELocalObj) : + add(defined-objects, n(e)) + do(walk, e) + (e) : + do(walk, e) + for exp in exps(epackage) do : + walk(exp) + defined-objects + +defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntSet, IntSet] : + val defs = defs(epackage) + val methods = methods(epackage) + val q = Queue() + val visited? = IntSet() + val reachables = IntSet() + val defined-objects = IntSet() + val orig-defined-objects = find-defined-objects(epackage) + for root in roots(epackage) do : + add(q, root) + while not empty?(q) : + val def = pop(q) + walk-reachable-defs(def, q, defs, methods, visited?, reachables, defined-objects) + println("FOUND %_ REACHABLE OUT OF %_ DEFINED" % [length(reachables), length(defs)]) + println("FOUND %_ DEFINED-OBJECTS OUT OF %_ ORIG-DEFINED-OBJECTS" % [length(defined-objects), length(orig-defined-objects)]) + [reachables, orig-defined-objects, defined-objects] + +defn prune (packageio:PackageIO, defined?:IntSet) -> PackageIO : + PackageIO(package(packageio), imported-packages(packageio), + to-tuple $ for i in imports(packageio) filter : defined?[n(i)], + to-tuple $ for e in exports(packageio) filter : defined?[n(e)]) + +defn check-vars-defined (e:ETExp, odefs:IntTable, defs:IntTable, undefines:IntSet) : + let walk (e:ELItem = e) : + match(e) : + (e:EVar|EVarLoc) : + if key?(odefs, n(e)) and not key?(defs, n(e)) : + add(undefines, n(e)) + (e:ELItem) : + do(walk, e) + undefines + +defn check-vars-defined (epackage:EPackage, odefs:IntTable) : + val defs = defs(epackage) + val undefines = IntSet() + for e in exps(epackage) do : + check-vars-defined(e, odefs, defs, undefines) + println("UNDEFINES %_" % [to-tuple $ undefines]) + +defn prune (e:ETExp, reachables:IntSet) -> ETExp : + defn* walk (e:ELItem) -> ELItem : + match(e) : + (e:ELocalObj) : + val methods = for m in methods(e) filter : reachables[multi(m)] + sub-methods(e, to-tuple $ for m in methods seq : map(walk, m)) + (e) : + map(walk, e) + walk(e) as ETExp + +defn prune-package (epackage:EPackage, gvt:VarTable) -> EPackage : + spit("epackage1.txt", epackage) + + val [reachables, orig-defined-objects, defined-objects] = walk-reachable-defs(epackage) + defn undefined-objects? (e:EType) : + var undefined? = false + let walk (e:ELItem = e) : + match(e) : + (e:EVar) : + if orig-defined-objects[n(e)] and not defined-objects[n(e)] : + undefined? = true + (e) : + do(walk, e) + undefined? + defn defined? (e:ETExp) : + match(e) : + (e:EDefn|EDefClosure|EDefmulti) : + reachables[n(e)] + (e:EDefmethod) : + ;; val res = reachables[n(e)] and not any?(undefined-objects?, targs(e)) + ;; println("REMOVING %_" % [n(e)]) when not res + ;; res + reachables[n(e)] + (e:EDefGlobal|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : + true + (e) : + true + println(qsort $ reachables) + val etexps = for exp in exps(epackage) filter : defined?(exp) + val res = EPackage(prune(packageio(epackage), reachables), + to-tuple $ for e in etexps seq : prune(e, reachables)) + spit("epackage2.txt", res) + check-vars-defined(res, defs(epackage)) + res + defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : + println("LOWERING") + ;Reset id generation take-ids(epackage) @@ -65,6 +209,7 @@ defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : ;Dump input ;dump(cur-package, "logs", "input") + run-pass("Prune Package", prune-package, "prune", true) run-pass("Map Methods", map-methods, "mapped-methods", false) run-pass("Create Closures", create-closures, "closures", false) run-pass("Convert Mixes", convert-mixes, "mixes", false) @@ -2312,6 +2457,7 @@ defn MethodTable (epackage:EPackage) : ;Resolve through dag defn resolve-target (class-tree:DynTree, dag-table:IntTable, multi:Int, types:Tuple) -> Int|False : + println("RESOLVING MULTI %_ TYPES %_" % [multi, types]) val mdag = get?(dag-table, multi) match(mdag:MethodDag) : val args = map(etype-to-arg{class-tree, sf, _, false}, types) diff --git a/examples/calculus.stanza b/examples/calculus.stanza index 5796189d7..b671bb2cc 100644 --- a/examples/calculus.stanza +++ b/examples/calculus.stanza @@ -1,6 +1,5 @@ defpackage calculus : import core - import collections ; Automatic Differentiation ; ========================= From 6fb83ec29f318b26791997989948c563838cb96c Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Wed, 18 Dec 2019 15:08:14 -0800 Subject: [PATCH 02/23] first working version of tree-shaker --- compiler/stz-el.stanza | 355 ++++++++++++++++++++++++----------------- 1 file changed, 212 insertions(+), 143 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index 4c3968944..0095cbcb7 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -35,149 +35,8 @@ public defn lower-unoptimized (epackage:EPackage) -> EPackage : ;; how to print out id table? -defn roots (epackage:EPackage) -> Seqable : - for e in exps(epackage) filter : - ;; e is EInit|EDefGlobal|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject - e is EInit|EDefGlobal|EDefStruct|EDefType|EDefObject|EDefTypeObject - -defn methods (epackage:EPackage) -> IntTable> : - val tab = IntTable>(List()) - for e in exps(epackage) do : - match(e) : - (e:EDefmethod) : tab[multi(e)] = cons(n(e), tab[multi(e)]) - (e) : false - tab - -defn defs (epackage:EPackage) -> IntTable : - val tab = IntTable() - for e in exps(epackage) do : - match(e) : - (e:EDefGlobal|EDefn|EDefClosure|EDefmulti|EDefmethod) : tab[n(e)] = e - (e:EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : tab[n(e)] = e - (e) : false - tab - -defn walk-reachable-defs (e:ETExp, q:Queue, defs:IntTable, methods:IntTable>, visited?:IntSet, reachables:IntSet, defined-objects:IntSet) : - match(e) : - (e:EDefGlobal|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : add(reachables, n(e)) - (e) : false - let walk (e:ELItem = e) : - defn maybe-walk (n:Int) : - if add(visited?, n) : - if key?(defs, n) : - add(reachables, n) - add(q, defs[n]) - match(e) : - (e:EVar|EVarLoc) : - maybe-walk(n(e)) - if key?(methods, n(e)) : - for m in methods[n(e)] do : maybe-walk(m) - (e:EDefObject|ELocalObj) : - add(defined-objects, n(e)) - do(walk, e) - (e:ELItem) : - do(walk, e) - -defn find-defined-objects (epackage:EPackage) -> IntSet : - val defined-objects = IntSet() - defn* walk (e:ELItem) : - match(e) : - (e:EDefObject|ELocalObj) : - add(defined-objects, n(e)) - do(walk, e) - (e) : - do(walk, e) - for exp in exps(epackage) do : - walk(exp) - defined-objects - -defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntSet, IntSet] : - val defs = defs(epackage) - val methods = methods(epackage) - val q = Queue() - val visited? = IntSet() - val reachables = IntSet() - val defined-objects = IntSet() - val orig-defined-objects = find-defined-objects(epackage) - for root in roots(epackage) do : - add(q, root) - while not empty?(q) : - val def = pop(q) - walk-reachable-defs(def, q, defs, methods, visited?, reachables, defined-objects) - println("FOUND %_ REACHABLE OUT OF %_ DEFINED" % [length(reachables), length(defs)]) - println("FOUND %_ DEFINED-OBJECTS OUT OF %_ ORIG-DEFINED-OBJECTS" % [length(defined-objects), length(orig-defined-objects)]) - [reachables, orig-defined-objects, defined-objects] - -defn prune (packageio:PackageIO, defined?:IntSet) -> PackageIO : - PackageIO(package(packageio), imported-packages(packageio), - to-tuple $ for i in imports(packageio) filter : defined?[n(i)], - to-tuple $ for e in exports(packageio) filter : defined?[n(e)]) - -defn check-vars-defined (e:ETExp, odefs:IntTable, defs:IntTable, undefines:IntSet) : - let walk (e:ELItem = e) : - match(e) : - (e:EVar|EVarLoc) : - if key?(odefs, n(e)) and not key?(defs, n(e)) : - add(undefines, n(e)) - (e:ELItem) : - do(walk, e) - undefines - -defn check-vars-defined (epackage:EPackage, odefs:IntTable) : - val defs = defs(epackage) - val undefines = IntSet() - for e in exps(epackage) do : - check-vars-defined(e, odefs, defs, undefines) - println("UNDEFINES %_" % [to-tuple $ undefines]) - -defn prune (e:ETExp, reachables:IntSet) -> ETExp : - defn* walk (e:ELItem) -> ELItem : - match(e) : - (e:ELocalObj) : - val methods = for m in methods(e) filter : reachables[multi(m)] - sub-methods(e, to-tuple $ for m in methods seq : map(walk, m)) - (e) : - map(walk, e) - walk(e) as ETExp - -defn prune-package (epackage:EPackage, gvt:VarTable) -> EPackage : - spit("epackage1.txt", epackage) - - val [reachables, orig-defined-objects, defined-objects] = walk-reachable-defs(epackage) - defn undefined-objects? (e:EType) : - var undefined? = false - let walk (e:ELItem = e) : - match(e) : - (e:EVar) : - if orig-defined-objects[n(e)] and not defined-objects[n(e)] : - undefined? = true - (e) : - do(walk, e) - undefined? - defn defined? (e:ETExp) : - match(e) : - (e:EDefn|EDefClosure|EDefmulti) : - reachables[n(e)] - (e:EDefmethod) : - ;; val res = reachables[n(e)] and not any?(undefined-objects?, targs(e)) - ;; println("REMOVING %_" % [n(e)]) when not res - ;; res - reachables[n(e)] - (e:EDefGlobal|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : - true - (e) : - true - println(qsort $ reachables) - val etexps = for exp in exps(epackage) filter : defined?(exp) - val res = EPackage(prune(packageio(epackage), reachables), - to-tuple $ for e in etexps seq : prune(e, reachables)) - spit("epackage2.txt", res) - check-vars-defined(res, defs(epackage)) - res defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : - println("LOWERING") - ;Reset id generation take-ids(epackage) @@ -209,7 +68,10 @@ defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : ;Dump input ;dump(cur-package, "logs", "input") - run-pass("Prune Package", prune-package, "prune", true) + if optimize? : + spit("epackage1.txt", cur-package) when trace? + run-pass("Prune Package", prune-package, "prune", true) + spit("epackage2.txt", cur-package) when trace? run-pass("Map Methods", map-methods, "mapped-methods", false) run-pass("Create Closures", create-closures, "closures", false) run-pass("Convert Mixes", convert-mixes, "mixes", false) @@ -230,6 +92,9 @@ defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : run-pass("Lift Objects", lift-objects, "objlifted", true) if optimize? : run-pass("Resolve Methods", resolve-methods, "resolved-methods", false) + spit("epackage3.txt", cur-package) when trace? + run-pass("Prune Package 2", prune-package, "prune2", true) + spit("epackage4.txt", cur-package) when trace? run-pass("Lift Closures", lift-closures, "closurelifted", false) run-pass("Lift Type Objects", lift-type-objects, "typelifted", false) @@ -275,6 +140,211 @@ defn ensure-unique-identifiers! (epackage:EPackage) : print(o, "The following identifiers are declared more than once: %," % [non-unique]) print(o, "\nThe program is as follows:\n%_" % [epackage]) +;============================================================ +;======================= Pruning ============================ +;============================================================ + +val trace? = false + +;; roots that need to be traced +;; globals are there because their values might have side effects +defn roots (epackage:EPackage) -> Seqable : + for e in exps(epackage) filter : + match(e) : + (e:EInit|EDefGlobal|EDefStruct|EExternFn|EDefType|EDefObject|EDefTypeObject) : true ;; EExtern| + (e) : false + +;; definitions that need to be in runtime and aren't traceable from roots +defn predefines (epackage:EPackage) -> Seqable : + val iotable = IOTable(packageio(epackage)) + val predefines-ids = [ + CORE-LIVENESS-TRACKER-ID + CORE-ARITY-ERROR-ID + CORE-NO-METHOD-ERROR-ID + CORE-AMB-METHOD-ERROR-ID + CORE-TUPLE-LENGTH-ERROR-ID + CORE-NO-BRANCH-ERROR-ID + CORE-AMB-BRANCH-ERROR-ID + CORE-CAST-ERROR-ID + CORE-VARIABLE-UNINITIALIZED-ERROR-ID + CORE-INVALID-RETURN-ID + CORE-VOID-TUPLE-ID + CORE-INIT-CONSTS-ID + CORE-EXTEND-HEAP-ID + CORE-EXTEND-STACK-ID + CORE-PRINT-STACK-TRACE-ID + CORE-COLLECT-GARBAGE-ID + CORE-CLASS-NAME-ID + CORE-MAKE-STRING-ID + CORE-EXECUTE-TOPLEVEL-COMMAND-ID] + val predefines = to-intset $ for i in predefines-ids seq : n(iotable, i) + generate : + for e in exps(epackage) do : + match(e) : + (e:EDefn) : yield(e) when predefines[n(e)] + (e) : false + +;; top level definition dictionary by global id +defn defs (epackage:EPackage) -> IntTable : + val tab = IntTable() + for e in exps(epackage) do : + match(e) : + (e:EDefGlobal|EDefn|EDefClosure|EDefmulti|EDefmethod) : tab[n(e)] = e + (e:EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : tab[n(e)] = e + (e) : false + tab + +;; methods for each multi using ids +defn methods (epackage:EPackage) -> IntTable> : + val tab = IntTable>(List()) + for e in exps(epackage) do : + match(e) : + (e:EDefmethod) : tab[multi(e)] = cons(n(e), tab[multi(e)]) + (e) : false + tab + +;; main function for tracing produces set of reachables and defined-objects +defn walk-reachable-defs (e:ETExp, q:Queue, defs:IntTable, methods:IntTable>, visited?:IntSet, reachables:IntSet, defined-objects:IntSet, parents:IntTable) : + match(e) : + (e:EDefGlobal|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : add(reachables, n(e)) + (e) : false + val pid = match(e) : + (e:EDefGlobal|EDefn|EDefClosure|EDefmulti|EDefmethod|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : n(e) + (e) : -1 + let walk (e:ELItem = e) : + defn maybe-walk (id:Int) : + if add(visited?, id) : + if key?(defs, id) : + add(reachables, id) + parents[id] = pid + add(q, defs[id]) + match(e) : + (e:EVar|EVarLoc) : + maybe-walk(n(e)) + if key?(methods, n(e)) : ;; is a multi? + for m in methods[n(e)] do : maybe-walk(m) + (e:EDefObject|ELocalObj) : + add(defined-objects, n(e)) + do(walk, e) + (e:EDefmethod) : + maybe-walk(multi(e)) ;; compiler needs multi to be there if method is reachable + do(walk, e) + (e:ELItem) : + do(walk, e) + +;; find all defined-objects inside package +defn find-defined-objects (epackage:EPackage) -> IntSet : + val defined-objects = IntSet() + defn* walk (e:ELItem) : + match(e) : + (e:EDefObject|ELocalObj) : + add(defined-objects, n(e)) + do(walk, e) + (e) : + do(walk, e) + for exp in exps(epackage) do : + walk(exp) + defined-objects + +;; find all reachable defs and defined-objects and also returns complete set of defined-objects +defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntSet, IntSet, IntTable] : + val defs = defs(epackage) + val methods = methods(epackage) + val q = Queue() + val visited? = IntSet() + val reachables = IntSet() + val parents = IntTable(-1) + val defined-objects = IntSet() + val orig-defined-objects = find-defined-objects(epackage) + for root in roots(epackage) do : + add(q, root) + for predef in predefines(epackage) do : + add(reachables, n(predef)) + add(q, predef) + while not empty?(q) : + val def = pop(q) + walk-reachable-defs(def, q, defs, methods, visited?, reachables, defined-objects, parents) + println("FOUND %_ REACHABLE OUT OF %_ DEFINED" % [length(reachables), length(defs)]) when trace? + println("FOUND %_ DEFINED-OBJECTS OUT OF %_ ORIG-DEFINED-OBJECTS" % [length(defined-objects), length(orig-defined-objects)]) when trace? + [reachables, orig-defined-objects, defined-objects, parents] + +;; walk all top level forms and check that all vars are defined +defn check-vars-defined (epackage:EPackage, odefs:IntTable) : + defn check (e:ETExp, odefs:IntTable, defs:IntTable, undefines:IntSet) : + let walk (e:ELItem = e) : + match(e) : + (e:EVar|EVarLoc) : + if key?(odefs, n(e)) and not key?(defs, n(e)) : + add(undefines, n(e)) + (e:ELItem) : + do(walk, e) + undefines + val defs = defs(epackage) + val undefines = IntSet() + for e in exps(epackage) do : + check(e, odefs, defs, undefines) + println("UNDEFINES %_" % [to-tuple $ qsort $ undefines]) when not empty?(undefines) + +defn prune-package (epackage:EPackage, gvt:VarTable) -> EPackage : + val iotable = IOTable(packageio(epackage)) + val [reachables, orig-defined-objects, defined-objects, parents] = walk-reachable-defs(epackage) + + defn undefined-objects? (e:EType) : + var undefined? = false + let walk (e:ELItem = e) : + match(e) : + (e:EVar) : + if orig-defined-objects[n(e)] and not defined-objects[n(e)] : + undefined? = true + (e) : + do(walk, e) + undefined? + + defn defined? (e:ETExp) : + match(e) : + (e:EDefn|EDefClosure|EDefmulti) : + reachables[n(e)] + (e:EDefmethod) : + val res = reachables[n(e)] and not any?(undefined-objects?, targs(e)) + println("REMOVING %_ -> %_ %_" % [n(e), iotable[multi(e)], targs(e)]) when (trace? and not res) + res + ;; reachables[n(e)] + (e:EDefGlobal|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : + true + (e) : + true + + println(qsort $ reachables) when trace? + + ;; remove all unreachable defs from package + defn prune (e:ETExp, reachables:IntSet) -> ETExp : + defn* walk (e:ELItem) -> ELItem : + match(e) : + (e:ELocalObj) : + val methods = for m in methods(e) filter : reachables[multi(m)] + sub-methods(e, to-tuple $ for m in methods seq : map(walk, m)) + (e) : + map(walk, e) + walk(e) as ETExp + + defn prune-packageio (packageio:PackageIO, defined?:IntSet) -> PackageIO : + PackageIO(package(packageio), imported-packages(packageio), + to-tuple $ for i in imports(packageio) filter : defined?[n(i)], + to-tuple $ for e in exports(packageio) filter : defined?[n(e)]) + + val etexps = for exp in exps(epackage) filter : defined?(exp) + val new-packageio = prune-packageio(packageio(epackage), reachables) + val res = EPackage(new-packageio, to-tuple $ for e in etexps seq : prune(e, reachables)) + val defs = defs(epackage) + val tree = to-list $ for kv in parents seq : List(key(kv), value(kv)) + val symbol-table = to-list $ for ie in cat(imports(new-packageio), exports(new-packageio)) seq : + val n = n(ie) + val rec = rec(ie) + List(n, List(package(id(rec)), name(id(rec)))) + spit("parents.txt", List(symbol-table, tree)) + check-vars-defined(res, defs) + res + ;============================================================ ;===================== Collapsing =========================== ;============================================================ @@ -2457,7 +2527,6 @@ defn MethodTable (epackage:EPackage) : ;Resolve through dag defn resolve-target (class-tree:DynTree, dag-table:IntTable, multi:Int, types:Tuple) -> Int|False : - println("RESOLVING MULTI %_ TYPES %_" % [multi, types]) val mdag = get?(dag-table, multi) match(mdag:MethodDag) : val args = map(etype-to-arg{class-tree, sf, _, false}, types) From b921f4720830be11a1004b3a312a01602a8aa066 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Fri, 20 Dec 2019 15:53:59 -0800 Subject: [PATCH 03/23] remove unused global vars, comment code --- compiler/stz-el.stanza | 180 ++++++++++++++++++++++------------------- 1 file changed, 95 insertions(+), 85 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index 0095cbcb7..a1c257035 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -144,40 +144,28 @@ defn ensure-unique-identifiers! (epackage:EPackage) : ;======================= Pruning ============================ ;============================================================ -val trace? = false +val trace? = true ;; roots that need to be traced ;; globals are there because their values might have side effects +;; +;; the roots and their traceable fields are +;; EInit -> body:EBody +;; EDefStruct -> parent:False|EType, base:Tuple, items:EType|False +;; EEXternFn -> func|EFn +;; EDefType -> parent:False|EType +;; EDefTypeObject -> type:EType +;; EDefGlobal -> type:EType defn roots (epackage:EPackage) -> Seqable : for e in exps(epackage) filter : match(e) : - (e:EInit|EDefGlobal|EDefStruct|EExternFn|EDefType|EDefObject|EDefTypeObject) : true ;; EExtern| + (e:EInit|EDefStruct|EExternFn|EDefType|EDefTypeObject|EDefGlobal) : true ;; EExtern|EDefObject (e) : false ;; definitions that need to be in runtime and aren't traceable from roots defn predefines (epackage:EPackage) -> Seqable : val iotable = IOTable(packageio(epackage)) - val predefines-ids = [ - CORE-LIVENESS-TRACKER-ID - CORE-ARITY-ERROR-ID - CORE-NO-METHOD-ERROR-ID - CORE-AMB-METHOD-ERROR-ID - CORE-TUPLE-LENGTH-ERROR-ID - CORE-NO-BRANCH-ERROR-ID - CORE-AMB-BRANCH-ERROR-ID - CORE-CAST-ERROR-ID - CORE-VARIABLE-UNINITIALIZED-ERROR-ID - CORE-INVALID-RETURN-ID - CORE-VOID-TUPLE-ID - CORE-INIT-CONSTS-ID - CORE-EXTEND-HEAP-ID - CORE-EXTEND-STACK-ID - CORE-PRINT-STACK-TRACE-ID - CORE-COLLECT-GARBAGE-ID - CORE-CLASS-NAME-ID - CORE-MAKE-STRING-ID - CORE-EXECUTE-TOPLEVEL-COMMAND-ID] - val predefines = to-intset $ for i in predefines-ids seq : n(iotable, i) + val predefines = to-intset $ for i in core-ids() seq : n(iotable, i) generate : for e in exps(epackage) do : match(e) : @@ -203,8 +191,40 @@ defn methods (epackage:EPackage) -> IntTable> : (e) : false tab -;; main function for tracing produces set of reachables and defined-objects -defn walk-reachable-defs (e:ETExp, q:Queue, defs:IntTable, methods:IntTable>, visited?:IntSet, reachables:IntSet, defined-objects:IntSet, parents:IntTable) : +;; find global vars with that are not read but may be initialized +defn unused-global-vars (epackage:EPackage) -> IntSet : + val iotable = IOTable(packageio(epackage)) + val num-reads = IntTable(0) + val num-writes = IntTable(0) + val globals = IntSet() + for e in exps(epackage) do : + match(e) : + (e:EDefGlobal) : add(globals, n(e)) + (e) : false + defn* walk (e:ELItem) : + match(e) : + (e:EStore) : + match(loc(e)) : + (l:EVarLoc) : if globals[n(l)] : num-writes[n(l)] = num-writes[n(l)] + 1 + (l) : false + walk(y(e)) + (e:EVar|EVarLoc) : if globals[n(e)] : num-reads[n(e)] = num-reads[n(e)] + 1 + (e) : do(walk, e) + for e in exps(epackage) do : walk(e) + val unused-globals = IntSet() + for gv in to-list $ globals do : + if num-reads[gv] == 0 and num-writes[gv] <= 1 : + ;; println("%_ %_: %_ READS %_ WRITES" % [gv, iotable[gv], num-reads[gv], num-writes[gv]]) when trace? + add(unused-globals, gv) + unused-globals + +;; main function for tracing produces set of reachables and parents for tracing where +;; reachables are traced from program/data roots and +;; parents are ids of parents in trace useful for determining why something is reachable +;; walks +;; o methods when ref to multi is traversed +;; o multi when defmethod is traversed +defn walk-reachable-defs (e:ETExp, q:Queue, defs:IntTable, methods:IntTable>, visited?:IntSet, reachables:IntSet, parents:IntTable) : match(e) : (e:EDefGlobal|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : add(reachables, n(e)) (e) : false @@ -213,49 +233,29 @@ defn walk-reachable-defs (e:ETExp, q:Queue, defs:IntTable, methods (e) : -1 let walk (e:ELItem = e) : defn maybe-walk (id:Int) : - if add(visited?, id) : - if key?(defs, id) : - add(reachables, id) - parents[id] = pid - add(q, defs[id]) + if add(visited?, id) and key?(defs, id) : + add(reachables, id) + parents[id] = pid + add(q, defs[id]) match(e) : (e:EVar|EVarLoc) : maybe-walk(n(e)) if key?(methods, n(e)) : ;; is a multi? for m in methods[n(e)] do : maybe-walk(m) - (e:EDefObject|ELocalObj) : - add(defined-objects, n(e)) - do(walk, e) (e:EDefmethod) : maybe-walk(multi(e)) ;; compiler needs multi to be there if method is reachable do(walk, e) (e:ELItem) : do(walk, e) -;; find all defined-objects inside package -defn find-defined-objects (epackage:EPackage) -> IntSet : - val defined-objects = IntSet() - defn* walk (e:ELItem) : - match(e) : - (e:EDefObject|ELocalObj) : - add(defined-objects, n(e)) - do(walk, e) - (e) : - do(walk, e) - for exp in exps(epackage) do : - walk(exp) - defined-objects - -;; find all reachable defs and defined-objects and also returns complete set of defined-objects -defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntSet, IntSet, IntTable] : +;; find all reachable defs and parents +defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : val defs = defs(epackage) val methods = methods(epackage) val q = Queue() val visited? = IntSet() val reachables = IntSet() val parents = IntTable(-1) - val defined-objects = IntSet() - val orig-defined-objects = find-defined-objects(epackage) for root in roots(epackage) do : add(q, root) for predef in predefines(epackage) do : @@ -263,10 +263,9 @@ defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntSet, IntSet, IntTabl add(q, predef) while not empty?(q) : val def = pop(q) - walk-reachable-defs(def, q, defs, methods, visited?, reachables, defined-objects, parents) + walk-reachable-defs(def, q, defs, methods, visited?, reachables, parents) println("FOUND %_ REACHABLE OUT OF %_ DEFINED" % [length(reachables), length(defs)]) when trace? - println("FOUND %_ DEFINED-OBJECTS OUT OF %_ ORIG-DEFINED-OBJECTS" % [length(defined-objects), length(orig-defined-objects)]) when trace? - [reachables, orig-defined-objects, defined-objects, parents] + [reachables, parents] ;; walk all top level forms and check that all vars are defined defn check-vars-defined (epackage:EPackage, odefs:IntTable) : @@ -285,38 +284,29 @@ defn check-vars-defined (epackage:EPackage, odefs:IntTable) : check(e, odefs, defs, undefines) println("UNDEFINES %_" % [to-tuple $ qsort $ undefines]) when not empty?(undefines) -defn prune-package (epackage:EPackage, gvt:VarTable) -> EPackage : +defn prune-package (epackage:EPackage) -> EPackage : val iotable = IOTable(packageio(epackage)) - val [reachables, orig-defined-objects, defined-objects, parents] = walk-reachable-defs(epackage) + val [reachables, parents] = walk-reachable-defs(epackage) - defn undefined-objects? (e:EType) : - var undefined? = false - let walk (e:ELItem = e) : - match(e) : - (e:EVar) : - if orig-defined-objects[n(e)] and not defined-objects[n(e)] : - undefined? = true - (e) : - do(walk, e) - undefined? + val unused-vars = unused-global-vars(epackage) + + if trace? : + val defs = defs(epackage) + for v in unused-vars do : + println("UNUSED %_ / %_" % [defs[v] iotable[v]]) defn defined? (e:ETExp) : match(e) : - (e:EDefn|EDefClosure|EDefmulti) : + (e:EDefn|EDefClosure|EDefmulti|EDefmethod) : reachables[n(e)] - (e:EDefmethod) : - val res = reachables[n(e)] and not any?(undefined-objects?, targs(e)) - println("REMOVING %_ -> %_ %_" % [n(e), iotable[multi(e)], targs(e)]) when (trace? and not res) - res - ;; reachables[n(e)] - (e:EDefGlobal|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : + (e:EDefGlobal) : + not unused-vars[n(e)] + (e:EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : true (e) : true - println(qsort $ reachables) when trace? - - ;; remove all unreachable defs from package + ;; remove all methods on unreachable multis from package defn prune (e:ETExp, reachables:IntSet) -> ETExp : defn* walk (e:ELItem) -> ELItem : match(e) : @@ -327,22 +317,42 @@ defn prune-package (epackage:EPackage, gvt:VarTable) -> EPackage : map(walk, e) walk(e) as ETExp + ;; remove unreachable imports/exports defn prune-packageio (packageio:PackageIO, defined?:IntSet) -> PackageIO : PackageIO(package(packageio), imported-packages(packageio), to-tuple $ for i in imports(packageio) filter : defined?[n(i)], to-tuple $ for e in exports(packageio) filter : defined?[n(e)]) + ;; remove inits corresponding to unused global vars + defn prune-inits (exps:Seqable) -> Seq : + defn remove-unused-stores (ins:Tuple) -> Seq : + defn unused-store? (i:EIns) -> True|False : + match(i) : + (i:EStore) : + match(loc(i)) : + (l:EVarLoc) : unused-vars[n(l)] + (l) : false + (e) : false + for i in ins filter : not unused-store?(i) + for e in exps seq : + match(e) : + (e:EInit) : sub-body(e, sub-ins(body(e), to-tuple $ remove-unused-stores $ ins(body(e)))) + (e) : e + val etexps = for exp in exps(epackage) filter : defined?(exp) val new-packageio = prune-packageio(packageio(epackage), reachables) - val res = EPackage(new-packageio, to-tuple $ for e in etexps seq : prune(e, reachables)) - val defs = defs(epackage) - val tree = to-list $ for kv in parents seq : List(key(kv), value(kv)) - val symbol-table = to-list $ for ie in cat(imports(new-packageio), exports(new-packageio)) seq : - val n = n(ie) - val rec = rec(ie) - List(n, List(package(id(rec)), name(id(rec)))) - spit("parents.txt", List(symbol-table, tree)) - check-vars-defined(res, defs) + val new-exps = for e in etexps seq : prune(e, reachables) + val res = EPackage(new-packageio, to-tuple $ prune-inits $ new-exps) + + if trace? : + val defs = defs(epackage) + val tree = to-list $ for kv in parents seq : List(key(kv), value(kv)) + val symbol-table = to-list $ for ie in cat(imports(new-packageio), exports(new-packageio)) seq : + val n = n(ie) + val rec = rec(ie) + List(n, List(package(id(rec)), name(id(rec)))) + spit("parents.txt", List(symbol-table, tree)) + check-vars-defined(res, defs) res ;============================================================ From 71a771b3d8c21510b953595742846563888c158f Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Thu, 2 Jan 2020 15:44:07 -0800 Subject: [PATCH 04/23] clean up global var pruning and simplify rest of tree shaker --- compiler/stz-el.stanza | 61 ++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index a1c257035..7abbd4fb6 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -144,7 +144,7 @@ defn ensure-unique-identifiers! (epackage:EPackage) : ;======================= Pruning ============================ ;============================================================ -val trace? = true +val trace? = false ;; roots that need to be traced ;; globals are there because their values might have side effects @@ -191,32 +191,24 @@ defn methods (epackage:EPackage) -> IntTable> : (e) : false tab -;; find global vars with that are not read but may be initialized +;; find global vars with that are not read but are written such as in initialization code defn unused-global-vars (epackage:EPackage) -> IntSet : val iotable = IOTable(packageio(epackage)) val num-reads = IntTable(0) - val num-writes = IntTable(0) val globals = IntSet() for e in exps(epackage) do : - match(e) : - (e:EDefGlobal) : add(globals, n(e)) - (e) : false + match(e:EDefGlobal) : add(globals, n(e)) defn* walk (e:ELItem) : match(e) : (e:EStore) : match(loc(e)) : - (l:EVarLoc) : if globals[n(l)] : num-writes[n(l)] = num-writes[n(l)] + 1 - (l) : false + (l:EVarLoc) : if not globals[n(l)] : walk(loc(e)) ;; avoid counting reads in a write + (l) : walk(loc(e)) walk(y(e)) (e:EVar|EVarLoc) : if globals[n(e)] : num-reads[n(e)] = num-reads[n(e)] + 1 (e) : do(walk, e) for e in exps(epackage) do : walk(e) - val unused-globals = IntSet() - for gv in to-list $ globals do : - if num-reads[gv] == 0 and num-writes[gv] <= 1 : - ;; println("%_ %_: %_ READS %_ WRITES" % [gv, iotable[gv], num-reads[gv], num-writes[gv]]) when trace? - add(unused-globals, gv) - unused-globals + to-intset $ for gv in globals filter : num-reads[gv] == 0 ;; main function for tracing produces set of reachables and parents for tracing where ;; reachables are traced from program/data roots and @@ -238,6 +230,9 @@ defn walk-reachable-defs (e:ETExp, q:Queue, defs:IntTable, methods parents[id] = pid add(q, defs[id]) match(e) : + (e:EStore) : + if loc(e) is-not EVarLoc : walk(loc(e)) + walk(y(e)) (e:EVar|EVarLoc) : maybe-walk(n(e)) if key?(methods, n(e)) : ;; is a multi? @@ -323,28 +318,36 @@ defn prune-package (epackage:EPackage) -> EPackage : to-tuple $ for i in imports(packageio) filter : defined?[n(i)], to-tuple $ for e in exports(packageio) filter : defined?[n(e)]) - ;; remove inits corresponding to unused global vars - defn prune-inits (exps:Seqable) -> Seq : - defn remove-unused-stores (ins:Tuple) -> Seq : - defn unused-store? (i:EIns) -> True|False : - match(i) : - (i:EStore) : - match(loc(i)) : - (l:EVarLoc) : unused-vars[n(l)] - (l) : false - (e) : false - for i in ins filter : not unused-store?(i) - for e in exps seq : + ;; remove store instructions that are on vars that are only written to not read from + defn remove-unused-var-stores (exps:Seqable) -> Seq : + defn* walk (e:ELItem) -> ELItem : + defn remove-unused-stores (ins:Tuple) -> Seq : + for i in ins filter : + match(i) : + (i:EStore) : + match(loc(i)) : + (l:EVarLoc) : not unused-vars[n(l)] + (l) : true + (e) : true match(e) : - (e:EInit) : sub-body(e, sub-ins(body(e), to-tuple $ remove-unused-stores $ ins(body(e)))) - (e) : e + (e:EType|EDefGlobal|EDefmulti|EDefStruct|EExtern|EDefType|EDefObject|EDefTypeObject) : + e + (e:EBody) : + EBody(map(walk, locals(e)) as Tuple, map(walk, localtypes(e)) as Tuple, + map(walk, localfns(e)) as Tuple, map(walk, localobjs(e)) as Tuple, + (to-tuple $ seq(walk, remove-unused-stores(ins(e)))) as Tuple) + (e) : + map(walk, e) + val res = for e in exps seq : walk(e) + res as Seq val etexps = for exp in exps(epackage) filter : defined?(exp) val new-packageio = prune-packageio(packageio(epackage), reachables) val new-exps = for e in etexps seq : prune(e, reachables) - val res = EPackage(new-packageio, to-tuple $ prune-inits $ new-exps) + val res = EPackage(new-packageio, to-tuple $ remove-unused-var-stores $ new-exps) if trace? : + ;; write out file that can be used to find trace path to variable to understand why it's held onto val defs = defs(epackage) val tree = to-list $ for kv in parents seq : List(key(kv), value(kv)) val symbol-table = to-list $ for ie in cat(imports(new-packageio), exports(new-packageio)) seq : From 1909fc5c9c062c64d3b004e0338544efab695011 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Thu, 2 Jan 2020 15:49:23 -0800 Subject: [PATCH 05/23] remove check because when methods are removed with tree-shaking, it could result in an empty dispatch --- compiler/stz-dispatch-dag.stanza | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/stz-dispatch-dag.stanza b/compiler/stz-dispatch-dag.stanza index 10bee7e62..0d0f38620 100644 --- a/compiler/stz-dispatch-dag.stanza +++ b/compiler/stz-dispatch-dag.stanza @@ -9,7 +9,7 @@ defpackage stz/dispatch-dag : public deftype Arg <: Hashable & Equalable public defstruct Top <: Arg public defstruct Nums <: Arg : - values: Tuple with: (ensure => non-empty!) + values: Tuple ;; with: (ensure => non-empty!) public defstruct Branch : args: Tuple From 644660e4e3a1f84ac64544ef64e87ed4ea3e7de9 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Thu, 2 Jan 2020 16:07:07 -0800 Subject: [PATCH 06/23] better comments on tree-shaker --- compiler/stz-el.stanza | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index 7abbd4fb6..f120fe608 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -213,9 +213,20 @@ defn unused-global-vars (epackage:EPackage) -> IntSet : ;; main function for tracing produces set of reachables and parents for tracing where ;; reachables are traced from program/data roots and ;; parents are ids of parents in trace useful for determining why something is reachable +;; inputs +;; o defs -- id to ETExp +;; o methods -- multi id to list of method ids ;; walks ;; o methods when ref to multi is traversed ;; o multi when defmethod is traversed +;; collects +;; o reachables -- all traced ids +;; o parents -- ids of parent ETExp in trace (for use in determining why something is held onto by tracing to roots) +;; using +;; o q -- for walking +;; o visited? -- for not walking things twice +;; where this is referenced is from +;; o EVar and EVarLoc or through walking rules above defn walk-reachable-defs (e:ETExp, q:Queue, defs:IntTable, methods:IntTable>, visited?:IntSet, reachables:IntSet, parents:IntTable) : match(e) : (e:EDefGlobal|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : add(reachables, n(e)) @@ -243,14 +254,17 @@ defn walk-reachable-defs (e:ETExp, q:Queue, defs:IntTable, methods (e:ELItem) : do(walk, e) -;; find all reachable defs and parents +;; find all reachable defs and parents using walk-reachable-defs +;; returns: +;; o set of reachable ids traced from roots of package +;; o parents of reachables for tracing path back to roots defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : val defs = defs(epackage) val methods = methods(epackage) val q = Queue() val visited? = IntSet() val reachables = IntSet() - val parents = IntTable(-1) + val parents = IntTable(-1) ;; parents ids in trace for root in roots(epackage) do : add(q, root) for predef in predefines(epackage) do : @@ -279,6 +293,7 @@ defn check-vars-defined (epackage:EPackage, odefs:IntTable) : check(e, odefs, defs, undefines) println("UNDEFINES %_" % [to-tuple $ qsort $ undefines]) when not empty?(undefines) +;; top level for tree shaking package removing all unreachable vars and all unread global vars defn prune-package (epackage:EPackage) -> EPackage : val iotable = IOTable(packageio(epackage)) val [reachables, parents] = walk-reachable-defs(epackage) From 72524c4b78bd2b7990547bfc910f8646c4993483 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Thu, 2 Jan 2020 16:38:55 -0800 Subject: [PATCH 07/23] use intset for reads on global vars --- compiler/stz-el.stanza | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index f120fe608..ee98d52e3 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -194,7 +194,7 @@ defn methods (epackage:EPackage) -> IntTable> : ;; find global vars with that are not read but are written such as in initialization code defn unused-global-vars (epackage:EPackage) -> IntSet : val iotable = IOTable(packageio(epackage)) - val num-reads = IntTable(0) + val read? = IntSet() val globals = IntSet() for e in exps(epackage) do : match(e:EDefGlobal) : add(globals, n(e)) @@ -202,13 +202,13 @@ defn unused-global-vars (epackage:EPackage) -> IntSet : match(e) : (e:EStore) : match(loc(e)) : - (l:EVarLoc) : if not globals[n(l)] : walk(loc(e)) ;; avoid counting reads in a write + (l:EVarLoc) : if not globals[n(l)] : walk(loc(e)) ;; avoid counting global var reads in a write (l) : walk(loc(e)) walk(y(e)) - (e:EVar|EVarLoc) : if globals[n(e)] : num-reads[n(e)] = num-reads[n(e)] + 1 + (e:EVar|EVarLoc) : add(read?, n(e)) when globals[n(e)] (e) : do(walk, e) for e in exps(epackage) do : walk(e) - to-intset $ for gv in globals filter : num-reads[gv] == 0 + to-intset $ for gv in globals filter : not read?[gv] ;; main function for tracing produces set of reachables and parents for tracing where ;; reachables are traced from program/data roots and From 54f75cea2b90d9b22b6a63243525984b1be656df Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Fri, 3 Jan 2020 12:00:01 -0800 Subject: [PATCH 08/23] cleanup up tree-shaker with consolidation and inlining and better docs --- compiler/stz-el.stanza | 206 +++++++++++++++++------------------------ 1 file changed, 87 insertions(+), 119 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index ee98d52e3..5b08e16f7 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -69,9 +69,9 @@ defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : ;Dump input ;dump(cur-package, "logs", "input") if optimize? : - spit("epackage1.txt", cur-package) when trace? + spit("epackage1.txt", cur-package) when prune-trace? run-pass("Prune Package", prune-package, "prune", true) - spit("epackage2.txt", cur-package) when trace? + spit("epackage2.txt", cur-package) when prune-trace? run-pass("Map Methods", map-methods, "mapped-methods", false) run-pass("Create Closures", create-closures, "closures", false) run-pass("Convert Mixes", convert-mixes, "mixes", false) @@ -92,9 +92,9 @@ defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : run-pass("Lift Objects", lift-objects, "objlifted", true) if optimize? : run-pass("Resolve Methods", resolve-methods, "resolved-methods", false) - spit("epackage3.txt", cur-package) when trace? + spit("epackage3.txt", cur-package) when prune-trace? run-pass("Prune Package 2", prune-package, "prune2", true) - spit("epackage4.txt", cur-package) when trace? + spit("epackage4.txt", cur-package) when prune-trace? run-pass("Lift Closures", lift-closures, "closurelifted", false) run-pass("Lift Type Objects", lift-type-objects, "typelifted", false) @@ -144,35 +144,16 @@ defn ensure-unique-identifiers! (epackage:EPackage) : ;======================= Pruning ============================ ;============================================================ -val trace? = false - -;; roots that need to be traced -;; globals are there because their values might have side effects -;; -;; the roots and their traceable fields are -;; EInit -> body:EBody -;; EDefStruct -> parent:False|EType, base:Tuple, items:EType|False -;; EEXternFn -> func|EFn -;; EDefType -> parent:False|EType -;; EDefTypeObject -> type:EType -;; EDefGlobal -> type:EType -defn roots (epackage:EPackage) -> Seqable : - for e in exps(epackage) filter : - match(e) : - (e:EInit|EDefStruct|EExternFn|EDefType|EDefTypeObject|EDefGlobal) : true ;; EExtern|EDefObject - (e) : false +val prune-trace? = false ;; definitions that need to be in runtime and aren't traceable from roots defn predefines (epackage:EPackage) -> Seqable : val iotable = IOTable(packageio(epackage)) val predefines = to-intset $ for i in core-ids() seq : n(iotable, i) generate : - for e in exps(epackage) do : - match(e) : - (e:EDefn) : yield(e) when predefines[n(e)] - (e) : false + for e in exps(epackage) do : match(e:EDefn) : yield(e) when predefines[n(e)] -;; top level definition dictionary by global id +;; ids to definitions table defn defs (epackage:EPackage) -> IntTable : val tab = IntTable() for e in exps(epackage) do : @@ -182,101 +163,71 @@ defn defs (epackage:EPackage) -> IntTable : (e) : false tab -;; methods for each multi using ids -defn methods (epackage:EPackage) -> IntTable> : - val tab = IntTable>(List()) - for e in exps(epackage) do : - match(e) : - (e:EDefmethod) : tab[multi(e)] = cons(n(e), tab[multi(e)]) - (e) : false - tab - -;; find global vars with that are not read but are written such as in initialization code -defn unused-global-vars (epackage:EPackage) -> IntSet : - val iotable = IOTable(packageio(epackage)) - val read? = IntSet() - val globals = IntSet() - for e in exps(epackage) do : - match(e:EDefGlobal) : add(globals, n(e)) - defn* walk (e:ELItem) : - match(e) : - (e:EStore) : - match(loc(e)) : - (l:EVarLoc) : if not globals[n(l)] : walk(loc(e)) ;; avoid counting global var reads in a write - (l) : walk(loc(e)) - walk(y(e)) - (e:EVar|EVarLoc) : add(read?, n(e)) when globals[n(e)] - (e) : do(walk, e) - for e in exps(epackage) do : walk(e) - to-intset $ for gv in globals filter : not read?[gv] - -;; main function for tracing produces set of reachables and parents for tracing where -;; reachables are traced from program/data roots and -;; parents are ids of parents in trace useful for determining why something is reachable -;; inputs -;; o defs -- id to ETExp -;; o methods -- multi id to list of method ids -;; walks -;; o methods when ref to multi is traversed -;; o multi when defmethod is traversed -;; collects -;; o reachables -- all traced ids -;; o parents -- ids of parent ETExp in trace (for use in determining why something is held onto by tracing to roots) -;; using -;; o q -- for walking -;; o visited? -- for not walking things twice -;; where this is referenced is from -;; o EVar and EVarLoc or through walking rules above -defn walk-reachable-defs (e:ETExp, q:Queue, defs:IntTable, methods:IntTable>, visited?:IntSet, reachables:IntSet, parents:IntTable) : - match(e) : - (e:EDefGlobal|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : add(reachables, n(e)) - (e) : false - val pid = match(e) : - (e:EDefGlobal|EDefn|EDefClosure|EDefmulti|EDefmethod|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : n(e) - (e) : -1 - let walk (e:ELItem = e) : - defn maybe-walk (id:Int) : - if add(visited?, id) and key?(defs, id) : - add(reachables, id) - parents[id] = pid - add(q, defs[id]) - match(e) : - (e:EStore) : - if loc(e) is-not EVarLoc : walk(loc(e)) - walk(y(e)) - (e:EVar|EVarLoc) : - maybe-walk(n(e)) - if key?(methods, n(e)) : ;; is a multi? - for m in methods[n(e)] do : maybe-walk(m) - (e:EDefmethod) : - maybe-walk(multi(e)) ;; compiler needs multi to be there if method is reachable - do(walk, e) - (e:ELItem) : - do(walk, e) - -;; find all reachable defs and parents using walk-reachable-defs +;; find all reachable defs from program/data roots of package and parents ;; returns: ;; o set of reachable ids traced from roots of package -;; o parents of reachables for tracing path back to roots +;; o parents of reachables for tracing path back to roots useful for determining why something is reachable +;; where is referenced is +;; o EVar and EVarLoc or through walking rules above +;; and in addition walks +;; o methods when ref to multi is traversed +;; o multi when defmethod is traversed +;; the roots and their traceable fields are +;; EInit -> body:EBody +;; EDefStruct -> parent:False|EType, base:Tuple, items:EType|False +;; EEXternFn -> func|EFn +;; EDefType -> parent:False|EType +;; EDefTypeObject -> type:EType +;; EDefGlobal -> type:EType ;; because inits might have side effects + defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : - val defs = defs(epackage) - val methods = methods(epackage) - val q = Queue() - val visited? = IntSet() - val reachables = IntSet() + val defs = defs(epackage) ;; id to ETExp's + val methods = IntTable>(List()) ;; multi ids to list of their method ids + for e in exps(epackage) do : + match(e:EDefmethod) : methods[multi(e)] = cons(n(e), methods[multi(e)]) + val q = Queue() ;; for walking todo list + val visited? = IntSet() ;; for not walking things twice + val reachables = IntSet() ;; all ids traced val parents = IntTable(-1) ;; parents ids in trace - for root in roots(epackage) do : - add(q, root) + defn walk-top (e:ETExp) : + match(e) : + (e:EDefGlobal|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : add(reachables, n(e)) + (e) : false + val pid = match(e) : + (e:EDefGlobal|EDefn|EDefClosure|EDefmulti|EDefmethod|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : n(e) + (e) : -1 + let walk (e:ELItem = e) : + defn maybe-walk (id:Int) : + if add(visited?, id) and key?(defs, id) : + add(reachables, id) + parents[id] = pid + add(q, defs[id]) + match(e) : + (e:EStore) : + if loc(e) is-not EVarLoc : walk(loc(e)) + walk(y(e)) + (e:EVar|EVarLoc) : + maybe-walk(n(e)) + if key?(methods, n(e)) : ;; is a multi? + for m in methods[n(e)] do : maybe-walk(m) + (e:ELItem) : + match(e:EDefmethod) : maybe-walk(multi(e)) ;; compiler needs multi to be there if method is reachable + do(walk, e) + val roots = for e in exps(epackage) filter : + match(e) : + (e:EInit|EDefStruct|EExternFn|EDefType|EDefTypeObject|EDefGlobal) : true ;; EExtern|EDefObject + (e) : false + for root in roots do : add(q, root) for predef in predefines(epackage) do : add(reachables, n(predef)) add(q, predef) while not empty?(q) : val def = pop(q) - walk-reachable-defs(def, q, defs, methods, visited?, reachables, parents) - println("FOUND %_ REACHABLE OUT OF %_ DEFINED" % [length(reachables), length(defs)]) when trace? + walk-top(def) + println("FOUND %_ REACHABLE OUT OF %_ DEFINED" % [length(reachables), length(defs)]) when prune-trace? [reachables, parents] -;; walk all top level forms and check that all vars are defined +;; walk all top level forms and check that all vars are defined -- only needed in debug mode defn check-vars-defined (epackage:EPackage, odefs:IntTable) : defn check (e:ETExp, odefs:IntTable, defs:IntTable, undefines:IntSet) : let walk (e:ELItem = e) : @@ -287,20 +238,37 @@ defn check-vars-defined (epackage:EPackage, odefs:IntTable) : (e:ELItem) : do(walk, e) undefines - val defs = defs(epackage) val undefines = IntSet() + val defs = defs(epackage) for e in exps(epackage) do : check(e, odefs, defs, undefines) println("UNDEFINES %_" % [to-tuple $ qsort $ undefines]) when not empty?(undefines) +;; find global vars with that are not read but are written such as in initialization code +defn unused-global-vars (epackage:EPackage) -> IntSet : + val iotable = IOTable(packageio(epackage)) + val read? = IntSet() + val globals = IntSet() + for e in exps(epackage) do : + match(e:EDefGlobal) : add(globals, n(e)) + defn* walk (e:ELItem) : + match(e) : + (e:EStore) : + match(loc(e)) : + (l:EVarLoc) : if not globals[n(l)] : walk(loc(e)) ;; avoid counting global var reads in a write + (l) : walk(loc(e)) + walk(y(e)) + (e:EVar|EVarLoc) : add(read?, n(e)) when globals[n(e)] + (e) : do(walk, e) + for e in exps(epackage) do : walk(e) + to-intset $ for gv in globals filter : not read?[gv] + ;; top level for tree shaking package removing all unreachable vars and all unread global vars defn prune-package (epackage:EPackage) -> EPackage : val iotable = IOTable(packageio(epackage)) val [reachables, parents] = walk-reachable-defs(epackage) - val unused-vars = unused-global-vars(epackage) - - if trace? : + if prune-trace? : val defs = defs(epackage) for v in unused-vars do : println("UNUSED %_ / %_" % [defs[v] iotable[v]]) @@ -317,7 +285,7 @@ defn prune-package (epackage:EPackage) -> EPackage : true ;; remove all methods on unreachable multis from package - defn prune (e:ETExp, reachables:IntSet) -> ETExp : + defn prune-methods (e:ETExp, reachables:IntSet) -> ETExp : defn* walk (e:ELItem) -> ELItem : match(e) : (e:ELocalObj) : @@ -356,12 +324,12 @@ defn prune-package (epackage:EPackage) -> EPackage : val res = for e in exps seq : walk(e) res as Seq - val etexps = for exp in exps(epackage) filter : defined?(exp) + ;; top-level work val new-packageio = prune-packageio(packageio(epackage), reachables) - val new-exps = for e in etexps seq : prune(e, reachables) - val res = EPackage(new-packageio, to-tuple $ remove-unused-var-stores $ new-exps) + val new-exps = remove-unused-var-stores $ for e in filter(defined?, exps(epackage)) seq : prune-methods(e, reachables) + val res = EPackage(new-packageio, to-tuple $ new-exps) - if trace? : + if prune-trace? : ;; write out file that can be used to find trace path to variable to understand why it's held onto val defs = defs(epackage) val tree = to-list $ for kv in parents seq : List(key(kv), value(kv)) From b86e4148bd57eb80d5d376f18695d08596051365 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 4 Feb 2020 14:15:08 -0800 Subject: [PATCH 09/23] remove some comments debug code --- compiler/stz-el.stanza | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index 5b08e16f7..bc2648f56 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -33,9 +33,6 @@ public defn lower-unoptimized (epackage:EPackage) -> EPackage : ;========================= Lowering ========================= ;============================================================ -;; how to print out id table? - - defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : ;Reset id generation take-ids(epackage) @@ -69,9 +66,7 @@ defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : ;Dump input ;dump(cur-package, "logs", "input") if optimize? : - spit("epackage1.txt", cur-package) when prune-trace? run-pass("Prune Package", prune-package, "prune", true) - spit("epackage2.txt", cur-package) when prune-trace? run-pass("Map Methods", map-methods, "mapped-methods", false) run-pass("Create Closures", create-closures, "closures", false) run-pass("Convert Mixes", convert-mixes, "mixes", false) @@ -92,9 +87,7 @@ defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : run-pass("Lift Objects", lift-objects, "objlifted", true) if optimize? : run-pass("Resolve Methods", resolve-methods, "resolved-methods", false) - spit("epackage3.txt", cur-package) when prune-trace? run-pass("Prune Package 2", prune-package, "prune2", true) - spit("epackage4.txt", cur-package) when prune-trace? run-pass("Lift Closures", lift-closures, "closurelifted", false) run-pass("Lift Type Objects", lift-type-objects, "typelifted", false) @@ -3164,4 +3157,4 @@ defn has-tvar? (t:EType) : tvar? public defn select (xs:Tuple, mask:Tuple) -> Tuple : - to-tuple(filter(xs, mask)) \ No newline at end of file + to-tuple(filter(xs, mask)) From 3501e5fa59b65b56aa1923eb1e6c0ca179e1c926 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 4 Feb 2020 14:18:01 -0800 Subject: [PATCH 10/23] patrick edits --- compiler/stz-el.stanza | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index bc2648f56..ba82be50a 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -142,19 +142,17 @@ val prune-trace? = false ;; definitions that need to be in runtime and aren't traceable from roots defn predefines (epackage:EPackage) -> Seqable : val iotable = IOTable(packageio(epackage)) - val predefines = to-intset $ for i in core-ids() seq : n(iotable, i) - generate : - for e in exps(epackage) do : match(e:EDefn) : yield(e) when predefines[n(e)] + for e in exps(epackage) filter : + match(e:EDefn) : predefines[n(e)] ;; ids to definitions table defn defs (epackage:EPackage) -> IntTable : - val tab = IntTable() - for e in exps(epackage) do : - match(e) : - (e:EDefGlobal|EDefn|EDefClosure|EDefmulti|EDefmethod) : tab[n(e)] = e - (e:EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : tab[n(e)] = e - (e) : false - tab + to-inttable $ + for e in exps(epackage) seq? : + match(e:EDefGlobal|EDefn|EDefClosure|EDefmulti|EDefmethod| + EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : + One(n(e) => e) + else : None() ;; find all reachable defs from program/data roots of package and parents ;; returns: From 634afca1a92e45fefc3cbcd256f9414e7f5ed4fc Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Wed, 5 Feb 2020 17:34:17 -0800 Subject: [PATCH 11/23] add identifier? and clean up code --- compiler/stz-el.stanza | 51 +++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index ba82be50a..1360de21b 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -139,37 +139,48 @@ defn ensure-unique-identifiers! (epackage:EPackage) : val prune-trace? = false +;Retrieve identifier of top-level declaration if it has one. +defn identifier? (e:ETExp) -> Int|False : + match(e:EDefGlobal|EDefn|EDefClosure|EDefmulti|EDefmethod|EDefStruct| + EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : + n(e) + ;; definitions that need to be in runtime and aren't traceable from roots defn predefines (epackage:EPackage) -> Seqable : val iotable = IOTable(packageio(epackage)) + val predefines = to-intset $ for i in predefines-ids seq : n(iotable, i) for e in exps(epackage) filter : - match(e:EDefn) : predefines[n(e)] + val n = identifier?(e) + match(n:Int) : predefines[n] ;; ids to definitions table defn defs (epackage:EPackage) -> IntTable : to-inttable $ for e in exps(epackage) seq? : - match(e:EDefGlobal|EDefn|EDefClosure|EDefmulti|EDefmethod| - EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : - One(n(e) => e) + val n = identifier?(e) + match(n:Int) : One(n => e) else : None() -;; find all reachable defs from program/data roots of package and parents -;; returns: -;; o set of reachable ids traced from roots of package -;; o parents of reachables for tracing path back to roots useful for determining why something is reachable -;; where is referenced is -;; o EVar and EVarLoc or through walking rules above -;; and in addition walks -;; o methods when ref to multi is traversed -;; o multi when defmethod is traversed -;; the roots and their traceable fields are -;; EInit -> body:EBody -;; EDefStruct -> parent:False|EType, base:Tuple, items:EType|False -;; EEXternFn -> func|EFn -;; EDefType -> parent:False|EType -;; EDefTypeObject -> type:EType -;; EDefGlobal -> type:EType ;; because inits might have side effects +;Find all reachable defs from program/data roots of package and parents. +;Returns: +; - Set of reachable ids traced from roots of package. +; - Parent of each reachable id. (Used for determining why something was reached.) +; +;Algorithm for reachability: +; - All EVar and EVarLoc objects are reachable if they are contained within a +; reachable definition. +; - All methods are reachable when its multi is reachable. +; - A multi is reachable when any of its methods is reachable. (Necessary for after +; resolve-methods pass, as methods can be referred to directly). +;The following top-level declarations are considered roots, and their associated +;tracable fields are: +; EInit -> body:EBody (This is always evaluated.) +; EExternFn -> func|EFn +; --- +; EDefStruct -> parent:False|EType, base:Tuple, items:EType|False +; EDefType -> parent:False|EType +; EDefTypeObject -> type:EType +; EDefGlobal -> type:EType (Initial values are evaluated.) defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : val defs = defs(epackage) ;; id to ETExp's From 46adce0fa03b3ade80f78e79fbb56c591fd0eddb Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Thu, 6 Feb 2020 11:31:36 -0800 Subject: [PATCH 12/23] clean up code -- wip --- compiler/stz-el.stanza | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index 1360de21b..a66dd71cd 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -192,12 +192,9 @@ defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : val reachables = IntSet() ;; all ids traced val parents = IntTable(-1) ;; parents ids in trace defn walk-top (e:ETExp) : - match(e) : - (e:EDefGlobal|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : add(reachables, n(e)) - (e) : false - val pid = match(e) : - (e:EDefGlobal|EDefn|EDefClosure|EDefmulti|EDefmethod|EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : n(e) - (e) : -1 + val n = identifier?(e) + match(n:Int) : add(reachables, n) + val pid = match(n:Int) : n let walk (e:ELItem = e) : defn maybe-walk (id:Int) : if add(visited?, id) and key?(defs, id) : @@ -210,22 +207,16 @@ defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : walk(y(e)) (e:EVar|EVarLoc) : maybe-walk(n(e)) - if key?(methods, n(e)) : ;; is a multi? - for m in methods[n(e)] do : maybe-walk(m) + (e:EDefmulti) : + do(maybe-walk, methods[n(e)]) + (e:EDefmethod) : + maybe-walk(multi(e)) + do(walk, e) (e:ELItem) : - match(e:EDefmethod) : maybe-walk(multi(e)) ;; compiler needs multi to be there if method is reachable do(walk, e) - val roots = for e in exps(epackage) filter : - match(e) : - (e:EInit|EDefStruct|EExternFn|EDefType|EDefTypeObject|EDefGlobal) : true ;; EExtern|EDefObject - (e) : false - for root in roots do : add(q, root) - for predef in predefines(epackage) do : - add(reachables, n(predef)) - add(q, predef) - while not empty?(q) : - val def = pop(q) - walk-top(def) + do(walk-top, filter-by(exps(epackage))) + do(walk-top, predefines(epackage)) + while not empty?(q) : walk-top(pop(q)) println("FOUND %_ REACHABLE OUT OF %_ DEFINED" % [length(reachables), length(defs)]) when prune-trace? [reachables, parents] From 09a6edede7d6794bf92cf202d1b4fd7edb2ea184 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Thu, 6 Feb 2020 17:13:51 -0800 Subject: [PATCH 13/23] working with precise tracing of types and slimmer roots --- compiler/stz-el.stanza | 123 +++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 66 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index a66dd71cd..81dc80e64 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -66,7 +66,9 @@ defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : ;Dump input ;dump(cur-package, "logs", "input") if optimize? : + ;; spit("out0.txt", cur-package) run-pass("Prune Package", prune-package, "prune", true) + ;; spit("out1.txt", cur-package) run-pass("Map Methods", map-methods, "mapped-methods", false) run-pass("Create Closures", create-closures, "closures", false) run-pass("Convert Mixes", convert-mixes, "mixes", false) @@ -87,10 +89,13 @@ defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : run-pass("Lift Objects", lift-objects, "objlifted", true) if optimize? : run-pass("Resolve Methods", resolve-methods, "resolved-methods", false) + ;; spit("out2.txt", cur-package) run-pass("Prune Package 2", prune-package, "prune2", true) + ;; spit("out3.txt", cur-package) run-pass("Lift Closures", lift-closures, "closurelifted", false) run-pass("Lift Type Objects", lift-type-objects, "typelifted", false) + spit("out.txt", cur-package) ;Return processed package cur-package @@ -137,7 +142,7 @@ defn ensure-unique-identifiers! (epackage:EPackage) : ;======================= Pruning ============================ ;============================================================ -val prune-trace? = false +val prune-trace? = true ;Retrieve identifier of top-level declaration if it has one. defn identifier? (e:ETExp) -> Int|False : @@ -146,16 +151,16 @@ defn identifier? (e:ETExp) -> Int|False : n(e) ;; definitions that need to be in runtime and aren't traceable from roots -defn predefines (epackage:EPackage) -> Seqable : +defn predefines (epackage:EPackage) -> Seqable : val iotable = IOTable(packageio(epackage)) - val predefines = to-intset $ for i in predefines-ids seq : n(iotable, i) + val predefines = to-intset $ for id in core-ids() seq : n(iotable, id) for e in exps(epackage) filter : val n = identifier?(e) match(n:Int) : predefines[n] ;; ids to definitions table defn defs (epackage:EPackage) -> IntTable : - to-inttable $ + to-inttable $ for e in exps(epackage) seq? : val n = identifier?(e) match(n:Int) : One(n => e) @@ -183,18 +188,18 @@ defn defs (epackage:EPackage) -> IntTable : ; EDefGlobal -> type:EType (Initial values are evaluated.) defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : - val defs = defs(epackage) ;; id to ETExp's - val methods = IntTable>(List()) ;; multi ids to list of their method ids + val defs = defs(epackage) ;id to ETExp's + val method-ids = IntTable>(List()) ;multi ids to list of their method ids for e in exps(epackage) do : - match(e:EDefmethod) : methods[multi(e)] = cons(n(e), methods[multi(e)]) - val q = Queue() ;; for walking todo list - val visited? = IntSet() ;; for not walking things twice - val reachables = IntSet() ;; all ids traced - val parents = IntTable(-1) ;; parents ids in trace + match(e:EDefmethod) : method-ids[multi(e)] = cons(n(e), method-ids[multi(e)]) + val q = Queue() ;for walking todo list + val visited? = IntSet() ;for not walking things twice + val reachables = IntSet() ;all ids traced + val parents = IntTable(-1) ;parents ids in trace defn walk-top (e:ETExp) : - val n = identifier?(e) - match(n:Int) : add(reachables, n) - val pid = match(n:Int) : n + val pid = match(identifier?(e)) : + (n:Int) : (add(reachables, n), n) + (f:False) : -1 let walk (e:ELItem = e) : defn maybe-walk (id:Int) : if add(visited?, id) and key?(defs, id) : @@ -207,16 +212,28 @@ defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : walk(y(e)) (e:EVar|EVarLoc) : maybe-walk(n(e)) + (e:EStructT|EOf|ETVar|EField|ETVarLoc) : + maybe-walk(n(e)) + (e:ENew|EObject|EObject|EArray|EStruct|ETypeObject|ENewObject|EObjectGet|EObjectTGet| + EConstClosure|EClosureGet|EClosureTGet|EClosure) : + maybe-walk(n(e)) + do(walk, e) + (e:EDefClosure) : + maybe-walk(closure(e)) + do(walk, e) (e:EDefmulti) : - do(maybe-walk, methods[n(e)]) - (e:EDefmethod) : - maybe-walk(multi(e)) + do(maybe-walk, method-ids[n(e)]) do(walk, e) + ;; (e:EDefmethod|EMethod) : + ;; maybe-walk(multi(e)) + ;; do(walk, e) (e:ELItem) : do(walk, e) - do(walk-top, filter-by(exps(epackage))) + + do(walk-top, filter-by(exps(epackage))) ; |EDefStruct|EDefType|EDefTypeObject do(walk-top, predefines(epackage)) while not empty?(q) : walk-top(pop(q)) + println("FOUND %_ REACHABLE OUT OF %_ DEFINED" % [length(reachables), length(defs)]) when prune-trace? [reachables, parents] @@ -237,55 +254,25 @@ defn check-vars-defined (epackage:EPackage, odefs:IntTable) : check(e, odefs, defs, undefines) println("UNDEFINES %_" % [to-tuple $ qsort $ undefines]) when not empty?(undefines) -;; find global vars with that are not read but are written such as in initialization code -defn unused-global-vars (epackage:EPackage) -> IntSet : - val iotable = IOTable(packageio(epackage)) - val read? = IntSet() - val globals = IntSet() - for e in exps(epackage) do : - match(e:EDefGlobal) : add(globals, n(e)) - defn* walk (e:ELItem) : - match(e) : - (e:EStore) : - match(loc(e)) : - (l:EVarLoc) : if not globals[n(l)] : walk(loc(e)) ;; avoid counting global var reads in a write - (l) : walk(loc(e)) - walk(y(e)) - (e:EVar|EVarLoc) : add(read?, n(e)) when globals[n(e)] - (e) : do(walk, e) - for e in exps(epackage) do : walk(e) - to-intset $ for gv in globals filter : not read?[gv] - ;; top level for tree shaking package removing all unreachable vars and all unread global vars defn prune-package (epackage:EPackage) -> EPackage : val iotable = IOTable(packageio(epackage)) val [reachables, parents] = walk-reachable-defs(epackage) - val unused-vars = unused-global-vars(epackage) - if prune-trace? : - val defs = defs(epackage) - for v in unused-vars do : - println("UNUSED %_ / %_" % [defs[v] iotable[v]]) - defn defined? (e:ETExp) : - match(e) : - (e:EDefn|EDefClosure|EDefmulti|EDefmethod) : - reachables[n(e)] - (e:EDefGlobal) : - not unused-vars[n(e)] - (e:EDefStruct|EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : - true - (e) : - true + defn keep? (e:ETExp) : + match(identifier?(e)) : + (n:Int) : reachables[n] + (f:False) : true ;; remove all methods on unreachable multis from package defn prune-methods (e:ETExp, reachables:IntSet) -> ETExp : defn* walk (e:ELItem) -> ELItem : - match(e) : + match(map(walk,e)) : (e:ELocalObj) : val methods = for m in methods(e) filter : reachables[multi(m)] - sub-methods(e, to-tuple $ for m in methods seq : map(walk, m)) + sub-methods(e, to-tuple $ methods) (e) : - map(walk, e) + e walk(e) as ETExp ;; remove unreachable imports/exports @@ -294,32 +281,36 @@ defn prune-package (epackage:EPackage) -> EPackage : to-tuple $ for i in imports(packageio) filter : defined?[n(i)], to-tuple $ for e in exports(packageio) filter : defined?[n(e)]) + val globals = to-intset $ for e in exps(epackage) seq? : + match(e:EDefGlobal) : One(n(e)) + else : None() + ;; remove store instructions that are on vars that are only written to not read from defn remove-unused-var-stores (exps:Seqable) -> Seq : defn* walk (e:ELItem) -> ELItem : defn remove-unused-stores (ins:Tuple) -> Seq : for i in ins filter : - match(i) : - (i:EStore) : - match(loc(i)) : - (l:EVarLoc) : not unused-vars[n(l)] - (l) : true - (e) : true + match(i:EStore) : + match(loc(i)) : + (l:EVarLoc) : not globals[n(l)] or reachables[n(l)] + (l) : true + else: true match(e) : (e:EType|EDefGlobal|EDefmulti|EDefStruct|EExtern|EDefType|EDefObject|EDefTypeObject) : e (e:EBody) : - EBody(map(walk, locals(e)) as Tuple, map(walk, localtypes(e)) as Tuple, - map(walk, localfns(e)) as Tuple, map(walk, localobjs(e)) as Tuple, - (to-tuple $ seq(walk, remove-unused-stores(ins(e)))) as Tuple) + val e* = map(walk, e) + val ins* = to-tuple $ remove-unused-stores(ins(e*)) + sub-ins(e*, ins*) (e) : map(walk, e) - val res = for e in exps seq : walk(e) + val res = for e in exps seq : + walk(e) res as Seq ;; top-level work val new-packageio = prune-packageio(packageio(epackage), reachables) - val new-exps = remove-unused-var-stores $ for e in filter(defined?, exps(epackage)) seq : prune-methods(e, reachables) + val new-exps = remove-unused-var-stores $ for e in filter(keep?, exps(epackage)) seq : prune-methods(e, reachables) val res = EPackage(new-packageio, to-tuple $ new-exps) if prune-trace? : From 88afad81127f80d82473c9c2f8870788452f8d6b Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Thu, 6 Feb 2020 23:35:11 -0800 Subject: [PATCH 14/23] merge reachables and visited? --- compiler/stz-el.stanza | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index 81dc80e64..62df0494f 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -185,7 +185,6 @@ defn defs (epackage:EPackage) -> IntTable : ; EDefStruct -> parent:False|EType, base:Tuple, items:EType|False ; EDefType -> parent:False|EType ; EDefTypeObject -> type:EType -; EDefGlobal -> type:EType (Initial values are evaluated.) defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : val defs = defs(epackage) ;id to ETExp's @@ -194,16 +193,14 @@ defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : match(e:EDefmethod) : method-ids[multi(e)] = cons(n(e), method-ids[multi(e)]) val q = Queue() ;for walking todo list val visited? = IntSet() ;for not walking things twice - val reachables = IntSet() ;all ids traced val parents = IntTable(-1) ;parents ids in trace defn walk-top (e:ETExp) : val pid = match(identifier?(e)) : - (n:Int) : (add(reachables, n), n) + (n:Int) : (add(visited?, n), n) (f:False) : -1 let walk (e:ELItem = e) : defn maybe-walk (id:Int) : if add(visited?, id) and key?(defs, id) : - add(reachables, id) parents[id] = pid add(q, defs[id]) match(e) : @@ -224,18 +221,15 @@ defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : (e:EDefmulti) : do(maybe-walk, method-ids[n(e)]) do(walk, e) - ;; (e:EDefmethod|EMethod) : - ;; maybe-walk(multi(e)) - ;; do(walk, e) (e:ELItem) : do(walk, e) - do(walk-top, filter-by(exps(epackage))) ; |EDefStruct|EDefType|EDefTypeObject + do(walk-top, filter-by(exps(epackage))) do(walk-top, predefines(epackage)) while not empty?(q) : walk-top(pop(q)) - println("FOUND %_ REACHABLE OUT OF %_ DEFINED" % [length(reachables), length(defs)]) when prune-trace? - [reachables, parents] + println("FOUND %_ REACHABLE OUT OF %_ DEFINED" % [length(visited?), length(defs)]) when prune-trace? + [visited?, parents] ;; walk all top level forms and check that all vars are defined -- only needed in debug mode defn check-vars-defined (epackage:EPackage, odefs:IntTable) : From d24a5bd7266d23edd93954403985f3f8ab5edb6f Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Thu, 6 Feb 2020 23:39:24 -0800 Subject: [PATCH 15/23] only build method-table for multis that are still around after tree-shaking --- compiler/stz-stitcher.stanza | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/compiler/stz-stitcher.stanza b/compiler/stz-stitcher.stanza index 99d1f7110..23b226995 100644 --- a/compiler/stz-stitcher.stanza +++ b/compiler/stz-stitcher.stanza @@ -362,12 +362,15 @@ public defn Stitcher (packages:Collection, bindings:Bindings|False, s for p in packages do : val pkgids = package-ids[package(p)] for m in methods(p) do : - val multi* = global-id!(pkgids, multi(m)) - val fid* = global-id!(pkgids, fid(m)) - val lbl = lbl(global-props[fid*] as CodeProps) - val types* = map(resolve{pkgids, _}, types(m)) - val branch* = Branch(to-list(types*), M(lbl)) - add(method-table, multi*, branch*) + match(global-id(pkgids,multi(m))) : + (multi*:Int) : + val fid* = global-id!(pkgids, fid(m)) + val lbl = lbl(global-props[fid*] as CodeProps) + val types* = map(resolve{pkgids, _}, types(m)) + val branch* = Branch(to-list(types*), M(lbl)) + add(method-table, multi*, branch*) + (f:False) : + false ;Resolve a package-local TypeSet to use global ids defn resolve (pkgids:PackageIds, t:TypeSet) -> TypeSet : From 9fa13d089b64b80c6fe7e54aadcd4ee4c0a43dad Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Thu, 6 Feb 2020 23:50:24 -0800 Subject: [PATCH 16/23] cleanup --- compiler/stz-el.stanza | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index 62df0494f..fa695c737 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -66,9 +66,7 @@ defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : ;Dump input ;dump(cur-package, "logs", "input") if optimize? : - ;; spit("out0.txt", cur-package) run-pass("Prune Package", prune-package, "prune", true) - ;; spit("out1.txt", cur-package) run-pass("Map Methods", map-methods, "mapped-methods", false) run-pass("Create Closures", create-closures, "closures", false) run-pass("Convert Mixes", convert-mixes, "mixes", false) @@ -89,13 +87,10 @@ defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : run-pass("Lift Objects", lift-objects, "objlifted", true) if optimize? : run-pass("Resolve Methods", resolve-methods, "resolved-methods", false) - ;; spit("out2.txt", cur-package) run-pass("Prune Package 2", prune-package, "prune2", true) - ;; spit("out3.txt", cur-package) run-pass("Lift Closures", lift-closures, "closurelifted", false) run-pass("Lift Type Objects", lift-type-objects, "typelifted", false) - spit("out.txt", cur-package) ;Return processed package cur-package @@ -142,7 +137,7 @@ defn ensure-unique-identifiers! (epackage:EPackage) : ;======================= Pruning ============================ ;============================================================ -val prune-trace? = true +val prune-trace? = false ;Retrieve identifier of top-level declaration if it has one. defn identifier? (e:ETExp) -> Int|False : @@ -181,10 +176,6 @@ defn defs (epackage:EPackage) -> IntTable : ;tracable fields are: ; EInit -> body:EBody (This is always evaluated.) ; EExternFn -> func|EFn -; --- -; EDefStruct -> parent:False|EType, base:Tuple, items:EType|False -; EDefType -> parent:False|EType -; EDefTypeObject -> type:EType defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : val defs = defs(epackage) ;id to ETExp's From 78b5cba724d6dedcc581001c7b3207757a82959a Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Fri, 7 Feb 2020 00:43:59 -0800 Subject: [PATCH 17/23] go back to using reachables --- compiler/stz-el.stanza | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index fa695c737..43d39289b 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -184,10 +184,11 @@ defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : match(e:EDefmethod) : method-ids[multi(e)] = cons(n(e), method-ids[multi(e)]) val q = Queue() ;for walking todo list val visited? = IntSet() ;for not walking things twice + val reachables = IntSet() ;set of traceable etexps val parents = IntTable(-1) ;parents ids in trace defn walk-top (e:ETExp) : val pid = match(identifier?(e)) : - (n:Int) : (add(visited?, n), n) + (n:Int) : (add(reachables, n), n) (f:False) : -1 let walk (e:ELItem = e) : defn maybe-walk (id:Int) : @@ -219,8 +220,8 @@ defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : do(walk-top, predefines(epackage)) while not empty?(q) : walk-top(pop(q)) - println("FOUND %_ REACHABLE OUT OF %_ DEFINED" % [length(visited?), length(defs)]) when prune-trace? - [visited?, parents] + println("FOUND %_ REACHABLE OUT OF %_ DEFINED %_%%" % [length(reachables), length(defs), 100.0 * to-double(length(reachables)) / to-double(length(defs))]) when prune-trace? + [reachables, parents] ;; walk all top level forms and check that all vars are defined -- only needed in debug mode defn check-vars-defined (epackage:EPackage, odefs:IntTable) : From 3944a07fe7e255c6ba1ab2b8879542f283e3a519 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Fri, 7 Feb 2020 09:11:26 -0800 Subject: [PATCH 18/23] put tree-shaker behind dead-stripping? config flag --- compiler/stz-config.stanza | 2 ++ compiler/stz-el.stanza | 5 +++-- compiler/stz-params.stanza | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/compiler/stz-config.stanza b/compiler/stz-config.stanza index 701f78b63..164a69673 100644 --- a/compiler/stz-config.stanza +++ b/compiler/stz-config.stanza @@ -69,6 +69,8 @@ defsyntax stanza-config : STANZA-MAX-COMPILER-HEAP-SIZE = sz defrule entry = (experimental = ?b:#bool!) : EXPERIMENTAL = b + defrule entry = (dead-stripping? = ?b:#bool!) : + DEAD-STRIPPING? = b fail-if entry = () : PE(closest-info(), "Invalid configuration rule.") diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index 43d39289b..a26d13d8b 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -1,6 +1,7 @@ defpackage stz/el : import core import collections + import stz/params import stz/dl-ir import stz/el-ir import stz/utils @@ -66,7 +67,7 @@ defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : ;Dump input ;dump(cur-package, "logs", "input") if optimize? : - run-pass("Prune Package", prune-package, "prune", true) + run-pass("Prune Package", prune-package, "prune", true) when DEAD-STRIPPING? run-pass("Map Methods", map-methods, "mapped-methods", false) run-pass("Create Closures", create-closures, "closures", false) run-pass("Convert Mixes", convert-mixes, "mixes", false) @@ -87,7 +88,7 @@ defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : run-pass("Lift Objects", lift-objects, "objlifted", true) if optimize? : run-pass("Resolve Methods", resolve-methods, "resolved-methods", false) - run-pass("Prune Package 2", prune-package, "prune2", true) + run-pass("Prune Package 2", prune-package, "prune2", true) when DEAD-STRIPPING? run-pass("Lift Closures", lift-closures, "closurelifted", false) run-pass("Lift Type Objects", lift-type-objects, "typelifted", false) diff --git a/compiler/stz-params.stanza b/compiler/stz-params.stanza index 438ffc9ad..a77d2c181 100644 --- a/compiler/stz-params.stanza +++ b/compiler/stz-params.stanza @@ -20,6 +20,7 @@ public var OUTPUT-PLATFORM:Symbol = `platform public var STANZA-PKG-DIRS:List = List() public val STANZA-PROJ-FILES = Vector() public var EXPERIMENTAL:True|False = false +public var DEAD-STRIPPING?:True|False = false ;====== Compiler Configuration ===== public var STANZA-MAX-COMPILER-HEAP-SIZE = 4L * 1024L * 1024L * 1024L From 7fc3fc8ef13cc2861741b623d8046be59abfce73 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Fri, 7 Feb 2020 10:37:34 -0800 Subject: [PATCH 19/23] remove tracing TVars --- compiler/stz-el.stanza | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index a26d13d8b..4b1b57d9b 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -202,7 +202,7 @@ defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : walk(y(e)) (e:EVar|EVarLoc) : maybe-walk(n(e)) - (e:EStructT|EOf|ETVar|EField|ETVarLoc) : + (e:EStructT|EOf|EField) : maybe-walk(n(e)) (e:ENew|EObject|EObject|EArray|EStruct|ETypeObject|ENewObject|EObjectGet|EObjectTGet| EConstClosure|EClosureGet|EClosureTGet|EClosure) : From 24a62646310835e94d62c0c295645fb57b64d699 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Fri, 7 Feb 2020 10:58:33 -0800 Subject: [PATCH 20/23] remove debug code and add another flag for dead-stripping-debug --- compiler/stz-config.stanza | 2 ++ compiler/stz-el.stanza | 25 ++----------------------- compiler/stz-params.stanza | 1 + 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/compiler/stz-config.stanza b/compiler/stz-config.stanza index 164a69673..247241235 100644 --- a/compiler/stz-config.stanza +++ b/compiler/stz-config.stanza @@ -71,6 +71,8 @@ defsyntax stanza-config : EXPERIMENTAL = b defrule entry = (dead-stripping? = ?b:#bool!) : DEAD-STRIPPING? = b + defrule entry = (dead-stripping-debug? = ?b:#bool!) : + DEAD-STRIPPING-DEBUG? = b fail-if entry = () : PE(closest-info(), "Invalid configuration rule.") diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index 4b1b57d9b..203816d58 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -138,8 +138,6 @@ defn ensure-unique-identifiers! (epackage:EPackage) : ;======================= Pruning ============================ ;============================================================ -val prune-trace? = false - ;Retrieve identifier of top-level declaration if it has one. defn identifier? (e:ETExp) -> Int|False : match(e:EDefGlobal|EDefn|EDefClosure|EDefmulti|EDefmethod|EDefStruct| @@ -220,27 +218,8 @@ defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : do(walk-top, filter-by(exps(epackage))) do(walk-top, predefines(epackage)) while not empty?(q) : walk-top(pop(q)) - - println("FOUND %_ REACHABLE OUT OF %_ DEFINED %_%%" % [length(reachables), length(defs), 100.0 * to-double(length(reachables)) / to-double(length(defs))]) when prune-trace? [reachables, parents] -;; walk all top level forms and check that all vars are defined -- only needed in debug mode -defn check-vars-defined (epackage:EPackage, odefs:IntTable) : - defn check (e:ETExp, odefs:IntTable, defs:IntTable, undefines:IntSet) : - let walk (e:ELItem = e) : - match(e) : - (e:EVar|EVarLoc) : - if key?(odefs, n(e)) and not key?(defs, n(e)) : - add(undefines, n(e)) - (e:ELItem) : - do(walk, e) - undefines - val undefines = IntSet() - val defs = defs(epackage) - for e in exps(epackage) do : - check(e, odefs, defs, undefines) - println("UNDEFINES %_" % [to-tuple $ qsort $ undefines]) when not empty?(undefines) - ;; top level for tree shaking package removing all unreachable vars and all unread global vars defn prune-package (epackage:EPackage) -> EPackage : val iotable = IOTable(packageio(epackage)) @@ -300,7 +279,7 @@ defn prune-package (epackage:EPackage) -> EPackage : val new-exps = remove-unused-var-stores $ for e in filter(keep?, exps(epackage)) seq : prune-methods(e, reachables) val res = EPackage(new-packageio, to-tuple $ new-exps) - if prune-trace? : + if DEAD-STRIPPING-DEBUG? : ;; write out file that can be used to find trace path to variable to understand why it's held onto val defs = defs(epackage) val tree = to-list $ for kv in parents seq : List(key(kv), value(kv)) @@ -309,7 +288,7 @@ defn prune-package (epackage:EPackage) -> EPackage : val rec = rec(ie) List(n, List(package(id(rec)), name(id(rec)))) spit("parents.txt", List(symbol-table, tree)) - check-vars-defined(res, defs) + res ;============================================================ diff --git a/compiler/stz-params.stanza b/compiler/stz-params.stanza index a77d2c181..3767996f3 100644 --- a/compiler/stz-params.stanza +++ b/compiler/stz-params.stanza @@ -21,6 +21,7 @@ public var STANZA-PKG-DIRS:List = List() public val STANZA-PROJ-FILES = Vector() public var EXPERIMENTAL:True|False = false public var DEAD-STRIPPING?:True|False = false +public var DEAD-STRIPPING-DEBUG?:True|False = false ;====== Compiler Configuration ===== public var STANZA-MAX-COMPILER-HEAP-SIZE = 4L * 1024L * 1024L * 1024L From ab754957f711f3a20aa52c4d5a413f0c4224bd60 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Fri, 7 Feb 2020 11:02:59 -0800 Subject: [PATCH 21/23] change comments to adhere to standards --- compiler/stz-el.stanza | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index 203816d58..865badbc6 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -220,7 +220,7 @@ defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : while not empty?(q) : walk-top(pop(q)) [reachables, parents] -;; top level for tree shaking package removing all unreachable vars and all unread global vars +;Top level for tree shaking package removing all unreachable vars and all unread global vars. defn prune-package (epackage:EPackage) -> EPackage : val iotable = IOTable(packageio(epackage)) val [reachables, parents] = walk-reachable-defs(epackage) @@ -230,7 +230,7 @@ defn prune-package (epackage:EPackage) -> EPackage : (n:Int) : reachables[n] (f:False) : true - ;; remove all methods on unreachable multis from package + ;Remove all methods on unreachable multis from package. defn prune-methods (e:ETExp, reachables:IntSet) -> ETExp : defn* walk (e:ELItem) -> ELItem : match(map(walk,e)) : @@ -241,7 +241,7 @@ defn prune-package (epackage:EPackage) -> EPackage : e walk(e) as ETExp - ;; remove unreachable imports/exports + ;Remove unreachable imports/exports. defn prune-packageio (packageio:PackageIO, defined?:IntSet) -> PackageIO : PackageIO(package(packageio), imported-packages(packageio), to-tuple $ for i in imports(packageio) filter : defined?[n(i)], @@ -251,7 +251,7 @@ defn prune-package (epackage:EPackage) -> EPackage : match(e:EDefGlobal) : One(n(e)) else : None() - ;; remove store instructions that are on vars that are only written to not read from + ;Remove store instructions that are on vars that are only written to not read from. defn remove-unused-var-stores (exps:Seqable) -> Seq : defn* walk (e:ELItem) -> ELItem : defn remove-unused-stores (ins:Tuple) -> Seq : @@ -274,13 +274,13 @@ defn prune-package (epackage:EPackage) -> EPackage : walk(e) res as Seq - ;; top-level work + ;Top-level work. val new-packageio = prune-packageio(packageio(epackage), reachables) val new-exps = remove-unused-var-stores $ for e in filter(keep?, exps(epackage)) seq : prune-methods(e, reachables) val res = EPackage(new-packageio, to-tuple $ new-exps) if DEAD-STRIPPING-DEBUG? : - ;; write out file that can be used to find trace path to variable to understand why it's held onto + ;Write out file that can be used to find trace path to variable to understand why it's held onto. val defs = defs(epackage) val tree = to-list $ for kv in parents seq : List(key(kv), value(kv)) val symbol-table = to-list $ for ie in cat(imports(new-packageio), exports(new-packageio)) seq : From 36a9c41e5ab24dc02e8267ac8d1a4f506f13707e Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Fri, 7 Feb 2020 11:33:34 -0800 Subject: [PATCH 22/23] promote methods without multis to defns and update stitcher to old way of emitting method-tables --- compiler/stz-el.stanza | 24 ++++++++++++++++++------ compiler/stz-stitcher.stanza | 15 ++++++--------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index 865badbc6..ad0a2da50 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -67,7 +67,7 @@ defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : ;Dump input ;dump(cur-package, "logs", "input") if optimize? : - run-pass("Prune Package", prune-package, "prune", true) when DEAD-STRIPPING? + run-pass("Prune Package 1", prune-package, "prune1", true) when DEAD-STRIPPING? run-pass("Map Methods", map-methods, "mapped-methods", false) run-pass("Create Closures", create-closures, "closures", false) run-pass("Convert Mixes", convert-mixes, "mixes", false) @@ -163,6 +163,7 @@ defn defs (epackage:EPackage) -> IntTable : ;Find all reachable defs from program/data roots of package and parents. ;Returns: ; - Set of reachable ids traced from roots of package. +; - Id to ETExp table. ; - Parent of each reachable id. (Used for determining why something was reached.) ; ;Algorithm for reachability: @@ -176,7 +177,7 @@ defn defs (epackage:EPackage) -> IntTable : ; EInit -> body:EBody (This is always evaluated.) ; EExternFn -> func|EFn -defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : +defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable, IntTable] : val defs = defs(epackage) ;id to ETExp's val method-ids = IntTable>(List()) ;multi ids to list of their method ids for e in exps(epackage) do : @@ -218,12 +219,12 @@ defn walk-reachable-defs (epackage:EPackage) -> [IntSet, IntTable] : do(walk-top, filter-by(exps(epackage))) do(walk-top, predefines(epackage)) while not empty?(q) : walk-top(pop(q)) - [reachables, parents] + [reachables, defs, parents] ;Top level for tree shaking package removing all unreachable vars and all unread global vars. defn prune-package (epackage:EPackage) -> EPackage : val iotable = IOTable(packageio(epackage)) - val [reachables, parents] = walk-reachable-defs(epackage) + val [reachables, defs, parents] = walk-reachable-defs(epackage) defn keep? (e:ETExp) : match(identifier?(e)) : @@ -274,14 +275,25 @@ defn prune-package (epackage:EPackage) -> EPackage : walk(e) res as Seq + val multis = to-intset $ for id in reachables seq? : + val e = defs[id] + match(e:EDefmulti) : One(n(e)) + else : None() + + ;Change all methods without multis to defns. + defn promote-methods (exps:Seqable) -> Seq : + for e in exps seq : + match(e:EDefmethod) : e when multis[multi(e)] else EDefn(n(e), func(e), lostanza?(e)) + else : e + ;Top-level work. val new-packageio = prune-packageio(packageio(epackage), reachables) - val new-exps = remove-unused-var-stores $ for e in filter(keep?, exps(epackage)) seq : prune-methods(e, reachables) + val kept-exps = for e in filter(keep?, exps(epackage)) seq : prune-methods(e, reachables) + val new-exps = remove-unused-var-stores $ promote-methods $ kept-exps val res = EPackage(new-packageio, to-tuple $ new-exps) if DEAD-STRIPPING-DEBUG? : ;Write out file that can be used to find trace path to variable to understand why it's held onto. - val defs = defs(epackage) val tree = to-list $ for kv in parents seq : List(key(kv), value(kv)) val symbol-table = to-list $ for ie in cat(imports(new-packageio), exports(new-packageio)) seq : val n = n(ie) diff --git a/compiler/stz-stitcher.stanza b/compiler/stz-stitcher.stanza index 23b226995..a6f34027e 100644 --- a/compiler/stz-stitcher.stanza +++ b/compiler/stz-stitcher.stanza @@ -362,15 +362,12 @@ public defn Stitcher (packages:Collection, bindings:Bindings|False, s for p in packages do : val pkgids = package-ids[package(p)] for m in methods(p) do : - match(global-id(pkgids,multi(m))) : - (multi*:Int) : - val fid* = global-id!(pkgids, fid(m)) - val lbl = lbl(global-props[fid*] as CodeProps) - val types* = map(resolve{pkgids, _}, types(m)) - val branch* = Branch(to-list(types*), M(lbl)) - add(method-table, multi*, branch*) - (f:False) : - false + val multi* = global-id!(pkgids,multi(m)) + val fid* = global-id!(pkgids, fid(m)) + val lbl = lbl(global-props[fid*] as CodeProps) + val types* = map(resolve{pkgids, _}, types(m)) + val branch* = Branch(to-list(types*), M(lbl)) + add(method-table, multi*, branch*) ;Resolve a package-local TypeSet to use global ids defn resolve (pkgids:PackageIds, t:TypeSet) -> TypeSet : From 5f02162b325f909e5754c49b4160ba01c4f4c128 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Fri, 7 Feb 2020 15:08:39 -0800 Subject: [PATCH 23/23] fix couple more comments --- compiler/stz-el.stanza | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/stz-el.stanza b/compiler/stz-el.stanza index ad0a2da50..36da90eed 100644 --- a/compiler/stz-el.stanza +++ b/compiler/stz-el.stanza @@ -144,7 +144,7 @@ defn identifier? (e:ETExp) -> Int|False : EExternFn|EExtern|EDefType|EDefObject|EDefTypeObject) : n(e) -;; definitions that need to be in runtime and aren't traceable from roots +;Definitions that need to be in runtime and aren't traceable from roots. defn predefines (epackage:EPackage) -> Seqable : val iotable = IOTable(packageio(epackage)) val predefines = to-intset $ for id in core-ids() seq : n(iotable, id) @@ -152,7 +152,7 @@ defn predefines (epackage:EPackage) -> Seqable : val n = identifier?(e) match(n:Int) : predefines[n] -;; ids to definitions table +;Ids to definitions table. defn defs (epackage:EPackage) -> IntTable : to-inttable $ for e in exps(epackage) seq? :