From 64e9b87b56278d7155070741fc7fc01d0b8cfcfe Mon Sep 17 00:00:00 2001 From: Kevin Welsh Date: Sun, 24 Jun 2018 10:23:31 -0500 Subject: [PATCH 01/11] some more operators --- TFFactory/GraphBuilder.py | 14 ++++++++++++-- setup.py | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/TFFactory/GraphBuilder.py b/TFFactory/GraphBuilder.py index fc24eca..b665fbb 100644 --- a/TFFactory/GraphBuilder.py +++ b/TFFactory/GraphBuilder.py @@ -16,6 +16,7 @@ MOCKED_FUNCTIONS = [ 'tensorflow.placeholder', 'tensorflow.Variable', + 'tensorflow.abs', 'tensorflow.add', 'tensorflow.subtract', 'tensorflow.multiply', @@ -146,25 +147,34 @@ def asDict(self): } return d + def __neg__(self): + return multiply(-1, self) + def __pos__(self): + return abs(self) + def __add__(self, other): return add(self, other) def __radd__(self, other): return add(other, self) + __iadd__ = __add__ def __sub__(self, other): return subtract(self, other) def __rsub__(self, other): return subtract(other, self) + __isub__ = __sub__ def __mul__(self, other): return multiply(self, other) def __rmul__(self, other): return multiply(other, self) + __imul__ = __mul__ - def __div__(self, other): + def __truediv__(self, other): return divide(self, other) - def __rdiv__(self, other): + def __rtruediv__(self, other): return divide(other, self) + __itruediv__ = __truediv__ def __str__(self): return json.dumps(self.asDict()) diff --git a/setup.py b/setup.py index 2a15234..ded15c1 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="TFFactory", - version="0.0.1", + version="0.1.3", author="Kevin Welsh", author_email="kevinwelsh132@gmail.com", description="A mock tensorflow package that constructs JSON objects instead of tensors.", From 4f29b5030ed37541cc897888d5f8b93a4bbf3c3c Mon Sep 17 00:00:00 2001 From: Kevin Welsh Date: Sun, 24 Jun 2018 10:24:24 -0500 Subject: [PATCH 02/11] version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ded15c1..7af636b 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="TFFactory", - version="0.1.3", + version="0.1.4", author="Kevin Welsh", author_email="kevinwelsh132@gmail.com", description="A mock tensorflow package that constructs JSON objects instead of tensors.", From 691d22ca373b8e28093799ebb3820a1c56beae8a Mon Sep 17 00:00:00 2001 From: Kevin Welsh Date: Sun, 24 Jun 2018 10:26:57 -0500 Subject: [PATCH 03/11] Forgot reduce_mean --- TFFactory/GraphBuilder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TFFactory/GraphBuilder.py b/TFFactory/GraphBuilder.py index b665fbb..5d387be 100644 --- a/TFFactory/GraphBuilder.py +++ b/TFFactory/GraphBuilder.py @@ -25,6 +25,7 @@ 'tensorflow.tanh', 'tensorflow.tensordot', 'tensorflow.reduce_sum', + 'tensorflow.reduce_mean', 'tensorflow.shape', 'tensorflow.transpose', 'tensorflow.expand_dims', From dce4eaa3670c362de9081b59ead866936cead5b2 Mon Sep 17 00:00:00 2001 From: Kevin Welsh Date: Sun, 24 Jun 2018 10:27:22 -0500 Subject: [PATCH 04/11] version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7af636b..3e56397 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="TFFactory", - version="0.1.4", + version="0.1.5", author="Kevin Welsh", author_email="kevinwelsh132@gmail.com", description="A mock tensorflow package that constructs JSON objects instead of tensors.", From bbb131e51fd717223665a44a893e38d35af580c7 Mon Sep 17 00:00:00 2001 From: Kevin Welsh Date: Sun, 24 Jun 2018 10:38:20 -0500 Subject: [PATCH 05/11] square and sqrt --- TFFactory/GraphBuilder.py | 2 ++ setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/TFFactory/GraphBuilder.py b/TFFactory/GraphBuilder.py index 5d387be..c7e670a 100644 --- a/TFFactory/GraphBuilder.py +++ b/TFFactory/GraphBuilder.py @@ -24,6 +24,8 @@ 'tensorflow.log', 'tensorflow.tanh', 'tensorflow.tensordot', + 'tensorflow.square', + 'tensorflow.sqrt', 'tensorflow.reduce_sum', 'tensorflow.reduce_mean', 'tensorflow.shape', diff --git a/setup.py b/setup.py index 3e56397..3095870 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="TFFactory", - version="0.1.5", + version="0.1.5001", author="Kevin Welsh", author_email="kevinwelsh132@gmail.com", description="A mock tensorflow package that constructs JSON objects instead of tensors.", From 30456c718b22a7b37508243cef5a3b351a67190c Mon Sep 17 00:00:00 2001 From: Kevin Welsh Date: Sun, 24 Jun 2018 11:14:42 -0500 Subject: [PATCH 06/11] more functions --- TFFactory/GraphBuilder.py | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/TFFactory/GraphBuilder.py b/TFFactory/GraphBuilder.py index c7e670a..fdd14f4 100644 --- a/TFFactory/GraphBuilder.py +++ b/TFFactory/GraphBuilder.py @@ -38,6 +38,14 @@ 'tensorflow.layers.batch_normalization', 'tensorflow.layers.dense', 'tensorflow.distributions.Dirichlet', + 'tensorflow.summary.tensor_summary', + 'tensorflow.summary.scalar', + 'tensorflow.summary.histogram', + 'tensorflow.summary.image', + 'tensorflow.summary.text', + 'tensorflow.summary.audio', + 'tensorflow.summary.merge', + 'tensorflow.summary.merge_all' ] SERIALIZE_MAP = { diff --git a/setup.py b/setup.py index 3095870..66eb158 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="TFFactory", - version="0.1.5001", + version="0.1.5002", author="Kevin Welsh", author_email="kevinwelsh132@gmail.com", description="A mock tensorflow package that constructs JSON objects instead of tensors.", From 1f5c1997ed0b35de9f25516b51d7e432070ec925 Mon Sep 17 00:00:00 2001 From: Kevin Welsh Date: Thu, 28 Jun 2018 20:44:34 -0500 Subject: [PATCH 07/11] Its better! --- TFFactory/Encoding.py | 75 ++++++++ TFFactory/Factory.py | 294 +++++++++++++++----------------- TFFactory/GraphBuilder.py | 222 ++++++++++++------------ TFFactory/Node.py | 33 ++++ TFFactory/Pointer.py | 7 + TFFactory/PythonNode.py | 60 +++++++ TFFactory/SupportedFunctions.py | 18 ++ TFFactory/TFNode.py | 23 +++ TFFactory/Utilities.py | 12 ++ test.py | 19 ++- test_temp.py | 71 ++++++++ 11 files changed, 560 insertions(+), 274 deletions(-) create mode 100644 TFFactory/Encoding.py create mode 100644 TFFactory/Node.py create mode 100644 TFFactory/Pointer.py create mode 100644 TFFactory/PythonNode.py create mode 100644 TFFactory/TFNode.py create mode 100644 TFFactory/Utilities.py create mode 100644 test_temp.py diff --git a/TFFactory/Encoding.py b/TFFactory/Encoding.py new file mode 100644 index 0000000..6898f82 --- /dev/null +++ b/TFFactory/Encoding.py @@ -0,0 +1,75 @@ +import json + +TENSORFLOW_MAP = { + tensorflow.float16 : 'tensorflow.float16', + tensorflow.float32 : 'tensorflow.float32', + tensorflow.float64 : 'tensorflow.float64', + tensorflow.int8 : 'tensorflow.int8', + tensorflow.int16 : 'tensorflow.int16', + tensorflow.int32 : 'tensorflow.int32', + tensorflow.int64 : 'tensorflow.int64' +} + +class Encoder(json.JSONEncoder): + def EncodeGraph(self, graph): + d = {} + for n in graph: + d.update({ + n.ID : { + 'type' : n.Type, + 'inputs' : n.Inputs + } + }) + return self.encode(d) + + def default(self, obj): + if isinstance(obj, Pointer): + return { + 'value' : obj.ID, + '_type' : 'pointer' + } + elif isinstance(obj, Hashable) and obj in TENSORFLOW_MAP: + return { + 'value' : SERIALIZE_MAP[obj], + '_type' : 'tensorflow' + } + + return json.JSONEncoder.default(self, obj) + +DESERIALIZE_MAP = { + 'tensorflow.float16': tensorflow.float16, + 'tensorflow.float32': tensorflow.float32, + 'tensorflow.float64': tensorflow.float64, + 'tensorflow.int8': tensorflow.int8, + 'tensorflow.int16': tensorflow.int16, + 'tensorflow.int32': tensorflow.int32, + 'tensorflow.int64': tensorflow.int64 +} + +class Decoder(json.JSONDecoder): + def __init__(self, *args, **kwargs): + json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) + return + + def object_hook(self, obj): + if '_type' in obj: + t = obj['_type'] + v = obj['value'] + if t == 'tensorflow' : + return DESERIALIZE_MAP[v] + if t == 'pointer': + return Pointer(str(v)) + return obj + +if __name__ == '__main__': + obj = { + 'dict' : { + 'string' : 'value', + 'int' : 2 + }, + 'string' : 'value', + 'int' : 1, + 'array' : [JSONNode(1), JSONNode(2)], + 'obj' : JSONNode(3) + } + print(json.dumps(obj, cls = Encoder, indent = 2)) \ No newline at end of file diff --git a/TFFactory/Factory.py b/TFFactory/Factory.py index 4146e56..9d7ba33 100644 --- a/TFFactory/Factory.py +++ b/TFFactory/Factory.py @@ -1,81 +1,24 @@ import tensorflow import numpy as np import json +import TFFactory.GraphBuilder as GraphBuilder +import TFFactory.SupportedFunctions as SupportedFunctions from .TFFactoryException import TFFactoryException -from .DataLoader import DataLoader -import TFFactory.GraphBuilder as GB - - -class Node: - EvalContext = None - def __init__(self, id, backingVariable = None, evalFunc = None, positionalArgs = [], dictParams = {}, needtoFeed = {}): - if backingVariable is None and evalFunc is None: - raise AssertionError('Node cannot be created without a backing function or backing variable') - assert id is not None, 'Nodes need a unique ID' - - self.BackVariable = backingVariable - self.EvalFunc = evalFunc - self.PositionalArgs = positionalArgs - self.DictParams = dictParams - self.NeedtoFeed = needtoFeed - self.LastContext = {} - self.JSONRep = {} - self.ID = id - return - - def eval(self, session = None, feed_dict = {}, newContext = True): - if self.ID in feed_dict: - return feed_dict[self.ID] - elif self.ID in self.NeedtoFeed: - raise TFFactoryException('Node {} was not fed during execution.'.format(self.ID)) - - if newContext: - Node.EvalContext = np.random.random() - if Node.EvalContext not in self.LastContext: - self.LastContext = {} - if Node.EvalContext is not None: - if Node.EvalContext in self.LastContext: - return self.LastContext[Node.EvalContext] - - val = None - if self.EvalFunc is not None: - args = [] - for a in self.PositionalArgs: - if isinstance(a, Node): - args.append(a.eval(session = session, feed_dict = feed_dict, newContext = False)) - else: - args.append(a) - _feed_dict = {} - for name, node in self.NeedtoFeed.items(): - _feed_dict[name] = node.eval(session = session, feed_dict = feed_dict, newContext = False) - val = self.EvalFunc(*args, **self.DictParams, **_feed_dict) - elif self.BackVariable is not None: - if len(self.NeedtoFeed) == 0: - val = self.BackVariable.eval(session = session) - else: - _feed_dict = {} - # Swap out placeholders with node eval function results. - for key, node in self.NeedtoFeed.items(): - _feed_dict[node.BackVariable] = node.eval(session = session, feed_dict = feed_dict, newContext = False) - val = self.BackVariable.eval(feed_dict = _feed_dict, session = session) - if Node.EvalContext is not None: - self.LastContext[Node.EvalContext] = val - if val is None: raise AssertionError('Good luck with this one.') - return val - - def __str__(self): - return json.dumps(self.JSONRep) - +from .Pointer import Pointer +from .PythonNode import PythonNode +from .TFNode import TFNode +from .Utilities import findAndApply +from functools import partial def CreateTFGraph(graph): nodes = {} - - for key in graph: + for key, node in graph.items(): + print(node) __buildBranch(graph, key, nodes) return nodes -def __buildBranch(graph, key, allNodes, needtofeed = None): +def __buildBranch(graph, key, allNodes): """ @graph -- The full json graph for traversal/reference @key -- the key to access the desired eval end point in @graph @@ -83,103 +26,112 @@ def __buildBranch(graph, key, allNodes, needtofeed = None): @needtofeed -- the keys of the nodes that need to be supplied to the feed_dict for evaluation @allNodes[key] will hold a reference to the node. """ - if needtofeed is None: - needtofeed = {} - if key in allNodes: # Just be done, but update the feeding dependencies for the caller. - if allNodes[key].EvalFunc is not None: # If it a non-TF op, it has a non-TF feed dict, and needs to be fed. Also it needs to have a backing variable. - needtofeed.update({key : allNodes[key]}) - else: # If it is a TF op, then compile its child dependencies. - needtofeed.update(**allNodes[key].NeedtoFeed) - return - - key = str(key) - graphNode = graph[key] # Stop it with the data type key errors... - type = graphNode['type'] - inputs = graphNode['inputs'] - node = None + if key in allNodes: + return allNodes[key] - tfOp = __resolveTFRef(type) - # Build a function reference to eval when it comes time. - if type in GB.PYTHON_FUNCTIONS: - func = GB.PYTHON_FUNCTIONS[type] - dictParams = {} - args = [] - childNeeds = {} - # Parse the provided kwargs - for name, value in inputs.get('kwargs', {}).items(): - v, isRef = GB.Deserialize(value) - if isRef: - __buildBranch(graph, v, allNodes) - childNeeds[name] = allNodes[v] - else: - dictParams[name] = v - # Parse the provided args - for p in inputs.get('args', []): - v, isRef = GB.Deserialize(p) - if isRef: - __buildBranch(graph, v, allNodes) - childNeeds[name] = allNodes[v] - args.append(allNodes[v]) - else: - args.append(p['value']) - - placeholder = None - if 'Shape' in inputs: - placeholder = tensorflow.placeholder(tensorflow.float32, - shape = inputs['Shape']['value'], - name = 'Placeholder_{}'.format(key)) - # Here the childNeeds is a dict of [param name] : node. - # This is because we don't actually call it recursively here, and still need to - # actually call the function. - node = Node(key, - placeholder, - evalFunc = func, - positionalParams = args, - dictParams = dictParams, - needtoFeed = childNeeds) - if placeholder is not None: - needtofeed[key] = node - allNodes[key] = node - # Apply TF functions to get a reference to a tensor - elif tfOp is not None: - params = {} - args = [] - childNeeds = {} - # Parse kwargs - for name, p in inputs.get('kwargs', {}).items(): - v, isRef = GB.Deserialize(p) - if isRef: - __buildBranch(graph, v, allNodes, childNeeds) - params[name] = allNodes[v].BackVariable - else: - params[name] = v - # Parse args - for p in inputs.get('args', []): - v, isRef = GB.Deserialize(p) - if isRef: - __buildBranch(graph, v, allNodes, childNeeds) - args.append(allNodes[v].BackVariable) - else: - args.append(v) - # We don't need the name of a parameter here, because we have already called the function - # If there is something we need to feed, it has a tensor attached, and we will feed that in - # at runtime. We just need to know which nodes need evaluation/replacing. - node = Node(key, tfOp(*args, **params), needtoFeed = childNeeds) - needtofeed.update(**childNeeds) - if tfOp == tensorflow.placeholder: - needtofeed.update({key : node}) - # It can't eval itself without being fed. So hack that in, I guess? - # Will cause an infinite loop if things break. Wont if they dont! - node.NeedtoFeed.update({key : node}) + _type = graph[key].get('_type') + node = None + if _type == 'pythonNode': + node = __buildPythonNode(key, graph, allNodes) + elif _type == 'tensorflowNode': + node = __buildTFNode(key, graph, allNodes) + if node is None: - raise AssertionError('Unsupported node type: {}'.format(type)) - node.JSONRep = {key : graphNode} + raise AssertionError('Unsupported node type: {}'.format(_type)) allNodes[key] = node - return + return node -def __resolveTFRef(ref): - if 'tensorflow' not in ref: +def __buildPythonNode(key, graph, allNodes): + graphNode = graph[key] + funcName = graphNode['funcName'] + inputs = graphNode['inputs'] + func = __resolveRef(funcName) + needToFeed = {} + + allArgs = { + 'args' : inputs.get('args', []), + 'kwargs' : inputs.get('kwargs', {}) + } + dependencies = set() + allArgs = findAndApply(allArgs, Pointer.IsInstance, + partial(__markPythonDependency, + dependencies = dependencies, + grpah = graph, + allNodes = allNodes)) + + args = allArgs['args'] + kwargs = allArgs['kwargs'] + for d in dependencies: + needToFeed[d.ID] = d + + placeholder = None + if '_shape' in inputs: + placeholder = tensorflow.placeholder(tensorflow.float32, + shape = inputs['_shape'], + name = 'Placeholder_{}'.format(key)) + + node = PythonNode(key, + tensor = placeholder, + evalFunc = func, + args = args, + kwargs = kwargs, + needToFeed = needToFeed) + + allNodes[key] = node + return node + +def __markPythonDependency(pointer, dependencies, graph, allNodes): + node = __buildBranch(graph, pointer.Ref, allNodes) + dependencies.add(node) + return pointer + +def __buildTFNode(key, graph, allNodes): + graphNode = graph[key] + funcName = graphNode['funcName'] + inputs = graphNode['inputs'] + tfOp = __resolveRef(funcName) + needToFeed = {} + + allArgs = { + 'args' : inputs.get('args', []), + 'kwargs' : inputs.get('kwargs', {}) + } + dependencies = set() + allArgs = findAndApply(allArgs, Pointer.IsInstance, + partial(__markTFDependency, + dependencies = dependencies, + graph = graph, + allNodes = allNodes)) + + print('AllArgs: {}'.format(allArgs)) + args = allArgs['args'] + kwargs = allArgs['kwargs'] + for d in dependencies: + if isinstance(d, PythonNode): + # Add any python node dependencies directly to the list. + needToFeed[d.ID] = d + elif isinstance(d, TFNode): + # For all of the TF node dependencies + # Add all of their dependencies to the list. + needToFeed.update(**d.NeedToFeed) + node = TFNode(key, tfOp(*args, **kwargs), needToFeed = needToFeed) + if tfOp == tensorflow.placeholder: + # It can't eval itself without being fed. So hack that in, I guess? + # Will cause an infinite loop if things break. Wont if they dont! + node.NeedToFeed.update({key : node}) + return node + +def __markTFDependency(pointer, dependencies, graph, allNodes): + node = __buildBranch(graph, pointer.Ref, allNodes) + if node not in dependencies: + dependencies.add(node) + if isinstance(node, TFNode) or isinstance(node, PythonNode): + return node.Tensor + return None + +def __resolveRef(ref): + if 'tensorflow' not in ref and 'SupportedFunctions' not in ref: return None try: obj = eval(ref) @@ -190,3 +142,27 @@ def __resolveTFRef(ref): return obj +def __findPointers(obj, pointers): + if isinstance(obj, list): + for i, o in enumerate(obj): + next = {} + pointer = __findPointers(o, next) + if pointer is not None: + pointers[i] = pointer + elif len(next.keys()) > 0: + pointers[i] = next + elif isinstance(obj, dict): + if obj.get('_type') == 'pointer': + value = obj.get('value') + return Pointer(value) + else: + for k, v in obj.items(): + next = {} + pointer = __findPointers(v, next) + if pointer is not None: + pointers[k] = pointer + elif len(next.keys()) > 0: + pointers[k] = next + return None + + diff --git a/TFFactory/GraphBuilder.py b/TFFactory/GraphBuilder.py index fdd14f4..01470be 100644 --- a/TFFactory/GraphBuilder.py +++ b/TFFactory/GraphBuilder.py @@ -3,15 +3,58 @@ import json from collections import defaultdict, Hashable from types import ModuleType -from TFFactory.SupportedFunctions import * +import TFFactory.SupportedFunctions as functions +from TFFactory.Pointer import Pointer +from .Utilities import findAndApply ID_COUNTER = defaultdict(int) CURRENT_GRAPH = {} -PYTHON_FUNCTIONS = { - 'fileSource' : lambda FilePath, NRows : readFile(FilePath, ',', NRows), - 'parser' : splitFile -} +def NewGraph(): + global ID_COUNTER, CURRENT_GRAPH + ID_COUNTER = defaultdict(int) + CURRENT_GRAPH = {} + return + +def _assignFunctions(this, functions, type): + for f in functions: + attrs = f.split('.') + curObj = this + for v in attrs[1:-1]: + if not hasattr(curObj, v): + setattr(curObj, v, ModuleType(v)) + curObj = getattr(curObj, v) + setattr(curObj, attrs[-1], _mockFunction(f, type)) + +def _mockFunction(funcName, type): + def MockedFunction(*args, **kwargs): + global ID_COUNTER, CURRENT_GRAPH + name = kwargs.get('name','Variable') + shape = kwargs.pop('_shape', None) + count = ID_COUNTER[name] + ID_COUNTER[name] += 1 + if count > 0: + name = '{}_{}'.format(name, count) + n = JSONNode(name, funcName, list(args), kwargs, shape, type) + CURRENT_GRAPH.update({n.ID : n}) + return n + return MockedFunction + + +this = sys.modules[__name__] +PYTHON_FUNCTIONS = [ + 'SupportedFunctions.fileSource', + 'SupportedFunctions.parser', + 'SupportedFunctions.testAdd' +] +_assignFunctions(this, PYTHON_FUNCTIONS, 'pythonNode') + +COMPOSITE_FUNCTIONS = [ + 'SupportedFunctions.AdamOptimizer', + 'SupportedFunctions.MomentumOptimizer', + 'SupportedFunctions.GradientDescentOptimizer' +] +_assignFunctions(this, COMPOSITE_FUNCTIONS, 'tensorflowNode') MOCKED_FUNCTIONS = [ 'tensorflow.placeholder', @@ -47,116 +90,82 @@ 'tensorflow.summary.merge', 'tensorflow.summary.merge_all' ] - -SERIALIZE_MAP = { - tensorflow.float16 : 'tensorflow.float16', - tensorflow.float32 : 'tensorflow.float32', - tensorflow.float64 : 'tensorflow.float64', - tensorflow.int8 : 'tensorflow.int8', - tensorflow.int16 : 'tensorflow.int16', - tensorflow.int32 : 'tensorflow.int32', - tensorflow.int64 : 'tensorflow.int64' -} - -DESERIALIZE_MAP = { - 'tensorflow.float16': tensorflow.float16, - 'tensorflow.float32': tensorflow.float32, - 'tensorflow.float64': tensorflow.float64, - 'tensorflow.int8': tensorflow.int8, - 'tensorflow.int16': tensorflow.int16, - 'tensorflow.int32': tensorflow.int32, - 'tensorflow.int64': tensorflow.int64 -} - - -def Serialize(value): - type = 'unknown' - if isinstance(value, JSONNode): - v = value.ID - type = 'ref' - elif isinstance(value, Hashable): - v = SERIALIZE_MAP.get(value, value) - else: - v = value - - d = { - 'value' : v, - 'type' : type +_assignFunctions(this, MOCKED_FUNCTIONS, 'tensorflowNode') + +class Encoder(json.JSONEncoder): + SERIALIZE_MAP = { + tensorflow.float16 : 'tensorflow.float16', + tensorflow.float32 : 'tensorflow.float32', + tensorflow.float64 : 'tensorflow.float64', + tensorflow.int8 : 'tensorflow.int8', + tensorflow.int16 : 'tensorflow.int16', + tensorflow.int32 : 'tensorflow.int32', + tensorflow.int64 : 'tensorflow.int64' } - - return d - -def Deserialize(value): - """ - Given the value object with format : - value = { - 'value' : <>, - 'type' : <> - } - - returns (the deserialized value, whether or not it is a node reference) - """ - v = value['value'] - if value['type'] == 'ref': - return (v, True) - if isinstance(v, Hashable): - v = DESERIALIZE_MAP.get(v, v) - - return (v, False) - -def NewGraph(): - global ID_COUNTER, CURRENT_GRAPH - ID_COUNTER = defaultdict(int) - CURRENT_GRAPH = {} - return - -def MockFunction(funcName): - def MockedFunction(*args, **kwargs): - global ID_COUNTER, CURRENT_GRAPH - name = kwargs.get('name','Variable') - count = ID_COUNTER[name] - ID_COUNTER[name] += 1 - if count > 0: - name = '{}_{}'.format(name, count) - n = JSONNode(name, funcName, args, kwargs) - CURRENT_GRAPH.update(n.asDict()) - return n - return MockedFunction + def default(self, obj): + if isinstance(obj, JSONNode): + return { + '_type' : obj.Type, + 'funcName' : obj.FuncName, + 'inputs' : obj.Inputs + } + elif isinstance(obj, Pointer): + return { + 'value' : obj.Ref, + '_type' : 'pointer' + } + elif isinstance(obj, Hashable) and obj in Encoder.SERIALIZE_MAP: + return { + 'value' : Encoder.SERIALIZE_MAP[obj], + '_type' : 'tensorflow' + } -this = sys.modules[__name__] -for f in MOCKED_FUNCTIONS: - attrs = f.split('.') - curObj = this - for v in attrs[1:-1]: - if not hasattr(curObj, v): - setattr(curObj, v, ModuleType(v)) - curObj = getattr(curObj, v) - setattr(curObj, attrs[-1], MockFunction(f)) + return json.JSONEncoder.default(self, obj) + +class Decoder(json.JSONDecoder): + DESERIALIZE_MAP = { + 'tensorflow.float16': tensorflow.float16, + 'tensorflow.float32': tensorflow.float32, + 'tensorflow.float64': tensorflow.float64, + 'tensorflow.int8': tensorflow.int8, + 'tensorflow.int16': tensorflow.int16, + 'tensorflow.int32': tensorflow.int32, + 'tensorflow.int64': tensorflow.int64 + } + def __init__(self, *args, **kwargs): + json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) + return + def object_hook(self, obj): + if '_type' in obj: + t = obj['_type'] + if t == 'tensorflow' : + return Decoder.DESERIALIZE_MAP[obj['value']] + if t == 'pointer': + return Pointer(obj['value']) + return obj class JSONNode: - def __init__(self, id, type, args, kwargs): + def __init__(self, id, funcName, args, kwargs, shape, type): self.ID = str(id) - self.type = str(type) + self.FuncName = funcName + self.Type = str(type) self.Inputs = { - 'args' : [], - 'kwargs' : {} + 'args' : args, + 'kwargs' : kwargs, + '_shape' : shape } - for name, value in kwargs.items(): - self.Inputs['kwargs'][name] = Serialize(value) - self.Inputs['args'] = [] - for value in args: - self.Inputs['args'].append(Serialize(value)) + self.Inputs = findAndApply(self.Inputs, self._shouldBePointer, self._replaceWithPointer) + return - def asDict(self): - d = { - self.ID : { - 'type' : self.type, - 'inputs' : self.Inputs - } - } - return d + @staticmethod + def _shouldBePointer(obj): + return isinstance(obj, JSONNode) + + @staticmethod + def _replaceWithPointer(obj): + return Pointer(obj.ID) def __neg__(self): return multiply(-1, self) @@ -187,9 +196,6 @@ def __rtruediv__(self, other): return divide(other, self) __itruediv__ = __truediv__ - def __str__(self): - return json.dumps(self.asDict()) - def __eq__(self, other): return other.ID == self.ID diff --git a/TFFactory/Node.py b/TFFactory/Node.py new file mode 100644 index 0000000..633bbb4 --- /dev/null +++ b/TFFactory/Node.py @@ -0,0 +1,33 @@ +import numpy as np + +class Node: + EvalContext = None + def __init__(self, id): + self.ID = id + self.LastContext = {} + return + + def Eval(self, session = None, feed_dict = {}, newContext = True): + if self.ID in feed_dict: + return feed_dict[self.ID] + + if newContext: + Node.EvalContext = np.random.random() + if Node.EvalContext not in self.LastContext: + self.LastContext = {} + if Node.EvalContext is not None: + if Node.EvalContext in self.LastContext: + return self.LastContext[Node.EvalContext] + + val = self._eval(session = session, feed_dict = feed_dict, newContext = newContext) + if Node.EvalContext is not None: + self.LastContext[Node.EvalContext] = val + return val + + def _eval(self, session = None, feed_dict = {}, newContext = True): + raise NotImplementedError + + def __hash__(self): + return hash(self.ID) + def __eq__(self, other): + return other.ID == self.ID \ No newline at end of file diff --git a/TFFactory/Pointer.py b/TFFactory/Pointer.py new file mode 100644 index 0000000..ad53a27 --- /dev/null +++ b/TFFactory/Pointer.py @@ -0,0 +1,7 @@ +class Pointer(): + def __init__(self, ref): + self.Ref = ref + + @classmethod + def IsInstance(cls, obj): + return isinstance(obj, cls) \ No newline at end of file diff --git a/TFFactory/PythonNode.py b/TFFactory/PythonNode.py new file mode 100644 index 0000000..8acd1b3 --- /dev/null +++ b/TFFactory/PythonNode.py @@ -0,0 +1,60 @@ +from .Node import Node +from .Utilities import findAndApply +from functools import partial +from .GraphBuilder import Encoder, Decoder +from .Pointer import Pointer + +class PythonNode(Node): + Encoder = Encoder() + Decoder = Decoder() + def __init__(self, id, evalFunc, tensor = None, + args = None, kwargs = None, pointerMap = None, needToFeed = None): + + self.EvalFunc = evalFunc + self.Tensor = tensor + self.Args = args if args is not None else [] + self.Kwargs = kwargs if kwargs is not None else {} + self.AllArgs = { + 'args' : self.Args, + 'kwargs' : self.Kwargs + } + self.NeedToFeed = needToFeed if needToFeed is not None else {} + self.PointerMap = pointerMap if pointerMap is not None else {} + return super().__init__(id) + + def _eval(self, session = None, feed_dict = {}, newContext = True): + pointers = PythonNode.Decoder.decode(PythonNode.Encoder.encode(self.PointerMap)) + print(pointers) + + pointers = findAndApply(pointers, Pointer.IsInstance, + partial(self._feed, + nodeMap = self.NeedToFeed, + session = session, + feed_dict = feed_dict, + newContext = newContext)) + self._mergeObj(self.AllArgs, pointers) + print(self.AllArgs) + val = self.EvalFunc(*self.AllArgs['args'], **self.AllArgs['kwargs']) + return val + + @staticmethod + def _feed(nodeMap, session, feed_dict, newContext, pointer): + return nodeMap[pointer.Ref].Eval(session, feed_dict, newContext) + + @staticmethod + def _mergeObj(o1, o2): + if o2 is None: + return o1 + if o1 is None: + return o2 + if isinstance(o2, dict): + for k, v in o2.items(): + o1[k] = mergeObj(o1[k], v) + return o1 + elif isinstance(o2, list): + for i, v in enumerate(o2): + o1[i] = mergeObj(o1[i], v) + return o1 + else: + return o2 + return None \ No newline at end of file diff --git a/TFFactory/SupportedFunctions.py b/TFFactory/SupportedFunctions.py index a5b28c2..2831165 100644 --- a/TFFactory/SupportedFunctions.py +++ b/TFFactory/SupportedFunctions.py @@ -9,3 +9,21 @@ def splitFile(Source, SegmentDelimeter, DataDelimeter, SegmentIndex, Shape): for row in Source: data.append(list(map(float, row.split(SegmentDelimeter)[SegmentIndex].split(DataDelimeter)))) return np.array(data).reshape((*Shape)) + +def testAdd(a, b): + return [a + b] + + +# TF ones +import tensorflow +def AdamOptimizer(loss, learningRate): + optimizer = tensorflow.train.AdamOptimizer(learningRate) + return optimizer.minimize(loss) + +def MomentumOptimizer(loss, learningRate, momentum): + optimizer = tensorflow.train.MomentumOptimizer(learningRate, momentum = momentum) + return optimizer.minimize(loss) + +def GradientDescentOptimizer(loss, learningRate): + optimizer = tensorflow.train.GradientDescentOptimizer(learningRate) + return optimizer.minimize(loss) diff --git a/TFFactory/TFNode.py b/TFFactory/TFNode.py new file mode 100644 index 0000000..5d17b6c --- /dev/null +++ b/TFFactory/TFNode.py @@ -0,0 +1,23 @@ +from .Node import Node +from .TFFactoryException import TFFactoryException + +class TFNode(Node): + def __init__(self, id, tensor, needToFeed = None): + + self.Tensor = tensor + self.NeedToFeed = needToFeed if needToFeed is not None else {} + return super().__init__(id) + + def _eval(self, session = None, feed_dict = {}, newContext = True): + if self.ID in self.NeedToFeed: + raise TFFactoryException('Node {} was not fed during evaluation.'.format(self.ID)) + + if len(self.NeedToFeed) == 0: + val = session.run(self.Tensor) + else: + tf_feed_dict = {} + # Swap out placeholders with node eval function results. + for key, node in self.NeedToFeed.items(): + tf_feed_dict[node.Tensor] = node.Eval(session, feed_dict, False) + val = session.run(self.Tensor, feed_dict = tf_feed_dict) + return val \ No newline at end of file diff --git a/TFFactory/Utilities.py b/TFFactory/Utilities.py new file mode 100644 index 0000000..3edf2c6 --- /dev/null +++ b/TFFactory/Utilities.py @@ -0,0 +1,12 @@ +# These just don't belong elsewhere + +def findAndApply(obj, found, apply): + if found(obj): + return apply(obj) + if isinstance(obj, list): + for i, o in enumerate(obj): + obj[i] = findAndApply(o, found, apply) + elif isinstance(obj, dict): + for k, v in obj.items(): + obj[k] = findAndApply(v, found, apply) + return obj \ No newline at end of file diff --git a/test.py b/test.py index bd6554f..d211a9c 100644 --- a/test.py +++ b/test.py @@ -4,19 +4,24 @@ import json if __name__ == '__main__': - placeHolder = tff.placeholder(tf.int32, shape = [3], name = 'input') - n = tff.Variable([-1, -2, -3], name = 'n1') - b = tff.Variable(initial_value = [4, 5, 6], name = 'b') - n = n + b + placeHolder - graph = json.dumps(tff.CURRENT_GRAPH) + pythonNode = tff.testAdd(1, -100, _shape = [1]) + + placeHolder = tff.placeholder(tf.float32, shape = [3], name = 'input') + n = tff.Variable([-1, -2, -3], name = 'n1', dtype = tf.float32) + b = tff.Variable(initial_value = [4, 5, 6], name = 'b', dtype = tf.float32) + n = n + b + placeHolder + pythonNode + n = tff.AdamOptimizer(n, 1.0) + + graph = json.dumps(tff.CURRENT_GRAPH, cls = tff.Encoder, indent = 2) print(graph) - graph = json.loads(graph) + graph = json.loads(graph, cls = tff.Decoder) + print('Compiling graph!') compiledGraph = factory.CreateTFGraph(graph) with tf.Session() as sess: tf.global_variables_initializer().run(session = sess) for k,v in compiledGraph.items(): - print('{} = {}'.format(k, v.eval(feed_dict = {'input' : [1,2,3]}))) + print('{} = {}'.format(k, v.Eval(session = sess, feed_dict = {'input' : [1,2,3]}))) diff --git a/test_temp.py b/test_temp.py new file mode 100644 index 0000000..3cac673 --- /dev/null +++ b/test_temp.py @@ -0,0 +1,71 @@ +from TFFactory.Pointer import Pointer + +def __findPointers(obj, pointers): + if isinstance(obj, list): + for i, o in enumerate(obj): + next = {} + pointer = __findPointers(o, next) + if pointer is not None: + pointers[i] = pointer + elif len(next.keys()) > 0: + pointers[i] = next + elif isinstance(obj, dict): + if obj.get('_type') == 'pointer': + value = obj.get('value') + return Pointer(value) + else: + for k, v in obj.items(): + next = {} + pointer = __findPointers(v, next) + if pointer is not None: + pointers[k] = pointer + elif len(next.keys()) > 0: + pointers[k] = next + return None + +def mergeObj(o1, o2): + if o2 is None: + return o1 + if o1 is None: + return o2 + if isinstance(o2, dict): + for k, v in o2.items(): + o1[k] = mergeObj(o1[k], v) + return o1 + elif isinstance(o2, list): + for i, v in enumerate(o2): + o1[i] = mergeObj(o1[i], v) + return o1 + else: + return o2 + return None + + +if __name__ == '__main__': + o1 = { + 'dict' : { + 'key1' : 'value', + 'key2' : 'more value', + 'list' : [ + { + 'value' : 'inAList', + '_type' : 'pointer' + }, + { + 'value' : 'inAList', + '_type' : 'pointer' + } + ] + }, + 'key' : 'value', + 'heres one' : { + 'value' : 'inAList', + '_type' : 'pointer' + } + } + + print(o1) + pointers = {} + __findPointers(o1, pointers) + print(pointers) + print(mergeObj(o1, pointers)) \ No newline at end of file From 936ccb3c33fe0807a90cd852d8d8a5be8bd82c31 Mon Sep 17 00:00:00 2001 From: Kevin Welsh Date: Thu, 28 Jun 2018 21:11:49 -0500 Subject: [PATCH 08/11] removed some useless files --- TFFactory/Encoding.py | 75 ------------------------------------------- test_temp.py | 71 ---------------------------------------- 2 files changed, 146 deletions(-) delete mode 100644 TFFactory/Encoding.py delete mode 100644 test_temp.py diff --git a/TFFactory/Encoding.py b/TFFactory/Encoding.py deleted file mode 100644 index 6898f82..0000000 --- a/TFFactory/Encoding.py +++ /dev/null @@ -1,75 +0,0 @@ -import json - -TENSORFLOW_MAP = { - tensorflow.float16 : 'tensorflow.float16', - tensorflow.float32 : 'tensorflow.float32', - tensorflow.float64 : 'tensorflow.float64', - tensorflow.int8 : 'tensorflow.int8', - tensorflow.int16 : 'tensorflow.int16', - tensorflow.int32 : 'tensorflow.int32', - tensorflow.int64 : 'tensorflow.int64' -} - -class Encoder(json.JSONEncoder): - def EncodeGraph(self, graph): - d = {} - for n in graph: - d.update({ - n.ID : { - 'type' : n.Type, - 'inputs' : n.Inputs - } - }) - return self.encode(d) - - def default(self, obj): - if isinstance(obj, Pointer): - return { - 'value' : obj.ID, - '_type' : 'pointer' - } - elif isinstance(obj, Hashable) and obj in TENSORFLOW_MAP: - return { - 'value' : SERIALIZE_MAP[obj], - '_type' : 'tensorflow' - } - - return json.JSONEncoder.default(self, obj) - -DESERIALIZE_MAP = { - 'tensorflow.float16': tensorflow.float16, - 'tensorflow.float32': tensorflow.float32, - 'tensorflow.float64': tensorflow.float64, - 'tensorflow.int8': tensorflow.int8, - 'tensorflow.int16': tensorflow.int16, - 'tensorflow.int32': tensorflow.int32, - 'tensorflow.int64': tensorflow.int64 -} - -class Decoder(json.JSONDecoder): - def __init__(self, *args, **kwargs): - json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) - return - - def object_hook(self, obj): - if '_type' in obj: - t = obj['_type'] - v = obj['value'] - if t == 'tensorflow' : - return DESERIALIZE_MAP[v] - if t == 'pointer': - return Pointer(str(v)) - return obj - -if __name__ == '__main__': - obj = { - 'dict' : { - 'string' : 'value', - 'int' : 2 - }, - 'string' : 'value', - 'int' : 1, - 'array' : [JSONNode(1), JSONNode(2)], - 'obj' : JSONNode(3) - } - print(json.dumps(obj, cls = Encoder, indent = 2)) \ No newline at end of file diff --git a/test_temp.py b/test_temp.py deleted file mode 100644 index 3cac673..0000000 --- a/test_temp.py +++ /dev/null @@ -1,71 +0,0 @@ -from TFFactory.Pointer import Pointer - -def __findPointers(obj, pointers): - if isinstance(obj, list): - for i, o in enumerate(obj): - next = {} - pointer = __findPointers(o, next) - if pointer is not None: - pointers[i] = pointer - elif len(next.keys()) > 0: - pointers[i] = next - elif isinstance(obj, dict): - if obj.get('_type') == 'pointer': - value = obj.get('value') - return Pointer(value) - else: - for k, v in obj.items(): - next = {} - pointer = __findPointers(v, next) - if pointer is not None: - pointers[k] = pointer - elif len(next.keys()) > 0: - pointers[k] = next - return None - -def mergeObj(o1, o2): - if o2 is None: - return o1 - if o1 is None: - return o2 - if isinstance(o2, dict): - for k, v in o2.items(): - o1[k] = mergeObj(o1[k], v) - return o1 - elif isinstance(o2, list): - for i, v in enumerate(o2): - o1[i] = mergeObj(o1[i], v) - return o1 - else: - return o2 - return None - - -if __name__ == '__main__': - o1 = { - 'dict' : { - 'key1' : 'value', - 'key2' : 'more value', - 'list' : [ - { - 'value' : 'inAList', - '_type' : 'pointer' - }, - { - 'value' : 'inAList', - '_type' : 'pointer' - } - ] - }, - 'key' : 'value', - 'heres one' : { - 'value' : 'inAList', - '_type' : 'pointer' - } - } - - print(o1) - pointers = {} - __findPointers(o1, pointers) - print(pointers) - print(mergeObj(o1, pointers)) \ No newline at end of file From 1e564135a96529ac7cb7f0c2bf1eebe05d881024 Mon Sep 17 00:00:00 2001 From: Kevin Welsh Date: Mon, 2 Jul 2018 19:54:10 -0500 Subject: [PATCH 09/11] Corrected things pylint catches --- TFFactory/Factory.py | 4 +--- TFFactory/PythonNode.py | 38 ++++++++++++++++----------------- TFFactory/SupportedFunctions.py | 17 ++++++++++++--- TFFactory/file.txt | 2 -- 4 files changed, 34 insertions(+), 27 deletions(-) delete mode 100644 TFFactory/file.txt diff --git a/TFFactory/Factory.py b/TFFactory/Factory.py index 9d7ba33..2f79ad4 100644 --- a/TFFactory/Factory.py +++ b/TFFactory/Factory.py @@ -12,8 +12,7 @@ def CreateTFGraph(graph): nodes = {} - for key, node in graph.items(): - print(node) + for key, _ in graph.items(): __buildBranch(graph, key, nodes) return nodes @@ -104,7 +103,6 @@ def __buildTFNode(key, graph, allNodes): graph = graph, allNodes = allNodes)) - print('AllArgs: {}'.format(allArgs)) args = allArgs['args'] kwargs = allArgs['kwargs'] for d in dependencies: diff --git a/TFFactory/PythonNode.py b/TFFactory/PythonNode.py index 8acd1b3..7300fe9 100644 --- a/TFFactory/PythonNode.py +++ b/TFFactory/PythonNode.py @@ -4,39 +4,39 @@ from .GraphBuilder import Encoder, Decoder from .Pointer import Pointer + class PythonNode(Node): Encoder = Encoder() Decoder = Decoder() - def __init__(self, id, evalFunc, tensor = None, - args = None, kwargs = None, pointerMap = None, needToFeed = None): - + + def __init__(self, id, evalFunc, tensor=None, + args=None, kwargs=None, pointerMap=None, needToFeed=None): + self.EvalFunc = evalFunc self.Tensor = tensor self.Args = args if args is not None else [] self.Kwargs = kwargs if kwargs is not None else {} self.AllArgs = { - 'args' : self.Args, - 'kwargs' : self.Kwargs + 'args': self.Args, + 'kwargs': self.Kwargs } self.NeedToFeed = needToFeed if needToFeed is not None else {} self.PointerMap = pointerMap if pointerMap is not None else {} - return super().__init__(id) - def _eval(self, session = None, feed_dict = {}, newContext = True): - pointers = PythonNode.Decoder.decode(PythonNode.Encoder.encode(self.PointerMap)) - print(pointers) + def _eval(self, session=None, feed_dict={}, newContext=True): + pointers = PythonNode.Decoder.decode( + PythonNode.Encoder.encode(self.PointerMap)) - pointers = findAndApply(pointers, Pointer.IsInstance, + pointers = findAndApply(pointers, Pointer.IsInstance, partial(self._feed, - nodeMap = self.NeedToFeed, - session = session, - feed_dict = feed_dict, - newContext = newContext)) + nodeMap=self.NeedToFeed, + session=session, + feed_dict=feed_dict, + newContext=newContext)) self._mergeObj(self.AllArgs, pointers) - print(self.AllArgs) val = self.EvalFunc(*self.AllArgs['args'], **self.AllArgs['kwargs']) return val - + @staticmethod def _feed(nodeMap, session, feed_dict, newContext, pointer): return nodeMap[pointer.Ref].Eval(session, feed_dict, newContext) @@ -49,12 +49,12 @@ def _mergeObj(o1, o2): return o2 if isinstance(o2, dict): for k, v in o2.items(): - o1[k] = mergeObj(o1[k], v) + o1[k] = PythonNode._mergeObj(o1[k], v) return o1 elif isinstance(o2, list): for i, v in enumerate(o2): - o1[i] = mergeObj(o1[i], v) + o1[i] = PythonNode._mergeObj(o1[i], v) return o1 else: return o2 - return None \ No newline at end of file + return None diff --git a/TFFactory/SupportedFunctions.py b/TFFactory/SupportedFunctions.py index 2831165..c50054e 100644 --- a/TFFactory/SupportedFunctions.py +++ b/TFFactory/SupportedFunctions.py @@ -1,29 +1,40 @@ from .DataLoader import DataLoader +import numpy as np DL = DataLoader() + def readFile(fp, delim, nRows): - return DL.sampleFile(fp, nRows, delim, caching = True) + global DL + return DL.sampleFile(fp, nRows, caching=True) + def splitFile(Source, SegmentDelimeter, DataDelimeter, SegmentIndex, Shape): data = [] for row in Source: - data.append(list(map(float, row.split(SegmentDelimeter)[SegmentIndex].split(DataDelimeter)))) + data.append(list(map(float, row.split(SegmentDelimeter) + [SegmentIndex].split(DataDelimeter)))) return np.array(data).reshape((*Shape)) + def testAdd(a, b): return [a + b] # TF ones import tensorflow + + def AdamOptimizer(loss, learningRate): optimizer = tensorflow.train.AdamOptimizer(learningRate) return optimizer.minimize(loss) + def MomentumOptimizer(loss, learningRate, momentum): - optimizer = tensorflow.train.MomentumOptimizer(learningRate, momentum = momentum) + optimizer = tensorflow.train.MomentumOptimizer( + learningRate, momentum=momentum) return optimizer.minimize(loss) + def GradientDescentOptimizer(loss, learningRate): optimizer = tensorflow.train.GradientDescentOptimizer(learningRate) return optimizer.minimize(loss) diff --git a/TFFactory/file.txt b/TFFactory/file.txt deleted file mode 100644 index 6138637..0000000 --- a/TFFactory/file.txt +++ /dev/null @@ -1,2 +0,0 @@ -1,2,3,4,5;2,3,4,5,6 -3,4,5,6,7;7,6,3,2,1 \ No newline at end of file From cd9f45c3618874429b332beec6f897383adc7988 Mon Sep 17 00:00:00 2001 From: Kevin Welsh Date: Tue, 3 Jul 2018 10:52:58 -0500 Subject: [PATCH 10/11] Suppports dirichlet + tensor slicing --- TFFactory/DataLoader.py | 5 +- TFFactory/GraphBuilder.py | 85 ++++++++++++++++++++++----------- TFFactory/Node.py | 14 ++++-- TFFactory/PythonNode.py | 1 + TFFactory/SupportedFunctions.py | 12 +++++ TFFactory/TFNode.py | 19 ++++---- test.py | 22 ++++----- 7 files changed, 102 insertions(+), 56 deletions(-) diff --git a/TFFactory/DataLoader.py b/TFFactory/DataLoader.py index 04a5634..86fb714 100644 --- a/TFFactory/DataLoader.py +++ b/TFFactory/DataLoader.py @@ -1,11 +1,12 @@ import numpy as np + class DataLoader(object): def __init__(self): self.FileCache = {} return - def sampleFile(self, filePath, nRows, caching = True): + def sampleFile(self, filePath, nRows, caching=True): data = [] if caching: if filePath in self.FileCache: @@ -27,5 +28,3 @@ def sampleFile(self, filePath, nRows, caching = True): data[r] = row self.FileCache[filePath] = data return data - - diff --git a/TFFactory/GraphBuilder.py b/TFFactory/GraphBuilder.py index 01470be..7fe0cc2 100644 --- a/TFFactory/GraphBuilder.py +++ b/TFFactory/GraphBuilder.py @@ -10,12 +10,14 @@ ID_COUNTER = defaultdict(int) CURRENT_GRAPH = {} + def NewGraph(): global ID_COUNTER, CURRENT_GRAPH ID_COUNTER = defaultdict(int) CURRENT_GRAPH = {} return + def _assignFunctions(this, functions, type): for f in functions: attrs = f.split('.') @@ -26,17 +28,18 @@ def _assignFunctions(this, functions, type): curObj = getattr(curObj, v) setattr(curObj, attrs[-1], _mockFunction(f, type)) + def _mockFunction(funcName, type): def MockedFunction(*args, **kwargs): global ID_COUNTER, CURRENT_GRAPH - name = kwargs.get('name','Variable') + name = kwargs.get('name', 'Variable') shape = kwargs.pop('_shape', None) count = ID_COUNTER[name] ID_COUNTER[name] += 1 if count > 0: name = '{}_{}'.format(name, count) n = JSONNode(name, funcName, list(args), kwargs, shape, type) - CURRENT_GRAPH.update({n.ID : n}) + CURRENT_GRAPH.update({n.ID: n}) return n return MockedFunction @@ -52,11 +55,13 @@ def MockedFunction(*args, **kwargs): COMPOSITE_FUNCTIONS = [ 'SupportedFunctions.AdamOptimizer', 'SupportedFunctions.MomentumOptimizer', - 'SupportedFunctions.GradientDescentOptimizer' + 'SupportedFunctions.GradientDescentOptimizer', + 'SupportedFunctions.SampleDirichlet', + 'SupportedFunctions.GetItem' ] _assignFunctions(this, COMPOSITE_FUNCTIONS, 'tensorflowNode') -MOCKED_FUNCTIONS = [ +MOCKED_FUNCTIONS = [ 'tensorflow.placeholder', 'tensorflow.Variable', 'tensorflow.abs', @@ -92,36 +97,44 @@ def MockedFunction(*args, **kwargs): ] _assignFunctions(this, MOCKED_FUNCTIONS, 'tensorflowNode') + class Encoder(json.JSONEncoder): SERIALIZE_MAP = { - tensorflow.float16 : 'tensorflow.float16', - tensorflow.float32 : 'tensorflow.float32', - tensorflow.float64 : 'tensorflow.float64', - tensorflow.int8 : 'tensorflow.int8', - tensorflow.int16 : 'tensorflow.int16', - tensorflow.int32 : 'tensorflow.int32', - tensorflow.int64 : 'tensorflow.int64' + tensorflow.float16: 'tensorflow.float16', + tensorflow.float32: 'tensorflow.float32', + tensorflow.float64: 'tensorflow.float64', + tensorflow.int8: 'tensorflow.int8', + tensorflow.int16: 'tensorflow.int16', + tensorflow.int32: 'tensorflow.int32', + tensorflow.int64: 'tensorflow.int64' } + def default(self, obj): if isinstance(obj, JSONNode): return { - '_type' : obj.Type, - 'funcName' : obj.FuncName, - 'inputs' : obj.Inputs + '_type': obj.Type, + 'funcName': obj.FuncName, + 'inputs': obj.Inputs } elif isinstance(obj, Pointer): return { - 'value' : obj.Ref, - '_type' : 'pointer' + 'value': obj.Ref, + '_type': 'pointer' + } + elif isinstance(obj, slice): + return { + 'value': [obj.start, obj.stop, obj.step], + '_type': 'slice' } elif isinstance(obj, Hashable) and obj in Encoder.SERIALIZE_MAP: return { - 'value' : Encoder.SERIALIZE_MAP[obj], - '_type' : 'tensorflow' + 'value': Encoder.SERIALIZE_MAP[obj], + '_type': 'tensorflow' } return json.JSONEncoder.default(self, obj) + class Decoder(json.JSONDecoder): DESERIALIZE_MAP = { 'tensorflow.float16': tensorflow.float16, @@ -130,35 +143,41 @@ class Decoder(json.JSONDecoder): 'tensorflow.int8': tensorflow.int8, 'tensorflow.int16': tensorflow.int16, 'tensorflow.int32': tensorflow.int32, - 'tensorflow.int64': tensorflow.int64 + 'tensorflow.int64': tensorflow.int64 } + def __init__(self, *args, **kwargs): - json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) + json.JSONDecoder.__init__( + self, object_hook=self.object_hook, *args, **kwargs) return def object_hook(self, obj): if '_type' in obj: t = obj['_type'] - if t == 'tensorflow' : + if t == 'tensorflow': return Decoder.DESERIALIZE_MAP[obj['value']] - if t == 'pointer': + elif t == 'pointer': return Pointer(obj['value']) + elif t == 'slice': + return slice(*obj['value']) return obj + class JSONNode: def __init__(self, id, funcName, args, kwargs, shape, type): self.ID = str(id) self.FuncName = funcName self.Type = str(type) self.Inputs = { - 'args' : args, - 'kwargs' : kwargs, - '_shape' : shape + 'args': args, + 'kwargs': kwargs, + '_shape': shape } - self.Inputs = findAndApply(self.Inputs, self._shouldBePointer, self._replaceWithPointer) + self.Inputs = findAndApply( + self.Inputs, self._shouldBePointer, self._replaceWithPointer) return - + @staticmethod def _shouldBePointer(obj): return isinstance(obj, JSONNode) @@ -166,32 +185,40 @@ def _shouldBePointer(obj): @staticmethod def _replaceWithPointer(obj): return Pointer(obj.ID) - + + def __getitem__(self, key): + return GetItem(self, key) + def __neg__(self): return multiply(-1, self) + def __pos__(self): return abs(self) def __add__(self, other): return add(self, other) + def __radd__(self, other): return add(other, self) __iadd__ = __add__ def __sub__(self, other): return subtract(self, other) + def __rsub__(self, other): return subtract(other, self) __isub__ = __sub__ def __mul__(self, other): return multiply(self, other) + def __rmul__(self, other): return multiply(other, self) __imul__ = __mul__ def __truediv__(self, other): return divide(self, other) + def __rtruediv__(self, other): return divide(other, self) __itruediv__ = __truediv__ @@ -200,4 +227,4 @@ def __eq__(self, other): return other.ID == self.ID def __hash__(self): - return hash(self.ID) \ No newline at end of file + return hash(self.ID) diff --git a/TFFactory/Node.py b/TFFactory/Node.py index 633bbb4..8c3e313 100644 --- a/TFFactory/Node.py +++ b/TFFactory/Node.py @@ -1,13 +1,15 @@ import numpy as np + class Node: EvalContext = None + def __init__(self, id): self.ID = id self.LastContext = {} return - def Eval(self, session = None, feed_dict = {}, newContext = True): + def Eval(self, session=None, feed_dict={}, newContext=True): if self.ID in feed_dict: return feed_dict[self.ID] @@ -18,16 +20,18 @@ def Eval(self, session = None, feed_dict = {}, newContext = True): if Node.EvalContext is not None: if Node.EvalContext in self.LastContext: return self.LastContext[Node.EvalContext] - - val = self._eval(session = session, feed_dict = feed_dict, newContext = newContext) + + val = self._eval(session=session, feed_dict=feed_dict, + newContext=newContext) if Node.EvalContext is not None: self.LastContext[Node.EvalContext] = val return val - def _eval(self, session = None, feed_dict = {}, newContext = True): + def _eval(self, session=None, feed_dict={}, newContext=True): raise NotImplementedError def __hash__(self): return hash(self.ID) + def __eq__(self, other): - return other.ID == self.ID \ No newline at end of file + return other.ID == self.ID diff --git a/TFFactory/PythonNode.py b/TFFactory/PythonNode.py index 7300fe9..495ade0 100644 --- a/TFFactory/PythonNode.py +++ b/TFFactory/PythonNode.py @@ -22,6 +22,7 @@ def __init__(self, id, evalFunc, tensor=None, } self.NeedToFeed = needToFeed if needToFeed is not None else {} self.PointerMap = pointerMap if pointerMap is not None else {} + super().__init__(id) def _eval(self, session=None, feed_dict={}, newContext=True): pointers = PythonNode.Decoder.decode( diff --git a/TFFactory/SupportedFunctions.py b/TFFactory/SupportedFunctions.py index c50054e..013d743 100644 --- a/TFFactory/SupportedFunctions.py +++ b/TFFactory/SupportedFunctions.py @@ -38,3 +38,15 @@ def MomentumOptimizer(loss, learningRate, momentum): def GradientDescentOptimizer(loss, learningRate): optimizer = tensorflow.train.GradientDescentOptimizer(learningRate) return optimizer.minimize(loss) + + +def SampleDirichlet(concentration, sampleShape, validateArgs=False, allowNanStats=True, **kwargs): + dist = tensorflow.distributions.Dirichlet(concentration, + validate_args=validateArgs, + allow_nan_stats=allowNanStats) + + return dist.sample(sampleShape, **kwargs) + + +def GetItem(tensor, key): + return tensor[key] diff --git a/TFFactory/TFNode.py b/TFFactory/TFNode.py index 5d17b6c..6dbd0c8 100644 --- a/TFFactory/TFNode.py +++ b/TFFactory/TFNode.py @@ -1,16 +1,18 @@ from .Node import Node from .TFFactoryException import TFFactoryException + class TFNode(Node): - def __init__(self, id, tensor, needToFeed = None): - + def __init__(self, id, tensor, needToFeed=None): + self.Tensor = tensor self.NeedToFeed = needToFeed if needToFeed is not None else {} - return super().__init__(id) + super().__init__(id) - def _eval(self, session = None, feed_dict = {}, newContext = True): + def _eval(self, session=None, feed_dict={}, newContext=True): if self.ID in self.NeedToFeed: - raise TFFactoryException('Node {} was not fed during evaluation.'.format(self.ID)) + raise TFFactoryException( + 'Node {} was not fed during evaluation.'.format(self.ID)) if len(self.NeedToFeed) == 0: val = session.run(self.Tensor) @@ -18,6 +20,7 @@ def _eval(self, session = None, feed_dict = {}, newContext = True): tf_feed_dict = {} # Swap out placeholders with node eval function results. for key, node in self.NeedToFeed.items(): - tf_feed_dict[node.Tensor] = node.Eval(session, feed_dict, False) - val = session.run(self.Tensor, feed_dict = tf_feed_dict) - return val \ No newline at end of file + tf_feed_dict[node.Tensor] = node.Eval( + session, feed_dict, False) + val = session.run(self.Tensor, feed_dict=tf_feed_dict) + return val diff --git a/test.py b/test.py index d211a9c..386d426 100644 --- a/test.py +++ b/test.py @@ -5,23 +5,23 @@ if __name__ == '__main__': - pythonNode = tff.testAdd(1, -100, _shape = [1]) + pythonNode = tff.testAdd(1, -100, _shape=[1]) - placeHolder = tff.placeholder(tf.float32, shape = [3], name = 'input') - n = tff.Variable([-1, -2, -3], name = 'n1', dtype = tf.float32) - b = tff.Variable(initial_value = [4, 5, 6], name = 'b', dtype = tf.float32) + placeHolder = tff.placeholder(tf.float32, shape=[3], name='input') + n = tff.Variable([-1, -2, -3], name='n1', dtype=tf.float32) + sliced = tff.SampleDirichlet([0.2, 0.8], [1, 5], name='dirichlet')[0, :] + b = tff.Variable(initial_value=[4, 5, 6], name='b', dtype=tf.float32) n = n + b + placeHolder + pythonNode n = tff.AdamOptimizer(n, 1.0) - graph = json.dumps(tff.CURRENT_GRAPH, cls = tff.Encoder, indent = 2) + graph = json.dumps(tff.CURRENT_GRAPH, cls=tff.Encoder, indent=2) print(graph) - graph = json.loads(graph, cls = tff.Decoder) + graph = json.loads(graph, cls=tff.Decoder) print('Compiling graph!') compiledGraph = factory.CreateTFGraph(graph) with tf.Session() as sess: - tf.global_variables_initializer().run(session = sess) - for k,v in compiledGraph.items(): - print('{} = {}'.format(k, v.Eval(session = sess, feed_dict = {'input' : [1,2,3]}))) - - + tf.global_variables_initializer().run(session=sess) + for k, v in compiledGraph.items(): + print('{} = {}'.format( + k, v.Eval(session=sess, feed_dict={'input': [1, 2, 3]}))) From e0f704e7aaa25b0938baecc0d6784fdb56e4e6b8 Mon Sep 17 00:00:00 2001 From: Kevin Welsh Date: Tue, 3 Jul 2018 19:15:45 -0500 Subject: [PATCH 11/11] Works better! --- TFFactory/Factory.py | 72 ++++++++++++++++++++++----------------- TFFactory/GraphBuilder.py | 6 ++-- 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/TFFactory/Factory.py b/TFFactory/Factory.py index 2f79ad4..e920e43 100644 --- a/TFFactory/Factory.py +++ b/TFFactory/Factory.py @@ -10,6 +10,7 @@ from .Utilities import findAndApply from functools import partial + def CreateTFGraph(graph): nodes = {} for key, _ in graph.items(): @@ -17,6 +18,7 @@ def CreateTFGraph(graph): return nodes + def __buildBranch(graph, key, allNodes): """ @graph -- The full json graph for traversal/reference @@ -25,7 +27,7 @@ def __buildBranch(graph, key, allNodes): @needtofeed -- the keys of the nodes that need to be supplied to the feed_dict for evaluation @allNodes[key] will hold a reference to the node. """ - if key in allNodes: + if key in allNodes: return allNodes[key] _type = graph[key].get('_type') @@ -35,12 +37,13 @@ def __buildBranch(graph, key, allNodes): node = __buildPythonNode(key, graph, allNodes) elif _type == 'tensorflowNode': node = __buildTFNode(key, graph, allNodes) - + if node is None: raise AssertionError('Unsupported node type: {}'.format(_type)) allNodes[key] = node return node + def __buildPythonNode(key, graph, allNodes): graphNode = graph[key] funcName = graphNode['funcName'] @@ -49,16 +52,16 @@ def __buildPythonNode(key, graph, allNodes): needToFeed = {} allArgs = { - 'args' : inputs.get('args', []), - 'kwargs' : inputs.get('kwargs', {}) + 'args': inputs.get('args', []), + 'kwargs': inputs.get('kwargs', {}) } dependencies = set() - allArgs = findAndApply(allArgs, Pointer.IsInstance, - partial(__markPythonDependency, - dependencies = dependencies, - grpah = graph, - allNodes = allNodes)) - + allArgs = findAndApply(allArgs, Pointer.IsInstance, + partial(__markPythonDependency, + dependencies=dependencies, + grpah=graph, + allNodes=allNodes)) + args = allArgs['args'] kwargs = allArgs['kwargs'] for d in dependencies: @@ -66,25 +69,27 @@ def __buildPythonNode(key, graph, allNodes): placeholder = None if '_shape' in inputs: - placeholder = tensorflow.placeholder(tensorflow.float32, - shape = inputs['_shape'], - name = 'Placeholder_{}'.format(key)) - + placeholder = tensorflow.placeholder(tensorflow.float32, + shape=inputs['_shape'], + name='Placeholder_{}'.format(key)) + node = PythonNode(key, - tensor = placeholder, - evalFunc = func, - args = args, - kwargs = kwargs, - needToFeed = needToFeed) + tensor=placeholder, + evalFunc=func, + args=args, + kwargs=kwargs, + needToFeed=needToFeed) allNodes[key] = node return node + def __markPythonDependency(pointer, dependencies, graph, allNodes): node = __buildBranch(graph, pointer.Ref, allNodes) dependencies.add(node) return pointer + def __buildTFNode(key, graph, allNodes): graphNode = graph[key] funcName = graphNode['funcName'] @@ -93,15 +98,15 @@ def __buildTFNode(key, graph, allNodes): needToFeed = {} allArgs = { - 'args' : inputs.get('args', []), - 'kwargs' : inputs.get('kwargs', {}) + 'args': inputs.get('args', []), + 'kwargs': inputs.get('kwargs', {}) } dependencies = set() - allArgs = findAndApply(allArgs, Pointer.IsInstance, - partial(__markTFDependency, - dependencies = dependencies, - graph = graph, - allNodes = allNodes)) + allArgs = findAndApply(allArgs, Pointer.IsInstance, + partial(__markTFDependency, + dependencies=dependencies, + graph=graph, + allNodes=allNodes)) args = allArgs['args'] kwargs = allArgs['kwargs'] @@ -111,15 +116,20 @@ def __buildTFNode(key, graph, allNodes): needToFeed[d.ID] = d elif isinstance(d, TFNode): # For all of the TF node dependencies - # Add all of their dependencies to the list. + # Add all of their dependencies to the list. needToFeed.update(**d.NeedToFeed) - node = TFNode(key, tfOp(*args, **kwargs), needToFeed = needToFeed) + try: + node = TFNode(key, tfOp(*args, **kwargs), needToFeed=needToFeed) + except Exception as e: + raise TFFactoryException( + 'Error occured while compiling node: {}'.format(str(graphNode))) from e if tfOp == tensorflow.placeholder: # It can't eval itself without being fed. So hack that in, I guess? # Will cause an infinite loop if things break. Wont if they dont! - node.NeedToFeed.update({key : node}) + node.NeedToFeed.update({key: node}) return node + def __markTFDependency(pointer, dependencies, graph, allNodes): node = __buildBranch(graph, pointer.Ref, allNodes) if node not in dependencies: @@ -128,6 +138,7 @@ def __markTFDependency(pointer, dependencies, graph, allNodes): return node.Tensor return None + def __resolveRef(ref): if 'tensorflow' not in ref and 'SupportedFunctions' not in ref: return None @@ -140,6 +151,7 @@ def __resolveRef(ref): return obj + def __findPointers(obj, pointers): if isinstance(obj, list): for i, o in enumerate(obj): @@ -162,5 +174,3 @@ def __findPointers(obj, pointers): elif len(next.keys()) > 0: pointers[k] = next return None - - diff --git a/TFFactory/GraphBuilder.py b/TFFactory/GraphBuilder.py index 7fe0cc2..a2ca5a5 100644 --- a/TFFactory/GraphBuilder.py +++ b/TFFactory/GraphBuilder.py @@ -32,12 +32,12 @@ def _assignFunctions(this, functions, type): def _mockFunction(funcName, type): def MockedFunction(*args, **kwargs): global ID_COUNTER, CURRENT_GRAPH - name = kwargs.get('name', 'Variable') + name = kwargs.get('name', 'unnamed') shape = kwargs.pop('_shape', None) count = ID_COUNTER[name] ID_COUNTER[name] += 1 if count > 0: - name = '{}_{}'.format(name, count) + name = '{}:{}'.format(name, count) n = JSONNode(name, funcName, list(args), kwargs, shape, type) CURRENT_GRAPH.update({n.ID: n}) return n @@ -190,7 +190,7 @@ def __getitem__(self, key): return GetItem(self, key) def __neg__(self): - return multiply(-1, self) + return multiply(-1.0, self) def __pos__(self): return abs(self)