From f008b67be927db63bb712a9808891bd05472c226 Mon Sep 17 00:00:00 2001 From: Purush Swaminathan Date: Sat, 11 Feb 2017 21:52:14 -0700 Subject: [PATCH 1/4] New functional resolver --- ref_resolver/resolver.py | 118 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 ref_resolver/resolver.py diff --git a/ref_resolver/resolver.py b/ref_resolver/resolver.py new file mode 100644 index 0000000..f8eb8aa --- /dev/null +++ b/ref_resolver/resolver.py @@ -0,0 +1,118 @@ +from urlparse import urlparse, urljoin +import simplejson as json +from os.path import isfile +import jsonpath_rw +import requests +import logging + + +logging.basicConfig(level=logging.DEBUG) + + +def debug(message): + logging.debug(message) + +def info(message): + logging.info(message) + +cache = {} + +class IdError(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class URLFragment(object): + + def __init__(self): + self.url_fragments = None + + def __init__(self, id_): + self.id = id_ + if id is not None: + self.url_fragments = urlparse(id_) + else: + self.url_fragments = None + + +def resolveInFile(fragment, fileObj): + path_expr = jsonpath_rw.parse("$" + ".".join(fragment.fragment.split("/"))) + matched_values = [match.value for match in path_expr.find(fileObj)] + return matched_values[0] if len(matched_values) > 0 else None + +#def parseAsFile(filename): + + +def parseAsHttp(url, ref_file): + debug("parseAsHttp:: " + url + ", "+ ref_file) + json_dump = None + if callable(requests.Response.json): + json_dump = requests.get(url).json() + else: + json_dump = requests.get(url).json + if 'id' in json_dump: + cache[ref_file] = resolve(json_dump) + return cache[ref_file] + else: + raise IdError("$ref-ed file has no `id`. Will not continue parsing anything. Go fix it!") + + +def parseRef(fragment, value): + ref_frag = urlparse(value) + ref_file = ref_frag.netloc + ref_frag.path + if ref_file in cache: + return cache[ref_file] + else: + if fragment.url_fragments.scheme in ['http', 'https']: + # http/https scheme retrieval of $refs + return parseAsHttp(urljoin(fragment.url_fragments.scheme, fragment.id, ref_file), ref_file) + #elif fragment.url_fragments.scheme == 'file' and isfile(ref_file): + # local file absolute and relative paths retrieval of $refs + # return parseAsFile(ref_file) + #elif fragment.url_fragments.scheme == "": + # same file internal $ref + # return parseAsFile(fragment.url_fragments.netloc + fragment.url_fragments.path) + else: + raise Exception("Scheme of resolution: " + fragment.url_fragments.scheme + " is currently not supported") + + +def update(fragment, key, value): + if "$ref" == key: + debug("update$ref:: " + key) + return parseRef(fragment, value) + else: + return {key: apropose(fragment, value)} + +def apropose(fragment, elem): + debug("apropose:: ") + if isinstance(elem, dict): + return parse(fragment, elem) + elif isinstance(elem, list): + return map(lambda x: apropose(fragment, x), elem) + else: + return elem + +def parse(fragment, json_dict): + mut_dict = {} + for (key,value) in json_dict.iteritems(): + debug("parse:: " + key) + mut_dict.update(update(fragment, key, value)) + return mut_dict + + +def resolve(json_obj): + """ + Resolves $ref in the `json_obj` and returns a `dict` that has inlined $ref elements. + Raises an `IdError` if the `id` key is not present in the `json_obj`. + """ + if 'id' not in json_obj: + raise IdError("No `id` field in passed parameter") + else: + debug("resolve::" + json_obj.get('id')) + return parse(URLFragment(json_obj.get("id")), json_obj) + + +if __name__ == "__main__": + ejson = json.load(open('test_schema.json')) + print resolve(ejson) From 6b680b65eac581688a68d9f04dfec653d353474f Mon Sep 17 00:00:00 2001 From: Purush Swaminathan Date: Sun, 12 Feb 2017 21:12:39 -0700 Subject: [PATCH 2/4] Now basic resolution with HTTP refs works --- ref_resolver/resolver.py | 51 +++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/ref_resolver/resolver.py b/ref_resolver/resolver.py index f8eb8aa..ced4da0 100644 --- a/ref_resolver/resolver.py +++ b/ref_resolver/resolver.py @@ -35,25 +35,33 @@ def __init__(self, id_): else: self.url_fragments = None + def __str__(self): + return repr(self.id + ", " + str(self.url_fragments)) + def resolveInFile(fragment, fileObj): - path_expr = jsonpath_rw.parse("$" + ".".join(fragment.fragment.split("/"))) + debug("resolveInFile:: fragment->" + fragment) + path_expr = jsonpath_rw.parse("$" + ".".join(fragment.split("/"))) matched_values = [match.value for match in path_expr.find(fileObj)] + debug("resolveInFile:: matches :: " + str(matched_values[0])) return matched_values[0] if len(matched_values) > 0 else None #def parseAsFile(filename): -def parseAsHttp(url, ref_file): - debug("parseAsHttp:: " + url + ", "+ ref_file) +def parseAsHttp(url): + debug("parseAsHttp::url -> " + url) json_dump = None if callable(requests.Response.json): json_dump = requests.get(url).json() + debug(json_dump) else: json_dump = requests.get(url).json if 'id' in json_dump: - cache[ref_file] = resolve(json_dump) - return cache[ref_file] + _id = json_dump.get('id') + if _id not in cache: + cache[_id] = resolve(json_dump) + return cache[_id] else: raise IdError("$ref-ed file has no `id`. Will not continue parsing anything. Go fix it!") @@ -61,26 +69,31 @@ def parseAsHttp(url, ref_file): def parseRef(fragment, value): ref_frag = urlparse(value) ref_file = ref_frag.netloc + ref_frag.path - if ref_file in cache: - return cache[ref_file] + debug(value) + # ref_frag ->ParseResult(scheme=u'http', netloc=u'localhost:3000', path=u'/ref_schema.json', params='', query='', fragment=u'/definitions/address') + + if ref_frag.scheme in ['http', 'https']: + # http/https scheme retrieval of $refs + _url = ref_frag.scheme + "://" + ref_file # -> http://localhost:3000/ref_schema.json + http_file_json = parseAsHttp(_url) + debug("resolveInFile :: " +ref_frag.fragment) + return resolveInFile(ref_frag.fragment, http_file_json) + #elif fragment.url_fragments.scheme == 'file' and isfile(ref_file): + # local file absolute and relative paths retrieval of $refs + # return parseAsFile(ref_file) + #elif fragment.url_fragments.scheme == "": + # same file internal $ref + # return parseAsFile(fragment.url_fragments.netloc + fragment.url_fragments.path) else: - if fragment.url_fragments.scheme in ['http', 'https']: - # http/https scheme retrieval of $refs - return parseAsHttp(urljoin(fragment.url_fragments.scheme, fragment.id, ref_file), ref_file) - #elif fragment.url_fragments.scheme == 'file' and isfile(ref_file): - # local file absolute and relative paths retrieval of $refs - # return parseAsFile(ref_file) - #elif fragment.url_fragments.scheme == "": - # same file internal $ref - # return parseAsFile(fragment.url_fragments.netloc + fragment.url_fragments.path) - else: - raise Exception("Scheme of resolution: " + fragment.url_fragments.scheme + " is currently not supported") + raise Exception("Scheme of resolution: " + fragment.url_fragments.scheme + " is currently not supported") def update(fragment, key, value): if "$ref" == key: debug("update$ref:: " + key) - return parseRef(fragment, value) + parsedResult = parseRef(fragment, value) + debug("update$ref to -> "+ str(parseRef(fragment, value))) + return parsedResult else: return {key: apropose(fragment, value)} From 333f87b4c0e49402f69f410975639e50cb504753 Mon Sep 17 00:00:00 2001 From: Purush Swaminathan Date: Sun, 12 Feb 2017 22:23:09 -0700 Subject: [PATCH 3/4] Now handles file:// format --- ref_resolver/resolver.py | 76 +++++++++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/ref_resolver/resolver.py b/ref_resolver/resolver.py index ced4da0..81b7f37 100644 --- a/ref_resolver/resolver.py +++ b/ref_resolver/resolver.py @@ -45,11 +45,27 @@ def resolveInFile(fragment, fileObj): matched_values = [match.value for match in path_expr.find(fileObj)] debug("resolveInFile:: matches :: " + str(matched_values[0])) return matched_values[0] if len(matched_values) > 0 else None - -#def parseAsFile(filename): + + +def resolveRefFile(json_dump): + if 'id' in json_dump: + _id = json_dump.get('id') + if _id not in cache: + cache[_id] = resolve(json_dump) + return cache[_id] + else: + raise IdError("$ref-ed file has no `id`. Will not continue parsing anything. Go fix it!") +def parseAsFile(filename): + json_dump = json.load(open(filename)) + return resolveRefFile(json_dump) + + def parseAsHttp(url): + """ + Use `requests` library to get json located at `url` and call `resolveRefFile` to resolve the $refs in the http-ed json further. + """ debug("parseAsHttp::url -> " + url) json_dump = None if callable(requests.Response.json): @@ -57,16 +73,13 @@ def parseAsHttp(url): debug(json_dump) else: json_dump = requests.get(url).json - if 'id' in json_dump: - _id = json_dump.get('id') - if _id not in cache: - cache[_id] = resolve(json_dump) - return cache[_id] - else: - raise IdError("$ref-ed file has no `id`. Will not continue parsing anything. Go fix it!") - + return resolveRefFile(json_dump) + def parseRef(fragment, value): + """ + Parse the $ref value and resolve using the scheme of resolution from `urlparse(value)`. + """ ref_frag = urlparse(value) ref_file = ref_frag.netloc + ref_frag.path debug(value) @@ -78,35 +91,50 @@ def parseRef(fragment, value): http_file_json = parseAsHttp(_url) debug("resolveInFile :: " +ref_frag.fragment) return resolveInFile(ref_frag.fragment, http_file_json) - #elif fragment.url_fragments.scheme == 'file' and isfile(ref_file): - # local file absolute and relative paths retrieval of $refs - # return parseAsFile(ref_file) + elif ref_frag.scheme == 'file': + # local file absolute and relative paths retrieval of $refs + if isfile(ref_file): + return resolveInFile(ref_frag.fragment,parseAsFile(ref_file)) + else: + raise Exception("FileNotFoundException:: "+ ref_file) #elif fragment.url_fragments.scheme == "": # same file internal $ref # return parseAsFile(fragment.url_fragments.netloc + fragment.url_fragments.path) else: raise Exception("Scheme of resolution: " + fragment.url_fragments.scheme + " is currently not supported") - + + +def resolveInnerElement(fragment, elem): + """ + Resolves inner elements of value attributes. Handles nesting gracefully with standard recursive procedures. + """ + if isinstance(elem, dict): + return parse(fragment, elem) + elif isinstance(elem, list): + return map(lambda x: resolveInnerElement(fragment, x), elem) + else: + return elem + def update(fragment, key, value): + """ + Either resolves the value part if the key is `$ref` or just resolves the inner element in value recursively. + """ if "$ref" == key: debug("update$ref:: " + key) parsedResult = parseRef(fragment, value) debug("update$ref to -> "+ str(parseRef(fragment, value))) return parsedResult else: - return {key: apropose(fragment, value)} + return {key: resolveInnerElement(fragment, value)} -def apropose(fragment, elem): - debug("apropose:: ") - if isinstance(elem, dict): - return parse(fragment, elem) - elif isinstance(elem, list): - return map(lambda x: apropose(fragment, x), elem) - else: - return elem - + def parse(fragment, json_dict): + """ + Looks at every key-value pair in `json_dict`, resolves $refs and returns new dictionary + containing resolutions. + This method is not purely functional. It creates a mutable dictionary and puts things into it. Yuck! + """ mut_dict = {} for (key,value) in json_dict.iteritems(): debug("parse:: " + key) From 5304793acc47b109f223afaec90d15dea222629b Mon Sep 17 00:00:00 2001 From: Purush Swaminathan Date: Mon, 13 Feb 2017 11:41:57 -0700 Subject: [PATCH 4/4] Ticking off self ref stuff --- ref_resolver/resolver.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/ref_resolver/resolver.py b/ref_resolver/resolver.py index 81b7f37..07e2916 100644 --- a/ref_resolver/resolver.py +++ b/ref_resolver/resolver.py @@ -11,23 +11,23 @@ def debug(message): logging.debug(message) + def info(message): logging.info(message) cache = {} + class IdError(Exception): def __init__(self, value): self.value = value def __str__(self): return repr(self.value) + class URLFragment(object): - def __init__(self): - self.url_fragments = None - def __init__(self, id_): self.id = id_ if id is not None: @@ -39,15 +39,19 @@ def __str__(self): return repr(self.id + ", " + str(self.url_fragments)) -def resolveInFile(fragment, fileObj): - debug("resolveInFile:: fragment->" + fragment) - path_expr = jsonpath_rw.parse("$" + ".".join(fragment.split("/"))) +def resolveInFile(jsonpath, fileObj): + """ + Run through the `fileObj` json dict for the json path in `fragment` and return the object under the path. + """ + path_expr = jsonpath_rw.parse("$" + ".".join(jsonpath.split("/"))) matched_values = [match.value for match in path_expr.find(fileObj)] - debug("resolveInFile:: matches :: " + str(matched_values[0])) return matched_values[0] if len(matched_values) > 0 else None def resolveRefFile(json_dump): + """ + Resolves a ref schema file. If the file is in the cache, returns the contents from the cache. + """ if 'id' in json_dump: _id = json_dump.get('id') if _id not in cache: @@ -58,6 +62,9 @@ def resolveRefFile(json_dump): def parseAsFile(filename): + """ + Opens and resolves a ref schema from file system. + """ json_dump = json.load(open(filename)) return resolveRefFile(json_dump) @@ -66,11 +73,9 @@ def parseAsHttp(url): """ Use `requests` library to get json located at `url` and call `resolveRefFile` to resolve the $refs in the http-ed json further. """ - debug("parseAsHttp::url -> " + url) json_dump = None if callable(requests.Response.json): json_dump = requests.get(url).json() - debug(json_dump) else: json_dump = requests.get(url).json return resolveRefFile(json_dump) @@ -82,14 +87,12 @@ def parseRef(fragment, value): """ ref_frag = urlparse(value) ref_file = ref_frag.netloc + ref_frag.path - debug(value) # ref_frag ->ParseResult(scheme=u'http', netloc=u'localhost:3000', path=u'/ref_schema.json', params='', query='', fragment=u'/definitions/address') if ref_frag.scheme in ['http', 'https']: # http/https scheme retrieval of $refs _url = ref_frag.scheme + "://" + ref_file # -> http://localhost:3000/ref_schema.json http_file_json = parseAsHttp(_url) - debug("resolveInFile :: " +ref_frag.fragment) return resolveInFile(ref_frag.fragment, http_file_json) elif ref_frag.scheme == 'file': # local file absolute and relative paths retrieval of $refs @@ -97,9 +100,9 @@ def parseRef(fragment, value): return resolveInFile(ref_frag.fragment,parseAsFile(ref_file)) else: raise Exception("FileNotFoundException:: "+ ref_file) - #elif fragment.url_fragments.scheme == "": + elif ref_frag.scheme == "": # same file internal $ref - # return parseAsFile(fragment.url_fragments.netloc + fragment.url_fragments.path) + return resolveInFile(ref_frag.fragment, json.load(open(fragment.url_fragments.path))) else: raise Exception("Scheme of resolution: " + fragment.url_fragments.scheme + " is currently not supported") @@ -121,9 +124,7 @@ def update(fragment, key, value): Either resolves the value part if the key is `$ref` or just resolves the inner element in value recursively. """ if "$ref" == key: - debug("update$ref:: " + key) parsedResult = parseRef(fragment, value) - debug("update$ref to -> "+ str(parseRef(fragment, value))) return parsedResult else: return {key: resolveInnerElement(fragment, value)} @@ -137,7 +138,6 @@ def parse(fragment, json_dict): """ mut_dict = {} for (key,value) in json_dict.iteritems(): - debug("parse:: " + key) mut_dict.update(update(fragment, key, value)) return mut_dict @@ -150,10 +150,10 @@ def resolve(json_obj): if 'id' not in json_obj: raise IdError("No `id` field in passed parameter") else: - debug("resolve::" + json_obj.get('id')) return parse(URLFragment(json_obj.get("id")), json_obj) if __name__ == "__main__": - ejson = json.load(open('test_schema.json')) - print resolve(ejson) + self_ref_json = json.load(open('self_ref_test.schema.json')) + #ejson = json.load(open('test_schema.json')) + print resolve(self_ref_json)