From 7e2e5aa5ece841f9e2fe39c4808354bed133036d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Alexandre?= Date: Thu, 23 Aug 2018 00:34:43 -0300 Subject: [PATCH 1/5] IpeaData: Package review --- .travis.yml | 2 +- ipeaData/ipeadata.py | 72 ---------------------------------- ipeadata/__init__.py | 1 + ipeadata/api.py | 93 ++++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- test/test_api.py | 15 +++---- 6 files changed, 104 insertions(+), 81 deletions(-) delete mode 100644 ipeaData/ipeadata.py create mode 100644 ipeadata/__init__.py create mode 100644 ipeadata/api.py diff --git a/.travis.yml b/.travis.yml index 377994b..4eaa1f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,5 +6,5 @@ install: - pip install -r requirements.txt # command to run tests script: - - pylint ipeaData/ipeadata.py + - pylint ipeadata/api.py - coverage run --source=. -m unittest discover -s test \ No newline at end of file diff --git a/ipeaData/ipeadata.py b/ipeaData/ipeadata.py deleted file mode 100644 index b83fd89..0000000 --- a/ipeaData/ipeadata.py +++ /dev/null @@ -1,72 +0,0 @@ -""" Module to get data from ipeadata.com.br """ -import requests as req -import pandas as pd - - -def basic_api_call(api): - """ - Function to make a call on ipedata api - :param api: url on api - :return: a dataFrame with data - """ - response = req.get(api) - if response.status_code == req.codes.ok: # pylint: disable=no-member - json_response = response.json() - if 'value' in json_response: - try: - data_frame = pd.DataFrame(json_response['value']) - return data_frame - except Exception: # pylint: disable=broad-except - return None - return None - -def get_sources(): - """ - Get sources from ipea web site - :return: a data frame with the information - """ - api = "http://ipeadata2-homologa.ipea.gov.br/api/v1/Fontes" - return basic_api_call(api) - -def get_metadata(serie=None): - """ - Return metadata of a serie - :param serie: serie to search for otherwise return metadata for all series - :return: a data frame - """ - url_final = "('%s')" % serie if serie is not None else '' - api = "http://ipeadata2-homologa.ipea.gov.br/api/v1/Metadados%s" % url_final - return basic_api_call(api) - - -def get_nivel_region(serie): - """ - Return region nivel of a serie - :param serie: serie to search for - :return: a data frame - """ - api = ("http://ipeadata2-homologa.ipea.gov.br/api/v1/Metadados('{}')" - "/Valores?$apply=groupby((NIVNOME))&$orderby=NIVNOME").format(serie) - return basic_api_call(api) - -# pylint: disable=invalid-name -def ipeadata(serie, groupby=None): - """ - Return the values from a given serie - :param serie: a serie to search for - :return: a data frame with the values - """ - if groupby is not None: - df = get_nivel_region(serie) - if df['NIVNOME'].isin([groupby]).any(): - api = ("http://ipeadata2-homologa.ipea.gov.br/api/v1/AnoValors" - "(SERCODIGO='{}',NIVNOME='{}')?$top=100&$skip=0&$orderby" - "=SERATUALIZACAO&$count=true").format(serie, groupby) - return basic_api_call(api) - return None - api = "http://ipeadata2-homologa.ipea.gov.br/api/v1/ValoresSerie(SERCODIGO='%s')" % serie - return basic_api_call(api) - - -if __name__ == "__main__": - print(ipeadata('ADMIS')) diff --git a/ipeadata/__init__.py b/ipeadata/__init__.py new file mode 100644 index 0000000..b67343b --- /dev/null +++ b/ipeadata/__init__.py @@ -0,0 +1 @@ +from ipeadata.api import get_data, get_region_level, get_metadata, get_sources diff --git a/ipeadata/api.py b/ipeadata/api.py new file mode 100644 index 0000000..596bc24 --- /dev/null +++ b/ipeadata/api.py @@ -0,0 +1,93 @@ +""" Module to get data from ipeadata.com.br """ +import requests as req +import pandas as pd + + +def basic_api_call(url): + """ + Calls IpeaData API. + + :param url: URL + + :return: DataFrame with requested data + :rtype: pandas DataFrame + """ + response = req.get(url) + + if response.status_code == req.codes.ok: # pylint: disable=no-member + + json_response = response.json() + if 'value' in json_response: + + df = pd.DataFrame(json_response['value']) + + # Raises an error if the DataFrame is empty + if df.empty: + raise FileNotFoundError('We were unable to find data according to the input parameters.') + + return df + + return None + + +def get_sources(): + """ + Lists sources from IpeaData containing the following fields: + + 1. FNTID: + 2. FNTSIGLA: + 3. MACRO: + 4. REGIONAL: + 5. SOCIAL: + + :return: DataFrame with sources attributes + :rtype: pandas DataFrame + """ + return basic_api_call("http://ipeadata2-homologa.ipea.gov.br/api/v1/Fontes") + + +def get_metadata(id): + """ + Returns metadata for the given time series. + + :param id: (str) time series id + + :return: Metadata for the given time series + :rtype: pandas DataFrame + """ + return basic_api_call("http://ipeadata2-homologa.ipea.gov.br/api/v1/Metadados('%s')" % id) + + +def get_region_level(id): + """ + Returns region level for the given time series. + + :param id: (str) time series id + + :return: Region level of a time series + :rtype: pandas DataFrame + """ + url = ("http://ipeadata2-homologa.ipea.gov.br/api/v1/Metadados('{}')/Valores?$apply=groupby((NIVNOME))&$orderby=NIV" + "NOME").format(id) + return basic_api_call(url) + + +# pylint: disable=invalid-name +def get_data(id, groupby=None): + """ + Returns data corresponding to the given time series. + + :param id: (str) time series id + + :return: Data for the given time series + :rtype: pandas DataFrame + """ + if groupby is not None: + df = get_region_level(id) + if df['NIVNOME'].isin([groupby]).any(): + url = ("http://ipeadata2-homologa.ipea.gov.br/api/v1/AnoValors(SERCODIGO='{}',NIVNOME='{}')?$top=100&$skip=" + "0&$orderby=SERATUALIZACAO&$count=true").format(id, groupby) + return basic_api_call(url) + return None + url = "http://ipeadata2-homologa.ipea.gov.br/api/v1/ValoresSerie(SERCODIGO='%s')" % id + return basic_api_call(url) diff --git a/setup.py b/setup.py index 3172503..98d5694 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ long_description = f.read() setup( - name='ipeaData', + name='ipeadata', version='0.0.4', packages=find_packages(exclude=["ipedata.tests"]), url='https://github.com/ipea/pyIpeaData', diff --git a/test/test_api.py b/test/test_api.py index 8b379e4..fd75464 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -1,16 +1,17 @@ -from ipeaData.ipeadata import * +from ipeadata.api import * import unittest + class TestApi(unittest.TestCase): def test_ipeadata(self): - self.assertIsNotNone(ipeadata('ADMIS')) + self.assertIsNotNone(get_data('ADMIS')) def test_ipeadata_groupby(self): - df = ipeadata('COMP', 'Estados') + df = get_data('COMP', 'Estados') self.assertIsNotNone(df) self.assertEqual(27, df.shape[0]) - df = ipeadata('COMP', 'nadanan') + df = get_data('COMP', 'nadanan') self.assertIsNone(df) def test_get_sources(self): @@ -18,12 +19,12 @@ def test_get_sources(self): def test_get_metadata(self): self.assertIsNotNone(get_metadata('ADMIS')) - self.assertIsNotNone(get_metadata()) def test_get_region(self): - df = get_nivel_region('QUANTLEITE') + df = get_region_level('QUANTLEITE') self.assertIsNotNone(df) self.assertEqual(13, df.shape[0]) + if __name__ == '__main__': - unittest.main(verbosity=3) \ No newline at end of file + unittest.main(verbosity=3) From 41fc0d4e976fedb6ca93b11942ea0faecb2d72f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Alexandre?= Date: Thu, 23 Aug 2018 00:59:00 -0300 Subject: [PATCH 2/5] IpeaData: Adding typing to basic_api_call docstring --- ipeadata/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipeadata/api.py b/ipeadata/api.py index 596bc24..88bcbc2 100644 --- a/ipeadata/api.py +++ b/ipeadata/api.py @@ -7,7 +7,7 @@ def basic_api_call(url): """ Calls IpeaData API. - :param url: URL + :param url: (str) URL :return: DataFrame with requested data :rtype: pandas DataFrame From b268a3b5b28d9609dfc2262a49766ed09f8223ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Alexandre?= Date: Thu, 23 Aug 2018 10:18:12 -0300 Subject: [PATCH 3/5] IpeaData: Fixing get_metadata() when no parameter is passed --- ipeadata/api.py | 8 +++++--- test/test_api.py | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ipeadata/api.py b/ipeadata/api.py index 88bcbc2..cc952b8 100644 --- a/ipeadata/api.py +++ b/ipeadata/api.py @@ -46,17 +46,19 @@ def get_sources(): return basic_api_call("http://ipeadata2-homologa.ipea.gov.br/api/v1/Fontes") -def get_metadata(id): +def get_metadata(id=None): """ Returns metadata for the given time series. + If no `id` parameter is passed, it returns metadata for all the time series in IpeaData. + :param id: (str) time series id :return: Metadata for the given time series :rtype: pandas DataFrame """ - return basic_api_call("http://ipeadata2-homologa.ipea.gov.br/api/v1/Metadados('%s')" % id) - + id = "('%s')" % id if id is not None else '' + return basic_api_call("http://ipeadata2-homologa.ipea.gov.br/api/v1/Metadados%s" % id) def get_region_level(id): """ diff --git a/test/test_api.py b/test/test_api.py index fd75464..5b231c8 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -19,6 +19,7 @@ def test_get_sources(self): def test_get_metadata(self): self.assertIsNotNone(get_metadata('ADMIS')) + self.assertIsNotNone(get_metadata()) def test_get_region(self): df = get_region_level('QUANTLEITE') From 078a4909ddda7bb16d47d56540eff06c849d380c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Alexandre?= Date: Thu, 23 Aug 2018 10:43:54 -0300 Subject: [PATCH 4/5] IpeaData: PEP8 compliance --- ipeadata/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ipeadata/api.py b/ipeadata/api.py index cc952b8..4fa03ee 100644 --- a/ipeadata/api.py +++ b/ipeadata/api.py @@ -60,6 +60,7 @@ def get_metadata(id=None): id = "('%s')" % id if id is not None else '' return basic_api_call("http://ipeadata2-homologa.ipea.gov.br/api/v1/Metadados%s" % id) + def get_region_level(id): """ Returns region level for the given time series. From e3d4151913bb87021ac3f23f5e71f9bd2d641bc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Alexandre?= Date: Tue, 28 Aug 2018 06:32:26 -0300 Subject: [PATCH 5/5] IpeaData: Changing package name and file name to meet PR review requests --- .travis.yml | 2 +- ipeaData/__init__.py | 1 + ipeadata/api.py => ipeaData/ipeadata.py | 0 ipeadata/__init__.py | 1 - setup.py | 2 +- test/test_api.py | 2 +- 6 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 ipeaData/__init__.py rename ipeadata/api.py => ipeaData/ipeadata.py (100%) delete mode 100644 ipeadata/__init__.py diff --git a/.travis.yml b/.travis.yml index 4eaa1f3..377994b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,5 +6,5 @@ install: - pip install -r requirements.txt # command to run tests script: - - pylint ipeadata/api.py + - pylint ipeaData/ipeadata.py - coverage run --source=. -m unittest discover -s test \ No newline at end of file diff --git a/ipeaData/__init__.py b/ipeaData/__init__.py new file mode 100644 index 0000000..3de044c --- /dev/null +++ b/ipeaData/__init__.py @@ -0,0 +1 @@ +from ipeaData.ipeadata import get_data, get_region_level, get_metadata, get_sources diff --git a/ipeadata/api.py b/ipeaData/ipeadata.py similarity index 100% rename from ipeadata/api.py rename to ipeaData/ipeadata.py diff --git a/ipeadata/__init__.py b/ipeadata/__init__.py deleted file mode 100644 index b67343b..0000000 --- a/ipeadata/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from ipeadata.api import get_data, get_region_level, get_metadata, get_sources diff --git a/setup.py b/setup.py index 98d5694..3172503 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ long_description = f.read() setup( - name='ipeadata', + name='ipeaData', version='0.0.4', packages=find_packages(exclude=["ipedata.tests"]), url='https://github.com/ipea/pyIpeaData', diff --git a/test/test_api.py b/test/test_api.py index 5b231c8..7ca5798 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -1,4 +1,4 @@ -from ipeadata.api import * +from ipeaData.ipeadata import * import unittest