diff --git a/astroquery/esasky/__init__.py b/astroquery/esasky/__init__.py index af4dcc29f1..398cff72fc 100644 --- a/astroquery/esasky/__init__.py +++ b/astroquery/esasky/__init__.py @@ -1,22 +1,80 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst +import warnings + from astropy import config as _config +from astropy.config import paths +from astropy.utils.exceptions import AstropyDeprecationWarning + +import os + +ESASKY_COMMON_SERVER = "https://sky.esa.int/esasky-tap/" + +ESASKY_TAP_COMMON = "tap" class Conf(_config.ConfigNamespace): """ Configuration parameters for `astroquery.esasky`. """ - urlBase = _config.ConfigItem( - 'https://sky.esa.int/esasky-tap', - 'ESASky base URL') - timeout = _config.ConfigItem( - 1000, - 'Time limit for connecting to template_module server.') + ESASKY_DOMAIN_SERVER = _config.ConfigItem(ESASKY_COMMON_SERVER, "ESASky TAP Common Server", + aliases=['astroquery.esasky.urlBase']) + ESASKY_TAP_SERVER = _config.ConfigItem(ESASKY_COMMON_SERVER + ESASKY_TAP_COMMON, "ESASky TAP Server") + ESASKY_DATA_SERVER = _config.ConfigItem(ESASKY_COMMON_SERVER + 'data?', "ESASky Data Server") + ESASKY_TABLES_SERVER = _config.ConfigItem(ESASKY_COMMON_SERVER + ESASKY_TAP_COMMON + "/tables", + "ESASky TAP Tables Server") + ESASKY_TARGET_ACTION = _config.ConfigItem("servlet/target-resolver?", "ESASky Target Resolver") + ESASKY_MESSAGES = _config.ConfigItem("notification?action=GetNotifications", "ESASky Messages") + ESASKY_LOGIN_SERVER = _config.ConfigItem(ESASKY_COMMON_SERVER + 'login', "ESASky Login Server") + ESASKY_LOGOUT_SERVER = _config.ConfigItem(ESASKY_COMMON_SERVER + 'logout', "ESASky Logout Server") + ESASKY_CONNECTION_TIMEOUT = _config.ConfigItem(1000, 'Time limit for connecting to a data product server.', + aliases=['astroquery.esasky.timeout']) + ESASKY_ROW_LIMIT = _config.ConfigItem(10000, 'Maximum number of rows returned (set to -1 for unlimited).', + aliases=['astroquery.esasky.row_limit']) + + @property + def urlBase(self): + return self.ESASKY_DOMAIN_SERVER + + @urlBase.setter + def urlBase(self, value): + warnings.warn( + "'urlBase' is deprecated and will be removed in a future version. " + "Use 'ESASKY_DOMAIN_SERVER' instead.", + AstropyDeprecationWarning, + stacklevel=2, + ) + self.ESASKY_DOMAIN_SERVER = value + + @property + def timeout(self): + return self.ESASKY_CONNECTION_TIMEOUT + + @timeout.setter + def timeout(self, value): + warnings.warn( + "'timeout' is deprecated and will be removed in a future version. " + "Use 'ESASKY_CONNECTION_TIMEOUT' instead.", + AstropyDeprecationWarning, + stacklevel=2, + ) + self.ESASKY_CONNECTION_TIMEOUT = value + + @property + def row_limit(self): + return self.ESASKY_ROW_LIMIT + + @row_limit.setter + def row_limit(self, value): + warnings.warn( + "'row_limit' is deprecated and will be removed in a future version. " + "Use 'ESASKY_ROW_LIMIT' instead.", + AstropyDeprecationWarning, + stacklevel=2, + ) + self.ESASKY_ROW_LIMIT = value - row_limit = _config.ConfigItem( - 10000, - 'Maximum number of rows returned (set to -1 for unlimited).') + cache_location = os.path.join(paths.get_cache_dir(), 'astroquery/esasky', ) conf = Conf() diff --git a/astroquery/esasky/core.py b/astroquery/esasky/core.py index 34755789b3..0f47cab752 100644 --- a/astroquery/esasky/core.py +++ b/astroquery/esasky/core.py @@ -1,5 +1,6 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst - +from astropy.utils.exceptions import AstropyDeprecationWarning +import astroquery.esa.utils.utils as esautils import json import os import tarfile as esatar @@ -14,14 +15,13 @@ from astropy.coordinates import Angle from astropy.io import fits from astropy.utils.console import ProgressBar +from astropy.utils import deprecated from astroquery import log from requests import HTTPError from requests import ConnectionError -from ..query import BaseQuery -from ..utils.tap.core import TapPlus +from astroquery.esa.utils import EsaTap from ..utils import commons -from ..utils import async_to_sync from . import conf from .. import version from astropy.coordinates.name_resolve import sesame_database @@ -33,12 +33,15 @@ esatar.TarFile.extraction_filter = staticmethod(esatar.fully_trusted_filter) -@async_to_sync -class ESASkyClass(BaseQuery): +class ESASkyClass(EsaTap): - URLbase = conf.urlBase - TIMEOUT = conf.timeout - DEFAULT_ROW_LIMIT = conf.row_limit + URLbase = conf.ESASKY_DOMAIN_SERVER + ESA_ARCHIVE_NAME = "ESASky" + TAP_URL = conf.ESASKY_TAP_SERVER + LOGIN_URL = conf.ESASKY_LOGIN_SERVER + LOGOUT_URL = conf.ESASKY_LOGOUT_SERVER + CONNECTION_TIMEOUT = conf.ESASKY_CONNECTION_TIMEOUT + DEFAULT_ROW_LIMIT = conf.ESASKY_ROW_LIMIT __FITS_STRING = ".fits" __FTZ_STRING = ".FTZ" @@ -79,21 +82,29 @@ class ESASkyClass(BaseQuery): SSO_TYPES = ['ALL', 'ASTEROID', 'COMET', 'SATELLITE', 'PLANET', 'DWARF_PLANET', 'SPACECRAFT', 'SPACEJUNK', 'EXOPLANET', 'STAR'] - def __init__(self, tap_handler=None): - super().__init__() + def __init__(self, *, tap_handler=None, show_messages=False, auth_session=None, tap_url=None): + super().__init__(auth_session=auth_session, tap_url=tap_url) + if tap_handler is not None: + warnings.warn( + "The 'tap_handler' parameter is deprecated and will be removed in a future version. " + "Use the ESASky instance directly for TAP queries (Using esa.utils.EsaTap and PyVO).", + AstropyDeprecationWarning, + stacklevel=2, + ) - if tap_handler is None: - self._tap = TapPlus(url=self.URLbase + "/tap") - else: - self._tap = tap_handler + if show_messages: + self.get_status_messages() - def query(self, query, *, output_file=None, output_format="votable", verbose=False): - """Launches a synchronous job to query the ESASky TAP + def query(self, query, *, async_job=False, output_file=None, output_format="votable", verbose=False): + """Launches a synchronous or asynchronous job to query the ESASky TAP Parameters ---------- query : str, mandatory query (adql) to be executed + async_job : bool, optional, default 'False' + executes the query (job) in asynchronous/synchronous mode (default + synchronous) output_file : str, optional, default None file name where the results are saved if dumpToFile is True. If this parameter is not provided, the jobid is used instead @@ -107,44 +118,10 @@ def query(self, query, *, output_file=None, output_format="votable", verbose=Fal ------- A table object """ - if not verbose: - with warnings.catch_warnings(): - commons.suppress_vo_warnings() - warnings.filterwarnings("ignore", category=u.UnitsWarning) - job = self._tap.launch_job(query=query, output_file=output_file, output_format=output_format, - verbose=False, dump_to_file=output_file is not None) - else: - job = self._tap.launch_job(query=query, output_file=output_file, output_format=output_format, - verbose=True, dump_to_file=output_file is not None) - return job.get_results() + return self.query_tap(query=query, async_job=async_job, output_file=output_file, output_format=output_format, + verbose=verbose) - def get_tables(self, *, only_names=True, verbose=False, cache=True): - """ - Get the available table in ESASky TAP service - - Parameters - ---------- - only_names : bool, optional, default 'True' - True to load table names only - verbose : bool, optional, default 'False' - flag to display information about the process - - Returns - ------- - A list of tables - """ - - if cache and self._cached_tables is not None: - tables = self._cached_tables - else: - tables = self._tap.load_tables(only_names=only_names, include_shared_tables=False, verbose=verbose) - self._cached_tables = tables - if only_names: - return [t.name for t in tables] - else: - return tables - - def get_columns(self, table_name, *, only_names=True, verbose=False): + def get_columns(self, table_name, *, only_names=True): """ Get the available columns for a table in ESASky TAP service @@ -154,30 +131,20 @@ def get_columns(self, table_name, *, only_names=True, verbose=False): table name of which, columns will be returned only_names : bool, optional, default 'True' True to load table names only - verbose : bool, optional, default 'False' - flag to display information about the process Returns ------- A list of columns """ - - tables = self.get_tables(only_names=False, verbose=verbose) - columns = None - for table in tables: - if str(table.name) == str(table_name): - columns = table.columns - break - - if columns is None: - raise ValueError("table name specified is not found in " - "ESASky TAP service") + columns = self.get_table(table=table_name).columns if only_names: return [c.name for c in columns] else: return columns + @deprecated(since="0.4.12", message="The ESASky module no longer uses the TapPlus module. Equivalent functionality" + "is available directly on the ESASky module (Using esa.utils.EsaTap and PyVO).") def get_tap(self): """ Get a TAP+ instance for the ESASky servers, which supports @@ -191,7 +158,7 @@ def get_tap(self): tap : `~astroquery.utils.tap.core.TapPlus` """ - return self._tap + return self def list_maps(self): """ @@ -944,9 +911,9 @@ def query_ids_catalogs(self, source_ids, *, catalogs=__ALL_STRING, row_limit=DEF Examples -------- - query_ids_catalogs(source_ids=['2CXO J090341.1-322609', '2CXO J090353.8-322642'], catalogs="CHANDRA-SC2") - query_ids_catalogs(source_ids='2CXO J090341.1-322609') - query_ids_catalogs(source_ids=['2CXO J090341.1-322609', '45057'], catalogs=["CHANDRA-SC2", "Hipparcos-2"]) + query_ids_catalogs(source_ids=['2CXO J031306.2-852820', '2CXO J031339.7-852543'], catalogs="CHANDRA-SC21") + query_ids_catalogs(source_ids='2CXO J031306.2-852820') + query_ids_catalogs(source_ids=['2CXO J031306.2-852820', '45057'], catalogs=["CHANDRA-SC21", "Hipparcos-2"]) """ sanitized_catalogs = self._sanitize_input_catalogs(catalogs) sanitized_row_limit = self._sanitize_input_row_limit(row_limit) @@ -1349,6 +1316,27 @@ def get_spectra_from_table(self, query_table_list, missions=__ALL_STRING, log.info("No spectra found.") return spectra + def get_status_messages(self): + """Retrieve the messages to inform users about the status of the ESASky TAP""" + + try: + esautils.execute_servlet_request( + url=conf.ESASKY_TAP_SERVER + "/" + conf.ESASKY_MESSAGES, + tap=self.tap, + query_params={}, + parser_method=self.parse_messages_response + ) + except OSError: + print("Status messages could not be retrieved") + + def parse_messages_response(self, response): + string_messages = [] + for line in response.iter_lines(): + string_message = line.decode("utf-8") + string_messages.append(string_message[string_message.index('=') + 1:]) + print(string_messages[len(string_messages)-1]) + return string_messages + def _sanitize_input_radius(self, radius): if isinstance(radius, (str, u.Quantity)): return radius @@ -1709,9 +1697,9 @@ def _query(self, name, descriptors, verbose=False, **kwargs): if not query: # Could not create query. The most common reason for this is a type mismatch between user specified ID and # data type of database column. - # For example query_ids_catalogs(source_ids=["2CXO J090341.1-322609"], mission=["CHANDRA", "HSC"]) + # For example query_ids_catalogs(source_ids=["2CXO J031306.2-852820"], mission=["CHANDRA", "HSC"]) # would be able to create a query for Chandra, but not for Hubble because the hubble source id column type - # is a number and "2CXO J090341.1-322609" cannot be converted to a number. + # is a number and "2CXO J031306.2-852820" cannot be converted to a number. return query return self.query(query, output_format="votable", verbose=verbose) @@ -1773,13 +1761,13 @@ def _build_id_query(self, ids, row_limit, descriptor): if id_column == "designation": id_column = "obsid" - data_type = None - for column in self.get_columns(table_name=descriptor['table_name'], only_names=False): + datatype = None + for column in self.get_table(table=descriptor['table_name']).columns: if column.name == id_column: - data_type = column.data_type + datatype = column.datatype valid_ids = ids - if data_type in self._NUMBER_DATA_TYPES: + if datatype in self._NUMBER_DATA_TYPES: valid_ids = [int(obs_id) for obs_id in ids if obs_id.isdigit()] if not valid_ids: raise ValueError(f"Could not construct query for mission {descriptor['mission']}. Database column " @@ -1850,7 +1838,7 @@ def _send_get_request(self, url_extension, request_payload, cache): return self._request('GET', url, params=request_payload, - timeout=self.TIMEOUT, + timeout=self.CONNECTION_TIMEOUT, cache=cache, headers=self._get_header()) @@ -1861,4 +1849,4 @@ def _get_header(self): return {'User-Agent': user_agent} -ESASky = ESASkyClass() +ESASky = ESASkyClass(show_messages=False) diff --git a/astroquery/esasky/tests/test_esasky_remote.py b/astroquery/esasky/tests/test_esasky_remote.py index 9f43ed294e..5fd8b6a504 100755 --- a/astroquery/esasky/tests/test_esasky_remote.py +++ b/astroquery/esasky/tests/test_esasky_remote.py @@ -6,9 +6,8 @@ import pytest from astropy.io.fits.hdu.hdulist import HDUList +from astroquery.exceptions import InputWarning from astroquery.utils.commons import TableList -from astroquery.utils.tap.model.tapcolumn import TapColumn -from astroquery.utils.tap.model.taptable import TapTableMeta from astroquery.esasky import ESASky @@ -34,18 +33,18 @@ def test_esasky_query_ids_maps(self): assert "1342221848" in result["HERSCHEL"].columns["observation_id"] def test_esasky_query_ids_catalogs(self): - result = ESASky.query_ids_catalogs(source_ids=["2CXO J090341.1-322609", "2CXO J090353.8-322642"], - catalogs="CHANDRA-SC2") + result = ESASky.query_ids_catalogs(source_ids=["2CXO J031306.2-852820", "2CXO J031339.7-852543"], + catalogs="CHANDRA-SC21") assert isinstance(result, TableList) - assert "2CXO J090341.1-322609" in result["CHANDRA-SC2"].columns["name"] - assert "2CXO J090353.8-322642" in result["CHANDRA-SC2"].columns["name"] + assert "2CXO J031306.2-852820" in result["CHANDRA-SC21"].columns["name"] + assert "2CXO J031339.7-852543" in result["CHANDRA-SC21"].columns["name"] - result = ESASky.query_ids_catalogs(source_ids=["2CXO J090341.1-322609", - "2CXO J090353.8-322642", "44899", "45057"], - catalogs=["CHANDRA-SC2", "Hipparcos-2"]) + result = ESASky.query_ids_catalogs(source_ids=["2CXO J031306.2-852820", + "2CXO J031339.7-852543", "44899", "45057"], + catalogs=["CHANDRA-SC21", "Hipparcos-2"]) assert isinstance(result, TableList) - assert "2CXO J090341.1-322609" in result["CHANDRA-SC2"].columns["name"] - assert "2CXO J090353.8-322642" in result["CHANDRA-SC2"].columns["name"] + assert "2CXO J031306.2-852820" in result["CHANDRA-SC21"].columns["name"] + assert "2CXO J031339.7-852543" in result["CHANDRA-SC21"].columns["name"] assert "44899" in result["HIPPARCOS-2"].columns["name"] assert "45057" in result["HIPPARCOS-2"].columns["name"] @@ -85,8 +84,7 @@ def test_esasky_get_images_obs_id(self, tmp_path, mission, obsid): assert isinstance(result[mission.upper()][0]["500"], HDUList) else: assert isinstance(result[mission.upper()][0], HDUList) - for hdu_list in result[mission.upper()]: - hdu_list.close() + self._close_hdu_lists(result, mission) @pytest.mark.parametrize("mission, observation_id", zip(["ISO-IR", "Chandra", "IUE", "XMM-NEWTON", @@ -118,30 +116,28 @@ def test_esasky_query_object_maps(self): 'ISO-IR', 'Herschel', 'Spitzer']) def test_esasky_get_images(self, tmp_path, mission): result = ESASky.get_images(position="M51", missions=mission, download_dir=tmp_path) - assert tmp_path.stat().st_size + assert any(p.is_file() for p in tmp_path.rglob("*")) if mission != "Herschel" and result: - for hdu_list in result[mission.upper()]: - hdu_list.close() + self._close_hdu_lists(result, mission) @pytest.mark.bigdata def test_esasky_get_images_for_erosita(self, tmp_path): mission = 'eROSITA' - result = ESASky.get_images(position="67.84 -61.44", missions=mission, download_dir=tmp_path) - assert tmp_path.stat().st_size - - for hdu_list in result[mission.upper()]: - hdu_list.close() + with pytest.warns(InputWarning): + result = ESASky.get_images(position="67.84 -61.44", missions=mission, download_dir=tmp_path) + assert any(p.is_file() for p in tmp_path.rglob("*")) + self._close_hdu_lists(result, mission) @pytest.mark.bigdata @pytest.mark.parametrize('mission, position', zip(['JWST-MID-IR', 'JWST-NEAR-IR'], ['340.50123388127435 -69.17904779241904', '225.6864099965157 -3.0315781490149467'])) def test_esasky_get_images_jwst(self, tmp_path, mission, position): - result = ESASky.get_images(position=position, missions=mission, download_dir=tmp_path) - assert tmp_path.stat().st_size - for hdu_list in result[mission.upper()]: - hdu_list.close() + with pytest.warns(InputWarning): + result = ESASky.get_images(position=position, missions=mission, download_dir=tmp_path) + assert any(p.is_file() for p in tmp_path.rglob("*")) + self._close_hdu_lists(result, mission) @pytest.mark.bigdata def test_esasky_get_images_hst(self, tmp_path): @@ -167,14 +163,12 @@ def test_esasky_get_maps(self, tmp_path): iso_maps[mission].remove_rows([0, 1]) result = ESASky.get_maps(iso_maps, download_dir=tmp_path) assert len(os.listdir(file_path)) == len(all_maps[mission]) - 2 - for hdu_list in result[mission]: - hdu_list.close() + self._close_hdu_lists(result, mission) iso_maps2 = dict({mission: all_maps[mission][:2]}) result = ESASky.get_maps(iso_maps2, download_dir=tmp_path) assert len(os.listdir(file_path)) == len(all_maps[mission]) - for hdu_list in result[mission]: - hdu_list.close() + self._close_hdu_lists(result, mission) def test_esasky_query_region_spectra(self): result = ESASky.query_region_spectra(position="M51", radius="5 arcmin") @@ -196,8 +190,7 @@ def test_esasky_get_spectra(self, tmp_path, mission): assert Path(tmp_path, mission.upper()).exists() if mission != "Herschel": - for hdu_list in result[mission.upper()]: - hdu_list.close() + self._close_hdu_lists(result, mission) def test_esasky_get_spectra_small(self, tmp_path): missions = ['HST-IR'] @@ -217,35 +210,24 @@ def test_esasky_get_spectra_from_table(self, tmp_path): # Remove a few maps, so the other list will have downloadable ones, too iso_spectra[mission].remove_rows([0, 1]) result = ESASky.get_spectra_from_table(query_table_list=iso_spectra, download_dir=tmp_path) - for hdu_list in result[mission]: - hdu_list.close() + self._close_hdu_lists(result, mission) assert len(os.listdir(file_path)) == len(all_spectra[mission]) - 2 iso_spectra2 = dict({mission: all_spectra[mission][:2]}) result = ESASky.get_spectra_from_table(query_table_list=iso_spectra2, download_dir=tmp_path) - for hdu_list in result[mission]: - hdu_list.close() + self._close_hdu_lists(result, mission) assert len(os.listdir(file_path)) == len(all_spectra[mission]) def test_query(self): result = ESASky.query(query="SELECT * from observations.mv_v_esasky_xmm_om_uv_fdw") - assert len(result) == 2000 # Default row limit is 2000 + assert len(result) > 0 def test_get_tables(self): table_names = ESASky.get_tables(only_names=True) assert len(table_names) > 70 tables = ESASky.get_tables(only_names=False) - assert isinstance(tables[0], TapTableMeta) assert len(table_names) == len(tables) - def test_get_columns(self): - column_names = ESASky.get_columns(table_name='observations.mv_v_esasky_xmm_om_uv_fdw', only_names=True) - assert len(column_names) == 17 - - columns = ESASky.get_columns(table_name='observations.mv_v_esasky_xmm_om_uv_fdw', only_names=False) - assert isinstance(columns[0], TapColumn) - assert len(column_names) == len(columns) - def test_esasky_query_sso(self): result = ESASky.query_sso(sso_name="ceres") assert isinstance(result, TableList) @@ -303,3 +285,8 @@ def test_esasky_get_images_sso(self, tmp_path): assert isinstance(fits_files["XMM"][0], HDUList) assert Path(tmp_path, "XMM").exists() + + def _close_hdu_lists(self, result, mission): + for hdu_list in result[mission.upper()]: + if hdu_list is not None: + hdu_list.close()