diff --git a/.github/workflows/ci_devtests.yml b/.github/workflows/ci_devtests.yml index cfb68e4b1c..509fc3e7ea 100644 --- a/.github/workflows/ci_devtests.yml +++ b/.github/workflows/ci_devtests.yml @@ -5,11 +5,13 @@ on: push: branches: - main + - develop tags: - '*' pull_request: branches: - main + - develop schedule: # run every Monday at 5am UTC - cron: '0 5 * * 1' diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index d0db0f48cc..0135d90b1e 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -5,11 +5,13 @@ on: push: branches: - main + - develop tags: - '*' pull_request: branches: - main + - develop schedule: # run every Monday at 5am UTC - cron: '0 5 * * 1' diff --git a/.github/workflows/sync_fork.yml b/.github/workflows/sync_fork.yml new file mode 100644 index 0000000000..18abb115e6 --- /dev/null +++ b/.github/workflows/sync_fork.yml @@ -0,0 +1,45 @@ +name: Sync Fork +run-name: Sync Fork +on: + schedule: + - cron: '58 23 * * *' # run every day - two minutes to midnight + workflow_dispatch: # to enable manual runs of the workflow + +jobs: + Get-Timestamp: + runs-on: ubuntu-latest + steps: + - run: date + + Sync-With-Upstream: + runs-on: ubuntu-latest + steps: + - run: echo "The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "This job is now running on a ${{ runner.os }} server hosted by GitHub" + - run: echo "Running on branch ${{ github.ref }}, repository ${{ github.repository }}." + - name: Check out repository code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - run: echo "The ${{ github.repository }} repository has been cloned to the runner." + - name: List files in the repository + run: | + ls ${{ github.workspace }} + - name: Sync repository with upstream + run: | + cd ${{ github.workspace }} + git config --global user.email "jcarmona@eso.org" + git config --global user.name "Nightly Sync" + git remote add upstream https://github.com/astropy/astroquery.git + git remote -v + git fetch upstream main + echo "--- upstream log: " + git log upstream/main --oneline -10 + echo "--- current branch log before merge: " + git log --oneline -10 + git merge upstream/main + echo "--- current branch log after merge: " + git log --oneline -10 + echo "--- push force with lease" + git push --force-with-lease + - run: echo "The job finished with status ${{ job.status }}." diff --git a/docs/eso/eso.rst b/docs/eso/eso.rst index bda39aa253..92060126e7 100644 --- a/docs/eso/eso.rst +++ b/docs/eso/eso.rst @@ -4,516 +4,174 @@ ESO Queries (`astroquery.eso`) ****************************** -.. warning:: - - **Backward Compatibility Notice** +Quick Start +=========== - **The WDB (Web DataBase) API is being deprecated and replaced by TAP (Table Access Protocol)**, - a standardized interface for querying astronomical datasets using ADQL (Astronomical Data Query Language). - While the Python interface remains the same, the values accepted by the ``columns`` and ``column_filters`` - parameters must reflect TAP field names and ADQL syntax. This means that, - although the structure of your code won't need to change, **the values you pass to the arguments** - ``columns`` **and** ``column_filters`` **must be revised** to comply with the new format. +.. note:: - In TAP, ``column_filters`` accepts ADQL expressions. For example: + Check if this is a version of ``astroquery.eso`` (0.4.12 or later) that includes TAP support + (see **⚠️ Backward Compatibility Notice ⚠️** below). - .. doctest-skip:: +This quick start example shows how to use the ``astroquery.eso`` interface to query the ESO +Science Archive for both raw and reduced observations, as well as catalogue data. - column_filters = { - 'some_int_column': "< 5", - 'some_float_column_2': ">= 1.23", - 'some_char_column': "like '%John%'", - 'some_generic_column': "in ('mango', 'apple', 'kiwi')", - 'other_generic_column': "between '2024-01-01' and '2024-12-31'" - } +Observations +------------ - Please review your queries carefully and update them accordingly to ensure compatibility with the new astroquery versions. - See section :ref:`column-filters-fix` - - -Getting started -=============== - -This is a python interface for querying the ESO archive web service. -For now, it supports the following: - -- listing available instruments -- listing available surveys (phase 3) -- searching INSTRUMENT SPECIFIC raw data (table ``ist.``) via the ESO TAP service* -- searching data products (phase 3; table ``ivoa.ObsCore``) via the ESO TAP service* -- searching raw data (table ``dbo.raw``) via the ESO TAP service* -- downloading data by dataset identifiers: http://archive.eso.org/cms/eso-data/eso-data-direct-retrieval.html - -\* ESO TAP web interface: https://archive.eso.org/programmatic/#TAP - -Requirements -============ - -The following packages are required for the use of this module: - -* keyring -* lxml -* requests >= 2.4.0 - - -Authentication with ESO User Portal -=================================== - -Most of the datasets in the ESO Science Archive are public and can be downloaded anonymously -without authenticating with the ESO User Portal (https://www.eso.org/sso/login). -Data with restricted access like datasets under proprietary period can be downloaded by authorised users -(for example PIs of the corresponding observing programmes and their delegates) -after authentication with the ESO User Portal. -This authentication is performed directly with the provided :meth:`~astroquery.query.QueryWithLogin.login` command, -as illustrated in the example below. This method uses your keyring to securely -store the password in your operating system. As such you should have to enter your -correct password only once, and later be able to use this package for automated -interaction with the ESO archive. +Here, we perform a cone search for `ESPRESSO `_ +spectra towards the star `HD 37903 `_, +and show how to download the corresponding data products. .. doctest-skip:: >>> from astroquery.eso import Eso >>> eso = Eso() - >>> # First example: TEST is not a valid username, it will fail - >>> eso.login(username="TEST") # doctest: +SKIP - WARNING: No password was found in the keychain for the provided username. [astroquery.query] - TEST, enter your password: - - INFO: Authenticating TEST on https://www.eso.org/sso ... [astroquery.eso.core] - ERROR: Authentication failed! [astroquery.eso.core] - >>> # Second example: pretend ICONDOR is a valid username - >>> eso.login(username="ICONDOR", store_password=True) # doctest: +SKIP - WARNING: No password was found in the keychain for the provided username. [astroquery.query] - ICONDOR, enter your password: - - INFO: Authenticating ICONDOR on https://www.eso.org/sso ... [astroquery.eso.core] - INFO: Authentication successful! [astroquery.eso.core] - >>> # After the first login, your password has been stored - >>> eso.login(username="ICONDOR") # doctest: +SKIP - INFO: Authenticating ICONDOR on https://www.eso.org/sso ... [astroquery.eso.core] - INFO: Authentication successful! [astroquery.eso.core] - - >>> # Successful download of a public file (with or without login) - >>> eso.retrieve_data('AMBER.2006-03-14T07:40:19.830') # doctest: +SKIP - INFO: Downloading file 1/1 https://dataportal.eso.org/dataPortal/file/AMBER.2006-03-14T07:40:19.830 - INFO: Successfully downloaded dataset AMBER.2006-03-14T07:40:19.830 - - >>> # Access denied to a restricted-access file (as anonymous user or as authenticated but not authorised user) - >>> eso.retrieve_data('ADP.2023-03-02T01:01:24.355') # doctest: +SKIP - INFO: Downloading file 1/1 https://dataportal.eso.org/dataPortal/file/ADP.2023-03-02T01:01:24.355 - ERROR: Access denied to https://dataportal.eso.org/dataPortal/file/ADP.2023-03-02T01:01:24.355 - - -.. The doctest-skip directive is not working for pytest within the note directive, thus - we leave the snippet below as a code-block - -.. note:: - - Even after logging with `~astroquery.eso.EsoClass.login()`, TAP queries are not authenticated by default. To issue - authenticated queries, the keyword argument ``authenticated`` must be set to `True` explicitly in the query functions. - For example: - - .. code-block:: python - - eso = Eso() - eso.ROW_LIMIT = None - eso.login(username="ICONDOR") - - # Some files are missing without `authenticated=True` - files_tbl = eso.query_instrument("nirps") - - # Get all files to which the authenticated user has access - files_tbl = eso.query_instrument("nirps", authenticated=True) - - Review your queries carefully and update them accordingly to ensure you are retrieving - all your expected results. If you don't stricly need to run an authenticated query, use - the default ``authenticated=False``, since it will retrieve your results much faster. + >>> from astropy.coordinates import SkyCoord + >>> import astropy.units as u + >>> coords = SkyCoord.from_name("HD 37903") + >>> ra = coords.ra.value + >>> dec = coords.dec.value + >>> r = (1*u.arcsec).to(u.deg).value -Automatic password ------------------- + >>> table_raw = eso.query_main("ESPRESSO", cone_ra=ra, cone_dec=dec, cone_radius=r) + >>> table_reduced = eso.query_surveys("ESPRESSO", cone_ra=ra, cone_dec=dec, cone_radius=r) + + >>> eso.retrieve_data(table_raw["dp_id"]) + >>> eso.retrieve_data(table_reduced["dp_id"]) -As shown above, your password can be stored by the `keyring -`_ module, if you -pass the argument ``store_password=True`` to ``Eso.login()``. -For security reason, storing the password is turned off by default. +Catalogues +------------ -MAKE SURE YOU TRUST THE MACHINE WHERE YOU USE THIS FUNCTIONALITY!!! - -NB: You can delete your password later with the command -``keyring.delete_password('astroquery:www.eso.org', 'username')``. - -Automatic login ---------------- - -You can further automate the authentication process by configuring a default username. -The astroquery configuration file, which can be found following the procedure -detailed in `astropy.config `_, -needs to be edited by adding ``username = ICONDOR`` in the ``[eso]`` section. - -When configured, the username in the ``login()`` method call can be omitted -as follows: +Catalogue queries operate on published survey tables and return column-based +results. For example, +to query the `KiDS DR4 `_ +catalogue and inspect a small subset of rows: .. doctest-skip:: - >>> from astroquery.eso import Eso - >>> eso = Eso() - >>> eso.login() # doctest: +SKIP - ICONDOR, enter your ESO password: - -NB: If an automatic login is configured, other Eso methods can log you in -automatically when needed. - - -Query the ESO archive for raw data -================================== - -Identifying available instrument-specific queries -------------------------------------------------- + >>> table_cat = eso.query_catalogue("KiDS_DR4_1_ugriZYJHKs_cat_fits", ROW_LIMIT=5) -The direct retrieval of datasets is better explained with a running example, continuing from the -authentication example above. The first thing to do is to identify the instrument to query. The -list of available instrument-specific queries can be obtained with the -:meth:`~astroquery.eso.EsoClass.list_instruments` method. +By default, queries are limited to returning a maximum of 1000 rows. This limit +can be modified by setting the ``ROW_LIMIT`` attribute. To disable truncation +and return all matching results (up to TAP limit of 15,000,000), set: -.. doctest-remote-data:: - - >>> from astroquery.eso import Eso - >>> eso = Eso() - >>> eso.list_instruments() - ['alpaca', 'amber', 'apex', 'crires', 'efosc', 'eris', 'espresso', 'fiat', - 'fors1', 'fors2', 'giraffe', 'gravity', 'harps', 'hawki', 'isaac', 'kmos', - 'matisse', 'midi', 'muse', 'naco', 'nirps', 'omegacam', 'pionier', 'sinfoni', - 'sofi', 'sphere', 'uves', 'vimos', 'vircam', 'visir', 'wlgsu', 'xshooter'] - -In the example above, the instruments listed correspond to those retrieved by running the -following query on the ESO **Programmatic Access** website (https://archive.eso.org/programmatic/#TAP): - -``select table_name from TAP_SCHEMA.tables where schema_name='ist' order by table_name`` - - -Inspecting available query options ----------------------------------- - -Once an instrument is chosen, ``midi`` for example, the columns available for that instrument can be -inspected by setting the ``help=True`` keyword of the :meth:`~astroquery.eso.EsoClass.query_instrument` -method. The list of columns contains its datatype and unit. The xtype is to be more specific, -as certain columns with datatype ``char`` actually define timestamps or regions in the sky. - -.. doctest-remote-data:: - - >>> eso.query_instrument('midi', help=True) # doctest: +IGNORE_OUTPUT - INFO: - Columns present in the table ist.midi: - column_name datatype xtype unit - ------------------- -------- ----------- ------ - access_estsize long kbyte - access_url char - datalink_url char - date_obs char - dec double deg - del_ft_sensor char - del_ft_status char - det_dit float s - det_ndit int - dimm_fwhm_avg float arcsec - dimm_fwhm_rms float arcsec - dp_cat char - dp_id char - ... ... - release_date char timestamp - s_region char adql:REGION - ... ... - telescope char - tpl_expno int - tpl_id char - tpl_name char - tpl_nexp int - tpl_start char - utc float s - - Number of records present in the table ist.midi: - 421764 - [astroquery.eso.core] - -**Note:** for a deeper description of each column, the following query can be issued -on the ESO **Programmatic Access** website (https://archive.eso.org/programmatic/#TAP): - -``select column_name, description from TAP_SCHEMA.columns where table_name = 'ist.midi'`` - -Querying with constraints -------------------------- - -It is now time to query the ``midi`` instrument for datasets. In the following example, observations of -target ``NGC 4151`` between ``2008-01-01`` and ``2009-05-12`` are searched, and the query is configured to -return two columns: the date of observation and the name of the object. - -.. doctest-remote-data:: - >>> table = eso.query_instrument( - ... 'midi', - ... column_filters={ - ... 'object':'NGC4151', - ... 'exp_start': "between '2008-01-01' and '2009-05-12'" - ... }, - ... columns=['object', 'date_obs'] - ... ) - >>> table - - object date_obs - ------- ----------------------- - NGC4151 2008-04-22T02:07:50.154 - NGC4151 2008-04-22T02:08:20.345 - NGC4151 2008-04-22T02:09:47.846 - NGC4151 2008-04-22T02:10:18.038 - ... ... - NGC4151 2009-05-11T01:39:09.750 - NGC4151 2009-05-11T01:40:24.235 - NGC4151 2009-05-11T01:41:38.742 - NGC4151 2009-05-11T01:42:08.432 - - -Querying all instruments ------------------------- - -The ESO database can also be queried without a specific instrument in mind. -This is what the method :meth:`~astroquery.eso.EsoClass.query_main` is for. -The associated table on the ESO **Programmatic Access** website (https://archive.eso.org/programmatic/#TAP) -is ``dbo.raw``, and the simplest query would be: ``select * from dbo.raw``. -Except for the keyword specifying the instrument,the behaviour of :meth:`~astroquery.eso.EsoClass.query_main` -is identical to :meth:`~astroquery.eso.EsoClass.query_instrument`. - -ESO instruments without a specific query interface can be queried with -:meth:`~astroquery.eso.EsoClass.query_main`, specifying the ``instrument`` constraint. -This is the case of e.g. ``harps``, ``feros`` or the all sky cameras APICAM and MASCOT. Here is an example to -query all-sky images from APICAM with ``luminance`` filter. - -.. doctest-remote-data:: - - >>> eso.maxrec = -1 # Return all results - # (i.e. do not truncate the query even if it is slow) - >>> table = eso.query_main( - ... column_filters={ - ... 'instrument': 'APICAM', - ... 'filter_path': 'LUMINANCE', - ... 'exp_start': "between '2019-04-26' and '2019-04-27'" - ... } - ... ) - >>> print(len(table)) - 215 - >>> print(table.columns) - - >>> table[["object", "ra", "dec", "date_obs", "prog_id"]].pprint(max_width=200) - object ra dec date_obs prog_id - deg deg - ------- ------------ ------------ ----------------------- ------------ - ALL SKY 145.29212694 -24.53624194 2019-04-26T00:08:49.000 60.A-9008(A) - ALL SKY 145.92251305 -24.53560305 2019-04-26T00:11:20.000 60.A-9008(A) - ALL SKY 146.55707 -24.53497111 2019-04-26T00:13:52.000 60.A-9008(A) - ALL SKY 147.18745 -24.53435388 2019-04-26T00:16:23.000 60.A-9008(A) - ALL SKY 147.81365305 -24.53375305 2019-04-26T00:18:53.000 60.A-9008(A) - ALL SKY 148.56509194 -24.533045 2019-04-26T00:21:53.000 60.A-9008(A) - ALL SKY 149.19963805 -24.53246 2019-04-26T00:24:25.000 60.A-9008(A) - ALL SKY 149.83418111 -24.53188611 2019-04-26T00:26:57.000 60.A-9008(A) - ALL SKY 150.46037194 -24.53133111 2019-04-26T00:29:27.000 60.A-9008(A) - ALL SKY 151.08656111 -24.53078805 2019-04-26T00:31:57.000 60.A-9008(A) - ALL SKY 151.85050805 -24.53014 2019-04-26T00:35:00.000 60.A-9008(A) - ALL SKY 152.48504 -24.529615 2019-04-26T00:37:32.000 60.A-9008(A) - ... ... ... ... ... - ALL SKY 289.40910694 -24.66412305 2019-04-26T09:44:00.000 60.A-9008(A) - ALL SKY 290.04024305 -24.66522194 2019-04-26T09:46:31.000 60.A-9008(A) - ALL SKY 290.67974305 -24.66633 2019-04-26T09:49:04.000 60.A-9008(A) - ALL SKY 291.30671 -24.66741111 2019-04-26T09:51:34.000 60.A-9008(A) - ALL SKY 291.93786305 -24.66849388 2019-04-26T09:54:05.000 60.A-9008(A) - ALL SKY 139.655775 -24.542425 2019-04-26T23:42:23.000 60.A-9008(A) - ALL SKY 140.282015 -24.54169694 2019-04-26T23:44:53.000 60.A-9008(A) - ALL SKY 140.91242694 -24.54097305 2019-04-26T23:47:24.000 60.A-9008(A) - ALL SKY 141.54283388 -24.54026 2019-04-26T23:49:55.000 60.A-9008(A) - ALL SKY 142.16906388 -24.53956194 2019-04-26T23:52:25.000 60.A-9008(A) - ALL SKY 142.93306 -24.53872388 2019-04-26T23:55:28.000 60.A-9008(A) - ALL SKY 143.56345694 -24.53804388 2019-04-26T23:57:59.000 60.A-9008(A) - Length = 215 rows - - -Query the ESO archive for reduced data -====================================== - -In addition to raw data, ESO makes available processed data. -In this section, we show how to obtain these processed survey data from the archive. - -Identify available surveys --------------------------- - -The list of available surveys can be obtained with :meth:`~astroquery.eso.EsoClass.list_surveys` as follows: - -.. doctest-remote-data:: - - >>> surveys = eso.list_surveys() +.. doctest-skip:: -Query a specific survey with constraints ----------------------------------------- + >>> eso.ROW_LIMIT = -1 -Let's assume that we work with the ``HARPS`` survey, and that we are interested in -target ``HD203608``. The archive can be queried as follows: +Getting Started +=============== -.. doctest-remote-data:: +This guide walks through logging in, running your first archive queries, inspecting +metadata, downloading products, and querying catalogues. - >>> table = eso.query_surveys(surveys='HARPS', target_name="HD203608") +Authentication +-------------- -The returned table has a ``dp_id`` column, which can be used to retrieve the datasets with -:meth:`~astroquery.eso.EsoClass.retrieve_data`: ``eso.retrieve_data(table["dp_id"][0])``. -More details about this method in the next section. +.. toctree:: + :maxdepth: 2 + eso_login -Obtaining extended information on data products -=============================================== +Observation Queries +------------------- -Only a small subset of the keywords present in the data products can be obtained -with :meth:`~astroquery.eso.EsoClass.query_instrument` or :meth:`~astroquery.eso.EsoClass.query_main`. -There is however a way to get the full primary header of the FITS data products, -using :meth:`~astroquery.eso.EsoClass.get_headers`. -This method is detailed in the example below. +.. toctree:: + :maxdepth: 2 + eso_obs_cone_search + eso_obs_raw_general + eso_obs_raw_instrument + eso_obs_reduced + eso_obs_tap + eso_obs_apex + eso_obs_header_info + eso_obs_download -.. doctest-remote-data:: +Catalogue Queries +----------------- - >>> table = eso.query_instrument('midi', - ... column_filters={ - ... 'object': 'NGC4151', - ... 'date_obs': "<='2008-01-01'" - ... }, - ... columns=['object', 'date_obs', 'dp_id']) - >>> table_headers = eso.get_headers(table["dp_id"]) - >>> len(table_headers.columns) - 336 - >>> table_headers.pprint() - DP.ID SIMPLE BITPIX ... HIERARCH ESO OCS EXPO7 FNAME2 HIERARCH ESO OCS EXPO8 FNAME1 HIERARCH ESO OCS EXPO8 FNAME2 - ---------------------------- ------ ------ ... --------------------------------- --------------------------------- --------------------------------- - MIDI.2007-02-07T07:01:51.000 True 16 ... - MIDI.2007-02-07T07:02:49.000 True 16 ... - MIDI.2007-02-07T07:03:30.695 True 16 ... - MIDI.2007-02-07T07:05:47.000 True 16 ... - MIDI.2007-02-07T07:06:28.695 True 16 ... - MIDI.2007-02-07T07:09:03.000 True 16 ... - MIDI.2007-02-07T07:09:44.695 True 16 ... - MIDI.2007-02-07T07:13:09.000 True 16 ... - MIDI.2007-02-07T07:13:50.695 True 16 ... - MIDI.2007-02-07T07:15:55.000 True 16 ... - MIDI.2007-02-07T07:16:36.694 True 16 ... - MIDI.2007-02-07T07:19:25.000 True 16 ... - MIDI.2007-02-07T07:20:06.695 True 16 ... MIDI.2007-02-07T07:20:06.695.fits - MIDI.2007-02-07T07:22:57.000 True 16 ... MIDI.2007-02-07T07:20:06.695.fits MIDI.2007-02-07T07:22:57.000.fits - MIDI.2007-02-07T07:23:38.695 True 16 ... MIDI.2007-02-07T07:20:06.695.fits MIDI.2007-02-07T07:22:57.000.fits MIDI.2007-02-07T07:23:38.695.fits +.. toctree:: + :maxdepth: 2 + eso_cat_cone_search + eso_cat_query + eso_cat_tap -As shown above, for each data product ID (``DP.ID``), the full header (336 columns in our case) of the archive -FITS file is collected. In the above table ``table_headers``, there are as many rows as in the column ``table['DP.ID']``. +Help +---- +.. toctree:: + :maxdepth: 2 -Downloading datasets from the archive -===================================== + eso_troubleshooting -Continuing from the query with constraints example, the first two datasets are selected, -using their data product IDs ``dp_id``, and retrieved from the ESO archive. +.. warning:: -.. doctest-skip:: + **⚠️ Backward Compatibility Notice ⚠️** - >>> data_files = eso.retrieve_data(table['dp_id'][:2]) - INFO: Downloading datasets ... [astroquery.eso.core] - INFO: Downloading 2 files ... [astroquery.eso.core] - INFO: Downloading file 1/2 https://dataportal.eso.org/dataPortal/file/MIDI.2007-02-07T07:01:51.000 to /Users/foobar/.astropy/cache/astroquery/Eso [astroquery.eso.core] - INFO: Successfully downloaded dataset MIDI.2007-02-07T07:01:51.000 to /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:01:51.000.fits.Z [astroquery.eso.core] - INFO: Downloading file 2/2 https://dataportal.eso.org/dataPortal/file/MIDI.2007-02-07T07:02:49.000 to /Users/foobar/.astropy/cache/astroquery/Eso [astroquery.eso.core] - INFO: Successfully downloaded dataset MIDI.2007-02-07T07:02:49.000 to /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:02:49.000.fits.Z [astroquery.eso.core] - INFO: Uncompressing file /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:01:51.000.fits.Z [astroquery.eso.core] - INFO: Uncompressing file /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:02:49.000.fits.Z [astroquery.eso.core] - INFO: Done! [astroquery.eso.core] - >>> data_files - ['/Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:01:51.000.fits', - '/Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:02:49.000.fits'] - -The file names, returned in data_files, points to the decompressed datasets -(without the .Z extension) that have been locally downloaded. -They are ready to be used with `~astropy.io.fits`. - -The default location (in the astropy cache) of the decompressed datasets can be adjusted by providing -a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. - -By default, if a requested dataset is already found, it is not downloaded again from the archive. -To force the retrieval of data that are present in the destination directory, use ``continuation=True`` -in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. - - -Troubleshooting -=============== + **The WDB (Web DataBase) API has been deprecated and replaced by TAP (Table Access Protocol)**, + a standardized interface for querying astronomical datasets using ADQL (Astronomical Data Query Language). + While the Python interface remains the same, the values accepted by the ``columns`` and ``column_filters`` + parameters must reflect TAP field names and ADQL syntax. This means that, + although the structure of your code won't need to change, **the values you pass to the arguments** + ``columns`` **and** ``column_filters`` **must be revised** to comply with the new format. -Clearing the cache ------------------- + In TAP, ``column_filters`` accepts ADQL expressions. For example: -If you are repeatedly getting failed queries, or bad/out-of-date results, try clearing your cache: + .. doctest-skip:: + + >>> column_filters = { + ... "some_int_column": "< 5", + ... "some_float_column_2": ">= 1.23", + ... "some_char_column": "like '%John%'", + ... "some_generic_column": "in ('mango', 'apple', 'kiwi')", + ... "other_generic_column": "between '2024-01-01' and '2024-12-31'" + ... } -.. code-block:: python + Please review your queries carefully and update them accordingly to ensure compatibility with the new astroquery versions. + See section :ref:`column-filters-fix` for more information and examples. - >>> from astroquery.eso import Eso - >>> Eso.clear_cache() +Introduction +============ -If this function is unavailable, upgrade your version of astroquery. -The ``clear_cache`` function was introduced in version 0.4.7.dev8479. +Overview of the ESO Archive +--------------------------- -.. _column-filters-fix: +The `ESO Science Archive Facility (SAF) `_ is one of the largest ground-based astronomical data repositories in the world. It contains, and provides access to, data from ESO telescopes at the `La Silla `_, `Paranal `_, as well as the (sub-)millimeter `APEX `_ telescope and `ALMA `_ array, and selected external datasets, such as raw WFCAM/UKIDSS data from the UKIRT facility in Hawaii. The archive includes both `raw data `_, and a wide range of `processed data products `_, either generated by ESO or contributed by the community (via `ESO Phase 3 `_). Processed data downloaded from the ESO Archive are assigned a Digital Object Identifier (DOI), where the list of available DOIs can be browsed in the `ESO DOI query page `_. -Using the correct ``column_filters`` ------------------------------------- +Ways to Access the Archive +-------------------------- -Two concrete and relevant examples of fields present in WDB but not present in TAP/ADQL -are ``stime`` and ``etime``. The following snippet shows how to adapt the filters to -the TAP / ADQL syntax: +Access to data follows ESO’s standard data policy: the Principal Investigator (PI) of an observing programme has exclusive access to their data during a proprietary period (typically one year), after which the data become publicly available. See the `ESO Data Access Policy `_ and `FAQ `_ for more information. Browsing the archive does not require authentication, but login is necessary to access proprietary datasets. Use of ESO archival data should be acknowledged in all publications. -.. doctest-skip:: +There are multiple ways to access the archive: - # The following filters won't work: - column_filters = { - 'stime': '2024-01-01' - 'etime': '2024-12-31' - } +- The `Raw Data query form `_ gives access to raw frames and their calibrations +- The `Science Portal `_ allows browsing and retrieval of processed (Phase 3) data +- The `Catalogue Facility `_ enables access to PI-contributed catalogue data +- The `Programmatic and Tools access layer `_ provides direct, scriptable access to raw and processed data, metadata, ambient conditions, and catalogues via Virtual Observatory (VO) protocols - # Replace by: - column_filters = { - 'exp_start': "between '2024-01-01' and '2024-12-31'" - } +This documentation focuses on the last method: accessing the archive programmatically using the ``astroquery.eso`` module. This Python interface allows users to search for both raw and reduced data, retrieve metadata, and download data products directly using dataset identifiers. Proprietary access is supported via authentication with the `ESO User Portal `_. - # --- # +Notebooks +========= - # The following filters won't work: - column_filters = { - 'stime': '2024-01-01' - } +A set of example Jupyter notebooks demonstrating the usage of the ``astroquery.eso`` module can be found in the `ESO GitHub astroquery examples +`_ repository, where a simple overview script can be found at `introduction.ipynb `_. - # Replace by: - column_filters = { - 'exp_start': "> '2024-01-01'" - } +Installation +============ - # --- # +Along with the methods to install the main ``astroquery`` package, the most up-to-date version of the ``astroquery.eso`` module can be installed directly from the `ESO GitHub astroquery `_ repository. - # The following filters won't work: - column_filters = { - 'etime': '2024-12-31' - } +Requirements +============ - # Replace by: - column_filters = { - 'exp_start': "< '2024-12-31'" - } +Along with the main requirements of the ``astroquery`` package, the following packages are required for the use of this module: +* `lxml `_ Reference/API ============= diff --git a/docs/eso/eso_cat_cone_search.rst b/docs/eso/eso_cat_cone_search.rst new file mode 100644 index 0000000000..4c05efbc2a --- /dev/null +++ b/docs/eso/eso_cat_cone_search.rst @@ -0,0 +1,198 @@ +************************ +Catalogues - Cone Search +************************ + +In the simplest case, catalogue queries can be constrained by sky +position (cone search). For catalogue tables, this is currently done by +building an ADQL cone-search expression and passing it through +``column_filters`` in ``eso.query_catalogue``. + +.. note:: + The ``cone_ra``, ``cone_dec``, and ``cone_radius`` arguments are not + currently implemented in ``eso.query_catalogue`` (as they are with e.g. ``eso.query_main``). + For catalogue cone searches, use a custom ADQL spatial filter as shown + below. + + +Identify Coordinate Columns +=========================== + +Before building a cone search, inspect the catalogue schema with +``help=True`` and identify the right ascension and declination columns from +their UCD values: + +- ``pos.eq.ra;meta.main`` for right ascension +- ``pos.eq.dec;meta.main`` for declination + +For example: + +.. doctest-skip:: + + >>> eso.query_catalogue(catalogue='KiDS_DR4_1_ugriZYJHKs_cat_fits', help=True) + ... + RAJ2000 DOUBLE deg pos.eq.ra;meta.main + DECJ2000 DOUBLE deg pos.eq.dec;meta.main + ... + +In this case the column names are ``RAJ2000`` and ``DECJ2000``. +However, different catalogues may use different coordinate column names (e.g. +``RA2000``/``DEC2000`` or ``RA_Spec``/``Dec_Spec``), so checking the UCD is the +most robust way to identify the correct columns. + +Query with Cone Filter +=========================== + +Catalogue cone searches are implemented using the ADQL spatial function +``CONTAINS(POINT, CIRCLE)``. Since ``eso.query_catalogue`` builds its +``WHERE`` clause from the ``column_filters`` dictionary, the spatial +constraint must be provided as a key with value ``1`` (corresponding to +``CONTAINS(...) = 1`` in ADQL). + +Under the hood, ``eso.query_catalogue`` submits an ADQL query via TAP. +For example, a cone search on the KiDS DR4 catalogue corresponds to an +ADQL statement of the form: + +.. code-block:: sql + + SELECT * + FROM KiDS_DR4_1_ugriZYJHKs_cat_fits + WHERE CONTAINS(POINT('', RAJ2000, DECJ2000), CIRCLE('', , , ))=1 + +Here: + +- ``POINT('', RAJ2000, DECJ2000)`` defines the source position + using the catalogue RA and Dec columns. +- ``CIRCLE('', , , )`` defines the + search region in degrees. +- ``CONTAINS`` evaluates to ``1`` when the source lies within + the specified region. + +When using ``column_filters``, the dictionary *key* is inserted as the +left-hand side of the ``WHERE`` clause, and the dictionary *value* +becomes the right-hand side. Therefore, providing a key like +``CONTAINS(...)`` with value ``1`` yields ``CONTAINS(...) = 1`` in ADQL. + +The helper function below constructs this automatically. + +Helper Function +--------------- + +Define target of interest and search radius: + +.. doctest-skip:: + + >>> from astroquery.eso import Eso + >>> from astropy.coordinates import SkyCoord + >>> import astropy.units as u + >>> eso = Eso() + + >>> coords = SkyCoord.from_name("NGC1097") + >>> radius = 3 * u.arcmin + +Helper function to construct ADQL cone search predicate: + +.. doctest-skip:: + + >>> def make_cone_filter( + ... ra, + ... dec, + ... radius, + ... ra_name="RAJ2000", + ... dec_name="DECJ2000", + ... ): + ... """ + ... Construct an ADQL cone-search predicate for catalogue queries. + ... + ... Parameters + ... ---------- + ... ra : astropy.units.Quantity + ... Right ascension of cone centre. + ... dec : astropy.units.Quantity + ... Declination of cone centre. + ... radius : astropy.units.Quantity + ... Search radius. + ... ra_name : str, optional + ... Name of the RA column in the catalogue + ... (default: 'RAJ2000'). + ... dec_name : str, optional + ... Name of the Dec column in the catalogue + ... (default: 'DECJ2000'). + ... + ... Returns + ... ------- + ... dict + ... Dictionary suitable for ``column_filters`` of the form + ... ``{predicate: 1}``, corresponding to + ... ``CONTAINS(...) = 1`` in ADQL. + ... """ + ... + ... ra_deg = ra.to_value(u.deg) + ... dec_deg = dec.to_value(u.deg) + ... rad_deg = radius.to_value(u.deg) + ... + ... predicate = ( + ... f"CONTAINS(" + ... f"POINT('', {ra_name}, {dec_name}), " + ... f"CIRCLE('', {ra_deg}, {dec_deg}, {rad_deg})" + ... f")" + ... ) + ... + ... return {predicate: 1} + +Run search with cone filter: + +.. doctest-skip:: + + >>> column_filters = make_cone_filter( + ... ra=coords.ra, + ... dec=coords.dec, + ... radius=radius, + ... ) + + >>> table = eso.query_catalogue( + ... catalogue="KiDS_DR4_1_ugriZYJHKs_cat_fits", + ... column_filters=column_filters, + ... ) + >>> table +
+ Level ALPHA_J2000 A_IMAGE A_WORLD ... Z_B_MAX Z_B_MIN Z_ML + count deg pixel deg ... + float32 float64 float64 float32 ... float64 float64 float64 + ---------- ----------- ------------------ ------------ ... ------------------ ------------------- ------------------- + 0.02452336 41.624103 3.264482 0.0001940674 ... 0.76 0.69 0.73 + 0.02450416 41.632288 1.199719 7.132157e-05 ... 1.35 0.52 1.87 + 0.02449745 41.634118 1.714497 0.0001019072 ... 1.03 0.41000000000000003 7.0 + 0.02455906 41.62684 1.2852190000000001 7.640447e-05 ... 1.62 0.98 1.65 + ... ... ... ... ... ... ... ... + 0.02458552 41.62298 1.037333 6.16569e-05 ... 1.6400000000000001 0.63 3.66 + 0.02453066 41.625326 1.191271 7.081894e-05 ... 1.38 0.53 0.24000000000000002 + 0.02451551 41.624227 1.582422 9.406044e-05 ... 0.63 0.43000000000000005 0.55 + 0.02454042 41.618984 2.15449 0.0001280547 ... 0.6799999999999999 0.17 0.11 + +Combining with Additional Column Filters +---------------------------------------- + +The cone filter returns a standard ``column_filters`` dictionary. +Additional constraints (e.g. magnitude limits) can be added using +normal dictionary updates. + +.. doctest:: + + >>> # Add a magnitude constraint + >>> column_filters.update({"MAG_AUTO": "<22"}) + + >>> table = eso.query_catalogue( + ... catalogue="KiDS_DR4_1_ugriZYJHKs_cat_fits", + ... column_filters=column_filters, + ... ) +
+ Level ALPHA_J2000 A_IMAGE A_WORLD Agaper ... Ypos Z_B Z_B_MAX Z_B_MIN Z_ML + count deg pixel deg arcsec ... pixel + float32 float64 float64 float32 float32 ... float32 float64 float64 float64 float64 + ---------- ----------- ------------------ ------------ ------- ... -------- ------- ------- ------------------- ------------------ + 0.02452336 41.624103 3.264482 0.0001940674 0.99 ... 9077.715 0.73 0.76 0.69 0.73 + 0.02450728 41.622675 3.695437 0.0002196531 1.06 ... 9153.268 0.51 0.54 0.48000000000000004 0.51 + 0.02450339 41.63336 2.607809 0.000155017 0.9 ... 9207.265 0.96 1.35 0.8200000000000001 1.6400000000000001 + 0.02452449 41.62697 2.9889900000000003 0.0001776759 0.95 ... 9217.702 0.72 0.75 0.6699999999999999 0.72 + 0.02452196 41.626775 2.787715 0.0001656992 0.92 ... 9311.3 0.56 0.59 0.53 0.56 + 0.0245021 41.629953 5.531547 0.0003288784 1.38 ... 9338.087 0.7 0.74 0.6599999999999999 0.7 \ No newline at end of file diff --git a/docs/eso/eso_cat_query.rst b/docs/eso/eso_cat_query.rst new file mode 100644 index 0000000000..d71698465d --- /dev/null +++ b/docs/eso/eso_cat_query.rst @@ -0,0 +1,151 @@ +************************************* +Catalogues - Query for Catalogue Data +************************************* + +In addition to raw and reduced observations, the ESO archive provides access to +science catalogues. This section shows how to +list the available catalogues and query one of them. + +Available Catalogues +==================== + +You can list the available catalogues with :meth:`~astroquery.eso.EsoClass.list_catalogues`. By default, +the ``all_versions=False`` option returns only the latest version of each +catalogue. + +.. doctest-skip:: + + >>> from astroquery.eso import Eso + >>> eso = Eso() + + >>> eso.list_catalogues(all_versions=False) + ['AMBRE_V1', + 'ATLASGAL_V1', + 'COSMOS2015_Laigle_v1_1b_latestV7_fits_V1', + 'EREBOS_cat_fits_V1', + 'EREBOS_RV_cat_fits_V1', + ... + 'KiDS_DR3_0_ugri_src_fits_V2', + 'KiDS_DR3_1_ugri_shear_fits_V1', + 'KiDS_DR4_1_ugriZYJHKs_cat_fits', + ... + 'vmc_dr7_yjks_back_V1', + 'vmc_dr7_yjks_varCat_V3'] + + >>> print(len(eso.list_catalogues(all_versions=False))) + 86 + +To include every available version of every catalogue, set +``all_versions=True``: + +.. doctest-skip:: + + >>> eso.list_catalogues(all_versions=True) + ['AMBRE_V1', + 'atlas_er3_ugriz_catMetaData_fits_V2', + 'ATLASGAL_V1', + 'COSMOS2015_Laigle_v1_1b_latestV7_fits_V1', + 'EREBOS_cat_fits_V1', + 'EREBOS_RV_cat_fits_V1', + ... + 'KiDS_DR3_0_ugri_src_fits_V2', + 'KiDS_DR3_1_ugri_shear_fits_V1', + 'KiDS_DR4_1_ugriZYJHKs_cat_fits', + ... + 'vmc_dr7_yjks_back_V1', + 'vmc_dr7_yjks_varCat_V3'] + + >>> print(len(eso.list_catalogues(all_versions=True))) + 129 + +Available Query Constraints +=========================== + +Before querying a specific catalogue, inspect its schema with +``eso.query_catalogue(..., help=True)``. This prints the available columns and +the total number of records in that table -- in this case the ``'KiDS_DR4_1_ugriZYJHKs_cat_fits'`` +table. To see more infomation on this specific version of the Kilo-Degree Survey (KiDS) catalogue, see the +`release documentation `_. + +.. doctest-skip:: + + >>> eso.query_catalogue(catalogue='KiDS_DR4_1_ugriZYJHKs_cat_fits', help=True) + INFO: + Columns present in the table safcat.KiDS_DR4_1_ugriZYJHKs_cat_fits: + column_name datatype unit ucd + ------------------- -------- ---------------- ------------------------------------- + ID CHAR meta.id;meta.main + KIDS_TILE CHAR meta.id + THELI_NAME CHAR meta.id + SeqNr INTEGER meta.id + SLID INTEGER meta.id + SID INTEGER meta.id + FLUX_AUTO REAL count phot.flux;em.opt.R + FLUXERR_AUTO REAL count stat.error;phot.flux;em.opt.R + MAG_AUTO REAL mag phot.mag;em.opt.R + ... + MASK INTEGER meta.code + COLOUR_GAAP_u_g REAL mag phot.color.reddFree;em.opt.U;em.opt.B + COLOUR_GAAP_g_r REAL mag phot.color.reddFree;em.opt.B;em.opt.R + COLOUR_GAAP_r_i REAL mag phot.color.reddFree;em.opt.R;em.opt.I + COLOUR_GAAP_i_Z REAL mag phot.color.reddFree;em.opt.I + COLOUR_GAAP_Z_Y REAL mag phot.color.reddFree;em.opt.I;em.IR.J + COLOUR_GAAP_Y_J REAL mag phot.color.reddFree;em.IR.J + COLOUR_GAAP_J_H REAL mag phot.color.reddFree;em.IR.J;em.IR.H + COLOUR_GAAP_H_Ks REAL mag phot.color.reddFree;em.IR.H;em.IR.K + Number of records present in the table safcat.KiDS_DR4_1_ugriZYJHKs_cat_fits: + 100350804 + [astroquery.eso.core] + +Query with Constraints +====================== + +The table can be queried using the :meth:`~astroquery.eso.EsoClass.query_catalogue` method. +Note, however, that catalogue tables can be very large, so it is often useful to start with a +small row limit, which can be set with e.g ``eso.ROW_LIMIT = 5`` or by passing +e.g. ``ROW_LIMIT=5`` to the query method: + +.. doctest-skip:: + + >>> table = eso.query_catalogue(catalogue='KiDS_DR4_1_ugriZYJHKs_cat_fits', ROW_LIMIT=5) + >>> table + WARNING: MaxResultsWarning: Results truncated to 5. To retrieve all the records set to None the ROW_LIMIT attribute [astroquery.eso.core] +
+ Level ALPHA_J2000 A_IMAGE A_WORLD ... Z_B_MAX Z_B_MIN Z_ML + count deg pixel deg ... + float32 float64 float64 float32 ... float64 float64 float64 + ---------- ----------- ------------------ ------------ ... ------- ------------------ ------------------ + 0.03316526 5.079035 1.023542 6.084906e-05 ... 0.97 0.6699999999999999 0.7799999999999999 + 0.03316413 5.127818 2.087884 0.0001240919 ... 0.92 0.7899999999999999 0.8300000000000001 + 0.03320557 5.171088 2.6607149999999997 0.00015816 ... 0.89 0.7899999999999999 0.8600000000000001 + 0.02512704 5.912251 201.661499 0.0119862 ... 0.04 0.01 0.01 + 0.04201349 6.206854 3.289606 0.0001955687 ... 0.35 0.23 0.27 + +A larger number of rows can be returned by setting a higher row limit, or by disabling truncation +with ``eso.ROW_LIMIT = -1``. Be aware that the TAP service has a maximum limit of 15,000,000 rows +per query, so setting ``eso.ROW_LIMIT = -1`` will return all matching results up to that TAP limit. + +You can also combine selected columns with ADQL filters in ``column_filters``. +For example, to retrieve bright sources with r-band magnitude of ``MAG_AUTO < 10``, and only return a subset of +the available columns, you can do: + +.. doctest-skip:: + + >>> table = eso.query_catalogue( + ... catalogue='KiDS_DR4_1_ugriZYJHKs_cat_fits', + ... columns=["ID", "RAJ2000", "DECJ2000", "KIDS_TILE", "MAG_AUTO", "MAGERR_AUTO"], + ... column_filters={"MAG_AUTO": "<10"} + ... ) + >>> table +
+ ID RAJ2000 DECJ2000 ... MAG_AUTO MAGERR_AUTO + deg deg ... mag mag + object float64 float64 ... float32 float32 + ----------------------------- ------------------ ------------------- ... -------- ------------ + KiDSDR4 J092035.791+001110.92 140.14913 0.186369 ... 9.263525 3.264208e-05 + KiDSDR4 J110654.466+015628.34 166.726944 1.941208 ... 9.844576 4.391666e-05 + KiDSDR4 J225722.767-293856.66 344.34486300000003 -29.649074 ... 9.725844 0.0001094358 + ... + KiDSDR4 J223130.214-322011.16 337.875892 -32.336436 ... 9.379268 2.736172e-05 + KiDSDR4 J223131.492-322022.97 337.881217 -32.339715 ... 9.846291 3.40929e-05 + KiDSDR4 J032045.699-263559.19 50.190415 -26.599775 ... 9.857556 3.780224e-05 \ No newline at end of file diff --git a/docs/eso/eso_cat_tap.rst b/docs/eso/eso_cat_tap.rst new file mode 100644 index 0000000000..8a23f007eb --- /dev/null +++ b/docs/eso/eso_cat_tap.rst @@ -0,0 +1,265 @@ +***************************** +Catalogues - Query with TAP +***************************** + +The ESO TAP service can also be used to query catalogue tables directly with +free ADQL, using :meth:`~astroquery.eso.EsoClass.query_tap`. By default, +``query_tap(query)`` targets the observations TAP service (equivalent to +``which_tap="tap_obs"``). To query catalogue content, pass +``which_tap="tap_cat"`` -- i.e. ``eso.query_tap(query, which_tap="tap_cat")``. + +Basic Usage +=========== + +.. doctest-skip:: + + >>> from astroquery.eso import Eso + >>> eso = Eso() + >>> query = "SELECT table_name FROM TAP_SCHEMA.tables" + >>> table = eso.query_tap(query, which_tap="tap_cat") + +The examples below focus on common catalogue TAP workflows: schema discovery, +spatial filtering, metadata-driven joins, and light-curve retrieval. + +Schema Discovery Examples +========================= + +Inspect Columns in a Specific Catalogue Table +--------------------------------------------- + +Use this to list columns and data types for a specific catalogue table. +``TAP_SCHEMA.columns`` contains column names, units, UCDs, and additional +metadata for all published ESO tables. + +.. doctest-skip:: + + >>> query = """ + ... SELECT column_name, datatype, unit, ucd + ... FROM TAP_SCHEMA.columns + ... WHERE table_name='COSMOS2015_Laigle_v1_1b_latestV7_fits_V1' + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +Find the Main RA/Dec Columns via UCD +------------------------------------ + +Use this to identify which columns carry the main sky coordinates in a table. +Here, the UCD values mark the primary ``RA`` and ``Dec`` fields -- in this case +``TRANSIENT_RAJ2000`` and ``TRANSIENT_DECJ2000``. + +.. doctest-skip:: + + >>> query = """ + ... SELECT column_name + ... FROM TAP_SCHEMA.columns + ... WHERE table_name='PESSTO_TRAN_CAT_fits_V2' + ... AND (ucd = 'pos.eq.ra;meta.main' OR ucd = 'pos.eq.dec;meta.main') + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +List VMC Tables with Coordinate and Source-ID Columns +----------------------------------------------------- + +Use this to list coordinate and source-ID columns for all tables in the ``VMC`` +collection, ordered by publication date. + +.. doctest-skip:: + + >>> query = """ + ... SELECT publication_date, T.table_name, column_name, ucd + ... FROM TAP_SCHEMA.columns AS C, TAP_SCHEMA.tables AS T + ... WHERE T.table_name = C.table_name + ... AND cat_id IN ( + ... SELECT cat_id + ... FROM TAP_SCHEMA.tables + ... WHERE collection='VMC' + ... ) + ... AND ( + ... ucd = 'pos.eq.ra;meta.main' + ... OR ucd = 'pos.eq.dec;meta.main' + ... OR ucd = 'meta.id;meta.main' + ... ) + ... ORDER BY publication_date DESC, 2, 4 + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +List Coordinate and Source-ID Columns Across All Collections +------------------------------------------------------------ + +Use this to discover coordinate and source-ID columns across all published +catalogues, ordered by publication date and collection. + +.. doctest-skip:: + + >>> query = """ + ... SELECT publication_date, collection, T.table_name, column_name, ucd + ... FROM TAP_SCHEMA.columns AS C, TAP_SCHEMA.tables AS T + ... WHERE T.table_name = C.table_name + ... AND ( + ... ucd = 'pos.eq.ra;meta.main' + ... OR ucd = 'pos.eq.dec;meta.main' + ... OR ucd = 'meta.id;meta.main' + ... ) + ... ORDER BY publication_date DESC, 2, 3, 5 + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +Spatial Query Example +===================== + +Cone Search for PESSTO Transients Around ESO 154-10 +--------------------------------------------------- + +Search the PESSTO master catalogue for transients in a circle of ``0.04 deg`` +(``2.4 arcmin``) around galaxy ``ESO 154-10`` (ICRS position +``41.2863, -55.7406``). + +.. doctest-skip:: + + >>> query = """ + ... SELECT host_id, transient_id, transient_classification, + ... transient_raj2000, transient_decj2000 + ... FROM pessto_tran_cat_fits_V2 + ... WHERE CONTAINS( + ... POINT('', transient_raj2000, transient_decj2000), + ... CIRCLE('', 41.2863, -55.7406, 0.04) + ... ) = 1 + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +The target coordinates can also be resolved online using a name resolver and then included in +the query, as e.g.: + +.. doctest-skip:: + + >>> import astropy.units as u + >>> from astropy.coordinates import SkyCoord + + >>> target = SkyCoord.from_name("ESO 154-10") + >>> ra_deg = target.icrs.ra.to_value(u.deg) + >>> dec_deg = target.icrs.dec.to_value(u.deg) + >>> radius_deg = (2.4 * u.arcmin).to_value(u.deg) + + >>> query = f""" + ... SELECT host_id, transient_id, transient_classification, + ... transient_raj2000, transient_decj2000 + ... FROM pessto_tran_cat_fits_V2 + ... WHERE CONTAINS( + ... POINT('ICRS', transient_raj2000, transient_decj2000), + ... CIRCLE('ICRS', {ra_deg}, {dec_deg}, {radius_deg}) + ... ) = 1 + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +Metadata Join Example +===================== + +Discover Join Keys for VVV Tables +--------------------------------- + +Use this to discover how ``VVV_CAT_V2`` can be joined with related VVV tables. +The resulting table shows join pairs as: + +- ``from_table`` via ``from_column`` +- ``target_table`` via ``target_column`` + +.. doctest-skip:: + + >>> query = """ + ... SELECT k.from_table, kc.from_column, k.target_table, kc.target_column + ... FROM TAP_SCHEMA.columns AS c, TAP_SCHEMA.keys AS k, TAP_SCHEMA.key_columns AS kc + ... WHERE c.table_name='VVV_CAT_V2' + ... AND ( + ... (k.from_table = c.table_name AND kc.from_column = c.column_name) + ... OR + ... (k.target_table = c.table_name AND kc.target_column = c.column_name) + ... ) + ... AND k.key_id = kc.key_id + ... ORDER BY table_name, column_name, 1, 3, 2 + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +For example, after identifying join keys (in this case the ``VVV_MPHOT_Ks_V2`` and ``VVV_VAR_V2`` tables), +you can build a light-curve (see below). + +Light-Curve Query Examples +========================== + +Get a VVV Source Light Curve and Variability Metrics +---------------------------------------------------- + +Retrieve multi-epoch photometry and variability metrics for a known VVV source. + +.. doctest-skip:: + + >>> query = """ + ... SELECT TOP 10 VVV.IAUNAME, VVV.SOURCEID, VVV.RA2000, VVV.DEC2000, + ... M.MJD, M.KSMAG, M.KSERR, + ... V.KSMEANMAG, V.KSAMPL, KSPROBVAR + ... FROM VVV_CAT_V2 AS VVV + ... JOIN VVV_MPHOT_Ks_V2 AS M ON VVV.SOURCEID = M.SOURCEID + ... JOIN VVV_VAR_V2 AS V ON VVV.SOURCEID = V.SOURCEID + ... WHERE VVV.IAUNAME='VVV J135433.73-594836.43' + ... ORDER BY 1, 5 + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +In this query: + +- ``VVV`` is the VVV master catalogue alias. +- ``M`` is the multi-epoch photometry table alias. +- ``V`` is the variability table alias. + + +Get PESSTO Light-Curve Magnitudes in a Sky Region +------------------------------------------------- + +Retrieve selected magnitudes from the PESSTO light-curve catalogue for +transients within ``0.05 deg`` (``3 arcmin``) around ESO 154-10. + +.. doctest-skip:: + + >>> query = """ + ... SELECT host_id, transient_id, transient_classification, + ... lc.B_VEGA_MAG, lc.B_VEGA_MAGERR, + ... lc.R_AB_MAG, lc.R_AB_MAGERR, + ... lc.R_VEGA_MAG, lc.R_VEGA_MAGERR + ... FROM safcat.PESSTO_TRAN_CAT_V3 AS master + ... JOIN safcat.PESSTO_MPHOT_V3 AS lc ON lc.SOURCE_ID = master.TRANSIENT_ID + ... WHERE CONTAINS( + ... POINT('', transient_raj2000, transient_decj2000), + ... CIRCLE('', 41.2863, -55.7406, 0.05) + ... ) = 1 + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +The light-curve table does not provide coordinates directly, so it is joined to +the PESSTO master catalogue for spatial filtering. To retrieve all available +magnitude columns instead of only ``B`` and ``R``, use ``lc.*`` in the +``SELECT`` clause. + +This can also be done using a name resolver for the target coordinates, as previously shown above. + +.. doctest-skip:: + + >>> import astropy.units as u + >>> from astropy.coordinates import SkyCoord + + >>> target = SkyCoord.from_name("ESO 154-10") + >>> ra_deg = target.icrs.ra.to_value(u.deg) + >>> dec_deg = target.icrs.dec.to_value(u.deg) + >>> radius_deg = (3 * u.arcmin).to_value(u.deg) + + >>> query = f""" + ... SELECT host_id, transient_id, transient_classification, + ... lc.B_VEGA_MAG, lc.B_VEGA_MAGERR, + ... lc.R_AB_MAG, lc.R_AB_MAGERR, + ... lc.R_VEGA_MAG, lc.R_VEGA_MAGERR + ... FROM safcat.PESSTO_TRAN_CAT_V3 AS master + ... JOIN safcat.PESSTO_MPHOT_V3 AS lc ON lc.SOURCE_ID = master.TRANSIENT_ID + ... WHERE CONTAINS( + ... POINT('', transient_raj2000, transient_decj2000), + ... CIRCLE('', {ra_deg}, {dec_deg}, {radius_deg}) + ... ) = 1 + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") \ No newline at end of file diff --git a/docs/eso/eso_login.rst b/docs/eso/eso_login.rst new file mode 100644 index 0000000000..16cca4140f --- /dev/null +++ b/docs/eso/eso_login.rst @@ -0,0 +1,110 @@ + +**************************** +Authenticated Access (Login) +**************************** + +Most datasets in the ESO Science Archive are publicly available and can be downloaded anonymously without requiring authentication. However, access to proprietary datasets—such as those under active proprietary periods—is restricted to authorised users (e.g., PIs of observing programmes and their designated delegates). These users must authenticate via the `ESO User Portal `_. + +Authentication is handled using the :meth:`~astroquery.query.QueryWithLogin.login` method. This command initiates a secure login session via the ESO Single Sign-On (SSO) service. It integrates with the `keyring `_ module to securely store your password in your system’s credential manager. After the first login, you should not need to re-enter your credentials for subsequent sessions on the same machine. + +Authentication is required when: + +- Downloading proprietary files (not yet public) +- Accessing services that require ESO user privileges + +Login Examples +============== + +The following examples show typical login and data retrieval workflows: + +**First example:** ``"TEST"`` is not a valid username, it will fail: + +.. doctest-skip:: + + >>> eso.login(username="TEST") # doctest: +SKIP + WARNING: No password was found in the keychain for the provided username. [astroquery.query] + TEST, enter your password: + + INFO: Authenticating TEST on 'www.eso.org' ... [astroquery.eso.core] + ERROR: Authentication failed! [astroquery.eso.core] + +**Second example:** pretend ``"ICONDOR"`` is a valid username: + +.. doctest-skip:: + + >>> eso.login(username="ICONDOR", store_password=True) # doctest: +SKIP + WARNING: No password was found in the keychain for the provided username. [astroquery.query] + ICONDOR, enter your password: + + INFO: Authenticating ICONDOR on 'www.eso.org' ... [astroquery.eso.core] + INFO: Authentication successful! [astroquery.eso.core] + +After the first login, your password has been stored: + +.. doctest-skip:: + + >>> eso.login(username="ICONDOR") # doctest: +SKIP + INFO: Authenticating ICONDOR on 'www.eso.org' ... [astroquery.eso.core] + INFO: Authentication successful! [astroquery.eso.core] + + +Successful download of a public file (with or without login): + +.. doctest-skip:: + + >>> eso.retrieve_data("ADP.2023-03-02T01:01:24.355") # doctest: +SKIP + INFO: Downloading datasets ... [astroquery.eso.core] + INFO: Downloading 1 files ... [astroquery.eso.core] + INFO: Downloading file 1/1 https://dataportal.eso.org/dataPortal/file/ADP.2023-03-02T01:01:24.355 to ... [astroquery.eso.core] + INFO: Successfully downloaded dataset ADP.2023-03-02T01:01:24.355 to ../ADP.2023-03-02T01:01:24.355.fits [astroquery.eso.core] + INFO: Done! [astroquery.eso.core] + +Access denied to a restricted-access file (as anonymous user or as authenticated but not authorised user): + +.. doctest-skip:: + + >>> eso.retrieve_data("FORS2.2010-10-23T23:38:41.912") # doctest: +SKIP + INFO: Downloading datasets ... [astroquery.eso.core] + INFO: Downloading 1 files ... [astroquery.eso.core] + INFO: Downloading file 1/1 https://dataportal.eso.org/dataPortal/file/FORS2.2010-10-23T23:38:41.912 to ... [astroquery.eso.core] + ERROR: Access denied to https://dataportal.eso.org/dataPortal/file/FORS2.2010-10-23T23:38:41.912 [astroquery.eso.core] + INFO: Done! [astroquery.eso.core] + +Password Storage +================ + +As shown above, your password can be stored securely using the `keyring `_ module by passing the argument ``store_password=True`` to ``Eso.login()``. For security reasons, password storage is disabled by default. + +.. warning:: + + **MAKE SURE YOU TRUST THE MACHINE WHERE YOU USE THIS FUNCTIONALITY!!!** + + When using the ``store_password=True`` option, your password is stored in your system’s keyring. This provides secure local storage, but only do this on machines you fully trust. + +**Note:** You can delete your stored password at any time with the following command: + +.. doctest-skip:: + + >>> import keyring + >>> keyring.delete_password("astroquery:www.eso.org", "your_username") + +Automatic Login +=============== + +To avoid having to enter your username every session, you can configure a default username in the Astroquery configuration file. This file is located as described in the `astropy.config documentation `_. + +Add the following to the ``[eso]`` section of your config file: + +.. doctest-skip:: + + [eso] + username = ICONDOR + +Once set, you can simply call ``eso.login()`` without specifying a username: + +.. doctest-skip:: + + >>> eso.login() # doctest: +SKIP + ICONDOR, enter your ESO password: + +**Note:** If automatic login is configured and the password is stored, other ``Eso`` methods (e.g. ``retrieve_data()``) can log you in automatically when needed. diff --git a/docs/eso/eso_obs_apex.rst b/docs/eso/eso_obs_apex.rst new file mode 100644 index 0000000000..3c553acdc6 --- /dev/null +++ b/docs/eso/eso_obs_apex.rst @@ -0,0 +1,190 @@ + +********************************** +Observations - Query for APEX Data +********************************** + +APEX observations can be discovered via the standard +archive query interfaces for reduced and raw data: + +- reduced data: :meth:`~astroquery.eso.EsoClass.query_surveys` +- raw data: :meth:`~astroquery.eso.EsoClass.query_main` +- raw (instrument-specific): :meth:`~astroquery.eso.EsoClass.query_instrument` + +In addition, :meth:`~astroquery.eso.EsoClass.query_apex_quicklooks` provides a +dedicated interface to retrieve APEX Quick Look products. These Quick Looks are +distributed as ``.tar`` bundles that typically include diagnostic material +(e.g. plots and logs) and ``class`` (``.apex``) files; for heterodyne +observations, the calibrated ``.class`` product is often the most practical +starting point, and the accompanying uncalibrated ``.fits`` files are usually +not required. + +The archive contains APEX observations from July 11, 2005 onward. This notebook +walks through a practical workflow to identify, query, and download APEX Quick +Look products. As an example, we use APEX observations from the ALCOHOLS survey +(12CO(3–2) line emission in the Milky Way) available in the ESO Archive. We +first locate the reduced survey products, then use an instrument-specific query +to identify the corresponding raw datasets, and finally retrieve the associated +Quick Look bundles. This indirection is sometimes necessary because Quick Look +queries may require the APEX project ID (distinct from the ESO proposal - aka +programme - ID). If the APEX project ID is known (e.g. for proprietary data), +Quick Look products can be queried directly. + +Query Reduced APEX data +======================= + +We first query for the reduced data from the **ALCOHOLS** survey, and retrieve +the ESO proposal ID. + +.. doctest-remote-data:: + + >>> # query the ESO archive for the ALCOHOLS survey + >>> table_reduced = eso.query_surveys("ALCOHOLS") + + >>> # extract unique ESO run IDs from the query result + >>> # (the survey table actually contains ESO run IDs) + >>> run_ids = list(set(table_reduced["proposal_id"])) + + >>> # check if we have a single run ID or multiple + >>> if len(run_ids) == 1: + ... run_id = run_ids[0] + ... else: + ... print("Warning: Multiple proposal IDs found...") + ... run_id = run_ids[0] + + >>> # Extract the proposal (aka programme) ID from the run ID + >>> def proposal_remove_run(run_id): + ... proposal_id = ".".join(run_id.split(".", 2)[:2]) + ... proposal_id = ".".join(proposal_id.split("(")[:1]) + ... return proposal_id + + >>> proposal_id = proposal_remove_run(run_id) + >>> print(f"Proposal ID: {proposal_id}") + Proposal ID: 094.C-0935 + +Note that multiple values of ``proposal_id`` may be returned, +which would require minor changes to the above example to loop through all +relevant IDs. + +Available Query Constraints +=========================== + +As always, it is good practice to check the available columns to search in the instrument-specific query. + +.. doctest-remote-data:: + + >>> eso.query_instrument("APEX", help=True) + INFO: + Columns present in the table ist.APEX: + column_name datatype xtype unit + ----------------- -------- ----------- ----- + access_estsize long kbyte + access_url char + bwid float GHz + channels int + datalink_url char + date_obs char + ... + tel_airm_end float + tel_airm_start float + tel_alt float deg + tel_az float deg + telescope char + wobcycle float s + wobthrow float deg + wobused char + Number of records present in the table ist.APEX: + 913029 + [astroquery.eso.core] + +Query Raw APEX data +=================== + +We now query for raw data from the APEX instrument, using the ESO proposal ID +retrieved from the previous query. + +.. doctest-remote-data:: + + >>> # query the APEX instrument for data related to the ESO proposal ID + >>> table_raw = eso.query_instrument("APEX", column_filters={"prog_id": f"like '{proposal_id}%'"}) + + >>> # extract unique APEX project IDs from the raw data query + >>> project_id = list(set(table_raw["project_id"])) + + >>> # assuming we only have one project ID + >>> project_id = project_id[0] + >>> print(f"Project ID: {project_id}") + Project ID: E-094.C-0935A-2014 + +In this case, we know there is only **one** APEX project ID, but if there were +multiple IDs, we would need to loop through them. + +.. tip:: + In the :meth:`~astroquery.eso.EsoClass.query_surveys` query, the + ``"proposal_id"`` column refers to the ESO program/run ID. Whereas, in + :meth:`~astroquery.eso.EsoClass.query_instrument`, it is the ``"prog_id"`` + column that refers to the ESO program/run ID (the proposal ID can be + extracted from the run ID), and ``"project_id"`` refers to the APEX project + ID. This is the value used to identify APEX Quick Look products (see + below). + +Query APEX Quick Look products +============================== + +We can check the available columns to search in the query. + +.. doctest-remote-data:: + + >>> eso.query_apex_quicklooks(help=True) + INFO: + Columns present in the table ist.apex_quicklooks: + column_name datatype xtype unit + --------------- -------- --------- ----- + access_estsize long kbyte + access_url char + instrument char + instrument_type char + partner char + pi_coi char + prog_id char + prog_title char + prog_type char + project_id char + quicklook_id char + release_date char timestamp + Number of records present in the table ist.apex_quicklooks: + 282296 + [astroquery.eso.core] + +And now, use :meth:`~astroquery.eso.EsoClass.query_apex_quicklooks` to query +for the APEX Quick Look products using the APEX project ID +(``project_id``) we retrieved from the previous query. + +.. doctest-remote-data:: + + >>> table_quicklooks = eso.query_apex_quicklooks(project_id) + >>> table_quicklooks +
+ access_estsize access_url instrument instrument_type partner ... prog_title prog_type project_id quicklook_id release_date + kbyte ... + int64 object object object object ... object object object object object + -------------- ---------------------------------------------------------------------- ---------- --------------- ------- ... ------------------------------------------------------------------------------- --------- ------------------ --------------------------- ------------------------ + 846755 https://dataportal.eso.org/dataPortal/file/E-094.C-0935A.2014DEC10.TAR APEXHET Heterodyne ESO ... The APEX Large CO Heterodyne Outflow Legacy Supercam survey of Orion (ALCOHOLS) Normal E-094.C-0935A-2014 E-094.C-0935A.2014DEC10.TAR 2014-12-10T07:05:44.397Z + ... ... ... ... ... ... ... ... ... ... ... + 40963041 https://dataportal.eso.org/dataPortal/file/E-094.C-0935A.2015AUG07.TAR APEXHET Heterodyne ESO ... The APEX Large CO Heterodyne Outflow Legacy Supercam survey of Orion (ALCOHOLS) Normal E-094.C-0935A-2014 E-094.C-0935A.2015AUG07.TAR 2015-04-25T18:41:53.900Z + 6389 https://dataportal.eso.org/dataPortal/file/E-094.C-0935A.2015AUG22.TAR APEXHET Heterodyne ESO ... The APEX Large CO Heterodyne Outflow Legacy Supercam survey of Orion (ALCOHOLS) Normal E-094.C-0935A-2014 E-094.C-0935A.2015AUG22.TAR 2015-04-25T18:41:53.900Z + + +As can be seen from the output above, there is one APEX Quick Look product +available per UT date, per APEX project ID. Also note that the APEX Quick Look +products are available in ``.tar`` format (e.g. +``E-094.C-0935A.2014DEC10.TAR``), which can be downloaded and extracted (see +below). + +Download APEX Quick Look products +================================= + +Finally, we can download the APEX Quick Look products using the :meth:`~astroquery.eso.EsoClass.retrieve_data` method: + +.. doctest-remote-data:: + + >>> eso.retrieve_data(table_quicklooks[0]["quicklook_id"]) diff --git a/docs/eso/eso_obs_cone_search.rst b/docs/eso/eso_obs_cone_search.rst new file mode 100644 index 0000000000..d42dcd3981 --- /dev/null +++ b/docs/eso/eso_obs_cone_search.rst @@ -0,0 +1,97 @@ + +************************** +Observations - Cone Search +************************** + +In the simplest case, archive queries can be performed using a positional +(cone) search, which returns all data products within a given angular radius +around a specified sky position. This approach is particularly useful when +searching for reduced (Phase 3) data products around well-known targets or +regions on the sky. + +.. note:: + This is typically done within the ``astroquery`` using the ``query_region`` or ``query_object`` methods. However, these methods are not yet implemented for ``astroquery.eso``. Instead, cone searches can be performed by passing the appropriate parameters to the existing query methods, such as :meth:`~astroquery.eso.EsoClass.query_surveys`, :meth:`~astroquery.eso.EsoClass.query_instrument`, and :meth:`~astroquery.eso.EsoClass.query_main`. + +Query for Reduced Data +====================== + +This example demonstrates how to perform a basic cone search for publicly available **HAWK-I reduced data products** around the Galactic Center (Sgr A\*) using :meth:`~astroquery.eso.EsoClass.query_surveys`. Sgr A\* is located at right ascension 266.41683° and declination –29.00781°. We perform a search within a 0.05-degree radius (~3 arcminutes). + +.. doctest-remote-data:: + + >>> import astropy.units as u + + >>> ra = 266.41683 *u.deg # degrees + >>> dec = -29.00781 *u.deg # degrees + >>> radius = 0.05 *u.deg # degrees + +If coordinates are not known, you can use the :class:`~astropy.coordinates.SkyCoord` class to create them. Here, we create a SkyCoord object for Sgr A\*. + +.. doctest-remote-data:: + + >>> from astropy.coordinates import SkyCoord + >>> import astropy.units as u + + >>> coords = SkyCoord.from_name("Sgr A*") + >>> ra = coords.ra + >>> dec = coords.dec + >>> radius = 0.05 *u.deg + +With the defined coordinates and radius, we can now perform the cone search using :meth:`~astroquery.eso.EsoClass.query_surveys`. This method allows us to filter results based on specific columns, such as ``instrument_name``. + +.. doctest-remote-data:: + + >>> table = eso.query_surveys( + ... cone_ra=ra.value, + ... cone_dec=dec.value, + ... cone_radius=radius.to(u.deg).value, + ... column_filters={"instrument_name": "HAWKI"} + ... ) + +Note that ``cone_ra``, ``cone_dec``, and ``cone_radius`` parameters are specified in degrees, but as float values (so we use ``.value`` to extract the float from the ``astropy.units.Quantity``). + +Query for Raw Data +================== + +Similar cone search functionality is also available through :meth:`~astroquery.eso.EsoClass.query_instrument` and :meth:`~astroquery.eso.EsoClass.query_main` by passing the same ``cone_ra``, ``cone_dec``, and ``cone_radius`` arguments. + +Instrument-Specific Cone Search +------------------------------- + +Cone search for raw data products can be performed using the instrument-specific method, such as :meth:`~astroquery.eso.EsoClass.query_instrument`. This example searches for HAWK-I raw data products around the same coordinates as above. + +.. doctest-remote-data:: + + >>> table = eso.query_instrument( + ... "HAWKI", + ... cone_ra=ra.value, + ... cone_dec=dec.value, + ... cone_radius=radius.to(u.deg).value + ... ) + +Generic Cone Search +------------------- + +Cone search for raw data products can also be performed using the more generic method, :meth:`~astroquery.eso.EsoClass.query_main`. This allows you to search across all instruments without specifying one, with the following example searching for HAWK-I raw data products around the same coordinates as above. + +.. doctest-remote-data:: + + >>> table = eso.query_main( + ... "HAWKI", + ... cone_ra=ra.value, + ... cone_dec=dec.value, + ... cone_radius=radius.to(u.deg).value + ... ) + +Download Data +============= + +To download the data returned by the query, you can use the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes a list of data product IDs (``dp_id``) and downloads the corresponding files from the ESO archive. + +.. doctest-remote-data:: + >>> eso.retrieve_data(table["dp_id"]) + +The ``data_files`` list points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. + +.. doctest-skip:: + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") \ No newline at end of file diff --git a/docs/eso/eso_obs_download.rst b/docs/eso/eso_obs_download.rst new file mode 100644 index 0000000000..6f13e28de2 --- /dev/null +++ b/docs/eso/eso_obs_download.rst @@ -0,0 +1,51 @@ + +**************************** +Observations - Download Data +**************************** + +Assuming you have already performed a query (e.g., using :meth:`~astroquery.eso.EsoClass.query_surveys`, :meth:`~astroquery.eso.EsoClass.query_instrument`, or :meth:`~astroquery.eso.EsoClass.query_main`) and have a resulting table of data products, you can download the actual datasets using the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes as input a list of data product IDs (``dp_id``) obtained from the query results. + +.. doctest-skip:: + + >>> data_files = eso.retrieve_data(table['dp_id'][:2]) + INFO: Downloading datasets ... [astroquery.eso.core] + INFO: Downloading 2 files ... [astroquery.eso.core] + INFO: Downloading file 1/2 https://dataportal.eso.org/dataPortal/file/MIDI.2007-02-07T07:01:51.000 to /Users/foobar/.astropy/cache/astroquery/Eso [astroquery.eso.core] + INFO: Successfully downloaded dataset MIDI.2007-02-07T07:01:51.000 to /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:01:51.000.fits.Z [astroquery.eso.core] + INFO: Downloading file 2/2 https://dataportal.eso.org/dataPortal/file/MIDI.2007-02-07T07:02:49.000 to /Users/foobar/.astropy/cache/astroquery/Eso [astroquery.eso.core] + INFO: Successfully downloaded dataset MIDI.2007-02-07T07:02:49.000 to /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:02:49.000.fits.Z [astroquery.eso.core] + INFO: Uncompressing file /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:01:51.000.fits.Z [astroquery.eso.core] + INFO: Uncompressing file /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:02:49.000.fits.Z [astroquery.eso.core] + INFO: Done! [astroquery.eso.core] + + >>> data_files + ['/Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:01:51.000.fits', + '/Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:02:49.000.fits'] + +The file names returned in ``data_files`` point to the decompressed datasets +(without the .Z extension) that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. + +.. doctest-skip:: + >>> data_files = eso.retrieve_data(table['dp_id'][:2], destination="./eso_data/") + +By default, if a requested dataset is already found, it is not downloaded again from the archive. +To force the retrieval of data that are present in the destination directory, use ``continuation=True`` in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. + +When downloading datasets, you can optionally retrieve associated calibration files by using the ``with_calib`` argument. +This makes use of the ESO CalSelector service (see the `CalSelector information page `_). + +Available options: + +- ``None`` (default): Download only the requested science data. +- ``"raw"``: Include raw calibration files associated with each dataset. +- ``"processed"``: Include processed calibration files (i.e., reduced/calibrated). + +.. doctest-skip:: + + >>> # Download science data with raw calibrations + >>> data_files = eso.retrieve_data(table['dp_id'][:2], with_calib="raw") + +.. doctest-skip:: + + >>> # Download science data with processed calibrations + >>> data_files = eso.retrieve_data(table['dp_id'][:2], with_calib="processed") diff --git a/docs/eso/eso_obs_header_info.rst b/docs/eso/eso_obs_header_info.rst new file mode 100644 index 0000000000..813725e381 --- /dev/null +++ b/docs/eso/eso_obs_header_info.rst @@ -0,0 +1,42 @@ + +********************************* +Observations - Header Information +********************************* + +Only a small subset of the keywords present in the data products can be obtained with the :meth:`~astroquery.eso.EsoClass.query_instrument`, :meth:`~astroquery.eso.EsoClass.query_main`, :meth:`~astroquery.eso.EsoClass.query_surveys`, or :meth:`~astroquery.eso.EsoClass.query_tap` methods. + +There is, however, a way to get the full primary header of the FITS data products using :meth:`~astroquery.eso.EsoClass.get_headers`. + +.. doctest-remote-data:: + + >>> table = eso.query_instrument("midi", + ... column_filters={ + ... "date_obs": "between '2008-01-01' and '2008-01-02'" + ... }, + ... columns=["object", "date_obs", "dp_id"] + ... ) + +While the archive query interfaces expose the most commonly used metadata fields, +they do not return the complete FITS headers associated with each data product. +For applications that require access to instrument- or pipeline-specific +keywords, the full primary FITS header can be retrieved directly from the +archive using a dedicated helper method. + +.. doctest-remote-data:: + >>> table_headers = eso.get_headers(table["dp_id"][:5]) + >>> table_headers +
+ DP.ID SIMPLE BITPIX ... HIERARCH ESO OCS EXPO8 FNAME1 HIERARCH ESO OCS EXPO9 FNAME1 + str28 bool int64 ... str33 str33 + ---------------------------- ------ ------ ... --------------------------------- --------------------------------- + MIDI.2008-01-01T08:29:16.388 True 16 ... + MIDI.2008-01-01T08:39:38.000 True 16 ... + MIDI.2008-01-01T08:47:35.000 True 16 ... + MIDI.2008-01-01T08:52:00.623 True 8 ... MIDI.2008-01-01T08:51:42.000.fits MIDI.2008-01-01T08:52:00.623.fits + MIDI.2008-01-01T09:00:10.000 True 16 ... MIDI.2008-01-01T09:00:10.000.fits + +As shown above, for each data product ID (``DP.ID``; note that this is equivalent to ``dp_id`` in ``table``), the full primary header (336 columns in our case) of the archive FITS file is collected. In the above table ``table_headers``, there are as many rows as there are entries in the ``table['dp_id']`` column. + +.. note:: + + At present, astroquery returns only the primary header; the rest of the FITS header is not accessible through astroquery yet. Support for returning the entire header is planned for a future version. diff --git a/docs/eso/eso_obs_raw_general.rst b/docs/eso/eso_obs_raw_general.rst new file mode 100644 index 0000000000..9ecaa66fda --- /dev/null +++ b/docs/eso/eso_obs_raw_general.rst @@ -0,0 +1,243 @@ + +******************************************* +Observations - Query for Raw Data (Generic) +******************************************* + +.. tip:: + The `astroquery.eso` module provides several ways to search for data in + the ESO Science Archive for raw data. This section focuses on the + **generic query interface** for raw data + :meth:`~astroquery.eso.EsoClass.query_main`, while in + :doc:`the instrument-specific raw-data query section ` + we describe **instrument-specific queries** using + :meth:`~astroquery.eso.EsoClass.query_instrument`. + +You may want to query the ESO Archive **without targeting a specific instrument**. This is exactly what the :meth:`~astroquery.eso.EsoClass.query_main` method is designed for. It allows access to the **global raw data table**, which combines metadata across all instruments. +Internally, this method queries the ``dbo.raw`` table via ESO's `TAP service `_ (or using the :meth:`~astroquery.eso.EsoClass.query_tap` function), which you could also access directly using ADQL with a simple statement like: ``SELECT * FROM dbo.raw``. + +Available Query Constraints +=========================== + +We start by inspecting the available columns that can be queried by :meth:`~astroquery.eso.EsoClass.query_main` with the ``help=True`` keyword. +This will return a list of all columns available, along with their data types, units, and any additional metadata such as ``xtype`` information. + +The output includes column names, data types, units, and, where applicable, +``xtype`` information to indicate more specific column content. The ``xtype`` +attribute defines how a string-valued column should be interpreted during +querying. For example, a column with datatype ``char`` may represent a standard +string (``xtype = null``), a timestamp (``xtype = timestamp``), or a sky region +footprint usable in ADQL spatial queries (``xtype = adql:REGION``), with this +interpretation reflected in the ``xtype`` field. + +.. doctest-remote-data:: + + >>> eso.query_main(help=True) + INFO: + Columns present in the table dbo.raw: + column_name datatype xtype unit + ------------------- -------- ----------- ------ + access_estsize long kbyte + access_url char + datalink_url char + date_obs char + dec double deg + dec_pnt double deg + ... + tpl_start char + Number of records present in the table dbo.raw: + 36404227 + [astroquery.eso.core] + +Query with Constraints (All Instruments) +======================================== + +To perform a genuinely generic raw-data query, you can combine a cone search +with additional constraints in ``column_filters``. The example below retrieves +all data products from any instrument within 30 arcsec of ``SN2013am``, with a +time constraint specified down to the second. + +.. doctest-skip:: + + >>> from astropy.coordinates import SkyCoord + >>> import astropy.units as u + + >>> coords = SkyCoord.from_name("SN2013am") # The Prawn Nebula + >>> ra = coords.ra.value + >>> dec = coords.dec.value + >>> r = (30 * u.arcsec).to(u.deg).value + + >>> table = eso.query_main( + ... cone_ra=ra, + ... cone_dec=dec, + ... cone_radius=r, + ... column_filters={ + ... "exp_start": "between '2013-04-12' and '2013-04-12T04:52:06'" + ... }, + ... columns=["object", "ra", "dec", "date_obs", "instrument", "prog_id"], + ... ) + >>> table +
+ object ra dec date_obs prog_id instrument + deg deg + object float64 float64 object object object + -------- --------- -------- ------------------------ ------------- ---------- + SN2013AM 169.737 13.05858 2013-04-12T04:12:05.4817 188.D-3003(S) SOFI + SN2013AM 169.737 13.05858 2013-04-12T04:22:05.6191 188.D-3003(S) SOFI + SN2013AM 169.73717 13.06367 2013-04-12T02:40:17.0744 188.D-3003(S) SOFI + SKY 169.73717 13.06922 2013-04-12T02:47:39.0284 188.D-3003(S) SOFI + SKY 169.73717 13.06367 2013-04-12T02:48:19.3074 188.D-3003(S) SOFI + SKY 169.737 13.0672 2013-04-12T02:49:02.2986 188.D-3003(S) SOFI + SN2013AM 169.737 13.05887 2013-04-12T02:51:34.0298 188.D-3003(S) SOFI + SN2013AM 169.737 13.06029 2013-04-12T03:09:58.1859 188.D-3003(S) SOFI + SN2013AM 169.737 13.06029 2013-04-12T03:15:59.6815 188.D-3003(S) SOFI + SN2013AM 169.737 13.05682 2013-04-12T03:34:23.8352 188.D-3003(S) SOFI + SN2013AM 169.737 13.05887 2013-04-12T03:41:45.3991 188.D-3003(S) SOFI + SN2013AM 169.73717 13.06367 2013-04-12T02:31:32.9840 188.D-3003(S) SOFI + SN2013AM 169.73717 13.06367 2013-04-12T02:34:32.7246 188.D-3003(S) SOFI + +Query with Constraints (Specific Instrument) +============================================ + +If needed, you can still narrow a generic query to a single instrument by +passing ``instruments=...`` to :meth:`~astroquery.eso.EsoClass.query_main`. +For example, to retrieve only ``MIDI`` data products within some time range: + +.. doctest-remote-data:: + + >>> eso.ROW_LIMIT = -1 # Return all results without truncation + >>> table = eso.query_main( + ... instruments="midi", + ... column_filters={ + ... "exp_start": "between '2008-01-01' and '2009-05-12'"} + ... ) + >>> print(len(table)) + 58810 + >>> table.colnames + ['object', 'ra', 'dec', 'dp_id', 'date_obs', 'prog_id', + 'access_estsize', 'access_url', 'datalink_url', ... 'tpl_start'] + >>> table[["object", "ra", "dec", "date_obs", "prog_id"]] +
+ object ra dec date_obs prog_id + deg deg + object float64 float64 object object + -------------- ------------- ------------- ----------------------- ------------ + FLAT -596.52323555 -596.52323555 2008-09-28T06:49:15.000 60.A-9224(A) + FLAT -596.52323555 -596.52323555 2008-09-28T06:54:23.000 60.A-9224(A) + FLAT -596.52323555 -596.52323555 2008-09-28T06:59:48.000 60.A-9224(A) + FLAT -596.52323555 -596.52323555 2008-09-28T07:02:29.000 60.A-9224(A) + ... ... ... ... ... + WAVE,SPECTEMPL -596.52323555 -596.52323555 2009-01-20T10:21:43.000 60.A-9224(A) + WAVE,SPECTEMPL -596.52323555 -596.52323555 2009-01-20T10:22:36.000 60.A-9224(A) + OTHER -596.52323555 -596.52323555 2009-01-20T10:25:35.128 60.A-9224(A) + WAVE,SPECTEMPL -596.52323555 -596.52323555 2009-01-20T10:26:48.000 60.A-9224(A) + WAVE,SPECTEMPL -596.52323555 -596.52323555 2009-01-20T10:29:37.000 60.A-9224(A) + + +Suppose that we have a large query; it may be useful to limit the columns +returned in the results table. This can be done using the ``columns`` argument. +For example, to return only the target name, right ascension, declination, +observation date, and program ID: + +.. doctest-remote-data:: + + >>> table = eso.query_main( + ... instruments="midi", + ... column_filters={ + ... "exp_start": "between '2008-01-01' and '2009-05-12'"}, + ... columns=["object", "ra", "dec", "date_obs", "prog_id"] + ... ) + >>> table +
+ object ra dec date_obs prog_id + deg deg + object float64 float64 object object + ----------------------- ------------- ------------- ----------------------- ------------ + SEARCH,OBJECT,DISPERSED 0.49041805 -6.01429 2008-10-27T03:27:44.000 60.A-9224(A) + PHOTOMETRY,OBJECT 0.49041805 -6.01429 2008-10-27T03:36:04.000 60.A-9224(A) + COARSE,OBJECT 0.49041805 -6.01429 2008-10-27T03:45:32.000 60.A-9224(A) + BIAS -596.52323555 -596.52323555 2008-10-27T06:22:12.388 60.A-9224(A) + ... ... ... ... ... + FLAT -596.52323555 -596.52323555 2009-03-27T09:28:52.000 60.A-9224(A) + OTHER -596.52323555 -596.52323555 2009-03-27T09:33:31.378 60.A-9224(A) + FLAT -596.52323555 -596.52323555 2009-03-27T09:34:46.000 60.A-9224(A) + FLAT -596.52323555 -596.52323555 2009-03-27T09:43:46.000 60.A-9224(A) + FLAT -596.52323555 -596.52323555 2009-03-27T09:47:37.000 60.A-9224(A) + + +Query with Constraints (Multiple Instruments) +============================================= + +We can also query for raw data from multiple instruments in a single call to +:meth:`~astroquery.eso.EsoClass.query_main`. For example, to retrieve raw data +from the interferometric instruments ``MIDI``, ``GRAVITY``, and ``PIONIER`` within 10 arcsec of +``ETA CAR``: + +.. doctest-remote-data:: + + >>> coords = SkyCoord.from_name("ETA CAR") # Eta Carinae + >>> ra = coords.ra.value + >>> dec = coords.dec.value + >>> r = (10 * u.arcsec).to(u.deg).value + + >>> table = eso.query_main( + ... cone_ra=ra, + ... cone_dec=dec, + ... cone_radius=r, + ... instruments=["midi", "gravity", "pionier"], + ... column_filters={ + ... columns=["instrument", "object", "ra", "dec", "date_obs", "prog_id"] + ... ) + >>> print(len(table)) + 622 + >>> table[["object", "ra", "dec", "date_obs", "prog_id"]] +
+ object ra dec date_obs prog_id instrument + deg deg + object float64 float64 object object object + ------------ ------------ --------- ------------------------ ------------- ---------- + ETA CARINAE 161.2603 -59.68441 2016-02-25T05:14:21 60.A-9102(A) GRAVITY + ETA CARINAE 161.2603 -59.68441 2016-02-25T06:56:58 60.A-9102(A) GRAVITY + ETA CARINAE 161.2603 -59.68441 2016-02-25T09:36:24 60.A-9102(A) GRAVITY + ETA_CAR 161.26028194 -59.68442 2017-02-20T08:38:01 098.D-0488(A) GRAVITY + ... ... ... ... ... ... + KAPPA,OBJECT 161.26358805 -59.68441 2015-12-07T07:28:26.1927 60.A-9209(A) PIONIER + KAPPA,OBJECT 161.26358805 -59.68441 2015-12-07T07:29:14.3074 60.A-9209(A) PIONIER + KAPPA,OBJECT 161.26358805 -59.68441 2015-12-07T07:29:38.4820 60.A-9209(A) PIONIER + ETA_CAR 161.26358805 -59.68441 2015-12-07T07:48:38.6025 60.A-9209(A) PIONIER + ETA_CAR 161.26358805 -59.68441 2015-12-07T07:52:06.9548 60.A-9209(A) PIONIER + +.. tip:: + + Use :meth:`~astroquery.eso.EsoClass.query_main` when you want to search + **across all instruments**. Use + :meth:`~astroquery.eso.EsoClass.query_instrument` when you want a more + **refined, instrument-specific search** (e.g. instrument modes, + configurations, or ambient conditions). + + .. doctest-remote-data:: + + >>> column_filters = { + ... "dp_cat": "SCIENCE", # Science data only + ... "ins_opt1_name": "HIGH_SENS", # High sensitivity mode + ... "night_flag": "night", # Nighttime observations only + ... "moon_illu": "< 0", # No moon (below horizon) + ... "lst": "between 0 and 6" # Local sidereal time early in the night + ... } + >>> table = eso.query_instrument("midi", column_filters=column_filters) + +Download Data +============= + +To download the data returned by the query, you can use the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes a list of data product IDs (``dp_id``) and downloads the corresponding files from the ESO archive. + +.. doctest-remote-data:: + >>> eso.retrieve_data(table["dp_id"]) + +The ``data_files`` list points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. + +.. doctest-skip:: + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") + +For raw data, you can also retrieve associated calibration files by +passing ``with_calib="raw"`` (or ``with_calib="processed"``). See +:doc:`eso_download` for examples and the related CalSelector terminology. diff --git a/docs/eso/eso_obs_raw_instrument.rst b/docs/eso/eso_obs_raw_instrument.rst new file mode 100644 index 0000000000..95c27b7edf --- /dev/null +++ b/docs/eso/eso_obs_raw_instrument.rst @@ -0,0 +1,149 @@ + +******************************************************* +Observations - Query for Raw Data (Instrument-Specific) +******************************************************* + +.. note:: + The `astroquery.eso` module provides several ways to search for raw data in the ESO Science Archive. This section focuses on the **instrument-specific query interface** for raw data :meth:`~astroquery.eso.EsoClass.query_instrument`, while in :doc:`the generic raw-data query section ` we describe **generic queries** using :meth:`~astroquery.eso.EsoClass.query_main`. + +In many cases, you will want to query the ESO Archive **for data from a specific instrument**. This is exactly what the :meth:`~astroquery.eso.EsoClass.query_instrument` method is designed for. It allows you to search instrument-specific tables, which expose metadata fields and filters unique to each instrument. Internally, this method queries the corresponding instrument table (e.g., ``ist.muse``) via ESO's `TAP service `_. This approach is ideal when you need precise control over your query, such as filtering by instrument configuration, mode, or observational setup. + +Available Instruments +===================== + +To begin retrieving raw data from the ESO Science Archive, you first need to identify the relevant instrument(s) for your search. Each instrument has its own dedicated query table accessible through the archive’s programmatic `TAP `_ interface. + +The list of all supported instruments can be retrieved using the :meth:`~astroquery.eso.EsoClass.list_instruments` method: + +.. doctest-remote-data:: + + >>> from astroquery.eso import Eso + >>> eso = Eso() + >>> eso.list_instruments() + ['alpaca', 'amber', 'apex', 'crires', 'efosc', 'eris', + 'espresso', 'fiat', 'fors1', 'fors2', 'giraffe', 'gravity', + 'harps', 'hawki', 'isaac', 'kmos', 'matisse', 'midi', + 'muse', 'naco', 'nirps', 'omegacam', 'pionier', 'sinfoni', + 'sofi', 'sphere', 'uves', 'vimos', 'vircam', 'visir', + 'wlgsu', 'xshooter'] + +This list corresponds to the instruments currently available for programmatic raw data queries in the ESO archive. + +The list is dynamically generated by querying the archive’s internal TAP service. It can also be reproduced directly by executing the following ADQL query on the `ESO TAP interface `_ or using the :meth:`~astroquery.eso.EsoClass.query_tap` function: + +.. doctest-skip:: + + SELECT table_name + FROM TAP_SCHEMA.tables + WHERE schema_name = 'ist' + ORDER BY table_name + +Once you have identified the instrument of interest, you can proceed with constructing your query and retrieving raw data products. + +Available Query Constraints +=========================== + +Once an instrument is selected — for example, ``midi`` — you can inspect the available queryable columns using the ``help=True`` keyword in the :meth:`~astroquery.eso.EsoClass.query_instrument` method. This is a useful first step to understand what metadata is available and how to structure your query. + +The output includes column names, data types, units, and, where applicable, +``xtype`` information to indicate more specific column content. The ``xtype`` +attribute defines how a string-valued column should be interpreted during +querying. For example, a column with datatype ``char`` may represent a standard +string (``xtype = null``), a timestamp (``xtype = timestamp``), or a sky region +footprint usable in ADQL spatial queries (``xtype = adql:REGION``), with this +interpretation reflected in the ``xtype`` field. + +.. doctest-remote-data:: + + >>> eso.query_instrument("midi", help=True) + INFO: + Columns present in the table ist.midi: + column_name datatype xtype unit + ------------------- -------- ------------ ----------- + access_estsize long kbyte + access_url char + datalink_url char + date_obs char + ... + exp_start char timestamp + ... + object char + ... + release_date char timestamp + s_region char adql:REGION + ... + utc float s + Number of records present in the table ist.midi: + 437577 + [astroquery.eso.core] + +Query with Constraints +====================== + +Once the available query columns have been inspected, you can construct a constrained query to retrieve relevant datasets. For example, suppose you want to retrieve MIDI observations that were taken between ``2008-01-01`` and ``2009-05-12``. + +The ``column_filters`` dictionary allows you to specify conditions for individual columns, using ADQL-compatible expressions under the hood. In this case, the filters apply to: + +- ``object``: the target name, matched as a string (case-insensitive) +- ``exp_start``: the observation start time, stored as a ``char`` column with ``timestamp`` ``xtype`` + +The ``columns`` argument controls which fields are returned in the results table. + +.. tip:: + + - Use only column names returned via ``help=True`` (in this case ``exp_start``, ``object``, ``prog_id``, ``exptime``, etc.). + - String filters (like ``object``) are matched case-insensitively. + - Temporal filters on fields like ``exp_start`` or ``release_date`` can use SQL-style syntax (e.g. ``between 'YYYY-MM-DD' and 'YYYY-MM-DD'``). + - Column names are case-sensitive in Python, so ensure they match exactly. + +.. doctest-remote-data:: + >>> table = eso.query_instrument( + ... instrument="midi", + ... column_filters={ + ... "exp_start": "between '2008-01-01' and '2009-05-12'"}, + ... columns=["object", "date_obs"] + ... ) + >>> table +
+ object date_obs + object object + ------------------------------ ----------------------- + WAVE,SPECTEMPL 2008-01-03T08:57:45.000 + OTHER 2008-01-03T08:58:07.803 + WAVE,SPECTEMPL 2008-01-03T08:59:37.000 + WAVE,SPECTEMPL 2008-01-03T09:00:38.000 + ... ... + PHOTOMETRY,OBJECT,SCIPHOT 2008-01-12T02:23:16.606 + +.. tip:: + + Use :meth:`~astroquery.eso.EsoClass.query_main` when you want to search **across all instruments**. Use :meth:`~astroquery.eso.EsoClass.query_instrument` when you want a more **refined, instrument-specific search** (e.g. instrument modes, configurations, or ambient conditions). + + .. doctest-remote-data:: + + >>> column_filters = { + ... "dp_cat": "SCIENCE", # Science data only + ... "ins_opt1_name": "HIGH_SENS", # High sensitivity mode + ... "night_flag": "night", # Nighttime observations only + ... "moon_illu": "< 0", # No moon (below horizon) + ... "lst": "between 0 and 6" # Local sidereal time early in the night + ... } + >>> table = eso.query_instrument("midi", column_filters=column_filters) + + +Download Data +============= + +To download the data returned by the query, you can use the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes a list of data product IDs (``dp_id``) and downloads the corresponding files from the ESO archive. + +.. doctest-remote-data:: + >>> eso.retrieve_data(table["dp_id"]) + +The ``data_files`` list points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. + +.. doctest-skip:: + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") + +For raw data, you can also retrieve associated calibration files by +passing ``with_calib="raw"`` (or ``with_calib="processed"``). See +:doc:`eso_download` for examples and the related CalSelector terminology. \ No newline at end of file diff --git a/docs/eso/eso_obs_reduced.rst b/docs/eso/eso_obs_reduced.rst new file mode 100644 index 0000000000..a0e912e47e --- /dev/null +++ b/docs/eso/eso_obs_reduced.rst @@ -0,0 +1,164 @@ + +************************************* +Observations - Query for Reduced Data +************************************* + +In addition to raw observational files, the ESO Science Archive provides access to a wide range of **processed (reduced) data products**. These include science-ready images, spectra, and datacubes that have been validated by ESO (through the `ESO Phase 3 `_ process). + +Available Surveys +================= + +The list of available surveys can be obtained with :meth:`~astroquery.eso.EsoClass.list_surveys` as follows: + +.. doctest-remote-data:: + + >>> surveys = eso.list_surveys() + >>> surveys + ['081.C-0827', '092.A-0472', '096.B-0054', '1100.A-0528', '1101.A-0127', '193.D-0232', + '195.B-0283', '196.B-0578', '196.D-0214', '197.A-0384', '198.A-0708', '60.A-9284H', + '60.A-9493', 'ADHOC', 'ALCOHOLS', 'ALLSMOG', 'ALMA', 'AMAZE', 'AMBRE', 'APEX-SciOps', + 'ARP_VST', 'ATLASGAL', 'CAFFEINE', 'ENTROPY', 'ePESSTOplus', 'ERIS-NIX', + 'ERIS-SPIFFIER', 'ESPRESSO', 'ESSENCE', 'FDS', 'FEROS', 'Fornax3D', 'FORS2-SPEC', + 'GAIAESO', 'GCAV', 'GIRAFFE', 'GOODS_FORS2', 'GOODS_ISAAC', 'GOODS_VIMOS_IMAG', + 'GOODS_VIMOS_SPEC', 'GW170817', 'HARPS', 'HAWKI', 'HUGS', 'INSPIRE', 'KIDS', 'KMOS', + 'LEGA-C', 'LESS', 'MAGIC', 'MUSE', 'MUSE-DEEP', 'MUSE-STD', 'MW-BULGE-PSFPHOT', + 'NGTS', 'NIRPS', 'OMEGACAM_INAF', 'PENELLOPE', 'PESSTO', 'PHANGS', 'PIONIER', + 'SHARKS', 'SPHERE', 'SUPER', 'UltraVISTA', 'UVES', 'UVES_SQUAD', 'VANDELS', 'VEGAS', + 'VEILS', 'VEXAS', 'VHS', 'VIDEO', 'VIKING', 'VIMOS', 'VINROUGE', 'VIPERS', 'VISIONS', + 'VMC', 'VPHASplus', 'VST-ATLAS', 'VVV', 'VVVX', 'XQ-100', 'XSGRB', 'XSHOOTER', + 'XShootU', 'XSL', 'ZCOSMOS'] + +Available Query Constraints +=========================== + +As before, list the possible columns in :meth:`~astroquery.eso.EsoClass.query_surveys` that can be queried with: + +.. doctest-remote-data:: + + >>> eso.query_surveys(help=True) # get help on the ESO query + INFO: + Columns present in the table ivoa.ObsCore: + column_name datatype xtype unit + ------------------- -------- ----------- ------ + abmaglim double mag + access_estsize long kbyte + access_format char + access_url char + bib_reference char + calib_level int + ... + target_name char + Number of records present in the table ivoa.ObsCore: + 4559928 + [astroquery.eso.core] + +.. note:: + Column names may differ between the tables returned by different query + methods. Users are encouraged to inspect closely the available columns and supported + filters using the ``help=True`` option. For example, for raw data queries + (:meth:`~astroquery.eso.EsoClass.query_main` and + :meth:`~astroquery.eso.EsoClass.query_instrument`), the target name is specified + using the ``object`` column, whereas for reduced data queries + (:meth:`~astroquery.eso.EsoClass.query_surveys`) the corresponding column is + named ``target_name``. + +Query with Constraints (Specific Survey) +======================================== + +Let's assume that we work with the `HARPS survey `_. The archive can be queried as follows: + +.. doctest-remote-data:: + + >>> table = eso.query_surveys(surveys="HARPS") + >>> table +
+ target_name s_ra s_dec dp_id proposal_id abmaglim access_estsize ... snr strehl t_exptime t_max t_min t_resolution t_xel + deg deg mag kbyte ... s d d s + object float64 float64 object object float64 int64 ... float64 float64 float64 float64 float64 float64 int64 + ------------ --------- --------- --------------------------- ------------- -------- -------------- ... ------- ------- --------- -------------- -------------- ------------ ----- + HD203608 321.61455 -65.36429 ADP.2014-09-16T11:03:30.940 077.D-0720(A) -- 5261 ... 60.9 -- 33.0 53956.24265204 53956.24227009 33.00048 -- + HD114613 198.0129 -37.80367 ADP.2014-09-16T11:03:30.947 072.C-0488(E) -- 5261 ... 267.2 -- 120.002 53765.36393677 53765.36254786 120.001824 -- + HIP5158 16.50838 -22.45455 ADP.2014-09-16T11:03:30.973 072.C-0488(E) -- 5261 ... 39.3 -- 711.599 53946.33177006 53946.32353395 711.599904 -- + ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... + HD203608 321.60939 -65.36484 ADP.2014-09-16T11:03:45.713 077.D-0720(A) -- 5261 ... 45.0 -- 33.0 53955.11202646 53955.11164451 33.00048 -- + +Suppose we want both `HARPS survey `_ and `NIRPS survey `_ data products. + +.. doctest-remote-data:: + + >>> table = eso.query_surveys(surveys=["HARPS", "NIRPS"]) + >>> table +
+ target_name s_ra s_dec dp_id proposal_id abmaglim access_estsize ... snr strehl t_exptime t_max t_min t_resolution t_xel + deg deg mag kbyte ... s d d s + object float64 float64 object object float64 int64 ... float64 float64 float64 float64 float64 float64 int64 + ------------ --------- --------- --------------------------- ------------- -------- -------------- ... ------- ------- --------- -------------- -------------- ------------ ----- + HD203608 321.61455 -65.36429 ADP.2014-09-16T11:03:30.940 077.D-0720(A) -- 5261 ... 60.9 -- 33.0 53956.24265204 53956.24227009 33.00048 -- + HD114613 198.0129 -37.80367 ADP.2014-09-16T11:03:30.947 072.C-0488(E) -- 5261 ... 267.2 -- 120.002 53765.36393677 53765.36254786 120.001824 -- + HIP5158 16.50838 -22.45455 ADP.2014-09-16T11:03:30.973 072.C-0488(E) -- 5261 ... 39.3 -- 711.599 53946.33177006 53946.32353395 711.599904 -- + ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... + HD203608 321.60939 -65.36484 ADP.2014-09-16T11:03:45.713 077.D-0720(A) -- 5261 ... 45.0 -- 33.0 53955.11202646 53955.11164451 33.00048 -- + +Now we see that this query is limited to 1000 results, so we can increase the row limit to retrieve all matching datasets and limit the columns returned: + +.. doctest-remote-data:: + + >>> eso.ROW_LIMIT = -1 + >>> table = eso.query_surveys(surveys=["HARPS", "NIRPS"], + ... columns=["target_name", "s_ra", "s_dec", "dp_id", "proposal_id"]) + >>> table +
+ target_name s_ra s_dec dp_id proposal_id + deg deg + object float64 float64 object object + --------------- ---------- --------- --------------------------- -------------- + HD17051 40.64236 -50.80053 ADP.2014-09-16T11:03:41.583 078.D-0067(A) + HD69830 124.59872 -12.6353 ADP.2014-09-16T11:03:41.610 072.C-0488(E) + HD63077 116.39514 -34.17365 ADP.2014-09-16T11:03:41.617 076.D-0103(A) + ... ... ... ... ... + TOI-2322 116.961088 -71.00243 ADP.2026-01-13T11:49:12.986 116.2974.001 + +The returned table has a ``dp_id`` column, which can be used to retrieve the datasets with +:meth:`~astroquery.eso.EsoClass.retrieve_data`: ``eso.retrieve_data(table["dp_id"])``. +Note that, in this example, there are 376,351 matching data products across both surveys so downloading +all of them may take a significant amount of time and disk space. +More details about this method are in the following section. + +Query with Constraints (Specific Instrument) +============================================ + +You can also query a specific instrument using the same method. For example, to retrieve **all** available HARPS data products regardless of the associated survey: + +.. doctest-remote-data:: + + >>> table = eso.query_surveys(column_filters={"instrument_name": "HARPS"}) + >>> table +
+ target_name s_ra s_dec dp_id proposal_id abmaglim access_estsize ... snr strehl t_exptime t_max t_min t_resolution t_xel + deg deg mag kbyte ... s d d s + object float64 float64 object object float64 int64 ... float64 float64 float64 float64 float64 float64 int64 + ------------ --------- --------- --------------------------- ------------- -------- -------------- ... ------- ------- --------- -------------- -------------- ------------ ----- + HD203608 321.61455 -65.36429 ADP.2014-09-16T11:03:30.940 077.D-0720(A) -- 5261 ... 60.9 -- 33.0 53956.24265204 53956.24227009 33.00048 -- + HD114613 198.0129 -37.80367 ADP.2014-09-16T11:03:30.947 072.C-0488(E) -- 5261 ... 267.2 -- 120.002 53765.36393677 53765.36254786 120.001824 -- + HIP5158 16.50838 -22.45455 ADP.2014-09-16T11:03:30.973 072.C-0488(E) -- 5261 ... 39.3 -- 711.599 53946.33177006 53946.32353395 711.599904 -- + ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... + HD203608 321.60939 -65.36484 ADP.2014-09-16T11:03:45.713 077.D-0720(A) -- 5261 ... 45.0 -- 33.0 53955.11202646 53955.11164451 33.00048 -- + +.. tip:: + + Keep in mind that the definition of a ``survey`` (also referred to as a **collection** in the ESO Science Archive) is not the same as the definition of an **instrument**. The ``instrument_name`` refers to the actual hardware that acquired the data (e.g., ``HARPS``, ``MUSE``), whereas the ``obs_collection`` identifies the scientific programme, survey, or processing pipeline associated with the data product. In many cases, survey names match the instrument name (e.g., ``HARPS``, ``MUSE``, ``XSHOOTER``), which typically indicates **products processed and curated by ESO**. However, when the collection name differs (e.g., ``AMBRE``, ``GAIAESO``, ``PHANGS``), it usually denotes **community-contributed data** from large collaborations or specific science teams. + + So, for example, querying for ``eso.query_surveys(column_filters={"instrument_name": "HARPS"})`` will return all products taken with the HARPS instrument, across all programmes and collections. In contrast, filtering on ``eso.query_surveys(surveys="HARPS")`` will return only the `HARPS data reduced by ESO `_. You can inspect the collection for each result via the ``obs_collection`` column in your results table. + +Download Data +============= + +To download the data returned by the query, you can use the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes a list of data product IDs (``dp_id``) and downloads the corresponding files from the ESO archive. + +.. doctest-remote-data:: + >>> eso.retrieve_data(table["dp_id"]) + +The ``data_files`` list points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. + +.. doctest-skip:: + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") diff --git a/docs/eso/eso_obs_tap.rst b/docs/eso/eso_obs_tap.rst new file mode 100644 index 0000000000..2ec7b23402 --- /dev/null +++ b/docs/eso/eso_obs_tap.rst @@ -0,0 +1,445 @@ + +*************************** +Query using the TAP Service +*************************** + +The ESO `TAP service `_ allows users to +submit custom `ADQL `_ (Astronomical Data +Query Language) queries against the archive metadata, providing fine-grained +control over complex searches. TAP queries can be executed against different +tables depending on the type of data required: + +- ``ivoa.ObsCore``: standardized metadata for **fully calibrated (Phase 3) data + products** corresponding to high-level queries, such as available via + :meth:`~astroquery.eso.EsoClass.query_surveys`. + +- ``dbo.raw``: metadata for **raw observational data** across all ESO + instruments, such as available via :meth:`~astroquery.eso.EsoClass.query_main`. + +- ``ist.`` (e.g. ``ist.muse``, ``ist.midi``): **instrument-specific + raw metadata** tables, such as available via + :meth:`~astroquery.eso.EsoClass.query_instrument`. + +While these query modes are covered by the high-level ``astroquery.eso`` API, +direct use of ADQL through TAP provides additional flexibility for constructing +advanced queries that combine constraints across multiple metadata fields or +tables. For more information about the TAP and how to write ADQL queries, refer to the following resources: + +- `ESO TAP documentation `_: Describes ESO's implementation of TAP and the available services. +- `IVOA TAP standard `_: The official specification from the International Virtual Observatory Alliance. +- `ADQL specification `_: Defines the query language used to interact with TAP services. + +Query for Raw Data (Generic) +============================ + +The following example demonstrates how to query the ``dbo.raw`` table for raw data products from the ``MUSE`` instrument: + +.. doctest-remote-data:: + + >>> query = """ + ... SELECT * + ... FROM dbo.raw + ... AND instrument = 'MUSE' + ... """ + >>> table = eso.query_tap(query) + +Query for Raw Data (Instrument-Specific) +======================================== + +The following example demonstrates how to query the ``ist.muse`` table for raw +data products from the ``MUSE`` instrument, selecting observations taken at +night with a Moon illumination below 30%, an LST range between 0 and 6 hours, +an average seeing (FWHM) below 1.0 arcsec, and an exposure time greater than +100 seconds: + +.. doctest-remote-data:: + + >>> query = """ + ... SELECT * + ... FROM ist.muse + ... WHERE night_flag = 'NIGHT' + ... AND moon_illu < 30 + ... AND dimm_fwhm_avg < 1.0 + ... AND exptime > 100 + ... AND lst between 0 and 6 + ... """ + >>> table = eso.query_tap(query) + +Query for Reduced Data Products +=============================== + +The following examples query the ``ivoa.ObsCore`` table to find enhanced data +products and perform common spatial selections using dataset footprints. Using +the TAP service directly allows access to functionality beyond what is available +via the high-level :meth:`~astroquery.eso.EsoClass.query_surveys` interface, such +as complex spatial constraints, joins, aggregations, and more flexible filtering +on ObsCore metadata. + +The same general approach applies when querying raw data products as described +above: while the specific metadata fields may differ, the use of TAP enables the +same general functionality as demonstrated in the examples below. + +Constrained query +----------------- + +The following example queries to find enhanced data products +(``calib_level = 3``; e.g. mosaics, resampled/drizzled images, or other heavily +processed survey fields) from the ``SPHERE`` survey. The query further restricts +the selection to multi-OB observations (``multi_ob = 'M'``) with spatial pixel +scales smaller than 0.2 arcsec: + +.. doctest-remote-data:: + + >>> query = """ + ... SELECT obs_collection, calib_level, multi_ob, filter, + ... s_pixel_scale, instrument_name + ... FROM ivoa.ObsCore + ... WHERE obs_collection = 'SPHERE' + ... AND calib_level = 3 + ... AND multi_ob = 'M' + ... AND s_pixel_scale < 0.2 + ... """ + >>> table = eso.query_tap(query) + >>> table +
+ obs_collection calib_level multi_ob filter s_pixel_scale instrument_name + arcsec + object int32 object object float64 object + -------------- ----------- -------- ------ ------------- --------------- + SPHERE 3 M K12 0.0122 SPHERE + SPHERE 3 M K12 0.0122 SPHERE + ... + SPHERE 3 M H 0.0122 SPHERE + +Cone search +----------- + +We can also use the TAP service to perform spatial queries based on dataset +footprints. Suppose you are looking for datasets closer than *2.5 arcmin* from *NGC 4666*. +Either you know the equatorial coordinates of your object, or you rely on a name +resolver such as SESAME (CDS) to obtain them. In the end you define a search +circle by three quantities: RA, Dec, and radius, all expressed in degrees. + +The cone-search constraint is typically implemented using the ADQL +``INTERSECTS`` operator, which takes two spatial footprints as input and returns +``1`` if they intersect in at least one point (and ``0`` otherwise). In a cone +search, one footprint is the input circle, while the other is the ``s_region`` +column, which represents the sky footprint of each dataset. + +.. doctest-remote-data:: + + >>> from astropy.coordinates import SkyCoord + >>> target = "NGC 4666" + >>> pos = SkyCoord.from_name(target) + >>> r = 2.5/60. # search radius of 2.5 arcmin, expressed in degrees + >>> query = """SELECT * + ... FROM ivoa.ObsCore + ... WHERE INTERSECTS(s_region, CIRCLE('', {ra:.6f}, {dec:.6f}, {r:.6f}))=1 + ... """.format(ra=pos.ra.degree, dec=pos.dec.degree, r=r) + +.. doctest-remote-data:: + + >>> table = eso.query_tap(query=query) + >>> print("Num matching datasets: %d" % (len(table))) + Num matching datasets: 219 + +.. note:: + **Cone search:** ``INTERSECTS`` or ``CONTAINS``? + + If you are looking specifically for spectra (whose footprint is typically a + point), you could use the stricter ``CONTAINS`` operator to ensure that only + footprints entirely contained within the defined circle are returned. + Remember: ``INTERSECTS`` is commutative, but the order of the operands in + ``CONTAINS`` matters and is defined as:: + + CONTAINS(contained, container) = 1 + + Try repeating the cone-search query after replacing ``INTERSECTS`` with + ``CONTAINS`` to see the difference. In practice, images (and measurements + derived from those images) may disappear from the results because their + footprints are larger than the 2.5 arcmin circle. + + +Point in footprint +------------------ + +Suppose you are interested in finding extended datasets that cover a specific +point on the sky (for example, datasets that could have imaged the progenitor of +a supernova). “Extended” here means datasets with a non-zero footprint area, excluding spectra +and visibilities whose footprint is effectively a single point. + +To find datasets that include a given sky position, you can use either +``INTERSECTS`` or ``CONTAINS``; for a point there is no practical difference. +Here we use ``CONTAINS`` and restrict the results to images and cubes. + +.. doctest-remote-data:: + + >>> query = """SELECT t_min, abmaglim, dataproduct_type as type, dp_id, obs_release_date + ... FROM ivoa.ObsCore + ... WHERE CONTAINS(POINT('', 193.815, 0.099819), s_region)=1 + ... AND dataproduct_type IN ('image', 'cube') + ... ORDER BY t_min ASC + ... """ + >>> table = eso.query_tap(query=query) + >>> table +
+ t_min abmaglim ... obs_release_date + d mag ... + float64 float64 ... object + ------------------ ------------------ ... ------------------------ + 56402.18240388 21.629 ... 2017-09-20T20:02:58.453Z + 56402.18240388 21.536 ... 2017-09-20T20:02:58.453Z + 56402.18390705 21.374 ... 2017-09-20T20:02:58.453Z + ... ... ... ... + 58787.638966122686 14.437809060706662 ... 2021-04-22T16:00:18Z + 58884.23649998 22.653 ... 2021-02-05T06:16:18.783Z + + +Region in footprint +------------------- + +If you want to ensure that matching datasets contain an entire region, you must +use ``CONTAINS``. The first operand is the region you want to be covered, and +the second operand is the dataset footprint in ``s_region``. The covered region +may be a simple circle or a more complex shape (e.g. a polygon). Here we show a +circle around NGC 253 and restrict to images and cubes. + +.. doctest-remote-data:: + + >>> query = """SELECT t_min, s_fov, dataproduct_type as type, dp_id, obs_release_date + ... FROM ivoa.ObsCore + ... WHERE CONTAINS(CIRCLE('', 11.888002, -25.288220, 0.21), s_region)=1 + ... AND dataproduct_type IN ('image', 'cube') + ... ORDER BY t_min ASC + ... """ + >>> table = eso.query_tap(query=query) + >>> table +
+ t_min s_fov type dp_id obs_release_date + d deg + float64 float64 object object object + -------------- ------------- ------ --------------------------- ------------------------ + 55864.21418374 1.15908954416 image ADP.2019-04-29T06:50:17.911 2019-04-29T13:08:36.937Z + 55893.13608187 1.45520652527 image ADP.2015-06-17T12:56:53.853 2015-06-18T11:00:50.587Z + 55897.15708812 1.45500126333 image ADP.2015-06-17T12:57:54.060 2015-06-18T05:46:38.167Z + 55927.08420409 1.15896704055 image ADP.2019-04-29T06:50:21.403 2019-04-29T18:06:01.943Z + 56132.3041874 1.15909942583 image ADP.2019-04-29T07:01:36.683 2019-05-02T17:40:17.620Z + 56147.27356555 1.15886452083 image ADP.2019-04-29T07:01:36.979 2019-05-02T17:40:17.620Z + 56180.34451683 1.15887357333 image ADP.2019-04-29T07:01:38.569 2019-05-02T18:00:17.460Z + +The above lists datasets whose footprints are large enough to entirely contain +the provided circular region. + +Search by polygon +----------------- + +You can also search for datasets intersecting a polygon. This is useful, for +example, when looking for optical/infrared/radio counterparts of a gravitational +wave (GW) event. GW spatial probability maps (e.g. via LIGO/Virgo skymaps) can be +converted into confidence contours at a given probability level, resulting in a +counterclockwise polygon suitable for ADQL spatial queries. + +The following example uses a polygon constructed for the GW170817 event and +returns matching datasets ordered by ``t_min``. + +.. doctest-remote-data:: + + >>> query = """SELECT t_min, snr, abmaglim, dataproduct_type as type, dp_id + ... FROM ivoa.ObsCore + ... WHERE INTERSECTS( + ... s_region, + ... POLYGON('J2000', + ... 196.8311,-23.5212, 196.7432,-23.3586, 196.6553,-23.1962, + ... 196.4795,-23.0339, 196.3916,-22.8719, 196.3037,-22.7100, + ... 196.2158,-22.5484, 196.1279,-22.3869, 196.0400,-22.2257, + ... 195.9521,-22.0646, 195.8643,-21.9037, 195.7764,-21.7429, + ... 195.7764,-21.5824, 195.6885,-21.4220, 195.6006,-21.2618, + ... 195.5127,-21.1018, 195.4248,-20.9420, 195.3369,-20.7823, + ... 195.3369,-20.6228, 195.2490,-20.4634, 195.1611,-20.3043, + ... 195.1611,-20.1452, 195.0732,-19.9864, 194.9854,-19.8277, + ... 194.8975,-19.6692, 194.8975,-19.5108, 194.8096,-19.3526, + ... 194.7217,-19.1945, 194.6338,-19.0366, 194.6338,-18.8788, + ... 194.5459,-18.7212, 194.4580,-18.5637, 194.4580,-18.4064, + ... 194.3701,-18.2492, 194.4580,-18.0922, 194.4580,-17.9353, + ... 194.6338,-18.0137, 194.8096,-18.1707, 194.9854,-18.3278, + ... 195.0732,-18.4851, 195.1611,-18.6425, 195.2490,-18.8000, + ... 195.3369,-18.9577, 195.4248,-19.1155, 195.5127,-19.2735, + ... 195.6006,-19.4317, 195.6885,-19.5900, 195.8643,-19.7484, + ... 195.9521,-19.9070, 196.1279,-20.0658, 196.2158,-20.2247, + ... 196.3916,-20.3838, 196.4795,-20.5431, 196.5674,-20.7025, + ... 196.6553,-20.8621, 196.7432,-21.0219, 196.8311,-21.1818, + ... 196.9189,-21.3419, 196.9189,-21.5022, 197.0068,-21.6626, + ... 197.0947,-21.8233, 197.1826,-21.9841, 197.2705,-22.1451, + ... 197.3584,-22.3063, 197.4463,-22.4676, 197.5342,-22.6292, + ... 197.6221,-22.7909, 197.7100,-22.9529, 197.7979,-23.1150, + ... 197.7979,-23.2773, 197.8857,-23.4399, 197.9736,-23.6026, + ... 196.9189,-23.6026 + ... ))=1 + ... ORDER BY t_min ASC + ... """ + >>> table = eso.query_tap(query=query) + >>> table +
+ t_min snr abmaglim type dp_id + d mag + float64 float64 float64 object object + -------------- ------- -------- ------------ --------------------------- + 52769.18223732 41.5 -- spectrum ADP.2020-08-04T16:44:55.149 + 53070.29890813 79.4 -- spectrum ADP.2016-09-20T08:08:13.622 + 53071.17154411 86.0 -- spectrum ADP.2016-09-20T08:08:12.820 + 53373.36714808 170.9 -- spectrum ADP.2016-09-21T07:05:31.370 + 54514.36343016 74.2 -- spectrum ADP.2020-06-28T16:34:31.221 + 54514.36347455 14.3 -- spectrum ADP.2020-06-28T16:34:31.235 + ... ... ... ... ... + 59355.20396096 6.9 -- spectrum ADP.2021-06-08T11:54:53.878 + 59759.00758508 -- -- visibility ADP.2025-12-18T13:15:42.554 + +Spatial joins +------------- + +Are you interested in finding images in different bands of the same sky region, +for photometric studies? The following example shows how to compose a spatial +join to find HAWKI source tables: + +* within 10 degrees from the Galactic plane, +* taken in the J and H bands (selected by wavelength constraints), +* where the J and H footprints overlap, and +* ensuring that the overlap covers at least 80% of the J-band footprint area. + +.. doctest-remote-data:: + + >>> query = """SELECT J.* FROM + ... (SELECT * FROM ivoa.ObsCore + ... WHERE dataproduct_subtype = 'srctbl' + ... AND obs_collection = 'HAWKI' + ... AND gal_lat < 10 AND gal_lat > -10 + ... AND em_min < 1.265E-6 AND em_max > 1.265E-6) J, + ... + ... (SELECT * FROM ivoa.ObsCore + ... WHERE dataproduct_subtype = 'srctbl' + ... AND obs_collection = 'HAWKI' + ... AND gal_lat < 10 AND gal_lat > -10 + ... AND em_min < 1.66E-6 AND em_max > 1.66E-6) H + ... + ... WHERE INTERSECTS(J.s_region, H.s_region)=1 + ... AND ESO_INTERSECTION(J.s_region, H.s_region) > 0.8 * AREA(J.s_region) + ... """ + >>> table = eso.query_tap(query=query) + >>> table +
+ abmaglim access_estsize access_format ... t_resolution t_xel target_name + float64 int64 object ... float64 int64 object + -------- -------------- ------------------------------------------ ... ------------ ----- ----------- + 23.029 2255 application/x-votable+xml;content=datalink ... 3580.209504 -- Lupus3-East + 23.054 2004 application/x-votable+xml;content=datalink ... 3580.209504 -- Lupus3-East + 23.035 1990 application/x-votable+xml;content=datalink ... 3580.94736 -- Lupus3-East + 23.01 2217 application/x-votable+xml;content=datalink ... 3580.94736 -- Lupus3-East + ... ... ... ... ... ... ... + 25.468 3081 application/x-votable+xml;content=datalink ... 280.788768 -- V U Car + 25.408 3093 application/x-votable+xml;content=datalink ... 280.788768 -- V U Car + +Notice the ``J.*`` in the ``SELECT`` clause: this returns only the metadata for +the J-band datasets. You can repeat the query selecting ``H.*`` to retrieve the +H-band metadata. This is useful, for example, when visualising results in tools +such as Aladin using different colours for different bands. Alternatively, you +can return both sets in one query (``SELECT *``) or select specific columns from +each side, e.g. ``SELECT J.dp_id, H.dp_id, ...``. + + +Query by wavelengths +-------------------- + +The IVOA ObsCore standard stores wavelengths in meters. You can rescale them in +the ``SELECT`` clause to display other units (e.g. nanometers), but you should +use meters when setting constraints on the ``em_min`` and ``em_max`` columns. +This applies even when the underlying data are naturally described in frequency +(e.g. radio observations), enabling uniform queries across observatories. + +.. doctest-remote-data:: + + >>> query = """SELECT obs_collection AS collection, + ... dataproduct_type AS type, + ... dataproduct_subtype AS subtype, + ... em_min*1E9 AS min_wavel_nm, + ... em_max*1E9 AS max_wavel_nm, + ... em_res_power + ... FROM ivoa.ObsCore + ... WHERE target_name = 'a370' + ... AND em_res_power < 3000""" + >>> table = eso.query_tap(query=query) + >>> table +
+ collection type subtype min_wavel_nm max_wavel_nm em_res_power + object object object float64 float64 float64 + ---------- ------------ -------- ------------------ ------------ ------------ + 092.A-0472 image 1981.9999999999998 2307.0 6.5985 + HAWKI measurements srctbl 1181.0 1335.0 8.1688 + HAWKI image pawprint 1181.0 1335.0 8.1688 + HAWKI image tile 1181.0 1335.0 8.1688 + ... ... ... ... ... ... + HAWKI image tile 1181.0 1335.0 8.1688 + +Query ESO Archive Stats +======================= + +The ESO TAP service can also be used to retrieve high-level statistics about the +contents of the ESO Science Archive. +The example below queries the total number of entries in the +``ivoa.ObsCore`` table, corresponding to the total number of data products +currently exposed through the ESO TAP service. + +.. doctest-remote-data:: + + >>> query = "SELECT COUNT(*) FROM ivoa.ObsCore" + >>> table = eso.query_tap(query) + >>> table +
+ COUNT_ALL + int32 + --------- + 4731624 + +More detailed statistics can be obtained by grouping on selected metadata +columns. The following example groups all data products by +``dataproduct_type`` and returns both the number of products in each category +and the total estimated archive volume, computed from the +``access_estsize`` column. The results are ordered by decreasing total data +volume. + +.. doctest-remote-data:: + + >>> query = """SELECT dataproduct_type, + ... COUNT(*) NumProducts, + ... SUM(access_estsize)/1000000000. TB + ... FROM ivoa.ObsCore + ... GROUP BY dataproduct_type + ... ORDER BY 3 DESC + ... """ + >>> table = eso.query_tap(query) + >>> table +
+ dataproduct_type numproducts tb + object int32 float64 + ---------------- ----------- ------------ + cube 690736 98.60936855 + image 969281 82.775606369 + measurements 675811 50.197075129 + spectrum 2377000 5.434112666 + visibility 18796 0.254378068 + +These are just some of the many possible ways to query for ESO Science Archive statistics +using the TAP service. For more examples on the available queries, +refer to the `TAP service `_. + +Download Data +============= + +As with the other functionality, to download the data returned by the above data queries (i.e. not the stats queries), you can use the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes a list of data product IDs (``dp_id``) and downloads the corresponding files from the ESO archive. + +.. doctest-remote-data:: + >>> eso.retrieve_data(table["dp_id"]) + +The ``data_files`` list points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. + +.. doctest-skip:: + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") diff --git a/docs/eso/eso_troubleshooting.rst b/docs/eso/eso_troubleshooting.rst new file mode 100644 index 0000000000..5ea8dcdcbd --- /dev/null +++ b/docs/eso/eso_troubleshooting.rst @@ -0,0 +1,82 @@ + +*************** +Troubleshooting +*************** + +Clearing Cache +============== + +If you encounter repeated query failures or see outdated or inconsistent results, you may be dealing with a stale or corrupted local cache. You can clear the entire Astropy cache for ESO with the following command: + +.. doctest-skip:: + + >>> from astroquery.eso import Eso + >>> Eso.clear_cache() + +This will remove all cached ESO queries and downloaded metadata files. Data products already downloaded will remain unless manually deleted. + +.. _column-filters-fix: + +Using Correct ``column_filters`` +================================ + +If your query fails or silently returns no results, it might be because you're using column names that are **accepted in the ESO web interface (WDB)** but **not supported by the TAP/ADQL interface** that is now used within ``astroquery.eso``. A common case involves using ``stime`` and ``etime``, which are not valid TAP fields. Instead, use ``exp_start``, the TAP-compliant column representing the observation start time. This field supports SQL-style date filtering. + +Below are examples of invalid filter usage and their corrected TAP-compatible versions. + +Filtering between two dates +--------------------------- + +.. doctest-skip:: + + # ❌ Invalid (WDB-specific fields, not recognized by TAP) + column_filters = { + "stime": "2024-01-01 12:00:00", + "etime": "2024-01-03 12:00:00" + } + +.. doctest-skip:: + + # ✅ Correct (TAP-compliant syntax using 'exp_start') + column_filters = { + "exp_start": "between '2024-01-01 12:00:00' and '2024-01-03 12:00:00'" + } + + # OR + column_filters = { + "exp_start": "between '2024-01-01T12:00:00' and '2024-01-03T12:00:00'" + } + +Filtering with only a start date +-------------------------------- + +.. doctest-skip:: + + # ❌ Invalid + column_filters = { + "stime": "2024-01-01 12:00:00" + } + +.. doctest-skip:: + + # ✅ Correct + column_filters = { + "exp_start": "> '2024-01-01 12:00:00'" + } + +Filtering with only an end date +------------------------------- + +.. doctest-skip:: + + # ❌ Invalid + column_filters = { + "etime": "2024-12-31 12:00:00" + } + +.. doctest-skip:: + + # ✅ Correct + column_filters = { + "exp_start": "< '2024-12-31 12:00:00'" + } diff --git a/docs/index.rst b/docs/index.rst index 64c3d35f66..074d19e8ca 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -339,6 +339,7 @@ for each source) :maxdepth: 1 alfalfa/alfalfa.rst + eso/eso.rst exoplanet_orbit_database/exoplanet_orbit_database.rst gama/gama.rst ipac/irsa/irsa_dust/irsa_dust.rst diff --git a/setup.cfg b/setup.cfg index ecdc423640..9d128fac73 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,9 @@ show-response = 1 [tool:pytest] minversion = 7.4 norecursedirs = build docs/_build astroquery/irsa astroquery/nasa_exoplanet_archive astroquery/ned astroquery/ibe astroquery/irsa_dust astroquery/cds astroquery/sha astroquery/dace -testpaths = astroquery docs +testpaths = + docs/eso + astroquery/eso doctest_plus = enabled astropy_header = true text_file_format = rst diff --git a/tox.ini b/tox.ini index 5c19109f91..896c0f8341 100644 --- a/tox.ini +++ b/tox.ini @@ -68,12 +68,12 @@ commands = devdeps: pip install -U --pre --no-deps --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy python -m pip freeze - !cov: pytest --pyargs astroquery {toxinidir}/docs {env:PYTEST_ARGS} {posargs} - cov: pytest --pyargs astroquery {toxinidir}/docs --cov astroquery --cov-config={toxinidir}/setup.cfg {env:PYTEST_ARGS} {posargs} + !cov: pytest --pyargs astroquery.eso {toxinidir}/docs/eso {env:PYTEST_ARGS} {posargs} + cov: pytest --pyargs astroquery.eso {toxinidir}/docs/eso --cov astroquery.eso --cov-config={toxinidir}/setup.cfg {env:PYTEST_ARGS} {posargs} # For remote tests, we re-run the failures to filter out at least some of the flaky ones. # We use a second pytest run with --last-failed as opposed to --rerun in order to rerun the # failed ones at the end rather than right away. - online: pytest --pyargs astroquery {toxinidir}/docs {env:PYTEST_ARGS_2} {posargs} + online: pytest --pyargs astroquery.eso {toxinidir}/docs/eso {env:PYTEST_ARGS_2} {posargs} cov: coverage xml -o {toxinidir}/coverage.xml pip_pre =