From 78b3e8d4e55ee8411d771ffa4f628f8e4b7f6bc7 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 12 Sep 2025 09:42:38 +0200 Subject: [PATCH 01/18] Setup auto sync fork for main branch --- .github/workflows/sync_fork.yml | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/sync_fork.yml diff --git a/.github/workflows/sync_fork.yml b/.github/workflows/sync_fork.yml new file mode 100644 index 0000000000..18abb115e6 --- /dev/null +++ b/.github/workflows/sync_fork.yml @@ -0,0 +1,45 @@ +name: Sync Fork +run-name: Sync Fork +on: + schedule: + - cron: '58 23 * * *' # run every day - two minutes to midnight + workflow_dispatch: # to enable manual runs of the workflow + +jobs: + Get-Timestamp: + runs-on: ubuntu-latest + steps: + - run: date + + Sync-With-Upstream: + runs-on: ubuntu-latest + steps: + - run: echo "The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "This job is now running on a ${{ runner.os }} server hosted by GitHub" + - run: echo "Running on branch ${{ github.ref }}, repository ${{ github.repository }}." + - name: Check out repository code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - run: echo "The ${{ github.repository }} repository has been cloned to the runner." + - name: List files in the repository + run: | + ls ${{ github.workspace }} + - name: Sync repository with upstream + run: | + cd ${{ github.workspace }} + git config --global user.email "jcarmona@eso.org" + git config --global user.name "Nightly Sync" + git remote add upstream https://github.com/astropy/astroquery.git + git remote -v + git fetch upstream main + echo "--- upstream log: " + git log upstream/main --oneline -10 + echo "--- current branch log before merge: " + git log --oneline -10 + git merge upstream/main + echo "--- current branch log after merge: " + git log --oneline -10 + echo "--- push force with lease" + git push --force-with-lease + - run: echo "The job finished with status ${{ job.status }}." From f03b75f72a9986192cbda54cc042115c680a6df2 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 11 Sep 2025 16:19:21 +0200 Subject: [PATCH 02/18] Setup custom CI/CD for ESO submodule --- .github/workflows/ci_devtests.yml | 2 ++ .github/workflows/ci_tests.yml | 2 ++ setup.cfg | 4 +++- tox.ini | 6 +++--- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_devtests.yml b/.github/workflows/ci_devtests.yml index cfb68e4b1c..509fc3e7ea 100644 --- a/.github/workflows/ci_devtests.yml +++ b/.github/workflows/ci_devtests.yml @@ -5,11 +5,13 @@ on: push: branches: - main + - develop tags: - '*' pull_request: branches: - main + - develop schedule: # run every Monday at 5am UTC - cron: '0 5 * * 1' diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index d0db0f48cc..0135d90b1e 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -5,11 +5,13 @@ on: push: branches: - main + - develop tags: - '*' pull_request: branches: - main + - develop schedule: # run every Monday at 5am UTC - cron: '0 5 * * 1' diff --git a/setup.cfg b/setup.cfg index ecdc423640..9d128fac73 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,9 @@ show-response = 1 [tool:pytest] minversion = 7.4 norecursedirs = build docs/_build astroquery/irsa astroquery/nasa_exoplanet_archive astroquery/ned astroquery/ibe astroquery/irsa_dust astroquery/cds astroquery/sha astroquery/dace -testpaths = astroquery docs +testpaths = + docs/eso + astroquery/eso doctest_plus = enabled astropy_header = true text_file_format = rst diff --git a/tox.ini b/tox.ini index 5c19109f91..896c0f8341 100644 --- a/tox.ini +++ b/tox.ini @@ -68,12 +68,12 @@ commands = devdeps: pip install -U --pre --no-deps --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy python -m pip freeze - !cov: pytest --pyargs astroquery {toxinidir}/docs {env:PYTEST_ARGS} {posargs} - cov: pytest --pyargs astroquery {toxinidir}/docs --cov astroquery --cov-config={toxinidir}/setup.cfg {env:PYTEST_ARGS} {posargs} + !cov: pytest --pyargs astroquery.eso {toxinidir}/docs/eso {env:PYTEST_ARGS} {posargs} + cov: pytest --pyargs astroquery.eso {toxinidir}/docs/eso --cov astroquery.eso --cov-config={toxinidir}/setup.cfg {env:PYTEST_ARGS} {posargs} # For remote tests, we re-run the failures to filter out at least some of the flaky ones. # We use a second pytest run with --last-failed as opposed to --rerun in order to rerun the # failed ones at the end rather than right away. - online: pytest --pyargs astroquery {toxinidir}/docs {env:PYTEST_ARGS_2} {posargs} + online: pytest --pyargs astroquery.eso {toxinidir}/docs/eso {env:PYTEST_ARGS_2} {posargs} cov: coverage xml -o {toxinidir}/coverage.xml pip_pre = From bfd19e0a87faf24bfb1ec9c7f4ac65f34d3fa154 Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Wed, 14 Jan 2026 14:00:48 +0100 Subject: [PATCH 03/18] initial idea for hierarchical docs --- docs/eso/eso.rst | 512 +++---------------------------- docs/eso/eso_apex.rst | 197 ++++++++++++ docs/eso/eso_cone_search.rst | 88 ++++++ docs/eso/eso_download.rst | 49 +++ docs/eso/eso_header_info.rst | 33 ++ docs/eso/eso_login.rst | 97 ++++++ docs/eso/eso_raw_general.rst | 154 ++++++++++ docs/eso/eso_raw_instrument.rst | 148 +++++++++ docs/eso/eso_reduced.rst | 126 ++++++++ docs/eso/eso_tap.rst | 95 ++++++ docs/eso/eso_troubleshooting.rst | 77 +++++ 11 files changed, 1114 insertions(+), 462 deletions(-) create mode 100644 docs/eso/eso_apex.rst create mode 100644 docs/eso/eso_cone_search.rst create mode 100644 docs/eso/eso_download.rst create mode 100644 docs/eso/eso_header_info.rst create mode 100644 docs/eso/eso_login.rst create mode 100644 docs/eso/eso_raw_general.rst create mode 100644 docs/eso/eso_raw_instrument.rst create mode 100644 docs/eso/eso_reduced.rst create mode 100644 docs/eso/eso_tap.rst create mode 100644 docs/eso/eso_troubleshooting.rst diff --git a/docs/eso/eso.rst b/docs/eso/eso.rst index bda39aa253..8180f11584 100644 --- a/docs/eso/eso.rst +++ b/docs/eso/eso.rst @@ -6,7 +6,7 @@ ESO Queries (`astroquery.eso`) .. warning:: - **Backward Compatibility Notice** + **⚠️ Backward Compatibility Notice ⚠️** **The WDB (Web DataBase) API is being deprecated and replaced by TAP (Table Access Protocol)**, a standardized interface for querying astronomical datasets using ADQL (Astronomical Data Query Language). @@ -28,495 +28,83 @@ ESO Queries (`astroquery.eso`) } Please review your queries carefully and update them accordingly to ensure compatibility with the new astroquery versions. - See section :ref:`column-filters-fix` + See section :ref:`column-filters-fix` at the end of this document. - -Getting started -=============== - -This is a python interface for querying the ESO archive web service. -For now, it supports the following: - -- listing available instruments -- listing available surveys (phase 3) -- searching INSTRUMENT SPECIFIC raw data (table ``ist.``) via the ESO TAP service* -- searching data products (phase 3; table ``ivoa.ObsCore``) via the ESO TAP service* -- searching raw data (table ``dbo.raw``) via the ESO TAP service* -- downloading data by dataset identifiers: http://archive.eso.org/cms/eso-data/eso-data-direct-retrieval.html - -\* ESO TAP web interface: https://archive.eso.org/programmatic/#TAP - -Requirements +Introduction ============ -The following packages are required for the use of this module: - -* keyring -* lxml -* requests >= 2.4.0 - - -Authentication with ESO User Portal -=================================== - -Most of the datasets in the ESO Science Archive are public and can be downloaded anonymously -without authenticating with the ESO User Portal (https://www.eso.org/sso/login). -Data with restricted access like datasets under proprietary period can be downloaded by authorised users -(for example PIs of the corresponding observing programmes and their delegates) -after authentication with the ESO User Portal. -This authentication is performed directly with the provided :meth:`~astroquery.query.QueryWithLogin.login` command, -as illustrated in the example below. This method uses your keyring to securely -store the password in your operating system. As such you should have to enter your -correct password only once, and later be able to use this package for automated -interaction with the ESO archive. - -.. doctest-skip:: - - >>> from astroquery.eso import Eso - >>> eso = Eso() - >>> # First example: TEST is not a valid username, it will fail - >>> eso.login(username="TEST") # doctest: +SKIP - WARNING: No password was found in the keychain for the provided username. [astroquery.query] - TEST, enter your password: - - INFO: Authenticating TEST on https://www.eso.org/sso ... [astroquery.eso.core] - ERROR: Authentication failed! [astroquery.eso.core] - >>> # Second example: pretend ICONDOR is a valid username - >>> eso.login(username="ICONDOR", store_password=True) # doctest: +SKIP - WARNING: No password was found in the keychain for the provided username. [astroquery.query] - ICONDOR, enter your password: - - INFO: Authenticating ICONDOR on https://www.eso.org/sso ... [astroquery.eso.core] - INFO: Authentication successful! [astroquery.eso.core] - >>> # After the first login, your password has been stored - >>> eso.login(username="ICONDOR") # doctest: +SKIP - INFO: Authenticating ICONDOR on https://www.eso.org/sso ... [astroquery.eso.core] - INFO: Authentication successful! [astroquery.eso.core] - - >>> # Successful download of a public file (with or without login) - >>> eso.retrieve_data('AMBER.2006-03-14T07:40:19.830') # doctest: +SKIP - INFO: Downloading file 1/1 https://dataportal.eso.org/dataPortal/file/AMBER.2006-03-14T07:40:19.830 - INFO: Successfully downloaded dataset AMBER.2006-03-14T07:40:19.830 - - >>> # Access denied to a restricted-access file (as anonymous user or as authenticated but not authorised user) - >>> eso.retrieve_data('ADP.2023-03-02T01:01:24.355') # doctest: +SKIP - INFO: Downloading file 1/1 https://dataportal.eso.org/dataPortal/file/ADP.2023-03-02T01:01:24.355 - ERROR: Access denied to https://dataportal.eso.org/dataPortal/file/ADP.2023-03-02T01:01:24.355 - - -.. The doctest-skip directive is not working for pytest within the note directive, thus - we leave the snippet below as a code-block - -.. note:: - - Even after logging with `~astroquery.eso.EsoClass.login()`, TAP queries are not authenticated by default. To issue - authenticated queries, the keyword argument ``authenticated`` must be set to `True` explicitly in the query functions. - For example: - - .. code-block:: python - - eso = Eso() - eso.ROW_LIMIT = None - eso.login(username="ICONDOR") - - # Some files are missing without `authenticated=True` - files_tbl = eso.query_instrument("nirps") - - # Get all files to which the authenticated user has access - files_tbl = eso.query_instrument("nirps", authenticated=True) - - Review your queries carefully and update them accordingly to ensure you are retrieving - all your expected results. If you don't stricly need to run an authenticated query, use - the default ``authenticated=False``, since it will retrieve your results much faster. - - -Automatic password ------------------- - -As shown above, your password can be stored by the `keyring -`_ module, if you -pass the argument ``store_password=True`` to ``Eso.login()``. -For security reason, storing the password is turned off by default. - -MAKE SURE YOU TRUST THE MACHINE WHERE YOU USE THIS FUNCTIONALITY!!! +Overview of the ESO Archive +--------------------------- -NB: You can delete your password later with the command -``keyring.delete_password('astroquery:www.eso.org', 'username')``. +The `ESO Science Archive Facility (SAF) `_ is one of the largest ground-based astronomical data repositories in the world. It contains, and provides access to, data from ESO telescopes at the `La Silla `_, `Paranal `_, and `APEX `_ observatories, as well as the submillimeter APEX telescope on Llano de Chajnantor and the `ALMA `_ array. The archive includes all raw data from La Silla Paranal Observatory, corresponding calibrations, and a wide range of processed data products, either generated by ESO or contributed by the community. It also hosts selected external datasets, such as raw WFCAM/UKIDSS data from the UKIRT facility in Hawaii. As of May 2025, the archive hosts over `4.6 million science products `_, including: -Automatic login ---------------- +- `>3.8 million products from La Silla, Paranal, and APEX `_ +- `>700,000 ALMA products `_ +- `~2.3 million spectra `_ (1D extracted, fully calibrated) +- `>900,000 imaging products `_ in optical, near-IR, and sub-mm bands +- `>500,000 spectral cubes `_ from IFU and radio/mm instruments +- `~700,000 catalog tiles `_ -You can further automate the authentication process by configuring a default username. -The astroquery configuration file, which can be found following the procedure -detailed in `astropy.config `_, -needs to be edited by adding ``username = ICONDOR`` in the ``[eso]`` section. +Processed data downloaded from the ESO Archive are assigned a **Digital Object Identifier (DOI)**. The list of available DOIs can be browsed in the `ESO DOI query page `_. -When configured, the username in the ``login()`` method call can be omitted -as follows: - -.. doctest-skip:: - - >>> from astroquery.eso import Eso - >>> eso = Eso() - >>> eso.login() # doctest: +SKIP - ICONDOR, enter your ESO password: - -NB: If an automatic login is configured, other Eso methods can log you in -automatically when needed. - - -Query the ESO archive for raw data -================================== - -Identifying available instrument-specific queries -------------------------------------------------- - -The direct retrieval of datasets is better explained with a running example, continuing from the -authentication example above. The first thing to do is to identify the instrument to query. The -list of available instrument-specific queries can be obtained with the -:meth:`~astroquery.eso.EsoClass.list_instruments` method. - -.. doctest-remote-data:: - - >>> from astroquery.eso import Eso - >>> eso = Eso() - >>> eso.list_instruments() - ['alpaca', 'amber', 'apex', 'crires', 'efosc', 'eris', 'espresso', 'fiat', - 'fors1', 'fors2', 'giraffe', 'gravity', 'harps', 'hawki', 'isaac', 'kmos', - 'matisse', 'midi', 'muse', 'naco', 'nirps', 'omegacam', 'pionier', 'sinfoni', - 'sofi', 'sphere', 'uves', 'vimos', 'vircam', 'visir', 'wlgsu', 'xshooter'] - -In the example above, the instruments listed correspond to those retrieved by running the -following query on the ESO **Programmatic Access** website (https://archive.eso.org/programmatic/#TAP): - -``select table_name from TAP_SCHEMA.tables where schema_name='ist' order by table_name`` - - -Inspecting available query options ----------------------------------- - -Once an instrument is chosen, ``midi`` for example, the columns available for that instrument can be -inspected by setting the ``help=True`` keyword of the :meth:`~astroquery.eso.EsoClass.query_instrument` -method. The list of columns contains its datatype and unit. The xtype is to be more specific, -as certain columns with datatype ``char`` actually define timestamps or regions in the sky. - -.. doctest-remote-data:: - - >>> eso.query_instrument('midi', help=True) # doctest: +IGNORE_OUTPUT - INFO: - Columns present in the table ist.midi: - column_name datatype xtype unit - ------------------- -------- ----------- ------ - access_estsize long kbyte - access_url char - datalink_url char - date_obs char - dec double deg - del_ft_sensor char - del_ft_status char - det_dit float s - det_ndit int - dimm_fwhm_avg float arcsec - dimm_fwhm_rms float arcsec - dp_cat char - dp_id char - ... ... - release_date char timestamp - s_region char adql:REGION - ... ... - telescope char - tpl_expno int - tpl_id char - tpl_name char - tpl_nexp int - tpl_start char - utc float s - - Number of records present in the table ist.midi: - 421764 - [astroquery.eso.core] - -**Note:** for a deeper description of each column, the following query can be issued -on the ESO **Programmatic Access** website (https://archive.eso.org/programmatic/#TAP): - -``select column_name, description from TAP_SCHEMA.columns where table_name = 'ist.midi'`` - -Querying with constraints -------------------------- - -It is now time to query the ``midi`` instrument for datasets. In the following example, observations of -target ``NGC 4151`` between ``2008-01-01`` and ``2009-05-12`` are searched, and the query is configured to -return two columns: the date of observation and the name of the object. - -.. doctest-remote-data:: - >>> table = eso.query_instrument( - ... 'midi', - ... column_filters={ - ... 'object':'NGC4151', - ... 'exp_start': "between '2008-01-01' and '2009-05-12'" - ... }, - ... columns=['object', 'date_obs'] - ... ) - >>> table - - object date_obs - ------- ----------------------- - NGC4151 2008-04-22T02:07:50.154 - NGC4151 2008-04-22T02:08:20.345 - NGC4151 2008-04-22T02:09:47.846 - NGC4151 2008-04-22T02:10:18.038 - ... ... - NGC4151 2009-05-11T01:39:09.750 - NGC4151 2009-05-11T01:40:24.235 - NGC4151 2009-05-11T01:41:38.742 - NGC4151 2009-05-11T01:42:08.432 - - -Querying all instruments ------------------------- - -The ESO database can also be queried without a specific instrument in mind. -This is what the method :meth:`~astroquery.eso.EsoClass.query_main` is for. -The associated table on the ESO **Programmatic Access** website (https://archive.eso.org/programmatic/#TAP) -is ``dbo.raw``, and the simplest query would be: ``select * from dbo.raw``. -Except for the keyword specifying the instrument,the behaviour of :meth:`~astroquery.eso.EsoClass.query_main` -is identical to :meth:`~astroquery.eso.EsoClass.query_instrument`. - -ESO instruments without a specific query interface can be queried with -:meth:`~astroquery.eso.EsoClass.query_main`, specifying the ``instrument`` constraint. -This is the case of e.g. ``harps``, ``feros`` or the all sky cameras APICAM and MASCOT. Here is an example to -query all-sky images from APICAM with ``luminance`` filter. - -.. doctest-remote-data:: - - >>> eso.maxrec = -1 # Return all results - # (i.e. do not truncate the query even if it is slow) - >>> table = eso.query_main( - ... column_filters={ - ... 'instrument': 'APICAM', - ... 'filter_path': 'LUMINANCE', - ... 'exp_start': "between '2019-04-26' and '2019-04-27'" - ... } - ... ) - >>> print(len(table)) - 215 - >>> print(table.columns) - - >>> table[["object", "ra", "dec", "date_obs", "prog_id"]].pprint(max_width=200) - object ra dec date_obs prog_id - deg deg - ------- ------------ ------------ ----------------------- ------------ - ALL SKY 145.29212694 -24.53624194 2019-04-26T00:08:49.000 60.A-9008(A) - ALL SKY 145.92251305 -24.53560305 2019-04-26T00:11:20.000 60.A-9008(A) - ALL SKY 146.55707 -24.53497111 2019-04-26T00:13:52.000 60.A-9008(A) - ALL SKY 147.18745 -24.53435388 2019-04-26T00:16:23.000 60.A-9008(A) - ALL SKY 147.81365305 -24.53375305 2019-04-26T00:18:53.000 60.A-9008(A) - ALL SKY 148.56509194 -24.533045 2019-04-26T00:21:53.000 60.A-9008(A) - ALL SKY 149.19963805 -24.53246 2019-04-26T00:24:25.000 60.A-9008(A) - ALL SKY 149.83418111 -24.53188611 2019-04-26T00:26:57.000 60.A-9008(A) - ALL SKY 150.46037194 -24.53133111 2019-04-26T00:29:27.000 60.A-9008(A) - ALL SKY 151.08656111 -24.53078805 2019-04-26T00:31:57.000 60.A-9008(A) - ALL SKY 151.85050805 -24.53014 2019-04-26T00:35:00.000 60.A-9008(A) - ALL SKY 152.48504 -24.529615 2019-04-26T00:37:32.000 60.A-9008(A) - ... ... ... ... ... - ALL SKY 289.40910694 -24.66412305 2019-04-26T09:44:00.000 60.A-9008(A) - ALL SKY 290.04024305 -24.66522194 2019-04-26T09:46:31.000 60.A-9008(A) - ALL SKY 290.67974305 -24.66633 2019-04-26T09:49:04.000 60.A-9008(A) - ALL SKY 291.30671 -24.66741111 2019-04-26T09:51:34.000 60.A-9008(A) - ALL SKY 291.93786305 -24.66849388 2019-04-26T09:54:05.000 60.A-9008(A) - ALL SKY 139.655775 -24.542425 2019-04-26T23:42:23.000 60.A-9008(A) - ALL SKY 140.282015 -24.54169694 2019-04-26T23:44:53.000 60.A-9008(A) - ALL SKY 140.91242694 -24.54097305 2019-04-26T23:47:24.000 60.A-9008(A) - ALL SKY 141.54283388 -24.54026 2019-04-26T23:49:55.000 60.A-9008(A) - ALL SKY 142.16906388 -24.53956194 2019-04-26T23:52:25.000 60.A-9008(A) - ALL SKY 142.93306 -24.53872388 2019-04-26T23:55:28.000 60.A-9008(A) - ALL SKY 143.56345694 -24.53804388 2019-04-26T23:57:59.000 60.A-9008(A) - Length = 215 rows - - -Query the ESO archive for reduced data -====================================== - -In addition to raw data, ESO makes available processed data. -In this section, we show how to obtain these processed survey data from the archive. - -Identify available surveys +Ways to Access the Archive -------------------------- -The list of available surveys can be obtained with :meth:`~astroquery.eso.EsoClass.list_surveys` as follows: - -.. doctest-remote-data:: - - >>> surveys = eso.list_surveys() - -Query a specific survey with constraints ----------------------------------------- - -Let's assume that we work with the ``HARPS`` survey, and that we are interested in -target ``HD203608``. The archive can be queried as follows: - -.. doctest-remote-data:: - - >>> table = eso.query_surveys(surveys='HARPS', target_name="HD203608") - -The returned table has a ``dp_id`` column, which can be used to retrieve the datasets with -:meth:`~astroquery.eso.EsoClass.retrieve_data`: ``eso.retrieve_data(table["dp_id"][0])``. -More details about this method in the next section. - - -Obtaining extended information on data products -=============================================== - -Only a small subset of the keywords present in the data products can be obtained -with :meth:`~astroquery.eso.EsoClass.query_instrument` or :meth:`~astroquery.eso.EsoClass.query_main`. -There is however a way to get the full primary header of the FITS data products, -using :meth:`~astroquery.eso.EsoClass.get_headers`. -This method is detailed in the example below. - - -.. doctest-remote-data:: +Access to data follows ESO’s standard data policy: the **Principal Investigator (PI)** of an observing program has exclusive access to their data during a proprietary period (typically one year), after which the data become publicly available. See the `ESO Data Access Policy `_ and `FAQ `_ for more information. Browsing the archive does not require authentication, but login is necessary to access proprietary datasets. Use of ESO archival data should be acknowledged in all publications. - >>> table = eso.query_instrument('midi', - ... column_filters={ - ... 'object': 'NGC4151', - ... 'date_obs': "<='2008-01-01'" - ... }, - ... columns=['object', 'date_obs', 'dp_id']) - >>> table_headers = eso.get_headers(table["dp_id"]) - >>> len(table_headers.columns) - 336 - >>> table_headers.pprint() - DP.ID SIMPLE BITPIX ... HIERARCH ESO OCS EXPO7 FNAME2 HIERARCH ESO OCS EXPO8 FNAME1 HIERARCH ESO OCS EXPO8 FNAME2 - ---------------------------- ------ ------ ... --------------------------------- --------------------------------- --------------------------------- - MIDI.2007-02-07T07:01:51.000 True 16 ... - MIDI.2007-02-07T07:02:49.000 True 16 ... - MIDI.2007-02-07T07:03:30.695 True 16 ... - MIDI.2007-02-07T07:05:47.000 True 16 ... - MIDI.2007-02-07T07:06:28.695 True 16 ... - MIDI.2007-02-07T07:09:03.000 True 16 ... - MIDI.2007-02-07T07:09:44.695 True 16 ... - MIDI.2007-02-07T07:13:09.000 True 16 ... - MIDI.2007-02-07T07:13:50.695 True 16 ... - MIDI.2007-02-07T07:15:55.000 True 16 ... - MIDI.2007-02-07T07:16:36.694 True 16 ... - MIDI.2007-02-07T07:19:25.000 True 16 ... - MIDI.2007-02-07T07:20:06.695 True 16 ... MIDI.2007-02-07T07:20:06.695.fits - MIDI.2007-02-07T07:22:57.000 True 16 ... MIDI.2007-02-07T07:20:06.695.fits MIDI.2007-02-07T07:22:57.000.fits - MIDI.2007-02-07T07:23:38.695 True 16 ... MIDI.2007-02-07T07:20:06.695.fits MIDI.2007-02-07T07:22:57.000.fits MIDI.2007-02-07T07:23:38.695.fits +There are multiple ways to access the archive: +- The `Raw Data query form `_ gives access to raw frames and their calibrations +- The `Science Portal `_ allows browsing and retrieval of processed (Phase 3) data +- The `Catalogue Facility `_ enables access to PI-contributed catalogue data +- The `Programmatic and Tools access layer `_ provides direct, scriptable access to raw and processed data, metadata, ambient conditions, and catalogues via Virtual Observatory (VO) protocols -As shown above, for each data product ID (``DP.ID``), the full header (336 columns in our case) of the archive -FITS file is collected. In the above table ``table_headers``, there are as many rows as in the column ``table['DP.ID']``. +This documentation focuses on the last method: accessing the archive programmatically using the ``astroquery.eso`` module. This Python interface allows users to search for both raw and reduced data, retrieve metadata, and download data products directly using dataset identifiers. Proprietary access is supported via authentication with the `ESO User Portal `_. - -Downloading datasets from the archive -===================================== - -Continuing from the query with constraints example, the first two datasets are selected, -using their data product IDs ``dp_id``, and retrieved from the ESO archive. +Quick Start +=========== .. doctest-skip:: - >>> data_files = eso.retrieve_data(table['dp_id'][:2]) - INFO: Downloading datasets ... [astroquery.eso.core] - INFO: Downloading 2 files ... [astroquery.eso.core] - INFO: Downloading file 1/2 https://dataportal.eso.org/dataPortal/file/MIDI.2007-02-07T07:01:51.000 to /Users/foobar/.astropy/cache/astroquery/Eso [astroquery.eso.core] - INFO: Successfully downloaded dataset MIDI.2007-02-07T07:01:51.000 to /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:01:51.000.fits.Z [astroquery.eso.core] - INFO: Downloading file 2/2 https://dataportal.eso.org/dataPortal/file/MIDI.2007-02-07T07:02:49.000 to /Users/foobar/.astropy/cache/astroquery/Eso [astroquery.eso.core] - INFO: Successfully downloaded dataset MIDI.2007-02-07T07:02:49.000 to /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:02:49.000.fits.Z [astroquery.eso.core] - INFO: Uncompressing file /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:01:51.000.fits.Z [astroquery.eso.core] - INFO: Uncompressing file /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:02:49.000.fits.Z [astroquery.eso.core] - INFO: Done! [astroquery.eso.core] - >>> data_files - ['/Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:01:51.000.fits', - '/Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:02:49.000.fits'] - -The file names, returned in data_files, points to the decompressed datasets -(without the .Z extension) that have been locally downloaded. -They are ready to be used with `~astropy.io.fits`. - -The default location (in the astropy cache) of the decompressed datasets can be adjusted by providing -a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. - -By default, if a requested dataset is already found, it is not downloaded again from the archive. -To force the retrieval of data that are present in the destination directory, use ``continuation=True`` -in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. - - -Troubleshooting -=============== - -Clearing the cache ------------------- - -If you are repeatedly getting failed queries, or bad/out-of-date results, try clearing your cache: - -.. code-block:: python - - >>> from astroquery.eso import Eso - >>> Eso.clear_cache() - -If this function is unavailable, upgrade your version of astroquery. -The ``clear_cache`` function was introduced in version 0.4.7.dev8479. + >>> from astroquery.eso import Eso # Import the ESO module + >>> eso = Eso() # Create an instance of the ESO class + >>> table_raw = eso.query_main("muse", column_filters={"object": "NGC300"}) # Query raw data + >>> table_reduced = eso.query_surveys("muse", column_filters={"target_name": "NGC300"}) # Query reduced data + >>> eso.retrieve_data(table_raw['dp_id']) # Download raw data products + >>> eso.retrieve_data(table_reduced['dp_id']) # Retrieve reduced data products -.. _column-filters-fix: - -Using the correct ``column_filters`` ------------------------------------- - -Two concrete and relevant examples of fields present in WDB but not present in TAP/ADQL -are ``stime`` and ``etime``. The following snippet shows how to adapt the filters to -the TAP / ADQL syntax: +By default, the maximum number of results returned by a query is by default set to 1000, but this can be adjusted as needed. To return all results without truncation, set the ``maxrec`` attribute to -1: .. doctest-skip:: - # The following filters won't work: - column_filters = { - 'stime': '2024-01-01' - 'etime': '2024-12-31' - } - - # Replace by: - column_filters = { - 'exp_start': "between '2024-01-01' and '2024-12-31'" - } - - # --- # + >>> eso.maxrec = -1 # Return all results - # The following filters won't work: - column_filters = { - 'stime': '2024-01-01' - } +Getting Started +=============== - # Replace by: - column_filters = { - 'exp_start': "> '2024-01-01'" - } +.. toctree:: + :maxdepth: 2 - # --- # + eso_login + eso_raw_general + eso_raw_instrument + eso_reduced + eso_apex + eso_tap + eso_cone_search + eso_header_info + eso_download + eso_troubleshooting - # The following filters won't work: - column_filters = { - 'etime': '2024-12-31' - } +Requirements +============ - # Replace by: - column_filters = { - 'exp_start': "< '2024-12-31'" - } +Along with the main requirements of the ``astroquery`` package, the following packages are required for the use of this module: +* `lxml `_ Reference/API ============= .. automodapi:: astroquery.eso - :no-inheritance-diagram: + :no-inheritance-diagram: \ No newline at end of file diff --git a/docs/eso/eso_apex.rst b/docs/eso/eso_apex.rst new file mode 100644 index 0000000000..5857dfb60a --- /dev/null +++ b/docs/eso/eso_apex.rst @@ -0,0 +1,197 @@ + +******************* +Query for APEX Data +******************* + +This notebook provides an example of how to query the ESO Science Archive for **APEX data**. + +APEX data are available within the archive and can be queried using any of the standard astroquery methods: +- :meth:`~astroquery.eso.EsoClass.query_survey` for **reduced data** +- :meth:`~astroquery.eso.EsoClass.query_main` for **raw data** +- :meth:`~astroquery.eso.EsoClass.query_instrument` for **raw data (instrument specific)** + +In addition, there is the option to use :meth:`~astroquery.eso.EsoClass.query_apex_quicklooks`, which is a dedicated query for retrieving **APEX Quick Look products**. + +The workflow for APEX data is somewhat different from that of other ESO-operated instruments and telescopes. The **APEX Quick Looks** aim to address some of these differences by providing users with ``.tar`` files that include additional data outputs — such as diagnostic plots, observing logs, and (in some cases) a reduced version of the dataset. These ``.tar`` files also include ``class`` (``.apex``) files that allow users to **re-run the data reduction**, if desired. This is often more useful than working directly with the ``.fits`` files available in the archive and can be **particularly helpful for heterodyne datasets**. For heterodyne observations, the calibrated ``.class`` file is typically included, while the corresponding uncalibrated ``.fits`` files are usually **not needed** and cannot be processed with standard reduction software. + +The APEX Science Archive provides access to raw observations and associated data products for observations performed with APEX since **July 11, 2005**. The Quick Look products are designed to help users preview and assess the data quality, providing useful summaries that complement the raw observations. + +This notebook will guide you through a step-by-step workflow to **identify, query, and download** APEX Quick Look data products from the ESO Science Archive. + +Here, we focus on APEX observations from the [**ALCOHOLS survey**](https://www.eso.org/rm/api/v1/public/releaseDescriptions/199) (12CO(3–2) line emission in the Milky Way) available in the ESO Archive. We will first search for the **reduced ALCOHOLS data products**, then identify the corresponding raw datasets using the **instrument-specific query**. Once the raw data is identified, we can use it to locate and retrieve the **associated APEX Quick Look products**. We follow this workflow because APEX Quick Looks typically require knowledge of the **APEX proposal ID**, which may not always be known in advance — this is **not** the same as the ESO programme ID. If you know your APEX proposal ID (for example, if you are querying your own data), you can search for APEX Quick Look files directly. + +Note that this workflow is **not specific to the ALCOHOLS survey** and can be used to query APEX Quick Look products for any project. + +Query Reduced APEX data +======================= + +We first query for the reduced data from the **ALCOHOLS** survey, and retrieve the (ESO) proposal ID. + +.. doctest-remote-data:: + + >>> table_reduced = eso.query_surveys("ALCOHOLS") # query the ESO archive for the ALCOHOLS survey + >>> proposal_id = list(set(table_reduced['proposal_id'])) # extract unique proposal IDs from the query result + + >>> # Check if we have a single proposal ID or multiple + >>> if len(proposal_id) == 1: + >>> proposal_id = proposal_id[0] + >>> else: + >>> print("Warning: Multiple proposal IDs found...") + + >>> proposal_id = proposal_id.split('(')[0] # extract the first part of the proposal ID before any parentheses (i.e. the run ID) + >>> print(f"Proposal ID: {proposal_id}") + Proposal ID: 094.C-0935 + +Note that multiple proposal IDs may be returned, which would require minor changes to the above example to loop through the projects. + +Available Query Constraints +=========================== + +As always, it is good practice to check the available columns to search in the instrument-specific query. + +.. doctest-remote-data:: + + >>> eso.query_instrument("APEX", help=True) + INFO: + Columns present in the table ist.APEX: + column_name datatype xtype unit + ----------------- -------- ----------- ----- + access_estsize long kbyte + access_url char + bwid float GHz + channels int + datalink_url char + date_obs char + dec double deg + dp_cat char + dp_id char + dp_tech char + dp_type char + ecl_lat double deg + ecl_lon double deg + exp_start char timestamp + exposure float s + exptime float s + febe char + freq float GHz + freqres float + gal_lat double deg + gal_lon double deg + instrument char + lambda_max double nm + lambda_min double nm + last_mod_date char timestamp + line char + lst float s + mjd_obs double d + npols int + nsubs short + ob_id int + object char + observer_initials char + observing_mode char + origfile char + period int + pi_coi char + prog_id char + prog_title char + prog_type char + project_id char + ra double deg + release_date char timestamp + restfreq float Hz + s_region char adql:REGION + scangeom char + scanmode char + scannum int + scantype char + skyfreq float Hz + tel_airm_end float + tel_airm_start float + tel_alt float deg + tel_az float deg + telescope char + wobcycle float s + wobthrow float deg + wobused char + + Number of records present in the table ist.APEX: + 913029 + [astroquery.eso.core] + +Query Raw APEX data +=================== + +We now query for raw data from the APEX instrument, using the proposal ID we retrieved from the previous query. + +.. doctest-remote-data:: + + >>> table_raw = eso.query_instrument("APEX", column_filters={"prog_id": f"like '{proposal_id}%'"}) # query the APEX instrument for data related to the proposal ID + >>> project_id = list(set(table_raw["project_id"])) # extract unique project IDs from the raw data query + >>> project_id = project_id[0] # Assuming we only have one project ID + >>> print(f"Project ID: {project_id}") + Project ID: E-094.C-0935A-2014 + +In this case, we know there is only **one** APEX proposal ID, but if there were multiple IDs, we would need to loop through them. + +.. tip:: + In the :meth:`~astroquery.eso.EsoClass.query_surveys` query, the ``"proposal_id"`` column refers to the **ESO programme ID**. In contrast, in an APEX-specific query using :meth:`~astroquery.eso.EsoClass.query_instrument`, the ``"prog_id"`` column also refers to the **ESO programme ID**, **not** the **APEX proposal ID**. The APEX proposal ID is instead found in the ``"project_id"`` column in the :meth:`~astroquery.eso.EsoClass.query_instrument` query—this is the value used to identify APEX Quick Look products. + +Query APEX Quick Look products +============================== + +We can check the available columns to search in the query. + +.. doctest-remote-data:: + + >>> eso.query_apex_quicklooks(help=True) + INFO: + Columns present in the table ist.apex_quicklooks: + column_name datatype xtype unit + --------------- -------- --------- ----- + access_estsize long kbyte + access_url char + instrument char + instrument_type char + partner char + pi_coi char + prog_id char + prog_title char + prog_type char + project_id char + quicklook_id char + release_date char timestamp + + Number of records present in the table ist.apex_quicklooks: + 282296 + [astroquery.eso.core] + +And now, query for the APEX Quick Look products using the APEX proposal ID (``project_id``) we retrieved from the previous query. + +.. doctest-remote-data:: + + >>> table_quicklooks = eso.query_apex_quicklooks(project_id) + >>> table_quicklooks +
+ access_estsize access_url instrument instrument_type partner ... prog_title prog_type project_id quicklook_id release_date + kbyte ... + int64 object object object object ... object object object object object + -------------- ---------------------------------------------------------------------- ---------- --------------- ------- ... ------------------------------------------------------------------------------- --------- ------------------ --------------------------- ------------------------ + 846755 https://dataportal.eso.org/dataPortal/file/E-094.C-0935A.2014DEC10.TAR APEXHET Heterodyne ESO ... The APEX Large CO Heterodyne Outflow Legacy Supercam survey of Orion (ALCOHOLS) Normal E-094.C-0935A-2014 E-094.C-0935A.2014DEC10.TAR 2014-12-10T07:05:44.397Z + ... ... ... ... ... ... ... ... ... ... ... + 40963041 https://dataportal.eso.org/dataPortal/file/E-094.C-0935A.2015AUG07.TAR APEXHET Heterodyne ESO ... The APEX Large CO Heterodyne Outflow Legacy Supercam survey of Orion (ALCOHOLS) Normal E-094.C-0935A-2014 E-094.C-0935A.2015AUG07.TAR 2015-04-25T18:41:53.900Z + 6389 https://dataportal.eso.org/dataPortal/file/E-094.C-0935A.2015AUG22.TAR APEXHET Heterodyne ESO ... The APEX Large CO Heterodyne Outflow Legacy Supercam survey of Orion (ALCOHOLS) Normal E-094.C-0935A-2014 E-094.C-0935A.2015AUG22.TAR 2015-04-25T18:41:53.900Z + + +As can be seen from the output above, there is one APEX Quick Look product available per UT date, per APEX proposal ID. + +Also note that the APEX Quick Look products are available in `.tar` (`.TAR`) format, which can be downloaded and extracted (see below). + +Download APEX Quick Look products +================================= + +Finally, we can download the APEX Quick Look products using the `eso.retrieve_data` function. + +.. doctest-remote-data:: + + >>> eso.retrieve_data(table_quicklooks[0]["quicklook_id"]) \ No newline at end of file diff --git a/docs/eso/eso_cone_search.rst b/docs/eso/eso_cone_search.rst new file mode 100644 index 0000000000..722c7ff276 --- /dev/null +++ b/docs/eso/eso_cone_search.rst @@ -0,0 +1,88 @@ + +****************** +Simple Cone Search +****************** + +Query for Reduced Data +====================== + +This example demonstrates how to perform a basic **cone search** for publicly available **HAWK-I reduced data products** around the Galactic Center (Sgr A\*) using :meth:`~astroquery.eso.EsoClass.query_surveys`. + +Sgr A\* is located at right ascension 266.41683° and declination –29.00781°. We perform a search within a 0.05-degree radius (~3 arcminutes). + +.. doctest-remote-data:: + + >> import astropy.units as u # import the astropy.units module + + >>> ra = 266.41683 *u.deg # degrees + >>> dec = -29.00781 *u.deg # degrees + >>> radius = 0.05 *u.deg # degrees + +If coordinates are not known, you can use the :class:`~astropy.coordinates.SkyCoord` class to create them. Here, we create a SkyCoord object for Sgr A\*. + +.. doctest-remote-data:: + + >>> from astropy.coordinates import SkyCoord # import the SkyCoord class from the astropy.coordinates module + >>> import astropy.units as u # import the astropy.units module + + >>> coords = SkyCoord.from_name("Sgr A*") + >>> ra = coords.ra + >>> dec = coords.dec + >>> radius = 0.05 *u.deg + +With the defined coordinates and radius, we can now perform the cone search using :meth:`~astroquery.eso.EsoClass.query_surveys`. This method allows us to filter results based on specific columns, such as ``instrument_name``. + +.. doctest-remote-data:: + + >>> # Cone search + >>> table = eso.query_surveys( + ... cone_ra=ra.value, + ... cone_dec=dec.value, + ... cone_radius=radius.to(u.deg).value, + ... column_filters={"instrument_name": "HAWKI"}) + +**Note**: The ``cone_ra``, ``cone_dec``, and ``cone_radius`` parameters are specified in degrees, but as float values (so we use ``.value`` to extract the float from the ``astropy.units.Quantity``). + +Query for Raw Data +================== + +Similar cone search functionality is also available through :meth:`~astroquery.eso.EsoClass.query_instrument` and :meth:`~astroquery.eso.EsoClass.query_main` by passing the same ``cone_ra``, ``cone_dec``, and ``cone_radius`` arguments. + +Instrument-Specific Cone Search +------------------------------- + +Cone search for raw data products can be performed using the instrument-specific method, such as :meth:`~astroquery.eso.EsoClass.query_instrument`. This example searches for HAWK-I raw data products around the same coordinates as above. + +.. doctest-remote-data:: + + >>> table = eso.query_instrument( + ... "HAWKI", + ... cone_ra=ra, + ... cone_dec=dec, + ... cone_radius=radius) + +Generic Cone Search +------------------- + +Cone search for raw data products can also be performed using the more generic method, :meth:`~astroquery.eso.EsoClass.query_main`. This allows you to search across all instruments without specifying one, with the following example searching for HAWK-I raw data products around the same coordinates as above. + +.. doctest-remote-data:: + + >>> table = eso.query_main( + ... "HAWKI", + ... cone_ra=ra, + ... cone_dec=dec, + ... cone_radius=radius + +Download Data +============= + +To download the data returned by the query, you can use the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes a list of data product IDs (``dp_id``) and downloads the corresponding files from the ESO archive. + +.. doctest-remote-data:: + >>> eso.retrieve_data(table["dp_id"]) + +The ``data_files`` points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. + +.. doctest-skip:: + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") \ No newline at end of file diff --git a/docs/eso/eso_download.rst b/docs/eso/eso_download.rst new file mode 100644 index 0000000000..0bcc3b1d3a --- /dev/null +++ b/docs/eso/eso_download.rst @@ -0,0 +1,49 @@ + +************* +Download Data +************* + +Continuing from the query with constraints example, the first two datasets are selected, +using their data product IDs ``dp_id``, and retrieved from the ESO archive. + +.. doctest-skip:: + + >>> data_files = eso.retrieve_data(table['dp_id'][:2]) + INFO: Downloading datasets ... [astroquery.eso.core] + INFO: Downloading 2 files ... [astroquery.eso.core] + INFO: Downloading file 1/2 https://dataportal.eso.org/dataPortal/file/MIDI.2007-02-07T07:01:51.000 to /Users/foobar/.astropy/cache/astroquery/Eso [astroquery.eso.core] + INFO: Successfully downloaded dataset MIDI.2007-02-07T07:01:51.000 to /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:01:51.000.fits.Z [astroquery.eso.core] + INFO: Downloading file 2/2 https://dataportal.eso.org/dataPortal/file/MIDI.2007-02-07T07:02:49.000 to /Users/foobar/.astropy/cache/astroquery/Eso [astroquery.eso.core] + INFO: Successfully downloaded dataset MIDI.2007-02-07T07:02:49.000 to /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:02:49.000.fits.Z [astroquery.eso.core] + INFO: Uncompressing file /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:01:51.000.fits.Z [astroquery.eso.core] + INFO: Uncompressing file /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:02:49.000.fits.Z [astroquery.eso.core] + INFO: Done! [astroquery.eso.core] + >>> data_files + ['/Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:01:51.000.fits', + '/Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:02:49.000.fits'] + +The file names, returned in data_files, points to the decompressed datasets +(without the .Z extension) that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. + +.. doctest-skip:: + >>> data_files = eso.retrieve_data(table['dp_id'][:2], destination='./eso_data/') + +By default, if a requested dataset is already found, it is not downloaded again from the archive. +To force the retrieval of data that are present in the destination directory, use ``continuation=True`` in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. + +When downloading datasets, you can optionally retrieve associated calibration files by using the with_calib argument. This allows you to obtain either raw or processed calibrations in addition to the science files. + +Available options: + • None (default): Download only the requested science data. + • 'raw': Include raw calibration files associated with each dataset. + • 'processed': Include processed calibration files (i.e., reduced/calibrated). + +.. doctest-skip:: + + >>> # Download science data with raw calibrations + >>> data_files = eso.retrieve_data(table['dp_id'][:2], with_calib="raw") + +.. doctest-skip:: + + >>> # Download science data with processed calibrations + >>> data_files = eso.retrieve_data(table['dp_id'][:2], with_calib="processed") \ No newline at end of file diff --git a/docs/eso/eso_header_info.rst b/docs/eso/eso_header_info.rst new file mode 100644 index 0000000000..0e2ab7fe41 --- /dev/null +++ b/docs/eso/eso_header_info.rst @@ -0,0 +1,33 @@ + +*************************** +Extended Header Information +*************************** + +Only a small subset of the keywords present in the data products can be obtained with :meth:`~astroquery.eso.EsoClass.query_instrument`, :meth:`~astroquery.eso.EsoClass.query_main`, :meth:`~astroquery.eso.EsoClass.query_surveys`, or :meth:`~astroquery.eso.EsoClass.query_tap_service`. + +There is however a way to get the full primary header of the FITS data products, using :meth:`~astroquery.eso.EsoClass.get_headers`. + +.. doctest-remote-data:: + + >>> table = eso.query_instrument('midi', + ... column_filters={ + ... 'object': 'NGC4151', + ... 'date_obs': "<='2008-01-01'" + ... }, + ... columns=['object', 'date_obs', 'dp_id']) + >>> table_headers = eso.get_headers(table["dp_id"]) + >>> len(table_headers.columns) + 336 + >>> table_headers + DP.ID SIMPLE BITPIX ... HIERARCH ESO OCS EXPO7 FNAME2 HIERARCH ESO OCS EXPO8 FNAME1 HIERARCH ESO OCS EXPO8 FNAME2 + ---------------------------- ------ ------ ... --------------------------------- --------------------------------- --------------------------------- + MIDI.2007-02-07T07:01:51.000 True 16 ... + MIDI.2007-02-07T07:02:49.000 True 16 ... + MIDI.2007-02-07T07:03:30.695 True 16 ... + ... + MIDI.2007-02-07T07:20:06.695 True 16 ... MIDI.2007-02-07T07:20:06.695.fits + MIDI.2007-02-07T07:22:57.000 True 16 ... MIDI.2007-02-07T07:20:06.695.fits MIDI.2007-02-07T07:22:57.000.fits + MIDI.2007-02-07T07:23:38.695 True 16 ... MIDI.2007-02-07T07:20:06.695.fits MIDI.2007-02-07T07:22:57.000.fits MIDI.2007-02-07T07:23:38.695.fits + + +As shown above, for each data product ID (``DP.ID``; note that this is equivalent to "dp_id" in ``table``), the full header (336 columns in our case) of the archive FITS file is collected. In the above table ``table_headers``, there are as many rows as in the column ``table['dp_id']``. \ No newline at end of file diff --git a/docs/eso/eso_login.rst b/docs/eso/eso_login.rst new file mode 100644 index 0000000000..9a06380560 --- /dev/null +++ b/docs/eso/eso_login.rst @@ -0,0 +1,97 @@ + +*********************** +Access Proprietary Data +*********************** + +Most datasets in the ESO Science Archive are publicly available and can be downloaded anonymously without requiring authentication. However, access to proprietary datasets—such as those under active proprietary periods—is restricted to authorized users (e.g., PIs of observing programs and their designated delegates). These users must authenticate via the `ESO User Portal `_. + +Authentication is handled using the :meth:`~astroquery.query.QueryWithLogin.login` method. This command initiates a secure login session via the ESO Single Sign-On (SSO) service. It integrates with the `keyring `_ module to securely store your password in your system’s credential manager. After the first login, you should not need to re-enter your credentials for subsequent sessions on the same machine. + +Authentication is required when: + +- Downloading proprietary files (not yet public) +- Accessing services that require ESO user privileges + +Login Examples +============== + +The following examples show typical login and data retrieval workflows: + +.. doctest-skip:: + + >>> # First example: TEST is not a valid username, it will fail + >>> eso.login(username="TEST") # doctest: +SKIP + WARNING: No password was found in the keychain for the provided username. [astroquery.query] + TEST, enter your password: + + INFO: Authenticating TEST on https://www.eso.org/sso ... [astroquery.eso.core] + ERROR: Authentication failed! [astroquery.eso.core] + +.. doctest-skip:: + + >>> # Second example: pretend ICONDOR is a valid username + >>> eso.login(username="ICONDOR", store_password=True) # doctest: +SKIP + WARNING: No password was found in the keychain for the provided username. [astroquery.query] + ICONDOR, enter your password: + + INFO: Authenticating ICONDOR on https://www.eso.org/sso ... [astroquery.eso.core] + INFO: Authentication successful! [astroquery.eso.core] + +.. doctest-skip:: + + >>> # After the first login, your password has been stored + >>> eso.login(username="ICONDOR") # doctest: +SKIP + INFO: Authenticating ICONDOR on https://www.eso.org/sso ... [astroquery.eso.core] + INFO: Authentication successful! [astroquery.eso.core] + +.. doctest-skip:: + + >>> # Successful download of a public file (with or without login) + >>> eso.retrieve_data("AMBER.2006-03-14T07:40:19.830") # doctest: +SKIP + INFO: Downloading file 1/1 https://dataportal.eso.org/dataPortal/file/AMBER.2006-03-14T07:40:19.830 + INFO: Successfully downloaded dataset AMBER.2006-03-14T07:40:19.830 + +.. doctest-skip:: + + >>> # Access denied to a restricted-access file (as anonymous user or as authenticated but not authorised user) + >>> eso.retrieve_data("ADP.2023-03-02T01:01:24.355") # doctest: +SKIP + INFO: Downloading file 1/1 https://dataportal.eso.org/dataPortal/file/ADP.2023-03-02T01:01:24.355 + ERROR: Access denied to https://dataportal.eso.org/dataPortal/file/ADP.2023-03-02T01:01:24.355 + +Password Storage +================ + +As shown above, your password can be stored securely using the `keyring `_ module by passing the argument ``store_password=True`` to ``Eso.login()``. For security reasons, password storage is disabled by default. + +.. warning:: + + **MAKE SURE YOU TRUST THE MACHINE WHERE YOU USE THIS FUNCTIONALITY!!!** + + When using the ``store_password=True`` option, your password is stored in your system’s keyring. This provides secure local storage, but only do this on machines you fully trust. + +**Note:** You can delete your stored password at any time with the following command: + +.. doctest-skip:: + + >>> keyring.delete_password("astroquery:www.eso.org", "your_username") + +Automatic Login +=============== + +To avoid having to enter your username every session, you can configure a default username in the Astroquery configuration file. This file is located as described in the `astropy.config documentation `_. + +Add the following to the ``[eso]`` section of your config file: + +.. doctest-skip:: + + [eso] + username = ICONDOR + +Once set, you can simply call ``eso.login()`` without specifying a username: + +.. doctest-skip:: + + >>> eso.login() # doctest: +SKIP + ICONDOR, enter your ESO password: + +**Note:** If automatic login is configured and the password is stored, other ``Eso`` methods (e.g. ``retrieve_data()``) can log you in automatically when needed. diff --git a/docs/eso/eso_raw_general.rst b/docs/eso/eso_raw_general.rst new file mode 100644 index 0000000000..01eeacbcae --- /dev/null +++ b/docs/eso/eso_raw_general.rst @@ -0,0 +1,154 @@ + +**************************** +Query for Raw Data (Generic) +**************************** + +The `astroquery.eso` module provides several ways to search for data in the ESO Science Archive for raw data. This section focuses on the **generic query interface** for raw data (:meth:`~astroquery.eso.EsoClass.query_main`), while in another section we describe **instrument-specific queries** (e.g., using :meth:`~astroquery.eso.EsoClass.query_instrument`). + +In some cases, you may want to query the ESO Archive **without targeting a specific instrument**. This is exactly what the :meth:`~astroquery.eso.EsoClass.query_main` method is designed for. It allows access to the **global raw data table**, which combines metadata across all instruments. Internally, this method queries the ``dbo.raw`` table via ESO's `TAP service `_, which you could also access directly using ADQL with a simple statement like: + +.. doctest-skip:: + SELECT * FROM dbo.raw + +Functionally, :meth:`~astroquery.eso.EsoClass.query_main` works the same way as :meth:`~astroquery.eso.EsoClass.query_instrument`, except you don’t need to specify an instrument table name directly. You can still apply column filters, control the columns returned, and limit result counts. + +Available Instruments +===================== + +To begin retrieving raw data from the ESO Science Archive, you may first want to know the relevant instrument(s) for your search. Each instrument has its own dedicated query table accessible through the archive’s programmatic `TAP `_ interface. + +The list of all supported instruments can be retrieved using the :meth:`~astroquery.eso.EsoClass.list_instruments` method: + +.. doctest-remote-data:: + + >>> from astroquery.eso import Eso + >>> eso = Eso() + >>> eso.list_instruments() + ['alpaca', 'amber', 'apex', 'crires', 'efosc', 'eris', + 'espresso', 'fiat', 'fors1', 'fors2', 'giraffe', 'gravity', + 'harps', 'hawki', 'isaac', 'kmos', 'matisse', 'midi', + 'muse', 'naco', 'nirps', 'omegacam', 'pionier', 'sinfoni', + 'sofi', 'sphere', 'uves', 'vimos', 'vircam', 'visir', + 'wlgsu', 'xshooter'] + +This list corresponds to the instruments currently available for programmatic raw data queries in the ESO archive. + +**Note:** The list is dynamically generated by querying the archive’s internal TAP service. It can also be reproduced directly by executing the following ADQL query on the `ESO TAP interface `_: + +.. doctest-skip:: + + SELECT table_name + FROM TAP_SCHEMA.tables + WHERE schema_name = 'ist' + ORDER BY table_name + +Once you have identified the instrument of interest, you can proceed with constructing your query and retrieving raw data products. + +Available Query Constraints +=========================== + +We start by inspecting the available columns that can be queried by ``query_main`` with the ``help=True`` keyword. This will return a list of all columns available in the ``dbo.raw`` table, along with their data types, units, and any additional metadata such as ``xtype`` information. + +.. doctest-remote-data:: + + >>> eso.query_main(help=True) + INFO: + Columns present in the table dbo.raw: + column_name datatype xtype unit + ------------------- -------- ----------- ------ + access_estsize long kbyte + access_url char + datalink_url char + date_obs char + dec double deg + dec_pnt double deg + ... + tpl_start char + + Number of records present in the table dbo.raw: + 34821254 + [astroquery.eso.core] + +Query with Constraints (Specific Instrument) +============================================ + +Now that we know which of the columns are available for queries, we can, for example, retrieving all-sky images from the ``APICAM`` instrument using the ``LUMINANCE`` filter, on a single night (i.e. 2019-04-26): + +.. doctest-remote-data:: + + >>> eso.maxrec = -1 # Return all results without truncation + >>> table = eso.query_main( + ... column_filters={ + ... "instrument": "APICAM", + ... "filter_path": "LUMINANCE", + ... "exp_start": "between '2019-04-26' and '2019-04-27'"}) + >>> print(len(table)) + 215 + >>> table.colnames + ['object', 'ra', 'dec', 'dp_id', 'date_obs', 'prog_id', + 'access_estsize', 'access_url', 'datalink_url', ... 'tpl_start'] + >>> table[["object", "ra", "dec", "date_obs", "prog_id"]] +
+ object ra dec date_obs prog_id + deg deg + object float64 float64 object object + ------- ------------ ------------ ----------------------- ------------ + ALL SKY 145.29212694 -24.53624194 2019-04-26T00:08:49.000 60.A-9008(A) + ALL SKY 145.92251305 -24.53560305 2019-04-26T00:11:20.000 60.A-9008(A) + ALL SKY 146.55707 -24.53497111 2019-04-26T00:13:52.000 60.A-9008(A) + ... + ALL SKY 143.56345694 -24.53804388 2019-04-26T23:57:59.000 60.A-9008(A) + + +Query with Constraints (All Instruments) +======================================== + +Alternatively, if you want to retrieve all available raw data products with the ``LUMINANCE`` filter, on a single night (i.e. 2019-04-26) without specifying the instrument, you can use the ``query_main`` method. + +.. doctest-remote-data:: + + >>> table = eso.query_main( + ... column_filters={ + ... "filter_path": "LUMINANCE", + ... "exp_start": "between '2019-04-26' and '2019-04-27'"}) + +Another useful example for ``eso.query_main`` is to query for all raw data for a specific target, such as ``NGC 3627``, across all instruments: + +.. doctest-remote-data:: + + >>> table = eso.query_main(column_filters={"object": "NGC 3627"}) + +.. tip:: + + Use ``query_main`` when you want to search **across all instruments**, for example to retrieve all observations of a specific source regardless of the instrument used. + + .. doctest-remote-data:: + + table = eso.query_main(column_filters={"object": "NGC 3627"}) + + Use ``query_instrument`` when you want a more **refined, instrument-specific search**, applying filters that are only available for a particular instrument (e.g. instrument modes, configurations, or ambient conditions). + + .. doctest-remote-data:: + + column_filters = { + "dp_cat": "SCIENCE", # Science data only + "ins_opt1_name": "HIGH_SENS", # High sensitivity mode + "night_flag": "night", # Nighttime observations only + "moon_illu": "< 0", # No moon (below horizon) + "lst": "between 0 and 6" # Local sidereal time early in the night + } + + table = eso.query_instrument("midi", column_filters=column_filters) + +Download Data +============= + +To download the data returned by the query, you can use the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes a list of data product IDs (``dp_id``) and downloads the corresponding files from the ESO archive. + +.. doctest-remote-data:: + >>> eso.retrieve_data(table["dp_id"]) + +The ``data_files`` points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. + +.. doctest-skip:: + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") \ No newline at end of file diff --git a/docs/eso/eso_raw_instrument.rst b/docs/eso/eso_raw_instrument.rst new file mode 100644 index 0000000000..563ae5c02c --- /dev/null +++ b/docs/eso/eso_raw_instrument.rst @@ -0,0 +1,148 @@ + +**************************************** +Query for Raw Data (Instrument-Specific) +**************************************** + +The `astroquery.eso` module provides several ways to search for data in the ESO Science Archive for raw data. This section focuses on the ***instrument-specific query interface** for raw data (:meth:`~astroquery.eso.EsoClass.query_instrument`), while in another section we describe **generic queries** (e.g., using :meth:`~astroquery.eso.EsoClass.query_main`). + +In many cases, you will want to query the ESO Archive **for data from a specific instrument**. This is exactly what the :meth:`~astroquery.eso.EsoClass.query_instrument` method is designed for. It allows you to search instrument-specific tables, which expose metadata fields and filters unique to each instrument. Internally, this method queries the corresponding instrument table (e.g., ``ist.muse``) via ESO's `TAP service `_. This approach is ideal when you need precise control over your query, such as filtering by instrument configuration, mode, or observational setup. + +Available Instruments +===================== + +To begin retrieving raw data from the ESO Science Archive, you first need to identify the relevant instrument(s) for your search. Each instrument has its own dedicated query table accessible through the archive’s programmatic `TAP `_ interface. + +The list of all supported instruments can be retrieved using the :meth:`~astroquery.eso.EsoClass.list_instruments` method: + +.. doctest-remote-data:: + + >>> from astroquery.eso import Eso + >>> eso = Eso() + >>> eso.list_instruments() + ['alpaca', 'amber', 'apex', 'crires', 'efosc', 'eris', + 'espresso', 'fiat', 'fors1', 'fors2', 'giraffe', 'gravity', + 'harps', 'hawki', 'isaac', 'kmos', 'matisse', 'midi', + 'muse', 'naco', 'nirps', 'omegacam', 'pionier', 'sinfoni', + 'sofi', 'sphere', 'uves', 'vimos', 'vircam', 'visir', + 'wlgsu', 'xshooter'] + +This list corresponds to the instruments currently available for programmatic raw data queries in the ESO archive. + +**Note:** The list is dynamically generated by querying the archive’s internal TAP service. It can also be reproduced directly by executing the following ADQL query on the `ESO TAP interface `_: + +.. doctest-skip:: + + SELECT table_name + FROM TAP_SCHEMA.tables + WHERE schema_name = 'ist' + ORDER BY table_name + +Once you have identified the instrument of interest, you can proceed with constructing your query and retrieving raw data products. + +Available Query Constraints +=========================== + +Once an instrument is selected—for example, ``midi`` — you can inspect the available queryable columns using the ``help=True`` keyword in the :meth:`~astroquery.eso.EsoClass.query_instrument` method. This is a useful first step to understand what metadata is available and how to structure your query. + +The output includes column names, data types, units, and, where applicable, ``xtype`` information to indicate more specific column content. For example, a column with datatype ``char`` may represent a timestamp or a sky region, which is reflected in the ``xtype`` field (e.g., ``timestamp`` or ``adql:REGION``). + +.. doctest-remote-data:: + + >>> eso.query_instrument("midi", help=True) + INFO: + Columns present in the table ist.midi: + column_name datatype xtype unit + ------------------- -------- ------------ ----------- + access_estsize long kbyte + access_url char + datalink_url char + date_obs char + ... + exp_start char timestamp + ... + object char + ... + release_date char timestamp + s_region char adql:REGION + ... + utc float s + + Number of records present in the table ist.midi: + 437577 + [astroquery.eso.core] + +**Note:** for a deeper description of each column, the following query can be issued +on the ESO `Programmatic Access `_ website (`see here `_): + +``select column_name, description from TAP_SCHEMA.columns where table_name = 'ist.midi'`` + +Query with Constraints +====================== + +Once the available query columns have been inspected (e.g., via ``help=True``), you can construct a constrained query to retrieve relevant datasets. For example, suppose you want to retrieve MIDI observations of the target ``NGC 4151`` that were taken between ``2008-01-01`` and ``2009-05-12``. + +The ``column_filters`` dictionary allows you to specify conditions for individual columns, using ADQL-compatible expressions under the hood. In this case, the filters apply to: + +- ``object``: the target name, matched as a string (case-insensitive) +- ``exp_start``: the observation start time, stored as a ``char`` column with ``timestamp`` ``xtype`` + +The ``columns`` argument controls which fields are returned in the results table. + +.. doctest-remote-data:: + >>> table = eso.query_instrument( + ... instrument="midi", + ... column_filters={ + ... "object":"NGC4151", + ... "exp_start": "between '2008-01-01' and '2009-05-12'"}, + ... columns=["object", "date_obs"]) + >>> table +
+ object date_obs + ------- ----------------------- + NGC4151 2008-04-22T02:07:50.154 + NGC4151 2008-04-22T02:08:20.345 + NGC4151 2008-04-22T02:09:47.846 + ... + NGC4151 2009-05-11T01:42:08.432 + +.. tip:: + + - Use only column names returned via ``help=True`` (e.g. ``exp_start``, ``object``, ``prog_id``, ``exptime``, etc.). + - String filters (like ``object``) are matched case-insensitively. + - Temporal filters on fields like ``exp_start`` or ``release_date`` can use SQL-style syntax (e.g. ``between 'YYYY-MM-DD' and 'YYYY-MM-DD'``). + - Column names are case-sensitive in Python, so ensure they match exactly. + +.. tip:: + + Use :meth:`~astroquery.eso.EsoClass.query_main` when you want to search **across all instruments**, for example to retrieve all observations of a specific source regardless of the instrument used. + + .. doctest-remote-data:: + + table = eso.query_main(column_filters={"object": "NGC 3627"}) + + Use ``query_instrument`` when you want a more **refined, instrument-specific search**, applying filters that are only available for a particular instrument (e.g. instrument modes, configurations, or ambient conditions). + + .. doctest-remote-data:: + + column_filters = { + "dp_cat": "SCIENCE", # Science data only + "ins_opt1_name": "HIGH_SENS", # High sensitivity mode + "night_flag": "night", # Nighttime observations only + "moon_illu": "< 0", # No moon (below horizon) + "lst": "between 0 and 6" # Local sidereal time early in the night + } + + table = eso.query_instrument("midi", column_filters=column_filters) + +Download Data +============= + +To download the data returned by the query, you can use the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes a list of data product IDs (``dp_id``) and downloads the corresponding files from the ESO archive. + +.. doctest-remote-data:: + >>> eso.retrieve_data(table["dp_id"]) + +The ``data_files`` points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. + +.. doctest-skip:: + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") \ No newline at end of file diff --git a/docs/eso/eso_reduced.rst b/docs/eso/eso_reduced.rst new file mode 100644 index 0000000000..f380c26c44 --- /dev/null +++ b/docs/eso/eso_reduced.rst @@ -0,0 +1,126 @@ + +********************** +Query for Reduced Data +********************** + +In addition to raw observational files, the ESO Science Archive provides access to a wide range of **processed (reduced) data products**, also known as **Phase 3** data. These include science-ready images, spectra, and datacubes that have been calibrated and validated by ESO or by contributing survey teams. + +This section demonstrates how to search for and retrieve these reduced products using ``astroquery.eso``. The examples focus on **Phase 3 survey data**, which are organized by instrument, observing program, and survey tile. + +Available Surveys +================= + +The list of available surveys can be obtained with :meth:`~astroquery.eso.EsoClass.list_surveys` as follows: + +.. doctest-remote-data:: + + >>> surveys = eso.list_surveys() + >>> surveys + ['081.C-0827', '092.A-0472', '096.B-0054', '1100.A-0528', '1101.A-0127', '193.D-0232', + '195.B-0283', '196.B-0578', '196.D-0214', '197.A-0384', '198.A-0708', '60.A-9284H', + '60.A-9493', 'ADHOC', 'ALCOHOLS', 'ALLSMOG', 'ALMA', 'AMAZE', 'AMBRE', 'APEX-SciOps', + 'ARP_VST', 'ATLASGAL', 'CAFFEINE', 'ENTROPY', 'ePESSTOplus', 'ERIS-NIX', + 'ERIS-SPIFFIER', 'ESPRESSO', 'ESSENCE', 'FDS', 'FEROS', 'Fornax3D', 'FORS2-SPEC', + 'GAIAESO', 'GCAV', 'GIRAFFE', 'GOODS_FORS2', 'GOODS_ISAAC', 'GOODS_VIMOS_IMAG', + 'GOODS_VIMOS_SPEC', 'GW170817', 'HARPS', 'HAWKI', 'HUGS', 'INSPIRE', 'KIDS', 'KMOS', + 'LEGA-C', 'LESS', 'MAGIC', 'MUSE', 'MUSE-DEEP', 'MUSE-STD', 'MW-BULGE-PSFPHOT', + 'NGTS', 'NIRPS', 'OMEGACAM_INAF', 'PENELLOPE', 'PESSTO', 'PHANGS', 'PIONIER', + 'SHARKS', 'SPHERE', 'SUPER', 'UltraVISTA', 'UVES', 'UVES_SQUAD', 'VANDELS', 'VEGAS', + 'VEILS', 'VEXAS', 'VHS', 'VIDEO', 'VIKING', 'VIMOS', 'VINROUGE', 'VIPERS', 'VISIONS', + 'VMC', 'VPHASplus', 'VST-ATLAS', 'VVV', 'VVVX', 'XQ-100', 'XSGRB', 'XSHOOTER', + 'XShootU', 'XSL', 'ZCOSMOS'] + +Available Query Constraints +=========================== + +As before, list the possible columns in :meth:`~astroquery.eso.EsoClass.query_surveys` that can be queried with: + +.. doctest-remote-data:: + + >>> eso.query_surveys(help=True) # get help on the ESO query + INFO: + Columns present in the table ivoa.ObsCore: + column_name datatype xtype unit + ------------------- -------- ----------- ------ + abmaglim double mag + access_estsize long kbyte + access_format char + access_url char + bib_reference char + calib_level int + ... + target_name char + + Number of records present in the table ivoa.ObsCore: + 4559928 + [astroquery.eso.core] + +Query with Constraints (Specific Survey) +======================================== + +Let's assume that we work with the ``HARPS`` survey, and that we are interested in +target ``HD203608``. The archive can be queried as follows: + +.. doctest-remote-data:: + + >>> table = eso.query_surveys(surveys="HARPS", + ... column_filters= {"target_name": "HD203608"}) + >>> table +
+ target_name s_ra s_dec dp_id proposal_id abmaglim access_estsize ... snr strehl t_exptime t_max t_min t_resolution t_xel + deg deg mag kbyte ... s d d s + object float64 float64 object object float64 int64 ... float64 float64 float64 float64 float64 float64 int64 + ----------- --------- --------- --------------------------- ------------- -------- -------------- ... ------- ------- --------- -------------- -------------- ------------ ----- + HD203608 321.61455 -65.36429 ADP.2014-09-16T11:03:30.940 077.D-0720(A) -- 5261 ... 60.9 -- 33.0 53956.24265204 53956.24227009 33.00048 -- + HD203608 321.61761 -65.36485 ADP.2014-09-16T11:03:31.020 077.D-0720(A) -- 5261 ... 87.0 -- 32.999 53953.36835125 53953.36796931 32.999616 -- + HD203608 321.60594 -65.36528 ADP.2014-09-16T11:03:31.067 077.D-0720(A) -- 5261 ... 73.9 -- 33.0 53956.15534682 53956.15496487 33.00048 -- + ... + HD203608 321.61113 -65.37211 ADP.2014-09-16T11:05:14.863 077.D-0720(A) -- 5261 ... 95.2 -- 32.999 53954.99642615 53954.99604421 32.999616 -- + +The returned table has a ``dp_id`` column, which can be used to retrieve the datasets with +:meth:`~astroquery.eso.EsoClass.retrieve_data`: ``eso.retrieve_data(table["dp_id"][0])``. +More details about this method in the following section. + +Query with Constraints (Specific Instrument) +============================================ + +You can also query a specific instrument using the same method (e.g., ``HARPS``). For example, to retrieve **all** available HARPS data products regardless of the associated survey towards ``HD203608`` is given the following query: + +.. doctest-remote-data:: + + >>> table = eso.query_surveys(column_filters={"instrument_name": "HARPS", + ... "target_name": "HD203608"}) + >>> table +
+ target_name s_ra s_dec dp_id proposal_id abmaglim access_estsize access_format ... s_xel2 snr strehl t_exptime t_max t_min t_resolution t_xel + deg deg mag kbyte ... s d d s + object float64 float64 object object float64 int64 object ... int64 float64 float64 float64 float64 float64 float64 int64 + ----------- --------- --------- --------------------------- ------------- -------- -------------- ------------------------------------------ ... ------ ------- ------- --------- -------------- -------------- ------------ ----- + HD203608 321.61455 -65.36429 ADP.2014-09-16T11:03:30.940 077.D-0720(A) -- 5261 application/x-votable+xml;content=datalink ... -- 60.9 -- 33.0 53956.24265204 53956.24227009 33.00048 -- + HD203608 321.61761 -65.36485 ADP.2014-09-16T11:03:31.020 077.D-0720(A) -- 5261 application/x-votable+xml;content=datalink ... -- 87.0 -- 32.999 53953.36835125 53953.36796931 32.999616 -- + HD203608 321.60594 -65.36528 ADP.2014-09-16T11:03:31.067 077.D-0720(A) -- 5261 application/x-votable+xml;content=datalink ... -- 73.9 -- 33.0 53956.15534682 53956.15496487 33.00048 -- + ... + HD203608 321.61113 -65.37211 ADP.2014-09-16T11:05:14.863 077.D-0720(A) -- 5261 application/x-votable+xml;content=datalink ... -- 95.2 -- 32.999 53954.99642615 53954.99604421 32.999616 -- + +.. tip:: + + Keep in mind that the definition of a ``survey`` (also referred to as a **collection** in the ESO Science Archive) is not the same as the definition of an **instrument**. The ``instrument_name`` refers to the actual hardware that acquired the data (e.g., ``HARPS``, ``MUSE``), whereas the ``obs_collection`` identifies the scientific program, survey, or processing pipeline associated with the data product. + + In many cases, survey names match the instrument name (e.g., ``HARPS``, ``MUSE``, ``XSHOOTER``), which typically indicates **Phase 3 products processed and curated by ESO**. However, when the collection name differs (e.g., ``AMBRE``, ``GAIAESO``, ``PHANGS``), it usually denotes **community-contributed data** from large collaborations or specific science teams. + + So, for example, querying for ``eso.query_surveys(column_filters={"instrument_name": "HARPS"})`` will return all products taken with the HARPS instrument, across all programs and collections. In contrast, filtering on ``eso.query_surveys(surveys="HARPS"}`` will return only the `HARPS data reduced by ESO `_. + + You can inspect the collection for each result via the ``obs_collection`` column in your results table. + +Download Data +============= + +To download the data returned by the query, you can use the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes a list of data product IDs (``dp_id``) and downloads the corresponding files from the ESO archive. + +.. doctest-remote-data:: + >>> eso.retrieve_data(table["dp_id"]) + +The ``data_files`` points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. + +.. doctest-skip:: + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") \ No newline at end of file diff --git a/docs/eso/eso_tap.rst b/docs/eso/eso_tap.rst new file mode 100644 index 0000000000..5256e40482 --- /dev/null +++ b/docs/eso/eso_tap.rst @@ -0,0 +1,95 @@ + +*********************************************** +Query Raw or Reduced Data using the TAP Service +*********************************************** + +The ESO `TAP service `_ allows you to issue custom `ADQL `_ (Astronomical Data Query Language) queries against the archive metadata, offering fine-grained control over your search. TAP queries can be issued against different tables, depending on the type of data you're interested in: + +- The ``ivoa.ObsCore`` table provides standardized metadata for **fully calibrated (Phase 3) data products**. +- The ``dbo.raw`` table provides access to **raw observational data** across all ESO instruments. +- The ``ist.`` tables (e.g. ``ist.midi``, ``ist.muse``) allow more detailed queries tailored to **instrument-specific raw metadata**. + +These various query options have also been demonstrated earlier in this documentation using high-level `astroquery.eso` interfaces such as :meth:`~astroquery.eso.EsoClass.query_instrument`, :meth:`~astroquery.eso.EsoClass.query_main`, and :meth:`~astroquery.eso.EsoClass.list_surveys`. Using ADQL directly through TAP enables greater flexibility when building complex queries that combine constraints across multiple metadata fields. + +Query for Raw Data (Generic) +============================ + +The following example demonstrates how to query the ``dbo.raw`` table for raw data products from the ``MUSE`` instrument, specifically looking for observations of the object ``NGC300``: + +.. doctest-remote-data:: + + >>> query = """ + ... SELECT * + ... FROM dbo.raw + ... WHERE object = 'NGC300' + ... AND instrument = 'muse' + ... """ + >>> result = eso.query_tap_service(query) + >>> result + +Query for Raw Data (Instrument-Specific) +======================================== + +The following example demonstrates how to query the ``ist.muse`` table for raw data products from the ``MUSE`` instrument, specifically looking for observations taken during the night with a moon illumination below 30% and an average seeing (FWHM) below 1.0 arcsec: + +.. doctest-remote-data:: + + >>> query = """ + ... SELECT * + ... FROM ist.muse + ... WHERE night_flag = 'NIGHT' + ... AND moon_illu < 30 + ... AND dimm_fwhm_avg < 1.0 + ... AND exptime > 100 + ... AND lst between 0 and 6 + ... """ + >>> result = eso.query_tap_service(query) + >>> result + +Query for Reduced Data Products +=============================== + +The following example queries the ``ivoa.ObsCore`` table to find fully calibrated (``calib_level=3``) multi-extension observations (``multi_ob='M'``) from the ``SPHERE`` survey, with spatial pixel scales smaller than 0.2 arcsec: + +.. doctest-remote-data:: + + >>> query = """ + ... SELECT obs_collection, calib_level, multi_ob, filter, s_pixel_scale, instrument_name + ... FROM ivoa.ObsCore + ... WHERE obs_collection IN ('sphere') + ... AND calib_level = 3 + ... AND multi_ob = 'M' + ... AND s_pixel_scale < 0.2 + ... """ + >>> result = eso.query_tap_service(query) + >>> result +
+ obs_collection calib_level multi_ob filter s_pixel_scale instrument_name + arcsec + object int32 object object float64 object + -------------- ----------- -------- ------ ------------- --------------- + SPHERE 3 M K12 0.0122 SPHERE + SPHERE 3 M K12 0.0122 SPHERE + ... + SPHERE 3 M H 0.0122 SPHERE + +.. tip:: + + For more information about the TAP and how to write ADQL queries, refer to the following resources: + + - `ESO TAP+ documentation `_: Describes ESO's implementation of TAP and the available services. + - `IVOA TAP standard `_: The official specification from the International Virtual Observatory Alliance. + - `ADQL specification `_: Defines the query language used to interact with TAP services. + +Download Data +============= + +To download the data returned by the query, you can use the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes a list of data product IDs (``dp_id``) and downloads the corresponding files from the ESO archive. + +.. doctest-remote-data:: + >>> eso.retrieve_data(table["dp_id"]) + +The ``data_files`` points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. + +.. doctest-skip:: + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") \ No newline at end of file diff --git a/docs/eso/eso_troubleshooting.rst b/docs/eso/eso_troubleshooting.rst new file mode 100644 index 0000000000..c5ea0bbd76 --- /dev/null +++ b/docs/eso/eso_troubleshooting.rst @@ -0,0 +1,77 @@ + +*************** +Troubleshooting +*************** + +Clearing Cache +============== + +If you encounter repeated query failures or see outdated or inconsistent results, you may be dealing with a stale or corrupted local cache. You can clear the entire Astropy cache for ESO with the following command: + +.. doctest-skip:: + + >>> from astroquery.eso import Eso + >>> Eso.clear_cache() + +This will remove all cached ESO queries and downloaded metadata files. Data products already downloaded will remain unless manually deleted. + +.. _column-filters-fix: + +Using Correct ``column_filters`` +================================ + +If your query fails or silently returns no results, it might be because you're using column names that are **accepted in the ESO web interface (WDB)** but **not supported by the TAP/ADQL interface** that now is used within ``astroquery.eso``. A common case involves using ``stime`` and ``etime``, which are not valid TAP fields. Instead, use ``exp_start``, the TAP-compliant column representing the observation start time. This field supports SQL-style date filtering. + +Below are examples of invalid filter usage and their corrected TAP-compatible versions. + +Filtering between two dates +--------------------------- + +.. doctest-skip:: + + # ❌ Invalid (WDB-specific fields, not recognized by TAP) + column_filters = { + "stime": "2024-01-01", + "etime": "2024-12-31" + } + +.. doctest-skip:: + + # ✅ Correct (TAP-compliant syntax using 'exp_start') + column_filters = { + "exp_start": "between '2024-01-01' and '2024-12-31'" + } + +Filtering with only a start date +-------------------------------- + +.. doctest-skip:: + + # ❌ Invalid + column_filters = { + "stime": "2024-01-01" + } + +.. doctest-skip:: + + # ✅ Correct + column_filters = { + "exp_start": "> '2024-01-01'" + } + +Filtering with only an end date +------------------------------- + +.. doctest-skip:: + + # ❌ Invalid + column_filters = { + "etime": "2024-12-31" + } + +.. doctest-skip:: + + # ✅ Correct + column_filters = { + "exp_start": "< '2024-12-31'" + } \ No newline at end of file From 390aed9c88974647f1f0493b2b60780da30ee750 Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Fri, 23 Jan 2026 12:58:47 +0100 Subject: [PATCH 04/18] updated main --- docs/eso/eso.rst | 98 ++++++++++++++++++++++++++---------------------- 1 file changed, 54 insertions(+), 44 deletions(-) diff --git a/docs/eso/eso.rst b/docs/eso/eso.rst index 8180f11584..e6ffba1730 100644 --- a/docs/eso/eso.rst +++ b/docs/eso/eso.rst @@ -4,11 +4,54 @@ ESO Queries (`astroquery.eso`) ****************************** +Quick Start +=========== + +This quick start example shows how to use the ``astroquery.eso`` interface to query the ESO +Science Archive for both raw and reduced `MUSE `_ +observations of the nearby galaxy `NGC 300 `_, +and how to download the corresponding data products using their dataset +identifiers (``dp_id``). + +.. doctest-skip:: + + >>> from astroquery.eso import Eso + >>> eso = Eso() + >>> table_raw = eso.query_main("MUSE", column_filters={"object": "NGC300"}) + >>> table_reduced = eso.query_surveys("MUSE", column_filters={"target_name": "NGC300"}) + >>> eso.retrieve_data(table_raw['dp_id']) + >>> eso.retrieve_data(table_reduced['dp_id']) + +By default, queries are limited to returning a maximum of 1000 rows. This limit +can be modified by setting the ``ROW_LIMIT`` attribute. To disable truncation +and return all matching results, set: + +.. doctest-skip:: + + >>> eso.ROW_LIMIT = -1 + +Getting Started +=============== + +.. toctree:: + :maxdepth: 2 + + eso_login + eso_raw_general + eso_raw_instrument + eso_reduced + eso_apex + eso_tap + eso_cone_search + eso_header_info + eso_download + eso_troubleshooting + .. warning:: **⚠️ Backward Compatibility Notice ⚠️** - **The WDB (Web DataBase) API is being deprecated and replaced by TAP (Table Access Protocol)**, + **The WDB (Web DataBase) API has been deprecated and replaced by TAP (Table Access Protocol)**, a standardized interface for querying astronomical datasets using ADQL (Astronomical Data Query Language). While the Python interface remains the same, the values accepted by the ``columns`` and ``column_filters`` parameters must reflect TAP field names and ADQL syntax. This means that, @@ -28,7 +71,7 @@ ESO Queries (`astroquery.eso`) } Please review your queries carefully and update them accordingly to ensure compatibility with the new astroquery versions. - See section :ref:`column-filters-fix` at the end of this document. + See section :ref:`column-filters-fix` for more information and examples. Introduction ============ @@ -36,21 +79,12 @@ Introduction Overview of the ESO Archive --------------------------- -The `ESO Science Archive Facility (SAF) `_ is one of the largest ground-based astronomical data repositories in the world. It contains, and provides access to, data from ESO telescopes at the `La Silla `_, `Paranal `_, and `APEX `_ observatories, as well as the submillimeter APEX telescope on Llano de Chajnantor and the `ALMA `_ array. The archive includes all raw data from La Silla Paranal Observatory, corresponding calibrations, and a wide range of processed data products, either generated by ESO or contributed by the community. It also hosts selected external datasets, such as raw WFCAM/UKIDSS data from the UKIRT facility in Hawaii. As of May 2025, the archive hosts over `4.6 million science products `_, including: - -- `>3.8 million products from La Silla, Paranal, and APEX `_ -- `>700,000 ALMA products `_ -- `~2.3 million spectra `_ (1D extracted, fully calibrated) -- `>900,000 imaging products `_ in optical, near-IR, and sub-mm bands -- `>500,000 spectral cubes `_ from IFU and radio/mm instruments -- `~700,000 catalog tiles `_ - -Processed data downloaded from the ESO Archive are assigned a **Digital Object Identifier (DOI)**. The list of available DOIs can be browsed in the `ESO DOI query page `_. +The `ESO Science Archive Facility (SAF) `_ is one of the largest ground-based astronomical data repositories in the world. It contains, and provides access to, data from ESO telescopes at the `La Silla `_, `Paranal `_, as well as the (sub-)millimeter `APEX `_ telescope and `ALMA `_ array, and selected external datasets, such as raw WFCAM/UKIDSS data from the UKIRT facility in Hawaii. The archive includes both `raw data `_, and a wide range of `processed data products `_, either generated by ESO or contributed by the community (via `ESO Phase 3 `_). Processed data downloaded from the ESO Archive are assigned a Digital Object Identifier (DOI), where the list of available DOIs can be browsed in the `ESO DOI query page `_. Ways to Access the Archive -------------------------- -Access to data follows ESO’s standard data policy: the **Principal Investigator (PI)** of an observing program has exclusive access to their data during a proprietary period (typically one year), after which the data become publicly available. See the `ESO Data Access Policy `_ and `FAQ `_ for more information. Browsing the archive does not require authentication, but login is necessary to access proprietary datasets. Use of ESO archival data should be acknowledged in all publications. +Access to data follows ESO’s standard data policy: the Principal Investigator (PI) of an observing program has exclusive access to their data during a proprietary period (typically one year), after which the data become publicly available. See the `ESO Data Access Policy `_ and `FAQ `_ for more information. Browsing the archive does not require authentication, but login is necessary to access proprietary datasets. Use of ESO archival data should be acknowledged in all publications. There are multiple ways to access the archive: @@ -61,40 +95,16 @@ There are multiple ways to access the archive: This documentation focuses on the last method: accessing the archive programmatically using the ``astroquery.eso`` module. This Python interface allows users to search for both raw and reduced data, retrieve metadata, and download data products directly using dataset identifiers. Proprietary access is supported via authentication with the `ESO User Portal `_. -Quick Start -=========== - -.. doctest-skip:: - - >>> from astroquery.eso import Eso # Import the ESO module - >>> eso = Eso() # Create an instance of the ESO class - >>> table_raw = eso.query_main("muse", column_filters={"object": "NGC300"}) # Query raw data - >>> table_reduced = eso.query_surveys("muse", column_filters={"target_name": "NGC300"}) # Query reduced data - >>> eso.retrieve_data(table_raw['dp_id']) # Download raw data products - >>> eso.retrieve_data(table_reduced['dp_id']) # Retrieve reduced data products - -By default, the maximum number of results returned by a query is by default set to 1000, but this can be adjusted as needed. To return all results without truncation, set the ``maxrec`` attribute to -1: - -.. doctest-skip:: +Notebooks +========= - >>> eso.maxrec = -1 # Return all results - -Getting Started -=============== +A set of example Jupyter notebooks demonstrating the usage of the ``astroquery.eso`` module can be found in the `ESO GitHub astroquery examples +`_ repository, where a simple overview script can be found at `introduction.ipynb `_. -.. toctree:: - :maxdepth: 2 +Installation +============ - eso_login - eso_raw_general - eso_raw_instrument - eso_reduced - eso_apex - eso_tap - eso_cone_search - eso_header_info - eso_download - eso_troubleshooting +Along with the methods to install the main ``astroquery`` package, the most up-to-date version of the ``astroquery.eso`` module can be installed directly from the `ESO GitHub astroquery `_ repository. Requirements ============ From e308f82c2f9a063c1e7561e6874e431b150bca36 Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Fri, 23 Jan 2026 13:01:01 +0100 Subject: [PATCH 05/18] updated login --- docs/eso/eso_login.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/eso/eso_login.rst b/docs/eso/eso_login.rst index 9a06380560..b24afa4eb9 100644 --- a/docs/eso/eso_login.rst +++ b/docs/eso/eso_login.rst @@ -1,7 +1,7 @@ -*********************** -Access Proprietary Data -*********************** +**************************** +Authenticated Access (login) +**************************** Most datasets in the ESO Science Archive are publicly available and can be downloaded anonymously without requiring authentication. However, access to proprietary datasets—such as those under active proprietary periods—is restricted to authorized users (e.g., PIs of observing programs and their designated delegates). These users must authenticate via the `ESO User Portal `_. From 3790c627bc89a28cf0119928fa834941035dd3f2 Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Fri, 23 Jan 2026 14:26:52 +0100 Subject: [PATCH 06/18] update raw docs --- docs/eso/eso_raw_general.rst | 27 ++++++++++++++------------- docs/eso/eso_raw_instrument.rst | 33 +++++++++++++++------------------ 2 files changed, 29 insertions(+), 31 deletions(-) diff --git a/docs/eso/eso_raw_general.rst b/docs/eso/eso_raw_general.rst index 01eeacbcae..20ce881440 100644 --- a/docs/eso/eso_raw_general.rst +++ b/docs/eso/eso_raw_general.rst @@ -3,15 +3,14 @@ Query for Raw Data (Generic) **************************** -The `astroquery.eso` module provides several ways to search for data in the ESO Science Archive for raw data. This section focuses on the **generic query interface** for raw data (:meth:`~astroquery.eso.EsoClass.query_main`), while in another section we describe **instrument-specific queries** (e.g., using :meth:`~astroquery.eso.EsoClass.query_instrument`). +.. tip:: + The `astroquery.eso` module provides several ways to search for data in the ESO Science Archive for raw data. This section focuses on the **generic query interface** for raw data :meth:`~astroquery.eso.EsoClass.query_main`, while in another section we describe **instrument-specific queries** using :meth:`~astroquery.eso.EsoClass.query_instrument`. Functionally, :meth:`~astroquery.eso.EsoClass.query_main` works the same way as :meth:`~astroquery.eso.EsoClass.query_instrument`, except you don’t need to specify an instrument table name directly. You can still apply column filters, control the columns returned, and limit result counts. -In some cases, you may want to query the ESO Archive **without targeting a specific instrument**. This is exactly what the :meth:`~astroquery.eso.EsoClass.query_main` method is designed for. It allows access to the **global raw data table**, which combines metadata across all instruments. Internally, this method queries the ``dbo.raw`` table via ESO's `TAP service `_, which you could also access directly using ADQL with a simple statement like: +You may want to query the ESO Archive **without targeting a specific instrument**. This is exactly what the :meth:`~astroquery.eso.EsoClass.query_main` method is designed for. It allows access to the **global raw data table**, which combines metadata across all instruments. Internally, this method queries the ``dbo.raw`` table via ESO's `TAP service `_, which you could also access directly using ADQL with a simple statement like: .. doctest-skip:: SELECT * FROM dbo.raw -Functionally, :meth:`~astroquery.eso.EsoClass.query_main` works the same way as :meth:`~astroquery.eso.EsoClass.query_instrument`, except you don’t need to specify an instrument table name directly. You can still apply column filters, control the columns returned, and limit result counts. - Available Instruments ===================== @@ -33,7 +32,7 @@ The list of all supported instruments can be retrieved using the :meth:`~astroqu This list corresponds to the instruments currently available for programmatic raw data queries in the ESO archive. -**Note:** The list is dynamically generated by querying the archive’s internal TAP service. It can also be reproduced directly by executing the following ADQL query on the `ESO TAP interface `_: +The list is dynamically generated by querying the archive’s internal TAP service. It can also be reproduced directly by executing the following ADQL query on the `ESO TAP interface `_: .. doctest-skip:: @@ -47,7 +46,7 @@ Once you have identified the instrument of interest, you can proceed with constr Available Query Constraints =========================== -We start by inspecting the available columns that can be queried by ``query_main`` with the ``help=True`` keyword. This will return a list of all columns available in the ``dbo.raw`` table, along with their data types, units, and any additional metadata such as ``xtype`` information. +We start by inspecting the available columns that can be queried by :meth:`~astroquery.eso.EsoClass.query_main` with the ``help=True`` keyword. This will return a list of all columns available, along with their data types, units, and any additional metadata such as ``xtype`` information. .. doctest-remote-data:: @@ -72,7 +71,7 @@ We start by inspecting the available columns that can be queried by ``query_main Query with Constraints (Specific Instrument) ============================================ -Now that we know which of the columns are available for queries, we can, for example, retrieving all-sky images from the ``APICAM`` instrument using the ``LUMINANCE`` filter, on a single night (i.e. 2019-04-26): +Now that we know which of the columns are available for queries, we can retrieve all-sky images from the ``APICAM`` instrument, with constraints on the ``LUMINANCE`` filter, on for a single night (i.e. 2019-04-26): .. doctest-remote-data:: @@ -81,7 +80,8 @@ Now that we know which of the columns are available for queries, we can, for exa ... column_filters={ ... "instrument": "APICAM", ... "filter_path": "LUMINANCE", - ... "exp_start": "between '2019-04-26' and '2019-04-27'"}) + ... "exp_start": "between '2019-04-26' and '2019-04-27'"} + ... ) >>> print(len(table)) 215 >>> table.colnames @@ -103,16 +103,17 @@ Now that we know which of the columns are available for queries, we can, for exa Query with Constraints (All Instruments) ======================================== -Alternatively, if you want to retrieve all available raw data products with the ``LUMINANCE`` filter, on a single night (i.e. 2019-04-26) without specifying the instrument, you can use the ``query_main`` method. +Alternatively, if you want to retrieve all available raw data products with the ``LUMINANCE`` filter, on a single night (i.e. 2019-04-26) without specifying the instrument, you can use: .. doctest-remote-data:: >>> table = eso.query_main( ... column_filters={ ... "filter_path": "LUMINANCE", - ... "exp_start": "between '2019-04-26' and '2019-04-27'"}) + ... "exp_start": "between '2019-04-26' and '2019-04-27'"} + ... ) -Another useful example for ``eso.query_main`` is to query for all raw data for a specific target, such as ``NGC 3627``, across all instruments: +Another useful example is to query for all raw data for a specific target, such as ``NGC 3627``: .. doctest-remote-data:: @@ -120,13 +121,13 @@ Another useful example for ``eso.query_main`` is to query for all raw data for a .. tip:: - Use ``query_main`` when you want to search **across all instruments**, for example to retrieve all observations of a specific source regardless of the instrument used. + Use :meth:`~astroquery.eso.EsoClass.query_main` when you want to search **across all instruments**, for example to retrieve all observations of a specific source regardless of the instrument used. .. doctest-remote-data:: table = eso.query_main(column_filters={"object": "NGC 3627"}) - Use ``query_instrument`` when you want a more **refined, instrument-specific search**, applying filters that are only available for a particular instrument (e.g. instrument modes, configurations, or ambient conditions). + Use :meth:`~astroquery.eso.EsoClass.query_instrument` when you want a more **refined, instrument-specific search** (e.g. instrument modes, configurations, or ambient conditions). .. doctest-remote-data:: diff --git a/docs/eso/eso_raw_instrument.rst b/docs/eso/eso_raw_instrument.rst index 563ae5c02c..09350e83e1 100644 --- a/docs/eso/eso_raw_instrument.rst +++ b/docs/eso/eso_raw_instrument.rst @@ -3,9 +3,10 @@ Query for Raw Data (Instrument-Specific) **************************************** -The `astroquery.eso` module provides several ways to search for data in the ESO Science Archive for raw data. This section focuses on the ***instrument-specific query interface** for raw data (:meth:`~astroquery.eso.EsoClass.query_instrument`), while in another section we describe **generic queries** (e.g., using :meth:`~astroquery.eso.EsoClass.query_main`). +.. note:: + The `astroquery.eso` module provides several ways to search for data in the ESO Science Archive for raw data. This section focuses on the **instrument-specific query interface** for raw data :meth:`~astroquery.eso.EsoClass.query_instrument`, while in another section we describe **generic queries** using :meth:`~astroquery.eso.EsoClass.query_main`. -In many cases, you will want to query the ESO Archive **for data from a specific instrument**. This is exactly what the :meth:`~astroquery.eso.EsoClass.query_instrument` method is designed for. It allows you to search instrument-specific tables, which expose metadata fields and filters unique to each instrument. Internally, this method queries the corresponding instrument table (e.g., ``ist.muse``) via ESO's `TAP service `_. This approach is ideal when you need precise control over your query, such as filtering by instrument configuration, mode, or observational setup. +In many cases, you will want to query the ESO Archive **for data from a specific instrument**. This is exactly what the :meth:`~astroquery.eso.EsoClass.query_instrument` method is designed for. It allows you to search instrument-specific tables, which expose metadata fields and filters unique to each instrument. Internally, this method queries the corresponding instrument table (e.g ``ist.muse``) via ESO's `TAP service `_. This approach is ideal when you need precise control over your query, such as filtering by instrument configuration, mode, or observational setup. Available Instruments ===================== @@ -28,7 +29,7 @@ The list of all supported instruments can be retrieved using the :meth:`~astroqu This list corresponds to the instruments currently available for programmatic raw data queries in the ESO archive. -**Note:** The list is dynamically generated by querying the archive’s internal TAP service. It can also be reproduced directly by executing the following ADQL query on the `ESO TAP interface `_: +The list is dynamically generated by querying the archive’s internal TAP service. It can also be reproduced directly by executing the following ADQL query on the `ESO TAP interface `_: .. doctest-skip:: @@ -42,7 +43,7 @@ Once you have identified the instrument of interest, you can proceed with constr Available Query Constraints =========================== -Once an instrument is selected—for example, ``midi`` — you can inspect the available queryable columns using the ``help=True`` keyword in the :meth:`~astroquery.eso.EsoClass.query_instrument` method. This is a useful first step to understand what metadata is available and how to structure your query. +Once an instrument is selected — for example, ``midi`` — you can inspect the available queryable columns using the ``help=True`` keyword in the :meth:`~astroquery.eso.EsoClass.query_instrument` method. This is a useful first step to understand what metadata is available and how to structure your query. The output includes column names, data types, units, and, where applicable, ``xtype`` information to indicate more specific column content. For example, a column with datatype ``char`` may represent a timestamp or a sky region, which is reflected in the ``xtype`` field (e.g., ``timestamp`` or ``adql:REGION``). @@ -71,15 +72,10 @@ The output includes column names, data types, units, and, where applicable, ``xt 437577 [astroquery.eso.core] -**Note:** for a deeper description of each column, the following query can be issued -on the ESO `Programmatic Access `_ website (`see here `_): - -``select column_name, description from TAP_SCHEMA.columns where table_name = 'ist.midi'`` - Query with Constraints ====================== -Once the available query columns have been inspected (e.g., via ``help=True``), you can construct a constrained query to retrieve relevant datasets. For example, suppose you want to retrieve MIDI observations of the target ``NGC 4151`` that were taken between ``2008-01-01`` and ``2009-05-12``. +Once the available query columns have been inspected, you can construct a constrained query to retrieve relevant datasets. For example, suppose you want to retrieve MIDI observations of the target ``NGC 4151`` that were taken between ``2008-01-01`` and ``2009-05-12``. The ``column_filters`` dictionary allows you to specify conditions for individual columns, using ADQL-compatible expressions under the hood. In this case, the filters apply to: @@ -88,13 +84,21 @@ The ``column_filters`` dictionary allows you to specify conditions for individua The ``columns`` argument controls which fields are returned in the results table. +.. tip:: + + - Use only column names returned via ``help=True`` (in this case ``exp_start``, ``object``, ``prog_id``, ``exptime``, etc.). + - String filters (like ``object``) are matched case-insensitively. + - Temporal filters on fields like ``exp_start`` or ``release_date`` can use SQL-style syntax (e.g. ``between 'YYYY-MM-DD' and 'YYYY-MM-DD'``). + - Column names are case-sensitive in Python, so ensure they match exactly. + .. doctest-remote-data:: >>> table = eso.query_instrument( ... instrument="midi", ... column_filters={ ... "object":"NGC4151", ... "exp_start": "between '2008-01-01' and '2009-05-12'"}, - ... columns=["object", "date_obs"]) + ... columns=["object", "date_obs"] + ... ) >>> table
object date_obs @@ -105,13 +109,6 @@ The ``columns`` argument controls which fields are returned in the results table ... NGC4151 2009-05-11T01:42:08.432 -.. tip:: - - - Use only column names returned via ``help=True`` (e.g. ``exp_start``, ``object``, ``prog_id``, ``exptime``, etc.). - - String filters (like ``object``) are matched case-insensitively. - - Temporal filters on fields like ``exp_start`` or ``release_date`` can use SQL-style syntax (e.g. ``between 'YYYY-MM-DD' and 'YYYY-MM-DD'``). - - Column names are case-sensitive in Python, so ensure they match exactly. - .. tip:: Use :meth:`~astroquery.eso.EsoClass.query_main` when you want to search **across all instruments**, for example to retrieve all observations of a specific source regardless of the instrument used. From 09626b9efe65fd88f8523f7474eacf82dea83d58 Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Fri, 23 Jan 2026 14:40:32 +0100 Subject: [PATCH 07/18] update reduced docs --- docs/eso/eso_reduced.rst | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/eso/eso_reduced.rst b/docs/eso/eso_reduced.rst index f380c26c44..4ae03fac00 100644 --- a/docs/eso/eso_reduced.rst +++ b/docs/eso/eso_reduced.rst @@ -3,9 +3,7 @@ Query for Reduced Data ********************** -In addition to raw observational files, the ESO Science Archive provides access to a wide range of **processed (reduced) data products**, also known as **Phase 3** data. These include science-ready images, spectra, and datacubes that have been calibrated and validated by ESO or by contributing survey teams. - -This section demonstrates how to search for and retrieve these reduced products using ``astroquery.eso``. The examples focus on **Phase 3 survey data**, which are organized by instrument, observing program, and survey tile. +In addition to raw observational files, the ESO Science Archive provides access to a wide range of **processed (reduced) data products**. These include science-ready images, spectra, and datacubes that have been validated by ESO (through the `ESO Phase 3 `_ process). Available Surveys ================= @@ -55,16 +53,27 @@ As before, list the possible columns in :meth:`~astroquery.eso.EsoClass.query_su 4559928 [astroquery.eso.core] +.. note:: + Column names may differ between the tables returned by different query + methods. Users are encouraged to inspect closly the available columns and supported + filters using the ``help=True`` option. For example, for raw data queries + (:meth:`~astroquery.eso.EsoClass.query_main` and + :meth:`~astroquery.eso.EsoClass.query_instrument`), the target name is specified + using the ``object`` column, whereas for reduced data queries + (:meth:`~astroquery.eso.EsoClass.query_surveys`) the corresponding column is + named ``target_name``. + Query with Constraints (Specific Survey) ======================================== -Let's assume that we work with the ``HARPS`` survey, and that we are interested in +Let's assume that we work with the `HARPS survey `_, and that we are interested in target ``HD203608``. The archive can be queried as follows: .. doctest-remote-data:: >>> table = eso.query_surveys(surveys="HARPS", - ... column_filters= {"target_name": "HD203608"}) + ... column_filters= {"target_name": "HD203608"} + ... ) >>> table
target_name s_ra s_dec dp_id proposal_id abmaglim access_estsize ... snr strehl t_exptime t_max t_min t_resolution t_xel @@ -104,13 +113,9 @@ You can also query a specific instrument using the same method (e.g., ``HARPS``) .. tip:: - Keep in mind that the definition of a ``survey`` (also referred to as a **collection** in the ESO Science Archive) is not the same as the definition of an **instrument**. The ``instrument_name`` refers to the actual hardware that acquired the data (e.g., ``HARPS``, ``MUSE``), whereas the ``obs_collection`` identifies the scientific program, survey, or processing pipeline associated with the data product. - - In many cases, survey names match the instrument name (e.g., ``HARPS``, ``MUSE``, ``XSHOOTER``), which typically indicates **Phase 3 products processed and curated by ESO**. However, when the collection name differs (e.g., ``AMBRE``, ``GAIAESO``, ``PHANGS``), it usually denotes **community-contributed data** from large collaborations or specific science teams. - - So, for example, querying for ``eso.query_surveys(column_filters={"instrument_name": "HARPS"})`` will return all products taken with the HARPS instrument, across all programs and collections. In contrast, filtering on ``eso.query_surveys(surveys="HARPS"}`` will return only the `HARPS data reduced by ESO `_. + Keep in mind that the definition of a ``survey`` (also referred to as a **collection** in the ESO Science Archive) is not the same as the definition of an **instrument**. The ``instrument_name`` refers to the actual hardware that acquired the data (e.g., ``HARPS``, ``MUSE``), whereas the ``obs_collection`` identifies the scientific program, survey, or processing pipeline associated with the data product. In many cases, survey names match the instrument name (e.g., ``HARPS``, ``MUSE``, ``XSHOOTER``), which typically indicates **products processed and curated by ESO**. However, when the collection name differs (e.g., ``AMBRE``, ``GAIAESO``, ``PHANGS``), it usually denotes **community-contributed data** from large collaborations or specific science teams. - You can inspect the collection for each result via the ``obs_collection`` column in your results table. + So, for example, querying for ``eso.query_surveys(column_filters={"instrument_name": "HARPS"})`` will return all products taken with the HARPS instrument, across all programs and collections. In contrast, filtering on ``eso.query_surveys(surveys="HARPS"}`` will return only the `HARPS data reduced by ESO `_. You can inspect the collection for each result via the ``obs_collection`` column in your results table. Download Data ============= From d58272bc152bcc23177901b26a5fd4f660420667 Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Fri, 23 Jan 2026 14:56:31 +0100 Subject: [PATCH 08/18] update apex docs --- docs/eso/eso_apex.rst | 123 ++++++++++++++++-------------------------- 1 file changed, 47 insertions(+), 76 deletions(-) diff --git a/docs/eso/eso_apex.rst b/docs/eso/eso_apex.rst index 5857dfb60a..032fe4bb56 100644 --- a/docs/eso/eso_apex.rst +++ b/docs/eso/eso_apex.rst @@ -3,24 +3,31 @@ Query for APEX Data ******************* -This notebook provides an example of how to query the ESO Science Archive for **APEX data**. - -APEX data are available within the archive and can be queried using any of the standard astroquery methods: -- :meth:`~astroquery.eso.EsoClass.query_survey` for **reduced data** -- :meth:`~astroquery.eso.EsoClass.query_main` for **raw data** -- :meth:`~astroquery.eso.EsoClass.query_instrument` for **raw data (instrument specific)** - -In addition, there is the option to use :meth:`~astroquery.eso.EsoClass.query_apex_quicklooks`, which is a dedicated query for retrieving **APEX Quick Look products**. - -The workflow for APEX data is somewhat different from that of other ESO-operated instruments and telescopes. The **APEX Quick Looks** aim to address some of these differences by providing users with ``.tar`` files that include additional data outputs — such as diagnostic plots, observing logs, and (in some cases) a reduced version of the dataset. These ``.tar`` files also include ``class`` (``.apex``) files that allow users to **re-run the data reduction**, if desired. This is often more useful than working directly with the ``.fits`` files available in the archive and can be **particularly helpful for heterodyne datasets**. For heterodyne observations, the calibrated ``.class`` file is typically included, while the corresponding uncalibrated ``.fits`` files are usually **not needed** and cannot be processed with standard reduction software. - -The APEX Science Archive provides access to raw observations and associated data products for observations performed with APEX since **July 11, 2005**. The Quick Look products are designed to help users preview and assess the data quality, providing useful summaries that complement the raw observations. - -This notebook will guide you through a step-by-step workflow to **identify, query, and download** APEX Quick Look data products from the ESO Science Archive. - -Here, we focus on APEX observations from the [**ALCOHOLS survey**](https://www.eso.org/rm/api/v1/public/releaseDescriptions/199) (12CO(3–2) line emission in the Milky Way) available in the ESO Archive. We will first search for the **reduced ALCOHOLS data products**, then identify the corresponding raw datasets using the **instrument-specific query**. Once the raw data is identified, we can use it to locate and retrieve the **associated APEX Quick Look products**. We follow this workflow because APEX Quick Looks typically require knowledge of the **APEX proposal ID**, which may not always be known in advance — this is **not** the same as the ESO programme ID. If you know your APEX proposal ID (for example, if you are querying your own data), you can search for APEX Quick Look files directly. - -Note that this workflow is **not specific to the ALCOHOLS survey** and can be used to query APEX Quick Look products for any project. +APEX observations can be discovered via the standard +archive query interfaces for reduced and raw data: + +- reduced data: :meth:`~astroquery.eso.EsoClass.query_surveys` +- raw data: :meth:`~astroquery.eso.EsoClass.query_main` +- raw (instrument-specific): :meth:`~astroquery.eso.EsoClass.query_instrument` + +In addition, :meth:`~astroquery.eso.EsoClass.query_apex_quicklooks` provides a +dedicated interface to retrieve APEX Quick Look products. These Quick Looks are +distributed as ``.tar`` bundles that typically include diagnostic material +(e.g. plots and logs) and ``class`` (``.apex``) files; for heterodyne +observations, the calibrated ``.class`` product is often the most practical +starting point, and the accompanying uncalibrated ``.fits`` files are usually +not required. + +The archive contains APEX observations from July 11, 2005 onward. This notebook +walks through a practical workflow to identify, query, and download APEX Quick +Look products. As an example, we use APEX observations from the ALCOHOLS survey +(12CO(3–2) line emission in the Milky Way) available in the ESO Archive. We +first locate the reduced survey products, then use an instrument-specific query +to identify the corresponding raw datasets, and finally retrieve the associated +Quick Look bundles. This indirection is sometimes necessary because Quick Look +queries may require the APEX proposal ID (distinct from the ESO programme ID). +If the APEX proposal ID is known (e.g. for proprietary data), Quick Look +products can be queried directly. Query Reduced APEX data ======================= @@ -29,20 +36,24 @@ We first query for the reduced data from the **ALCOHOLS** survey, and retrieve t .. doctest-remote-data:: - >>> table_reduced = eso.query_surveys("ALCOHOLS") # query the ESO archive for the ALCOHOLS survey - >>> proposal_id = list(set(table_reduced['proposal_id'])) # extract unique proposal IDs from the query result + >>> # query the ESO archive for the ALCOHOLS survey + >>> table_reduced = eso.query_surveys("ALCOHOLS") + + >>> # extract unique proposal IDs from the query result + >>> proposal_id = list(set(table_reduced['proposal_id'])) - >>> # Check if we have a single proposal ID or multiple + >>> # check if we have a single proposal ID or multiple >>> if len(proposal_id) == 1: >>> proposal_id = proposal_id[0] >>> else: >>> print("Warning: Multiple proposal IDs found...") - >>> proposal_id = proposal_id.split('(')[0] # extract the first part of the proposal ID before any parentheses (i.e. the run ID) + >>> # Extract the first part of the proposal ID before any parentheses (i.e. the run ID) + >>> proposal_id = proposal_id.split('(')[0] >>> print(f"Proposal ID: {proposal_id}") Proposal ID: 094.C-0935 -Note that multiple proposal IDs may be returned, which would require minor changes to the above example to loop through the projects. +Note that multiple values of ``proposal_id`` may be returned, which would require minor changes to the above example to loop through the projects. Available Query Constraints =========================== @@ -62,50 +73,7 @@ As always, it is good practice to check the available columns to search in the i channels int datalink_url char date_obs char - dec double deg - dp_cat char - dp_id char - dp_tech char - dp_type char - ecl_lat double deg - ecl_lon double deg - exp_start char timestamp - exposure float s - exptime float s - febe char - freq float GHz - freqres float - gal_lat double deg - gal_lon double deg - instrument char - lambda_max double nm - lambda_min double nm - last_mod_date char timestamp - line char - lst float s - mjd_obs double d - npols int - nsubs short - ob_id int - object char - observer_initials char - observing_mode char - origfile char - period int - pi_coi char - prog_id char - prog_title char - prog_type char - project_id char - ra double deg - release_date char timestamp - restfreq float Hz - s_region char adql:REGION - scangeom char - scanmode char - scannum int - scantype char - skyfreq float Hz + ... tel_airm_end float tel_airm_start float tel_alt float deg @@ -125,17 +93,22 @@ Query Raw APEX data We now query for raw data from the APEX instrument, using the proposal ID we retrieved from the previous query. .. doctest-remote-data:: + + >>> # query the APEX instrument for data related to the proposal ID + >>> table_raw = eso.query_instrument("APEX", column_filters={"prog_id": f"like '{proposal_id}%'"}) - >>> table_raw = eso.query_instrument("APEX", column_filters={"prog_id": f"like '{proposal_id}%'"}) # query the APEX instrument for data related to the proposal ID - >>> project_id = list(set(table_raw["project_id"])) # extract unique project IDs from the raw data query - >>> project_id = project_id[0] # Assuming we only have one project ID + >>> # extract unique project IDs from the raw data query + >>> project_id = list(set(table_raw["project_id"])) + + >>> # assuming we only have one project ID + >>> project_id = project_id[0] >>> print(f"Project ID: {project_id}") Project ID: E-094.C-0935A-2014 In this case, we know there is only **one** APEX proposal ID, but if there were multiple IDs, we would need to loop through them. .. tip:: - In the :meth:`~astroquery.eso.EsoClass.query_surveys` query, the ``"proposal_id"`` column refers to the **ESO programme ID**. In contrast, in an APEX-specific query using :meth:`~astroquery.eso.EsoClass.query_instrument`, the ``"prog_id"`` column also refers to the **ESO programme ID**, **not** the **APEX proposal ID**. The APEX proposal ID is instead found in the ``"project_id"`` column in the :meth:`~astroquery.eso.EsoClass.query_instrument` query—this is the value used to identify APEX Quick Look products. + In the :meth:`~astroquery.eso.EsoClass.query_surveys` query, the ``"proposal_id"`` column refers to the **ESO programme ID**. Whereas, in :meth:`~astroquery.eso.EsoClass.query_instrument`, the ``"prog_id"`` column refers to the **ESO programme ID** and ``"project_id"`` column refers to the **APEX proposal ID** - this is the value used to identify APEX Quick Look products (see below). Query APEX Quick Look products ============================== @@ -166,7 +139,7 @@ We can check the available columns to search in the query. 282296 [astroquery.eso.core] -And now, query for the APEX Quick Look products using the APEX proposal ID (``project_id``) we retrieved from the previous query. +And now, us :meth:`~astroquery.eso.EsoClass.query_apex_quicklooks` to query for the APEX Quick Look products using the APEX proposal ID (``project_id``) we retrieved from the previous query. .. doctest-remote-data:: @@ -183,14 +156,12 @@ And now, query for the APEX Quick Look products using the APEX proposal ID (``pr 6389 https://dataportal.eso.org/dataPortal/file/E-094.C-0935A.2015AUG22.TAR APEXHET Heterodyne ESO ... The APEX Large CO Heterodyne Outflow Legacy Supercam survey of Orion (ALCOHOLS) Normal E-094.C-0935A-2014 E-094.C-0935A.2015AUG22.TAR 2015-04-25T18:41:53.900Z -As can be seen from the output above, there is one APEX Quick Look product available per UT date, per APEX proposal ID. - -Also note that the APEX Quick Look products are available in `.tar` (`.TAR`) format, which can be downloaded and extracted (see below). +As can be seen from the output above, there is one APEX Quick Look product available per UT date, per APEX proposal ID. Also note that the APEX Quick Look products are available in ``.tar`` format (e.g. ``E-094.C-0935A.2014DEC10.TAR``), which can be downloaded and extracted (see below). Download APEX Quick Look products ================================= -Finally, we can download the APEX Quick Look products using the `eso.retrieve_data` function. +Finally, we can download the APEX Quick Look products using the :meth:`~astroquery.eso.EsoClass.retrieve_data` method: .. doctest-remote-data:: From f17898f8803ef6354ff82831e957efb4fdac8e07 Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Fri, 23 Jan 2026 15:05:48 +0100 Subject: [PATCH 09/18] update tap docs --- docs/eso/eso_tap.rst | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/docs/eso/eso_tap.rst b/docs/eso/eso_tap.rst index 5256e40482..5afa0db39b 100644 --- a/docs/eso/eso_tap.rst +++ b/docs/eso/eso_tap.rst @@ -3,13 +3,31 @@ Query Raw or Reduced Data using the TAP Service *********************************************** -The ESO `TAP service `_ allows you to issue custom `ADQL `_ (Astronomical Data Query Language) queries against the archive metadata, offering fine-grained control over your search. TAP queries can be issued against different tables, depending on the type of data you're interested in: +The ESO `TAP service `_ allows users to +submit custom `ADQL `_ (Astronomical Data +Query Language) queries against the archive metadata, providing fine-grained +control over complex searches. TAP queries can be executed against different +tables depending on the type of data required: -- The ``ivoa.ObsCore`` table provides standardized metadata for **fully calibrated (Phase 3) data products**. -- The ``dbo.raw`` table provides access to **raw observational data** across all ESO instruments. -- The ``ist.`` tables (e.g. ``ist.midi``, ``ist.muse``) allow more detailed queries tailored to **instrument-specific raw metadata**. +- ``ivoa.ObsCore``: standardized metadata for **fully calibrated (Phase 3) data + products** corresponding to high-level queries, such as available via + :meth:`~astroquery.eso.EsoClass.query_surveys`. -These various query options have also been demonstrated earlier in this documentation using high-level `astroquery.eso` interfaces such as :meth:`~astroquery.eso.EsoClass.query_instrument`, :meth:`~astroquery.eso.EsoClass.query_main`, and :meth:`~astroquery.eso.EsoClass.list_surveys`. Using ADQL directly through TAP enables greater flexibility when building complex queries that combine constraints across multiple metadata fields. +- ``dbo.raw``: metadata for **raw observational data** across all ESO + instruments, such as available via :meth:`~astroquery.eso.EsoClass.query_main`. + +- ``ist.`` (e.g. ``ist.muse``, ``ist.midi``): **instrument-specific + raw metadata** tables, such as available via + :meth:`~astroquery.eso.EsoClass.query_instrument`. + +While these query modes are covered by the high-level ``astroquery.eso`` API, +direct use of ADQL through TAP provides additional flexibility for constructing +advanced queries that combine constraints across multiple metadata fields or +tables. For more information about the TAP and how to write ADQL queries, refer to the following resources: + +- `ESO TAP+ documentation `_: Describes ESO's implementation of TAP and the available services. +- `IVOA TAP standard `_: The official specification from the International Virtual Observatory Alliance. +- `ADQL specification `_: Defines the query language used to interact with TAP services. Query for Raw Data (Generic) ============================ @@ -73,13 +91,6 @@ The following example queries the ``ivoa.ObsCore`` table to find fully calibrate ... SPHERE 3 M H 0.0122 SPHERE -.. tip:: - - For more information about the TAP and how to write ADQL queries, refer to the following resources: - - - `ESO TAP+ documentation `_: Describes ESO's implementation of TAP and the available services. - - `IVOA TAP standard `_: The official specification from the International Virtual Observatory Alliance. - - `ADQL specification `_: Defines the query language used to interact with TAP services. Download Data ============= From fcbbfb7163e7ec72c6bca613649b2c928bf265bd Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Fri, 23 Jan 2026 15:13:03 +0100 Subject: [PATCH 10/18] update cone search --- docs/eso/eso_cone_search.rst | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/docs/eso/eso_cone_search.rst b/docs/eso/eso_cone_search.rst index 722c7ff276..4c0099f46e 100644 --- a/docs/eso/eso_cone_search.rst +++ b/docs/eso/eso_cone_search.rst @@ -3,16 +3,23 @@ Simple Cone Search ****************** +In the simplest case, archive queries can be performed using a positional +(cone) search, which returns all data products within a given angular radius +around a specified sky position. This approach is particularly useful when +searching for reduced (Phase 3) data products around well-known targets or +regions on the sky. + +.. note:: + This is typically done within the ``astroquery`` using the ``query_region`` or ``query_object`` methods. However, these methods are not yet implemented for ``astroquery.eso``. Instead, cone searches can be performed by passing the appropriate parameters to the existing query methods, such as :meth:`~astroquery.eso.EsoClass.query_surveys`, :meth:`~astroquery.eso.EsoClass.query_instrument`, and :meth:`~astroquery.eso.EsoClass.query_main`. + Query for Reduced Data ====================== -This example demonstrates how to perform a basic **cone search** for publicly available **HAWK-I reduced data products** around the Galactic Center (Sgr A\*) using :meth:`~astroquery.eso.EsoClass.query_surveys`. - -Sgr A\* is located at right ascension 266.41683° and declination –29.00781°. We perform a search within a 0.05-degree radius (~3 arcminutes). +This example demonstrates how to perform a basic cone search for publicly available **HAWK-I reduced data products** around the Galactic Center (Sgr A\*) using :meth:`~astroquery.eso.EsoClass.query_surveys`. Sgr A\* is located at right ascension 266.41683° and declination –29.00781°. We perform a search within a 0.05-degree radius (~3 arcminutes). .. doctest-remote-data:: - >> import astropy.units as u # import the astropy.units module + >>> import astropy.units as u >>> ra = 266.41683 *u.deg # degrees >>> dec = -29.00781 *u.deg # degrees @@ -22,8 +29,8 @@ If coordinates are not known, you can use the :class:`~astropy.coordinates.SkyCo .. doctest-remote-data:: - >>> from astropy.coordinates import SkyCoord # import the SkyCoord class from the astropy.coordinates module - >>> import astropy.units as u # import the astropy.units module + >>> from astropy.coordinates import SkyCoord + >>> import astropy.units as u >>> coords = SkyCoord.from_name("Sgr A*") >>> ra = coords.ra @@ -34,14 +41,14 @@ With the defined coordinates and radius, we can now perform the cone search usin .. doctest-remote-data:: - >>> # Cone search >>> table = eso.query_surveys( - ... cone_ra=ra.value, - ... cone_dec=dec.value, - ... cone_radius=radius.to(u.deg).value, - ... column_filters={"instrument_name": "HAWKI"}) + ... cone_ra=ra.value, + ... cone_dec=dec.value, + ... cone_radius=radius.to(u.deg).value, + ... column_filters={"instrument_name": "HAWKI"} + ... ) -**Note**: The ``cone_ra``, ``cone_dec``, and ``cone_radius`` parameters are specified in degrees, but as float values (so we use ``.value`` to extract the float from the ``astropy.units.Quantity``). +Note that ``cone_ra``, ``cone_dec``, and ``cone_radius`` parameters are specified in degrees, but as float values (so we use ``.value`` to extract the float from the ``astropy.units.Quantity``). Query for Raw Data ================== @@ -59,7 +66,8 @@ Cone search for raw data products can be performed using the instrument-specific ... "HAWKI", ... cone_ra=ra, ... cone_dec=dec, - ... cone_radius=radius) + ... cone_radius=radius + ... ) Generic Cone Search ------------------- @@ -73,6 +81,7 @@ Cone search for raw data products can also be performed using the more generic m ... cone_ra=ra, ... cone_dec=dec, ... cone_radius=radius + ... ) Download Data ============= From 5472f215ab890232ea4b6dcd5486e0fab75349b1 Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Fri, 23 Jan 2026 15:17:07 +0100 Subject: [PATCH 11/18] updated header docs --- docs/eso/eso_header_info.rst | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/eso/eso_header_info.rst b/docs/eso/eso_header_info.rst index 0e2ab7fe41..d2e346406b 100644 --- a/docs/eso/eso_header_info.rst +++ b/docs/eso/eso_header_info.rst @@ -1,9 +1,9 @@ -*************************** -Extended Header Information -*************************** +****************** +Header Information +****************** -Only a small subset of the keywords present in the data products can be obtained with :meth:`~astroquery.eso.EsoClass.query_instrument`, :meth:`~astroquery.eso.EsoClass.query_main`, :meth:`~astroquery.eso.EsoClass.query_surveys`, or :meth:`~astroquery.eso.EsoClass.query_tap_service`. +Only a small subset of the keywords present in the data products can be obtained with the :meth:`~astroquery.eso.EsoClass.query_instrument`, :meth:`~astroquery.eso.EsoClass.query_main`, :meth:`~astroquery.eso.EsoClass.query_surveys`, or :meth:`~astroquery.eso.EsoClass.query_tap` methods. There is however a way to get the full primary header of the FITS data products, using :meth:`~astroquery.eso.EsoClass.get_headers`. @@ -14,10 +14,17 @@ There is however a way to get the full primary header of the FITS data products, ... 'object': 'NGC4151', ... 'date_obs': "<='2008-01-01'" ... }, - ... columns=['object', 'date_obs', 'dp_id']) + ... columns=['object', 'date_obs', 'dp_id'] + ... ) + +While the archive query interfaces expose the most commonly used metadata fields, +they do not return the complete FITS headers associated with each data product. +For applications that require access to instrument- or pipeline-specific +keywords, the full primary FITS header can be retrieved directly from the +archive using a dedicated helper method. + +.. doctest-remote-data:: >>> table_headers = eso.get_headers(table["dp_id"]) - >>> len(table_headers.columns) - 336 >>> table_headers DP.ID SIMPLE BITPIX ... HIERARCH ESO OCS EXPO7 FNAME2 HIERARCH ESO OCS EXPO8 FNAME1 HIERARCH ESO OCS EXPO8 FNAME2 ---------------------------- ------ ------ ... --------------------------------- --------------------------------- --------------------------------- @@ -30,4 +37,4 @@ There is however a way to get the full primary header of the FITS data products, MIDI.2007-02-07T07:23:38.695 True 16 ... MIDI.2007-02-07T07:20:06.695.fits MIDI.2007-02-07T07:22:57.000.fits MIDI.2007-02-07T07:23:38.695.fits -As shown above, for each data product ID (``DP.ID``; note that this is equivalent to "dp_id" in ``table``), the full header (336 columns in our case) of the archive FITS file is collected. In the above table ``table_headers``, there are as many rows as in the column ``table['dp_id']``. \ No newline at end of file +As shown above, for each data product ID (``DP.ID``; note that this is equivalent to ``dp_id`` in ``table``), the full header (336 columns in our case) of the archive FITS file is collected. In the above table ``table_headers``, there are as many rows as in the column ``table['dp_id']``. \ No newline at end of file From 2dc93cc24fe2720825ba2ce20174cd337c2ea7ae Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Fri, 23 Jan 2026 15:23:34 +0100 Subject: [PATCH 12/18] update download docs, and "" --- docs/eso/eso.rst | 14 +++++++------- docs/eso/eso_download.rst | 10 +++++----- docs/eso/eso_header_info.rst | 8 ++++---- docs/eso/eso_reduced.rst | 4 +++- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/docs/eso/eso.rst b/docs/eso/eso.rst index e6ffba1730..b7eec90152 100644 --- a/docs/eso/eso.rst +++ b/docs/eso/eso.rst @@ -19,8 +19,8 @@ identifiers (``dp_id``). >>> eso = Eso() >>> table_raw = eso.query_main("MUSE", column_filters={"object": "NGC300"}) >>> table_reduced = eso.query_surveys("MUSE", column_filters={"target_name": "NGC300"}) - >>> eso.retrieve_data(table_raw['dp_id']) - >>> eso.retrieve_data(table_reduced['dp_id']) + >>> eso.retrieve_data(table_raw["dp_id"]) + >>> eso.retrieve_data(table_reduced["dp_id"]) By default, queries are limited to returning a maximum of 1000 rows. This limit can be modified by setting the ``ROW_LIMIT`` attribute. To disable truncation @@ -63,11 +63,11 @@ Getting Started .. doctest-skip:: column_filters = { - 'some_int_column': "< 5", - 'some_float_column_2': ">= 1.23", - 'some_char_column': "like '%John%'", - 'some_generic_column': "in ('mango', 'apple', 'kiwi')", - 'other_generic_column': "between '2024-01-01' and '2024-12-31'" + "some_int_column": "< 5", + "some_float_column_2": ">= 1.23", + "some_char_column": "like '%John%'", + "some_generic_column": "in ('mango', 'apple', 'kiwi')", + "other_generic_column": "between '2024-01-01' and '2024-12-31'" } Please review your queries carefully and update them accordingly to ensure compatibility with the new astroquery versions. diff --git a/docs/eso/eso_download.rst b/docs/eso/eso_download.rst index 0bcc3b1d3a..a4e9b4e9a6 100644 --- a/docs/eso/eso_download.rst +++ b/docs/eso/eso_download.rst @@ -3,8 +3,7 @@ Download Data ************* -Continuing from the query with constraints example, the first two datasets are selected, -using their data product IDs ``dp_id``, and retrieved from the ESO archive. +Assuming you have already performed a query (e.g., using :meth:`~astroquery.eso.EsoClass.query_surveys`, :meth:`~astroquery.eso.EsoClass.query_instrument`, or :meth:`~astroquery.eso.EsoClass.query_main`) and have a resulting table of data products, you can download the actual datasets using the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes as input a list of data product IDs (``dp_id``) obtained from the query results. .. doctest-skip:: @@ -18,6 +17,7 @@ using their data product IDs ``dp_id``, and retrieved from the ESO archive. INFO: Uncompressing file /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:01:51.000.fits.Z [astroquery.eso.core] INFO: Uncompressing file /Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:02:49.000.fits.Z [astroquery.eso.core] INFO: Done! [astroquery.eso.core] + >>> data_files ['/Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:01:51.000.fits', '/Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:02:49.000.fits'] @@ -26,7 +26,7 @@ The file names, returned in data_files, points to the decompressed datasets (without the .Z extension) that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. .. doctest-skip:: - >>> data_files = eso.retrieve_data(table['dp_id'][:2], destination='./eso_data/') + >>> data_files = eso.retrieve_data(table['dp_id'][:2], destination="./eso_data/") By default, if a requested dataset is already found, it is not downloaded again from the archive. To force the retrieval of data that are present in the destination directory, use ``continuation=True`` in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. @@ -35,8 +35,8 @@ When downloading datasets, you can optionally retrieve associated calibration fi Available options: • None (default): Download only the requested science data. - • 'raw': Include raw calibration files associated with each dataset. - • 'processed': Include processed calibration files (i.e., reduced/calibrated). + • ``"raw"``: Include raw calibration files associated with each dataset. + • ``"processed"``: Include processed calibration files (i.e., reduced/calibrated). .. doctest-skip:: diff --git a/docs/eso/eso_header_info.rst b/docs/eso/eso_header_info.rst index d2e346406b..b8fae0be69 100644 --- a/docs/eso/eso_header_info.rst +++ b/docs/eso/eso_header_info.rst @@ -9,12 +9,12 @@ There is however a way to get the full primary header of the FITS data products, .. doctest-remote-data:: - >>> table = eso.query_instrument('midi', + >>> table = eso.query_instrument("midi", ... column_filters={ - ... 'object': 'NGC4151', - ... 'date_obs': "<='2008-01-01'" + ... "object": "NGC4151", + ... "date_obs": "<='2008-01-01'" ... }, - ... columns=['object', 'date_obs', 'dp_id'] + ... columns=["object", "date_obs", "dp_id"] ... ) While the archive query interfaces expose the most commonly used metadata fields, diff --git a/docs/eso/eso_reduced.rst b/docs/eso/eso_reduced.rst index 4ae03fac00..d3b9bb06a3 100644 --- a/docs/eso/eso_reduced.rst +++ b/docs/eso/eso_reduced.rst @@ -98,7 +98,9 @@ You can also query a specific instrument using the same method (e.g., ``HARPS``) .. doctest-remote-data:: >>> table = eso.query_surveys(column_filters={"instrument_name": "HARPS", - ... "target_name": "HD203608"}) + ... "target_name": "HD203608"} + ... ) + >>> table
target_name s_ra s_dec dp_id proposal_id abmaglim access_estsize access_format ... s_xel2 snr strehl t_exptime t_max t_min t_resolution t_xel From 743fc6457bf38fa880e3972fc3e567155879c993 Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Fri, 23 Jan 2026 15:56:38 +0100 Subject: [PATCH 13/18] minor grammar updates --- docs/eso/eso.rst | 6 +++--- docs/eso/eso_apex.rst | 4 ++-- docs/eso/eso_cone_search.rst | 4 ++-- docs/eso/eso_download.rst | 13 +++++++------ docs/eso/eso_header_info.rst | 4 ++-- docs/eso/eso_login.rst | 2 +- docs/eso/eso_raw_general.rst | 31 +++++++++++++++---------------- docs/eso/eso_raw_instrument.rst | 27 +++++++++++++-------------- docs/eso/eso_reduced.rst | 14 +++++++------- docs/eso/eso_tap.rst | 6 +++--- docs/eso/eso_troubleshooting.rst | 4 ++-- 11 files changed, 57 insertions(+), 58 deletions(-) diff --git a/docs/eso/eso.rst b/docs/eso/eso.rst index b7eec90152..35f60f99df 100644 --- a/docs/eso/eso.rst +++ b/docs/eso/eso.rst @@ -9,7 +9,7 @@ Quick Start This quick start example shows how to use the ``astroquery.eso`` interface to query the ESO Science Archive for both raw and reduced `MUSE `_ -observations of the nearby galaxy `NGC 300 `_, +observations of the nearby galaxy `NGC300 `_, and how to download the corresponding data products using their dataset identifiers (``dp_id``). @@ -84,7 +84,7 @@ The `ESO Science Archive Facility (SAF) `_ is one of th Ways to Access the Archive -------------------------- -Access to data follows ESO’s standard data policy: the Principal Investigator (PI) of an observing program has exclusive access to their data during a proprietary period (typically one year), after which the data become publicly available. See the `ESO Data Access Policy `_ and `FAQ `_ for more information. Browsing the archive does not require authentication, but login is necessary to access proprietary datasets. Use of ESO archival data should be acknowledged in all publications. +Access to data follows ESO’s standard data policy: the Principal Investigator (PI) of an observing programme has exclusive access to their data during a proprietary period (typically one year), after which the data become publicly available. See the `ESO Data Access Policy `_ and `FAQ `_ for more information. Browsing the archive does not require authentication, but login is necessary to access proprietary datasets. Use of ESO archival data should be acknowledged in all publications. There are multiple ways to access the archive: @@ -117,4 +117,4 @@ Reference/API ============= .. automodapi:: astroquery.eso - :no-inheritance-diagram: \ No newline at end of file + :no-inheritance-diagram: diff --git a/docs/eso/eso_apex.rst b/docs/eso/eso_apex.rst index 032fe4bb56..a466271d7a 100644 --- a/docs/eso/eso_apex.rst +++ b/docs/eso/eso_apex.rst @@ -139,7 +139,7 @@ We can check the available columns to search in the query. 282296 [astroquery.eso.core] -And now, us :meth:`~astroquery.eso.EsoClass.query_apex_quicklooks` to query for the APEX Quick Look products using the APEX proposal ID (``project_id``) we retrieved from the previous query. +And now, use :meth:`~astroquery.eso.EsoClass.query_apex_quicklooks` to query for the APEX Quick Look products using the APEX proposal ID (``project_id``) we retrieved from the previous query. .. doctest-remote-data:: @@ -165,4 +165,4 @@ Finally, we can download the APEX Quick Look products using the :meth:`~astroque .. doctest-remote-data:: - >>> eso.retrieve_data(table_quicklooks[0]["quicklook_id"]) \ No newline at end of file + >>> eso.retrieve_data(table_quicklooks[0]["quicklook_id"]) diff --git a/docs/eso/eso_cone_search.rst b/docs/eso/eso_cone_search.rst index 4c0099f46e..b6cfc6e183 100644 --- a/docs/eso/eso_cone_search.rst +++ b/docs/eso/eso_cone_search.rst @@ -91,7 +91,7 @@ To download the data returned by the query, you can use the :meth:`~astroquery.e .. doctest-remote-data:: >>> eso.retrieve_data(table["dp_id"]) -The ``data_files`` points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. +The ``data_files`` list points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. .. doctest-skip:: - >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") \ No newline at end of file + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") diff --git a/docs/eso/eso_download.rst b/docs/eso/eso_download.rst index a4e9b4e9a6..29ebb0a10f 100644 --- a/docs/eso/eso_download.rst +++ b/docs/eso/eso_download.rst @@ -22,7 +22,7 @@ Assuming you have already performed a query (e.g., using :meth:`~astroquery.eso. ['/Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:01:51.000.fits', '/Users/foobar/.astropy/cache/astroquery/Eso/MIDI.2007-02-07T07:02:49.000.fits'] -The file names, returned in data_files, points to the decompressed datasets +The file names returned in ``data_files`` point to the decompressed datasets (without the .Z extension) that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. .. doctest-skip:: @@ -31,12 +31,13 @@ The file names, returned in data_files, points to the decompressed datasets By default, if a requested dataset is already found, it is not downloaded again from the archive. To force the retrieval of data that are present in the destination directory, use ``continuation=True`` in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. -When downloading datasets, you can optionally retrieve associated calibration files by using the with_calib argument. This allows you to obtain either raw or processed calibrations in addition to the science files. +When downloading datasets, you can optionally retrieve associated calibration files by using the ``with_calib`` argument. This allows you to obtain either raw or processed calibrations in addition to the science files. Available options: - • None (default): Download only the requested science data. - • ``"raw"``: Include raw calibration files associated with each dataset. - • ``"processed"``: Include processed calibration files (i.e., reduced/calibrated). + +- ``None`` (default): Download only the requested science data. +- ``"raw"``: Include raw calibration files associated with each dataset. +- ``"processed"``: Include processed calibration files (i.e., reduced/calibrated). .. doctest-skip:: @@ -46,4 +47,4 @@ Available options: .. doctest-skip:: >>> # Download science data with processed calibrations - >>> data_files = eso.retrieve_data(table['dp_id'][:2], with_calib="processed") \ No newline at end of file + >>> data_files = eso.retrieve_data(table['dp_id'][:2], with_calib="processed") diff --git a/docs/eso/eso_header_info.rst b/docs/eso/eso_header_info.rst index b8fae0be69..739837b904 100644 --- a/docs/eso/eso_header_info.rst +++ b/docs/eso/eso_header_info.rst @@ -5,7 +5,7 @@ Header Information Only a small subset of the keywords present in the data products can be obtained with the :meth:`~astroquery.eso.EsoClass.query_instrument`, :meth:`~astroquery.eso.EsoClass.query_main`, :meth:`~astroquery.eso.EsoClass.query_surveys`, or :meth:`~astroquery.eso.EsoClass.query_tap` methods. -There is however a way to get the full primary header of the FITS data products, using :meth:`~astroquery.eso.EsoClass.get_headers`. +There is, however, a way to get the full primary header of the FITS data products using :meth:`~astroquery.eso.EsoClass.get_headers`. .. doctest-remote-data:: @@ -37,4 +37,4 @@ archive using a dedicated helper method. MIDI.2007-02-07T07:23:38.695 True 16 ... MIDI.2007-02-07T07:20:06.695.fits MIDI.2007-02-07T07:22:57.000.fits MIDI.2007-02-07T07:23:38.695.fits -As shown above, for each data product ID (``DP.ID``; note that this is equivalent to ``dp_id`` in ``table``), the full header (336 columns in our case) of the archive FITS file is collected. In the above table ``table_headers``, there are as many rows as in the column ``table['dp_id']``. \ No newline at end of file +As shown above, for each data product ID (``DP.ID``; note that this is equivalent to ``dp_id`` in ``table``), the full header (336 columns in our case) of the archive FITS file is collected. In the above table ``table_headers``, there are as many rows as there are entries in the ``table['dp_id']`` column. diff --git a/docs/eso/eso_login.rst b/docs/eso/eso_login.rst index b24afa4eb9..c12f5703ba 100644 --- a/docs/eso/eso_login.rst +++ b/docs/eso/eso_login.rst @@ -3,7 +3,7 @@ Authenticated Access (login) **************************** -Most datasets in the ESO Science Archive are publicly available and can be downloaded anonymously without requiring authentication. However, access to proprietary datasets—such as those under active proprietary periods—is restricted to authorized users (e.g., PIs of observing programs and their designated delegates). These users must authenticate via the `ESO User Portal `_. +Most datasets in the ESO Science Archive are publicly available and can be downloaded anonymously without requiring authentication. However, access to proprietary datasets—such as those under active proprietary periods—is restricted to authorised users (e.g., PIs of observing programmes and their designated delegates). These users must authenticate via the `ESO User Portal `_. Authentication is handled using the :meth:`~astroquery.query.QueryWithLogin.login` method. This command initiates a secure login session via the ESO Single Sign-On (SSO) service. It integrates with the `keyring `_ module to securely store your password in your system’s credential manager. After the first login, you should not need to re-enter your credentials for subsequent sessions on the same machine. diff --git a/docs/eso/eso_raw_general.rst b/docs/eso/eso_raw_general.rst index 20ce881440..441d46704b 100644 --- a/docs/eso/eso_raw_general.rst +++ b/docs/eso/eso_raw_general.rst @@ -71,11 +71,11 @@ We start by inspecting the available columns that can be queried by :meth:`~astr Query with Constraints (Specific Instrument) ============================================ -Now that we know which of the columns are available for queries, we can retrieve all-sky images from the ``APICAM`` instrument, with constraints on the ``LUMINANCE`` filter, on for a single night (i.e. 2019-04-26): +Now that we know which of the columns are available for queries, we can retrieve all-sky images from the ``APICAM`` instrument, with constraints on the ``LUMINANCE`` filter for a single night (i.e. 2019-04-26): .. doctest-remote-data:: - >>> eso.maxrec = -1 # Return all results without truncation + >>> eso.ROW_LIMIT = -1 # Return all results without truncation >>> table = eso.query_main( ... column_filters={ ... "instrument": "APICAM", @@ -113,11 +113,11 @@ Alternatively, if you want to retrieve all available raw data products with the ... "exp_start": "between '2019-04-26' and '2019-04-27'"} ... ) -Another useful example is to query for all raw data for a specific target, such as ``NGC 3627``: +Another useful example is to query for all raw data for a specific target, such as ``NGC3627``: .. doctest-remote-data:: - >>> table = eso.query_main(column_filters={"object": "NGC 3627"}) + >>> table = eso.query_main(column_filters={"object": "NGC3627"}) .. tip:: @@ -125,21 +125,20 @@ Another useful example is to query for all raw data for a specific target, such .. doctest-remote-data:: - table = eso.query_main(column_filters={"object": "NGC 3627"}) + >>> table = eso.query_main(column_filters={"object": "NGC3627"}) Use :meth:`~astroquery.eso.EsoClass.query_instrument` when you want a more **refined, instrument-specific search** (e.g. instrument modes, configurations, or ambient conditions). .. doctest-remote-data:: - column_filters = { - "dp_cat": "SCIENCE", # Science data only - "ins_opt1_name": "HIGH_SENS", # High sensitivity mode - "night_flag": "night", # Nighttime observations only - "moon_illu": "< 0", # No moon (below horizon) - "lst": "between 0 and 6" # Local sidereal time early in the night - } - - table = eso.query_instrument("midi", column_filters=column_filters) + >>> column_filters = { + ... "dp_cat": "SCIENCE", # Science data only + ... "ins_opt1_name": "HIGH_SENS", # High sensitivity mode + ... "night_flag": "night", # Nighttime observations only + ... "moon_illu": "< 0", # No moon (below horizon) + ... "lst": "between 0 and 6" # Local sidereal time early in the night + ... } + >>> table = eso.query_instrument("midi", column_filters=column_filters) Download Data ============= @@ -149,7 +148,7 @@ To download the data returned by the query, you can use the :meth:`~astroquery.e .. doctest-remote-data:: >>> eso.retrieve_data(table["dp_id"]) -The ``data_files`` points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. +The ``data_files`` list points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. .. doctest-skip:: - >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") \ No newline at end of file + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") diff --git a/docs/eso/eso_raw_instrument.rst b/docs/eso/eso_raw_instrument.rst index 09350e83e1..61a9c09933 100644 --- a/docs/eso/eso_raw_instrument.rst +++ b/docs/eso/eso_raw_instrument.rst @@ -6,7 +6,7 @@ Query for Raw Data (Instrument-Specific) .. note:: The `astroquery.eso` module provides several ways to search for data in the ESO Science Archive for raw data. This section focuses on the **instrument-specific query interface** for raw data :meth:`~astroquery.eso.EsoClass.query_instrument`, while in another section we describe **generic queries** using :meth:`~astroquery.eso.EsoClass.query_main`. -In many cases, you will want to query the ESO Archive **for data from a specific instrument**. This is exactly what the :meth:`~astroquery.eso.EsoClass.query_instrument` method is designed for. It allows you to search instrument-specific tables, which expose metadata fields and filters unique to each instrument. Internally, this method queries the corresponding instrument table (e.g ``ist.muse``) via ESO's `TAP service `_. This approach is ideal when you need precise control over your query, such as filtering by instrument configuration, mode, or observational setup. +In many cases, you will want to query the ESO Archive **for data from a specific instrument**. This is exactly what the :meth:`~astroquery.eso.EsoClass.query_instrument` method is designed for. It allows you to search instrument-specific tables, which expose metadata fields and filters unique to each instrument. Internally, this method queries the corresponding instrument table (e.g., ``ist.muse``) via ESO's `TAP service `_. This approach is ideal when you need precise control over your query, such as filtering by instrument configuration, mode, or observational setup. Available Instruments ===================== @@ -75,7 +75,7 @@ The output includes column names, data types, units, and, where applicable, ``xt Query with Constraints ====================== -Once the available query columns have been inspected, you can construct a constrained query to retrieve relevant datasets. For example, suppose you want to retrieve MIDI observations of the target ``NGC 4151`` that were taken between ``2008-01-01`` and ``2009-05-12``. +Once the available query columns have been inspected, you can construct a constrained query to retrieve relevant datasets. For example, suppose you want to retrieve MIDI observations of the target ``NGC4151`` that were taken between ``2008-01-01`` and ``2009-05-12``. The ``column_filters`` dictionary allows you to specify conditions for individual columns, using ADQL-compatible expressions under the hood. In this case, the filters apply to: @@ -115,21 +115,20 @@ The ``columns`` argument controls which fields are returned in the results table .. doctest-remote-data:: - table = eso.query_main(column_filters={"object": "NGC 3627"}) + >>> table = eso.query_main(column_filters={"object": "NGC3627"}) Use ``query_instrument`` when you want a more **refined, instrument-specific search**, applying filters that are only available for a particular instrument (e.g. instrument modes, configurations, or ambient conditions). .. doctest-remote-data:: - column_filters = { - "dp_cat": "SCIENCE", # Science data only - "ins_opt1_name": "HIGH_SENS", # High sensitivity mode - "night_flag": "night", # Nighttime observations only - "moon_illu": "< 0", # No moon (below horizon) - "lst": "between 0 and 6" # Local sidereal time early in the night - } - - table = eso.query_instrument("midi", column_filters=column_filters) + >>> column_filters = { + ... "dp_cat": "SCIENCE", # Science data only + ... "ins_opt1_name": "HIGH_SENS", # High sensitivity mode + ... "night_flag": "night", # Nighttime observations only + ... "moon_illu": "< 0", # No moon (below horizon) + ... "lst": "between 0 and 6" # Local sidereal time early in the night + ... } + >>> table = eso.query_instrument("midi", column_filters=column_filters) Download Data ============= @@ -139,7 +138,7 @@ To download the data returned by the query, you can use the :meth:`~astroquery.e .. doctest-remote-data:: >>> eso.retrieve_data(table["dp_id"]) -The ``data_files`` points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. +The ``data_files`` list points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. .. doctest-skip:: - >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") \ No newline at end of file + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") diff --git a/docs/eso/eso_reduced.rst b/docs/eso/eso_reduced.rst index d3b9bb06a3..5ca5d8215d 100644 --- a/docs/eso/eso_reduced.rst +++ b/docs/eso/eso_reduced.rst @@ -55,7 +55,7 @@ As before, list the possible columns in :meth:`~astroquery.eso.EsoClass.query_su .. note:: Column names may differ between the tables returned by different query - methods. Users are encouraged to inspect closly the available columns and supported + methods. Users are encouraged to inspect closely the available columns and supported filters using the ``help=True`` option. For example, for raw data queries (:meth:`~astroquery.eso.EsoClass.query_main` and :meth:`~astroquery.eso.EsoClass.query_instrument`), the target name is specified @@ -88,12 +88,12 @@ target ``HD203608``. The archive can be queried as follows: The returned table has a ``dp_id`` column, which can be used to retrieve the datasets with :meth:`~astroquery.eso.EsoClass.retrieve_data`: ``eso.retrieve_data(table["dp_id"][0])``. -More details about this method in the following section. +More details about this method are in the following section. Query with Constraints (Specific Instrument) ============================================ -You can also query a specific instrument using the same method (e.g., ``HARPS``). For example, to retrieve **all** available HARPS data products regardless of the associated survey towards ``HD203608`` is given the following query: +You can also query a specific instrument using the same method (e.g., ``HARPS``). For example, to retrieve **all** available HARPS data products regardless of the associated survey towards ``HD203608`` is given by the following query: .. doctest-remote-data:: @@ -115,9 +115,9 @@ You can also query a specific instrument using the same method (e.g., ``HARPS``) .. tip:: - Keep in mind that the definition of a ``survey`` (also referred to as a **collection** in the ESO Science Archive) is not the same as the definition of an **instrument**. The ``instrument_name`` refers to the actual hardware that acquired the data (e.g., ``HARPS``, ``MUSE``), whereas the ``obs_collection`` identifies the scientific program, survey, or processing pipeline associated with the data product. In many cases, survey names match the instrument name (e.g., ``HARPS``, ``MUSE``, ``XSHOOTER``), which typically indicates **products processed and curated by ESO**. However, when the collection name differs (e.g., ``AMBRE``, ``GAIAESO``, ``PHANGS``), it usually denotes **community-contributed data** from large collaborations or specific science teams. + Keep in mind that the definition of a ``survey`` (also referred to as a **collection** in the ESO Science Archive) is not the same as the definition of an **instrument**. The ``instrument_name`` refers to the actual hardware that acquired the data (e.g., ``HARPS``, ``MUSE``), whereas the ``obs_collection`` identifies the scientific programme, survey, or processing pipeline associated with the data product. In many cases, survey names match the instrument name (e.g., ``HARPS``, ``MUSE``, ``XSHOOTER``), which typically indicates **products processed and curated by ESO**. However, when the collection name differs (e.g., ``AMBRE``, ``GAIAESO``, ``PHANGS``), it usually denotes **community-contributed data** from large collaborations or specific science teams. - So, for example, querying for ``eso.query_surveys(column_filters={"instrument_name": "HARPS"})`` will return all products taken with the HARPS instrument, across all programs and collections. In contrast, filtering on ``eso.query_surveys(surveys="HARPS"}`` will return only the `HARPS data reduced by ESO `_. You can inspect the collection for each result via the ``obs_collection`` column in your results table. + So, for example, querying for ``eso.query_surveys(column_filters={"instrument_name": "HARPS"})`` will return all products taken with the HARPS instrument, across all programmes and collections. In contrast, filtering on ``eso.query_surveys(surveys="HARPS")`` will return only the `HARPS data reduced by ESO `_. You can inspect the collection for each result via the ``obs_collection`` column in your results table. Download Data ============= @@ -127,7 +127,7 @@ To download the data returned by the query, you can use the :meth:`~astroquery.e .. doctest-remote-data:: >>> eso.retrieve_data(table["dp_id"]) -The ``data_files`` points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. +The ``data_files`` list points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. .. doctest-skip:: - >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") \ No newline at end of file + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") diff --git a/docs/eso/eso_tap.rst b/docs/eso/eso_tap.rst index 5afa0db39b..3fe4e72e0b 100644 --- a/docs/eso/eso_tap.rst +++ b/docs/eso/eso_tap.rst @@ -59,7 +59,7 @@ The following example demonstrates how to query the ``ist.muse`` table for raw d ... AND moon_illu < 30 ... AND dimm_fwhm_avg < 1.0 ... AND exptime > 100 - ... AND lst between 0 and 6 + ... AND lst between 0 and 6 ... """ >>> result = eso.query_tap_service(query) >>> result @@ -100,7 +100,7 @@ To download the data returned by the query, you can use the :meth:`~astroquery.e .. doctest-remote-data:: >>> eso.retrieve_data(table["dp_id"]) -The ``data_files`` points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. +The ``data_files`` list points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. .. doctest-skip:: - >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") \ No newline at end of file + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") diff --git a/docs/eso/eso_troubleshooting.rst b/docs/eso/eso_troubleshooting.rst index c5ea0bbd76..7a54f98320 100644 --- a/docs/eso/eso_troubleshooting.rst +++ b/docs/eso/eso_troubleshooting.rst @@ -20,7 +20,7 @@ This will remove all cached ESO queries and downloaded metadata files. Data prod Using Correct ``column_filters`` ================================ -If your query fails or silently returns no results, it might be because you're using column names that are **accepted in the ESO web interface (WDB)** but **not supported by the TAP/ADQL interface** that now is used within ``astroquery.eso``. A common case involves using ``stime`` and ``etime``, which are not valid TAP fields. Instead, use ``exp_start``, the TAP-compliant column representing the observation start time. This field supports SQL-style date filtering. +If your query fails or silently returns no results, it might be because you're using column names that are **accepted in the ESO web interface (WDB)** but **not supported by the TAP/ADQL interface** that is now used within ``astroquery.eso``. A common case involves using ``stime`` and ``etime``, which are not valid TAP fields. Instead, use ``exp_start``, the TAP-compliant column representing the observation start time. This field supports SQL-style date filtering. Below are examples of invalid filter usage and their corrected TAP-compatible versions. @@ -74,4 +74,4 @@ Filtering with only an end date # ✅ Correct column_filters = { "exp_start": "< '2024-12-31'" - } \ No newline at end of file + } From c82bfc176795a28f0b94f369508da1f278eb2db4 Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Thu, 29 Jan 2026 14:48:21 +0100 Subject: [PATCH 14/18] minor updates --- docs/eso/eso.rst | 13 +++++++++++-- docs/eso/eso_login.rst | 2 +- docs/eso/eso_tap.rst | 6 +++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/eso/eso.rst b/docs/eso/eso.rst index 35f60f99df..bfa7b7d42f 100644 --- a/docs/eso/eso.rst +++ b/docs/eso/eso.rst @@ -17,8 +17,17 @@ identifiers (``dp_id``). >>> from astroquery.eso import Eso >>> eso = Eso() - >>> table_raw = eso.query_main("MUSE", column_filters={"object": "NGC300"}) - >>> table_reduced = eso.query_surveys("MUSE", column_filters={"target_name": "NGC300"}) + + >>> from astropy.coordinates import SkyCoord + >>> import astropy.units as u + >>> coords = SkyCoord.from_name("NGC 300") + >>> ra = coords.ra.value + >>> dec = coords.dec.value + >>> r = (30*u.arcsec).to(u.deg).value + + >>> table_raw = eso.query_main("MUSE", cone_ra=ra, cone_dec=dec, cone_radius=r) + >>> table_reduced = eso.query_surveys("MUSE", cone_ra=ra, cone_dec=dec, cone_radius=r) + >>> eso.retrieve_data(table_raw["dp_id"]) >>> eso.retrieve_data(table_reduced["dp_id"]) diff --git a/docs/eso/eso_login.rst b/docs/eso/eso_login.rst index c12f5703ba..7180d53c60 100644 --- a/docs/eso/eso_login.rst +++ b/docs/eso/eso_login.rst @@ -1,6 +1,6 @@ **************************** -Authenticated Access (login) +Authenticated Access (Login) **************************** Most datasets in the ESO Science Archive are publicly available and can be downloaded anonymously without requiring authentication. However, access to proprietary datasets—such as those under active proprietary periods—is restricted to authorised users (e.g., PIs of observing programmes and their designated delegates). These users must authenticate via the `ESO User Portal `_. diff --git a/docs/eso/eso_tap.rst b/docs/eso/eso_tap.rst index 3fe4e72e0b..22284907a8 100644 --- a/docs/eso/eso_tap.rst +++ b/docs/eso/eso_tap.rst @@ -42,7 +42,7 @@ The following example demonstrates how to query the ``dbo.raw`` table for raw da ... WHERE object = 'NGC300' ... AND instrument = 'muse' ... """ - >>> result = eso.query_tap_service(query) + >>> result = eso.query_tap(query) >>> result Query for Raw Data (Instrument-Specific) @@ -61,7 +61,7 @@ The following example demonstrates how to query the ``ist.muse`` table for raw d ... AND exptime > 100 ... AND lst between 0 and 6 ... """ - >>> result = eso.query_tap_service(query) + >>> result = eso.query_tap(query) >>> result Query for Reduced Data Products @@ -79,7 +79,7 @@ The following example queries the ``ivoa.ObsCore`` table to find fully calibrate ... AND multi_ob = 'M' ... AND s_pixel_scale < 0.2 ... """ - >>> result = eso.query_tap_service(query) + >>> result = eso.query_tap(query) >>> result
obs_collection calib_level multi_ob filter s_pixel_scale instrument_name From 0e5c5d2fa3a883c8e22d7b24c6608899e38ec411 Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Tue, 3 Feb 2026 12:04:39 +0100 Subject: [PATCH 15/18] updates after Alberto's feedback --- docs/eso/eso.rst | 28 ++- docs/eso/eso_apex.rst | 7 +- docs/eso/eso_cone_search.rst | 12 +- docs/eso/eso_header_info.rst | 28 ++- docs/eso/eso_login.rst | 41 ++-- docs/eso/eso_raw_general.rst | 54 +++-- docs/eso/eso_raw_instrument.rst | 42 ++-- docs/eso/eso_reduced.rst | 90 +++++--- docs/eso/eso_tap.rst | 379 ++++++++++++++++++++++++++++++-- 9 files changed, 548 insertions(+), 133 deletions(-) diff --git a/docs/eso/eso.rst b/docs/eso/eso.rst index bfa7b7d42f..510ec876d9 100644 --- a/docs/eso/eso.rst +++ b/docs/eso/eso.rst @@ -7,11 +7,21 @@ ESO Queries (`astroquery.eso`) Quick Start =========== +.. note:: + + Check if this is a version of ``astroquery.eso`` (0.4.13 or later) that includes TAP support (see warning below). + + .. doctest-skip:: + + >>> import astroquery + >>> astroquery.__version__ >= '0.4.13' + True + This quick start example shows how to use the ``astroquery.eso`` interface to query the ESO -Science Archive for both raw and reduced `MUSE `_ -observations of the nearby galaxy `NGC300 `_, -and how to download the corresponding data products using their dataset -identifiers (``dp_id``). +Science Archive for both raw and reduced observations. +The we can perform cone searches for `ESPRESSO `_ +spectra towards the star `HD 37903 `_, +and how to download the corresponding data products. .. doctest-skip:: @@ -20,20 +30,20 @@ identifiers (``dp_id``). >>> from astropy.coordinates import SkyCoord >>> import astropy.units as u - >>> coords = SkyCoord.from_name("NGC 300") + >>> coords = SkyCoord.from_name("HD 37903") >>> ra = coords.ra.value >>> dec = coords.dec.value - >>> r = (30*u.arcsec).to(u.deg).value + >>> r = (1*u.arcsec).to(u.deg).value - >>> table_raw = eso.query_main("MUSE", cone_ra=ra, cone_dec=dec, cone_radius=r) - >>> table_reduced = eso.query_surveys("MUSE", cone_ra=ra, cone_dec=dec, cone_radius=r) + >>> table_raw = eso.query_main("ESPRESSO", cone_ra=ra, cone_dec=dec, cone_radius=r) + >>> table_reduced = eso.query_surveys("ESPRESSO", cone_ra=ra, cone_dec=dec, cone_radius=r) >>> eso.retrieve_data(table_raw["dp_id"]) >>> eso.retrieve_data(table_reduced["dp_id"]) By default, queries are limited to returning a maximum of 1000 rows. This limit can be modified by setting the ``ROW_LIMIT`` attribute. To disable truncation -and return all matching results, set: +and return all matching results (up to TAP limit of 15,000,000), set: .. doctest-skip:: diff --git a/docs/eso/eso_apex.rst b/docs/eso/eso_apex.rst index a466271d7a..59bf0b4e31 100644 --- a/docs/eso/eso_apex.rst +++ b/docs/eso/eso_apex.rst @@ -49,7 +49,12 @@ We first query for the reduced data from the **ALCOHOLS** survey, and retrieve t >>> print("Warning: Multiple proposal IDs found...") >>> # Extract the first part of the proposal ID before any parentheses (i.e. the run ID) - >>> proposal_id = proposal_id.split('(')[0] + >>> def proposal_remove_run(proposal_id): + ... program = ".".join(proposal_id.split('.',2)[:2]) + ... program = ".".join(program.split('(')[:1]) + ... return program + + >>> proposal_id = proposal_remove_run(proposal_id) >>> print(f"Proposal ID: {proposal_id}") Proposal ID: 094.C-0935 diff --git a/docs/eso/eso_cone_search.rst b/docs/eso/eso_cone_search.rst index b6cfc6e183..dfed119d50 100644 --- a/docs/eso/eso_cone_search.rst +++ b/docs/eso/eso_cone_search.rst @@ -64,9 +64,9 @@ Cone search for raw data products can be performed using the instrument-specific >>> table = eso.query_instrument( ... "HAWKI", - ... cone_ra=ra, - ... cone_dec=dec, - ... cone_radius=radius + ... cone_ra=ra.value, + ... cone_dec=dec.value, + ... cone_radius=radius.to(u.deg).value ... ) Generic Cone Search @@ -78,9 +78,9 @@ Cone search for raw data products can also be performed using the more generic m >>> table = eso.query_main( ... "HAWKI", - ... cone_ra=ra, - ... cone_dec=dec, - ... cone_radius=radius + ... cone_ra=ra.value, + ... cone_dec=dec.value, + ... cone_radius=radius.to(u.deg).value ... ) Download Data diff --git a/docs/eso/eso_header_info.rst b/docs/eso/eso_header_info.rst index 739837b904..47589c7931 100644 --- a/docs/eso/eso_header_info.rst +++ b/docs/eso/eso_header_info.rst @@ -11,8 +11,7 @@ There is, however, a way to get the full primary header of the FITS data product >>> table = eso.query_instrument("midi", ... column_filters={ - ... "object": "NGC4151", - ... "date_obs": "<='2008-01-01'" + ... "date_obs": "between '2008-01-01' and '2008-01-02'" ... }, ... columns=["object", "date_obs", "dp_id"] ... ) @@ -24,17 +23,16 @@ keywords, the full primary FITS header can be retrieved directly from the archive using a dedicated helper method. .. doctest-remote-data:: - >>> table_headers = eso.get_headers(table["dp_id"]) + >>> table_headers = eso.get_headers(table["dp_id"][:5]) >>> table_headers - DP.ID SIMPLE BITPIX ... HIERARCH ESO OCS EXPO7 FNAME2 HIERARCH ESO OCS EXPO8 FNAME1 HIERARCH ESO OCS EXPO8 FNAME2 - ---------------------------- ------ ------ ... --------------------------------- --------------------------------- --------------------------------- - MIDI.2007-02-07T07:01:51.000 True 16 ... - MIDI.2007-02-07T07:02:49.000 True 16 ... - MIDI.2007-02-07T07:03:30.695 True 16 ... - ... - MIDI.2007-02-07T07:20:06.695 True 16 ... MIDI.2007-02-07T07:20:06.695.fits - MIDI.2007-02-07T07:22:57.000 True 16 ... MIDI.2007-02-07T07:20:06.695.fits MIDI.2007-02-07T07:22:57.000.fits - MIDI.2007-02-07T07:23:38.695 True 16 ... MIDI.2007-02-07T07:20:06.695.fits MIDI.2007-02-07T07:22:57.000.fits MIDI.2007-02-07T07:23:38.695.fits - - -As shown above, for each data product ID (``DP.ID``; note that this is equivalent to ``dp_id`` in ``table``), the full header (336 columns in our case) of the archive FITS file is collected. In the above table ``table_headers``, there are as many rows as there are entries in the ``table['dp_id']`` column. +
+ DP.ID SIMPLE BITPIX ... HIERARCH ESO OCS EXPO8 FNAME1 HIERARCH ESO OCS EXPO9 FNAME1 + str28 bool int64 ... str33 str33 + ---------------------------- ------ ------ ... --------------------------------- --------------------------------- + MIDI.2008-01-01T08:29:16.388 True 16 ... + MIDI.2008-01-01T08:39:38.000 True 16 ... + MIDI.2008-01-01T08:47:35.000 True 16 ... + MIDI.2008-01-01T08:52:00.623 True 8 ... MIDI.2008-01-01T08:51:42.000.fits MIDI.2008-01-01T08:52:00.623.fits + MIDI.2008-01-01T09:00:10.000 True 16 ... MIDI.2008-01-01T09:00:10.000.fits + +As shown above, for each data product ID (``DP.ID``; note that this is equivalent to ``dp_id`` in ``table``), the full primary header (336 columns in our case) of the archive FITS file is collected. In the above table ``table_headers``, there are as many rows as there are entries in the ``table['dp_id']`` column. diff --git a/docs/eso/eso_login.rst b/docs/eso/eso_login.rst index 7180d53c60..16cca4140f 100644 --- a/docs/eso/eso_login.rst +++ b/docs/eso/eso_login.rst @@ -17,46 +17,58 @@ Login Examples The following examples show typical login and data retrieval workflows: +**First example:** ``"TEST"`` is not a valid username, it will fail: + .. doctest-skip:: - >>> # First example: TEST is not a valid username, it will fail >>> eso.login(username="TEST") # doctest: +SKIP WARNING: No password was found in the keychain for the provided username. [astroquery.query] TEST, enter your password: - INFO: Authenticating TEST on https://www.eso.org/sso ... [astroquery.eso.core] + INFO: Authenticating TEST on 'www.eso.org' ... [astroquery.eso.core] ERROR: Authentication failed! [astroquery.eso.core] +**Second example:** pretend ``"ICONDOR"`` is a valid username: + .. doctest-skip:: - >>> # Second example: pretend ICONDOR is a valid username >>> eso.login(username="ICONDOR", store_password=True) # doctest: +SKIP WARNING: No password was found in the keychain for the provided username. [astroquery.query] ICONDOR, enter your password: - INFO: Authenticating ICONDOR on https://www.eso.org/sso ... [astroquery.eso.core] + INFO: Authenticating ICONDOR on 'www.eso.org' ... [astroquery.eso.core] INFO: Authentication successful! [astroquery.eso.core] +After the first login, your password has been stored: + .. doctest-skip:: - >>> # After the first login, your password has been stored >>> eso.login(username="ICONDOR") # doctest: +SKIP - INFO: Authenticating ICONDOR on https://www.eso.org/sso ... [astroquery.eso.core] + INFO: Authenticating ICONDOR on 'www.eso.org' ... [astroquery.eso.core] INFO: Authentication successful! [astroquery.eso.core] -.. doctest-skip:: - >>> # Successful download of a public file (with or without login) - >>> eso.retrieve_data("AMBER.2006-03-14T07:40:19.830") # doctest: +SKIP - INFO: Downloading file 1/1 https://dataportal.eso.org/dataPortal/file/AMBER.2006-03-14T07:40:19.830 - INFO: Successfully downloaded dataset AMBER.2006-03-14T07:40:19.830 +Successful download of a public file (with or without login): .. doctest-skip:: - >>> # Access denied to a restricted-access file (as anonymous user or as authenticated but not authorised user) >>> eso.retrieve_data("ADP.2023-03-02T01:01:24.355") # doctest: +SKIP - INFO: Downloading file 1/1 https://dataportal.eso.org/dataPortal/file/ADP.2023-03-02T01:01:24.355 - ERROR: Access denied to https://dataportal.eso.org/dataPortal/file/ADP.2023-03-02T01:01:24.355 + INFO: Downloading datasets ... [astroquery.eso.core] + INFO: Downloading 1 files ... [astroquery.eso.core] + INFO: Downloading file 1/1 https://dataportal.eso.org/dataPortal/file/ADP.2023-03-02T01:01:24.355 to ... [astroquery.eso.core] + INFO: Successfully downloaded dataset ADP.2023-03-02T01:01:24.355 to ../ADP.2023-03-02T01:01:24.355.fits [astroquery.eso.core] + INFO: Done! [astroquery.eso.core] + +Access denied to a restricted-access file (as anonymous user or as authenticated but not authorised user): + +.. doctest-skip:: + + >>> eso.retrieve_data("FORS2.2010-10-23T23:38:41.912") # doctest: +SKIP + INFO: Downloading datasets ... [astroquery.eso.core] + INFO: Downloading 1 files ... [astroquery.eso.core] + INFO: Downloading file 1/1 https://dataportal.eso.org/dataPortal/file/FORS2.2010-10-23T23:38:41.912 to ... [astroquery.eso.core] + ERROR: Access denied to https://dataportal.eso.org/dataPortal/file/FORS2.2010-10-23T23:38:41.912 [astroquery.eso.core] + INFO: Done! [astroquery.eso.core] Password Storage ================ @@ -73,6 +85,7 @@ As shown above, your password can be stored securely using the `keyring >> import keyring >>> keyring.delete_password("astroquery:www.eso.org", "your_username") Automatic Login diff --git a/docs/eso/eso_raw_general.rst b/docs/eso/eso_raw_general.rst index 441d46704b..1163163bf3 100644 --- a/docs/eso/eso_raw_general.rst +++ b/docs/eso/eso_raw_general.rst @@ -6,7 +6,7 @@ Query for Raw Data (Generic) .. tip:: The `astroquery.eso` module provides several ways to search for data in the ESO Science Archive for raw data. This section focuses on the **generic query interface** for raw data :meth:`~astroquery.eso.EsoClass.query_main`, while in another section we describe **instrument-specific queries** using :meth:`~astroquery.eso.EsoClass.query_instrument`. Functionally, :meth:`~astroquery.eso.EsoClass.query_main` works the same way as :meth:`~astroquery.eso.EsoClass.query_instrument`, except you don’t need to specify an instrument table name directly. You can still apply column filters, control the columns returned, and limit result counts. -You may want to query the ESO Archive **without targeting a specific instrument**. This is exactly what the :meth:`~astroquery.eso.EsoClass.query_main` method is designed for. It allows access to the **global raw data table**, which combines metadata across all instruments. Internally, this method queries the ``dbo.raw`` table via ESO's `TAP service `_, which you could also access directly using ADQL with a simple statement like: +You may want to query the ESO Archive **without targeting a specific instrument**. This is exactly what the :meth:`~astroquery.eso.EsoClass.query_main` method is designed for. It allows access to the **global raw data table**, which combines metadata across all instruments. Internally, this method queries the ``dbo.raw`` table via ESO's `TAP service `_ (or using the :meth:`~astroquery.eso.EsoClass.query_tap` function), which you could also access directly using ADQL with a simple statement like: .. doctest-skip:: SELECT * FROM dbo.raw @@ -32,7 +32,7 @@ The list of all supported instruments can be retrieved using the :meth:`~astroqu This list corresponds to the instruments currently available for programmatic raw data queries in the ESO archive. -The list is dynamically generated by querying the archive’s internal TAP service. It can also be reproduced directly by executing the following ADQL query on the `ESO TAP interface `_: +The list is dynamically generated by querying the archive’s internal TAP service. It can also be reproduced directly by executing the following ADQL query on the `ESO TAP interface `_ or using the :meth:`~astroquery.eso.EsoClass.query_tap` function: .. doctest-skip:: @@ -48,6 +48,14 @@ Available Query Constraints We start by inspecting the available columns that can be queried by :meth:`~astroquery.eso.EsoClass.query_main` with the ``help=True`` keyword. This will return a list of all columns available, along with their data types, units, and any additional metadata such as ``xtype`` information. +The output includes column names, data types, units, and, where applicable, +``xtype`` information to indicate more specific column content. The ``xtype`` +attribute defines how a string-valued column should be interpreted during +querying. For example, a column with datatype ``char`` may represent a standard +string (``xtype = null``), a timestamp (``xtype = timestamp``), or a sky region +footprint usable in ADQL spatial queries (``xtype = adql:REGION``), with this +interpretation reflected in the ``xtype`` field. + .. doctest-remote-data:: >>> eso.query_main(help=True) @@ -71,7 +79,7 @@ We start by inspecting the available columns that can be queried by :meth:`~astr Query with Constraints (Specific Instrument) ============================================ -Now that we know which of the columns are available for queries, we can retrieve all-sky images from the ``APICAM`` instrument, with constraints on the ``LUMINANCE`` filter for a single night (i.e. 2019-04-26): +Now that we know which of the columns are available for queries, we can retrieve all-sky images from the ``APICAM`` instrument for a single night (i.e. 2019-04-26): .. doctest-remote-data:: @@ -79,7 +87,6 @@ Now that we know which of the columns are available for queries, we can retrieve >>> table = eso.query_main( ... column_filters={ ... "instrument": "APICAM", - ... "filter_path": "LUMINANCE", ... "exp_start": "between '2019-04-26' and '2019-04-27'"} ... ) >>> print(len(table)) @@ -100,34 +107,43 @@ Now that we know which of the columns are available for queries, we can retrieve ALL SKY 143.56345694 -24.53804388 2019-04-26T23:57:59.000 60.A-9008(A) -Query with Constraints (All Instruments) -======================================== - -Alternatively, if you want to retrieve all available raw data products with the ``LUMINANCE`` filter, on a single night (i.e. 2019-04-26) without specifying the instrument, you can use: +Suppose that we have a large query, it may be useful to limit the columns returned in the results table. +This can be done using the ``columns`` argument. For example, to return only the target name, right ascension, declination, observation date, and program ID: .. doctest-remote-data:: >>> table = eso.query_main( ... column_filters={ - ... "filter_path": "LUMINANCE", - ... "exp_start": "between '2019-04-26' and '2019-04-27'"} + ... "instrument": "APICAM", + ... "exp_start": "between '2019-04-26' and '2019-04-27'"}, + ... columns=["object", "ra", "dec", "date_obs", "prog_id"] ... ) + >>> table +
+ object ra dec date_obs prog_id + deg deg + object float64 float64 object object + ------- ------------ ------------ ----------------------- ------------ + ALL SKY 145.29212694 -24.53624194 2019-04-26T00:08:49.000 60.A-9008(A) + ALL SKY 145.92251305 -24.53560305 2019-04-26T00:11:20.000 60.A-9008(A) + ALL SKY 146.55707 -24.53497111 2019-04-26T00:13:52.000 60.A-9008(A) + ... + ALL SKY 143.56345694 -24.53804388 2019-04-26T23:57:59.000 60.A-9008(A) -Another useful example is to query for all raw data for a specific target, such as ``NGC3627``: +Query with Constraints (All Instruments) +======================================== + +Alternatively, if you want to retrieve all available raw data products for all instruments observed on a specfic day. .. doctest-remote-data:: - >>> table = eso.query_main(column_filters={"object": "NGC3627"}) + >>> table = eso.query_main(column_filters={ + ... "exp_start": "between '2019-04-26' and '2019-04-27'" + ... }) .. tip:: - Use :meth:`~astroquery.eso.EsoClass.query_main` when you want to search **across all instruments**, for example to retrieve all observations of a specific source regardless of the instrument used. - - .. doctest-remote-data:: - - >>> table = eso.query_main(column_filters={"object": "NGC3627"}) - - Use :meth:`~astroquery.eso.EsoClass.query_instrument` when you want a more **refined, instrument-specific search** (e.g. instrument modes, configurations, or ambient conditions). + Use :meth:`~astroquery.eso.EsoClass.query_main` when you want to search **across all instruments**. Use :meth:`~astroquery.eso.EsoClass.query_instrument` when you want a more **refined, instrument-specific search** (e.g. instrument modes, configurations, or ambient conditions). .. doctest-remote-data:: diff --git a/docs/eso/eso_raw_instrument.rst b/docs/eso/eso_raw_instrument.rst index 61a9c09933..1f07782d33 100644 --- a/docs/eso/eso_raw_instrument.rst +++ b/docs/eso/eso_raw_instrument.rst @@ -4,7 +4,7 @@ Query for Raw Data (Instrument-Specific) **************************************** .. note:: - The `astroquery.eso` module provides several ways to search for data in the ESO Science Archive for raw data. This section focuses on the **instrument-specific query interface** for raw data :meth:`~astroquery.eso.EsoClass.query_instrument`, while in another section we describe **generic queries** using :meth:`~astroquery.eso.EsoClass.query_main`. + The `astroquery.eso` module provides several ways to search for raw data in the ESO Science Archive. This section focuses on the **instrument-specific query interface** for raw data :meth:`~astroquery.eso.EsoClass.query_instrument`, while in another section we describe **generic queries** using :meth:`~astroquery.eso.EsoClass.query_main`. In many cases, you will want to query the ESO Archive **for data from a specific instrument**. This is exactly what the :meth:`~astroquery.eso.EsoClass.query_instrument` method is designed for. It allows you to search instrument-specific tables, which expose metadata fields and filters unique to each instrument. Internally, this method queries the corresponding instrument table (e.g., ``ist.muse``) via ESO's `TAP service `_. This approach is ideal when you need precise control over your query, such as filtering by instrument configuration, mode, or observational setup. @@ -29,7 +29,7 @@ The list of all supported instruments can be retrieved using the :meth:`~astroqu This list corresponds to the instruments currently available for programmatic raw data queries in the ESO archive. -The list is dynamically generated by querying the archive’s internal TAP service. It can also be reproduced directly by executing the following ADQL query on the `ESO TAP interface `_: +The list is dynamically generated by querying the archive’s internal TAP service. It can also be reproduced directly by executing the following ADQL query on the `ESO TAP interface `_ or using the :meth:`~astroquery.eso.EsoClass.query_tap` function: .. doctest-skip:: @@ -45,7 +45,13 @@ Available Query Constraints Once an instrument is selected — for example, ``midi`` — you can inspect the available queryable columns using the ``help=True`` keyword in the :meth:`~astroquery.eso.EsoClass.query_instrument` method. This is a useful first step to understand what metadata is available and how to structure your query. -The output includes column names, data types, units, and, where applicable, ``xtype`` information to indicate more specific column content. For example, a column with datatype ``char`` may represent a timestamp or a sky region, which is reflected in the ``xtype`` field (e.g., ``timestamp`` or ``adql:REGION``). +The output includes column names, data types, units, and, where applicable, +``xtype`` information to indicate more specific column content. The ``xtype`` +attribute defines how a string-valued column should be interpreted during +querying. For example, a column with datatype ``char`` may represent a standard +string (``xtype = null``), a timestamp (``xtype = timestamp``), or a sky region +footprint usable in ADQL spatial queries (``xtype = adql:REGION``), with this +interpretation reflected in the ``xtype`` field. .. doctest-remote-data:: @@ -75,7 +81,7 @@ The output includes column names, data types, units, and, where applicable, ``xt Query with Constraints ====================== -Once the available query columns have been inspected, you can construct a constrained query to retrieve relevant datasets. For example, suppose you want to retrieve MIDI observations of the target ``NGC4151`` that were taken between ``2008-01-01`` and ``2009-05-12``. +Once the available query columns have been inspected, you can construct a constrained query to retrieve relevant datasets. For example, suppose you want to retrieve MIDI observations that were taken between ``2008-01-01`` and ``2009-05-12``. The ``column_filters`` dictionary allows you to specify conditions for individual columns, using ADQL-compatible expressions under the hood. In this case, the filters apply to: @@ -95,29 +101,24 @@ The ``columns`` argument controls which fields are returned in the results table >>> table = eso.query_instrument( ... instrument="midi", ... column_filters={ - ... "object":"NGC4151", ... "exp_start": "between '2008-01-01' and '2009-05-12'"}, ... columns=["object", "date_obs"] ... ) >>> table -
- object date_obs - ------- ----------------------- - NGC4151 2008-04-22T02:07:50.154 - NGC4151 2008-04-22T02:08:20.345 - NGC4151 2008-04-22T02:09:47.846 - ... - NGC4151 2009-05-11T01:42:08.432 +
+ object date_obs + object object + ------------------------------ ----------------------- + WAVE,SPECTEMPL 2008-01-03T08:57:45.000 + OTHER 2008-01-03T08:58:07.803 + WAVE,SPECTEMPL 2008-01-03T08:59:37.000 + WAVE,SPECTEMPL 2008-01-03T09:00:38.000 + ... ... + PHOTOMETRY,OBJECT,SCIPHOT 2008-01-12T02:23:16.606 .. tip:: - Use :meth:`~astroquery.eso.EsoClass.query_main` when you want to search **across all instruments**, for example to retrieve all observations of a specific source regardless of the instrument used. - - .. doctest-remote-data:: - - >>> table = eso.query_main(column_filters={"object": "NGC3627"}) - - Use ``query_instrument`` when you want a more **refined, instrument-specific search**, applying filters that are only available for a particular instrument (e.g. instrument modes, configurations, or ambient conditions). + Use :meth:`~astroquery.eso.EsoClass.query_main` when you want to search **across all instruments**. Use :meth:`~astroquery.eso.EsoClass.query_instrument` when you want a more **refined, instrument-specific search** (e.g. instrument modes, configurations, or ambient conditions). .. doctest-remote-data:: @@ -130,6 +131,7 @@ The ``columns`` argument controls which fields are returned in the results table ... } >>> table = eso.query_instrument("midi", column_filters=column_filters) + Download Data ============= diff --git a/docs/eso/eso_reduced.rst b/docs/eso/eso_reduced.rst index 5ca5d8215d..69788f3b3e 100644 --- a/docs/eso/eso_reduced.rst +++ b/docs/eso/eso_reduced.rst @@ -66,52 +66,84 @@ As before, list the possible columns in :meth:`~astroquery.eso.EsoClass.query_su Query with Constraints (Specific Survey) ======================================== -Let's assume that we work with the `HARPS survey `_, and that we are interested in -target ``HD203608``. The archive can be queried as follows: +Let's assume that we work with the `HARPS survey `_. The archive can be queried as follows: .. doctest-remote-data:: - >>> table = eso.query_surveys(surveys="HARPS", - ... column_filters= {"target_name": "HD203608"} - ... ) + >>> table = eso.query_surveys(surveys="HARPS") >>> table
- target_name s_ra s_dec dp_id proposal_id abmaglim access_estsize ... snr strehl t_exptime t_max t_min t_resolution t_xel - deg deg mag kbyte ... s d d s - object float64 float64 object object float64 int64 ... float64 float64 float64 float64 float64 float64 int64 - ----------- --------- --------- --------------------------- ------------- -------- -------------- ... ------- ------- --------- -------------- -------------- ------------ ----- - HD203608 321.61455 -65.36429 ADP.2014-09-16T11:03:30.940 077.D-0720(A) -- 5261 ... 60.9 -- 33.0 53956.24265204 53956.24227009 33.00048 -- - HD203608 321.61761 -65.36485 ADP.2014-09-16T11:03:31.020 077.D-0720(A) -- 5261 ... 87.0 -- 32.999 53953.36835125 53953.36796931 32.999616 -- - HD203608 321.60594 -65.36528 ADP.2014-09-16T11:03:31.067 077.D-0720(A) -- 5261 ... 73.9 -- 33.0 53956.15534682 53956.15496487 33.00048 -- - ... - HD203608 321.61113 -65.37211 ADP.2014-09-16T11:05:14.863 077.D-0720(A) -- 5261 ... 95.2 -- 32.999 53954.99642615 53954.99604421 32.999616 -- + target_name s_ra s_dec dp_id proposal_id abmaglim access_estsize ... snr strehl t_exptime t_max t_min t_resolution t_xel + deg deg mag kbyte ... s d d s + object float64 float64 object object float64 int64 ... float64 float64 float64 float64 float64 float64 int64 + ------------ --------- --------- --------------------------- ------------- -------- -------------- ... ------- ------- --------- -------------- -------------- ------------ ----- + HD203608 321.61455 -65.36429 ADP.2014-09-16T11:03:30.940 077.D-0720(A) -- 5261 ... 60.9 -- 33.0 53956.24265204 53956.24227009 33.00048 -- + HD114613 198.0129 -37.80367 ADP.2014-09-16T11:03:30.947 072.C-0488(E) -- 5261 ... 267.2 -- 120.002 53765.36393677 53765.36254786 120.001824 -- + HIP5158 16.50838 -22.45455 ADP.2014-09-16T11:03:30.973 072.C-0488(E) -- 5261 ... 39.3 -- 711.599 53946.33177006 53946.32353395 711.599904 -- + ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... + HD203608 321.60939 -65.36484 ADP.2014-09-16T11:03:45.713 077.D-0720(A) -- 5261 ... 45.0 -- 33.0 53955.11202646 53955.11164451 33.00048 -- + +Suppose we want both `HARPS survey `_ and `NIRPS survey `_ data products. + +.. doctest-remote-data:: + + >>> table = eso.query_surveys(surveys=["HARPS", "NIRPS"]) + >>> table +
+ target_name s_ra s_dec dp_id proposal_id abmaglim access_estsize ... snr strehl t_exptime t_max t_min t_resolution t_xel + deg deg mag kbyte ... s d d s + object float64 float64 object object float64 int64 ... float64 float64 float64 float64 float64 float64 int64 + ------------ --------- --------- --------------------------- ------------- -------- -------------- ... ------- ------- --------- -------------- -------------- ------------ ----- + HD203608 321.61455 -65.36429 ADP.2014-09-16T11:03:30.940 077.D-0720(A) -- 5261 ... 60.9 -- 33.0 53956.24265204 53956.24227009 33.00048 -- + HD114613 198.0129 -37.80367 ADP.2014-09-16T11:03:30.947 072.C-0488(E) -- 5261 ... 267.2 -- 120.002 53765.36393677 53765.36254786 120.001824 -- + HIP5158 16.50838 -22.45455 ADP.2014-09-16T11:03:30.973 072.C-0488(E) -- 5261 ... 39.3 -- 711.599 53946.33177006 53946.32353395 711.599904 -- + ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... + HD203608 321.60939 -65.36484 ADP.2014-09-16T11:03:45.713 077.D-0720(A) -- 5261 ... 45.0 -- 33.0 53955.11202646 53955.11164451 33.00048 -- + +Now we see that this query is limited to 1000 results, so we can increase the row limit to retrieve all matching datasets and limit the columns returned: + +.. doctest-remote-data:: + + >>> eso.ROW_LIMIT = -1 + >>> table = eso.query_surveys(surveys=["HARPS", "NIRPS"], + ... columns=["target_name", "s_ra", "s_dec", "dp_id", "proposal_id"]) + >>> table +
+ target_name s_ra s_dec dp_id proposal_id + deg deg + object float64 float64 object object + --------------- ---------- --------- --------------------------- -------------- + HD17051 40.64236 -50.80053 ADP.2014-09-16T11:03:41.583 078.D-0067(A) + HD69830 124.59872 -12.6353 ADP.2014-09-16T11:03:41.610 072.C-0488(E) + HD63077 116.39514 -34.17365 ADP.2014-09-16T11:03:41.617 076.D-0103(A) + ... ... ... ... ... + TOI-2322 116.961088 -71.00243 ADP.2026-01-13T11:49:12.986 116.2974.001 The returned table has a ``dp_id`` column, which can be used to retrieve the datasets with -:meth:`~astroquery.eso.EsoClass.retrieve_data`: ``eso.retrieve_data(table["dp_id"][0])``. +:meth:`~astroquery.eso.EsoClass.retrieve_data`: ``eso.retrieve_data(table["dp_id"])``. +Note that, in this example, there are 376,351 matching data products across both surveys so downloading +all of them may take a significant amount of time and disk space. More details about this method are in the following section. Query with Constraints (Specific Instrument) ============================================ -You can also query a specific instrument using the same method (e.g., ``HARPS``). For example, to retrieve **all** available HARPS data products regardless of the associated survey towards ``HD203608`` is given by the following query: +You can also query a specific instrument using the same method. For example, to retrieve **all** available HARPS data products regardless of the associated survey: .. doctest-remote-data:: - >>> table = eso.query_surveys(column_filters={"instrument_name": "HARPS", - ... "target_name": "HD203608"} - ... ) - + >>> table = eso.query_surveys(column_filters={"instrument_name": "HARPS"}) >>> table
- target_name s_ra s_dec dp_id proposal_id abmaglim access_estsize access_format ... s_xel2 snr strehl t_exptime t_max t_min t_resolution t_xel - deg deg mag kbyte ... s d d s - object float64 float64 object object float64 int64 object ... int64 float64 float64 float64 float64 float64 float64 int64 - ----------- --------- --------- --------------------------- ------------- -------- -------------- ------------------------------------------ ... ------ ------- ------- --------- -------------- -------------- ------------ ----- - HD203608 321.61455 -65.36429 ADP.2014-09-16T11:03:30.940 077.D-0720(A) -- 5261 application/x-votable+xml;content=datalink ... -- 60.9 -- 33.0 53956.24265204 53956.24227009 33.00048 -- - HD203608 321.61761 -65.36485 ADP.2014-09-16T11:03:31.020 077.D-0720(A) -- 5261 application/x-votable+xml;content=datalink ... -- 87.0 -- 32.999 53953.36835125 53953.36796931 32.999616 -- - HD203608 321.60594 -65.36528 ADP.2014-09-16T11:03:31.067 077.D-0720(A) -- 5261 application/x-votable+xml;content=datalink ... -- 73.9 -- 33.0 53956.15534682 53956.15496487 33.00048 -- - ... - HD203608 321.61113 -65.37211 ADP.2014-09-16T11:05:14.863 077.D-0720(A) -- 5261 application/x-votable+xml;content=datalink ... -- 95.2 -- 32.999 53954.99642615 53954.99604421 32.999616 -- + target_name s_ra s_dec dp_id proposal_id abmaglim access_estsize ... snr strehl t_exptime t_max t_min t_resolution t_xel + deg deg mag kbyte ... s d d s + object float64 float64 object object float64 int64 ... float64 float64 float64 float64 float64 float64 int64 + ------------ --------- --------- --------------------------- ------------- -------- -------------- ... ------- ------- --------- -------------- -------------- ------------ ----- + HD203608 321.61455 -65.36429 ADP.2014-09-16T11:03:30.940 077.D-0720(A) -- 5261 ... 60.9 -- 33.0 53956.24265204 53956.24227009 33.00048 -- + HD114613 198.0129 -37.80367 ADP.2014-09-16T11:03:30.947 072.C-0488(E) -- 5261 ... 267.2 -- 120.002 53765.36393677 53765.36254786 120.001824 -- + HIP5158 16.50838 -22.45455 ADP.2014-09-16T11:03:30.973 072.C-0488(E) -- 5261 ... 39.3 -- 711.599 53946.33177006 53946.32353395 711.599904 -- + ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... + HD203608 321.60939 -65.36484 ADP.2014-09-16T11:03:45.713 077.D-0720(A) -- 5261 ... 45.0 -- 33.0 53955.11202646 53955.11164451 33.00048 -- .. tip:: diff --git a/docs/eso/eso_tap.rst b/docs/eso/eso_tap.rst index 22284907a8..4e14767550 100644 --- a/docs/eso/eso_tap.rst +++ b/docs/eso/eso_tap.rst @@ -1,7 +1,7 @@ -*********************************************** -Query Raw or Reduced Data using the TAP Service -*********************************************** +*************************** +Query using the TAP Service +*************************** The ESO `TAP service `_ allows users to submit custom `ADQL `_ (Astronomical Data @@ -25,30 +25,32 @@ direct use of ADQL through TAP provides additional flexibility for constructing advanced queries that combine constraints across multiple metadata fields or tables. For more information about the TAP and how to write ADQL queries, refer to the following resources: -- `ESO TAP+ documentation `_: Describes ESO's implementation of TAP and the available services. +- `ESO TAP documentation `_: Describes ESO's implementation of TAP and the available services. - `IVOA TAP standard `_: The official specification from the International Virtual Observatory Alliance. -- `ADQL specification `_: Defines the query language used to interact with TAP services. +- `ADQL specification `_: Defines the query language used to interact with TAP services. Query for Raw Data (Generic) ============================ -The following example demonstrates how to query the ``dbo.raw`` table for raw data products from the ``MUSE`` instrument, specifically looking for observations of the object ``NGC300``: +The following example demonstrates how to query the ``dbo.raw`` table for raw data products from the ``MUSE`` instrument: .. doctest-remote-data:: >>> query = """ ... SELECT * ... FROM dbo.raw - ... WHERE object = 'NGC300' - ... AND instrument = 'muse' + ... AND instrument = 'MUSE' ... """ >>> result = eso.query_tap(query) - >>> result Query for Raw Data (Instrument-Specific) ======================================== -The following example demonstrates how to query the ``ist.muse`` table for raw data products from the ``MUSE`` instrument, specifically looking for observations taken during the night with a moon illumination below 30% and an average seeing (FWHM) below 1.0 arcsec: +The following example demonstrates how to query the ``ist.muse`` table for raw +data products from the ``MUSE`` instrument, selecting observations taken at +night with a Moon illumination below 30%, an LST range between 0 and 6 hours, +an average seeing (FWHM) below 1.0 arcsec, and an exposure time greater than +100 seconds: .. doctest-remote-data:: @@ -62,40 +64,377 @@ The following example demonstrates how to query the ``ist.muse`` table for raw d ... AND lst between 0 and 6 ... """ >>> result = eso.query_tap(query) - >>> result Query for Reduced Data Products =============================== -The following example queries the ``ivoa.ObsCore`` table to find fully calibrated (``calib_level=3``) multi-extension observations (``multi_ob='M'``) from the ``SPHERE`` survey, with spatial pixel scales smaller than 0.2 arcsec: +The following examples query the ``ivoa.ObsCore`` table to find enhanced data +products and perform common spatial selections using dataset footprints. Using +the TAP service directly allows access to functionality beyond what is available +via the high-level :meth:`~astroquery.eso.EsoClass.query_surveys` interface, such +as complex spatial constraints, joins, aggregations, and more flexible filtering +on ObsCore metadata. + +The same general approach applies when querying raw data products as described +above: while the specific metadata fields may differ, the use of TAP enables the +same general functionality as demonstrated in the examples below. + +Constrained query +----------------- + +The following example queries to find enhanced data products +(``calib_level = 3``; e.g. mosaics, resampled/drizzled images, or other heavily +processed survey fields) from the ``SPHERE`` survey. The query further restricts +the selection to multi-OB observations (``multi_ob = 'M'``) with spatial pixel +scales smaller than 0.2 arcsec: .. doctest-remote-data:: >>> query = """ - ... SELECT obs_collection, calib_level, multi_ob, filter, s_pixel_scale, instrument_name - ... FROM ivoa.ObsCore - ... WHERE obs_collection IN ('sphere') - ... AND calib_level = 3 - ... AND multi_ob = 'M' + ... SELECT obs_collection, calib_level, multi_ob, filter, + ... s_pixel_scale, instrument_name + ... FROM ivoa.ObsCore + ... WHERE obs_collection = 'SPHERE' + ... AND calib_level = 3 + ... AND multi_ob = 'M' ... AND s_pixel_scale < 0.2 ... """ >>> result = eso.query_tap(query) >>> result
obs_collection calib_level multi_ob filter s_pixel_scale instrument_name - arcsec - object int32 object object float64 object + arcsec + object int32 object object float64 object -------------- ----------- -------- ------ ------------- --------------- SPHERE 3 M K12 0.0122 SPHERE SPHERE 3 M K12 0.0122 SPHERE ... SPHERE 3 M H 0.0122 SPHERE +Cone search +----------- + +We can also use the TAP service to perform spatial queries based on dataset +footprints. Suppose you are looking for datasets closer than *2.5 arcmin* from *NGC 4666*. +Either you know the equatorial coordinates of your object, or you rely on a name +resolver such as SESAME (CDS) to obtain them. In the end you define a search +circle by three quantities: RA, Dec, and radius, all expressed in degrees. + +The cone-search constraint is typically implemented using the ADQL +``INTERSECTS`` operator, which takes two spatial footprints as input and returns +``1`` if they intersect in at least one point (and ``0`` otherwise). In a cone +search, one footprint is the input circle, while the other is the ``s_region`` +column, which represents the sky footprint of each dataset. + +.. doctest-remote-data:: + + >>> from astropy.coordinates import SkyCoord + >>> target = "NGC 4666" + >>> pos = SkyCoord.from_name(target) + >>> r = 2.5/60. # search radius of 2.5 arcmin, expressed in degrees + >>> query = """SELECT * + ... FROM ivoa.ObsCore + ... WHERE INTERSECTS(s_region, CIRCLE('', {ra:.6f}, {dec:.6f}, {r:.6f}))=1 + ... """.format(ra=pos.ra.degree, dec=pos.dec.degree, r=r) + +.. doctest-remote-data:: + + >>> result = eso.query_tap(query=query) + >>> print("Num matching datasets: %d" % (len(result))) + Num matching datasets: 219 + +.. note:: + **Cone search:** ``INTERSECTS`` or ``CONTAINS``? + + If you are looking specifically for spectra (whose footprint is typically a + point), you could use the stricter ``CONTAINS`` operator to ensure that only + footprints entirely contained within the defined circle are returned. + Remember: ``INTERSECTS`` is commutative, but the order of the operands in + ``CONTAINS`` matters and is defined as:: + + CONTAINS(contained, container) = 1 + + Try repeating the cone-search query after replacing ``INTERSECTS`` with + ``CONTAINS`` to see the difference. In practice, images (and measurements + derived from those images) may disappear from the results because their + footprints are larger than the 2.5 arcmin circle. + + +Point in footprint +------------------ + +Suppose you are interested in finding extended datasets that cover a specific +point on the sky (for example, datasets that could have imaged the progenitor of +a supernova). “Extended” here means datasets with a non-zero footprint area, excluding spectra +and visibilities whose footprint is effectively a single point. + +To find datasets that include a given sky position, you can use either +``INTERSECTS`` or ``CONTAINS``; for a point there is no practical difference. +Here we use ``CONTAINS`` and restrict the results to images and cubes. + +.. doctest-remote-data:: + + >>> query = """SELECT t_min, abmaglim, dataproduct_type as type, dp_id, obs_release_date + ... FROM ivoa.ObsCore + ... WHERE CONTAINS(POINT('', 193.815, 0.099819), s_region)=1 + ... AND dataproduct_type IN ('image', 'cube') + ... ORDER BY t_min ASC + ... """ + >>> result = eso.query_tap(query=query) + >>> result +
+ t_min abmaglim ... obs_release_date + d mag ... + float64 float64 ... object + ------------------ ------------------ ... ------------------------ + 56402.18240388 21.629 ... 2017-09-20T20:02:58.453Z + 56402.18240388 21.536 ... 2017-09-20T20:02:58.453Z + 56402.18390705 21.374 ... 2017-09-20T20:02:58.453Z + ... ... ... ... + 58787.638966122686 14.437809060706662 ... 2021-04-22T16:00:18Z + 58884.23649998 22.653 ... 2021-02-05T06:16:18.783Z + + +Region in footprint +------------------- + +If you want to ensure that matching datasets contain an entire region, you must +use ``CONTAINS``. The first operand is the region you want to be covered, and +the second operand is the dataset footprint in ``s_region``. The covered region +may be a simple circle or a more complex shape (e.g. a polygon). Here we show a +circle around NGC 253 and restrict to images and cubes. + +.. doctest-remote-data:: + + >>> query = """SELECT t_min, s_fov, dataproduct_type as type, dp_id, obs_release_date + ... FROM ivoa.ObsCore + ... WHERE CONTAINS(CIRCLE('', 11.888002, -25.288220, 0.21), s_region)=1 + ... AND dataproduct_type IN ('image', 'cube') + ... ORDER BY t_min ASC + ... """ + >>> result = eso.query_tap(query=query) + >>> result +
+ t_min s_fov type dp_id obs_release_date + d deg + float64 float64 object object object + -------------- ------------- ------ --------------------------- ------------------------ + 55864.21418374 1.15908954416 image ADP.2019-04-29T06:50:17.911 2019-04-29T13:08:36.937Z + 55893.13608187 1.45520652527 image ADP.2015-06-17T12:56:53.853 2015-06-18T11:00:50.587Z + 55897.15708812 1.45500126333 image ADP.2015-06-17T12:57:54.060 2015-06-18T05:46:38.167Z + 55927.08420409 1.15896704055 image ADP.2019-04-29T06:50:21.403 2019-04-29T18:06:01.943Z + 56132.3041874 1.15909942583 image ADP.2019-04-29T07:01:36.683 2019-05-02T17:40:17.620Z + 56147.27356555 1.15886452083 image ADP.2019-04-29T07:01:36.979 2019-05-02T17:40:17.620Z + 56180.34451683 1.15887357333 image ADP.2019-04-29T07:01:38.569 2019-05-02T18:00:17.460Z + +The above lists datasets whose footprints are large enough to entirely contain +the provided circular region. + +Search by polygon +----------------- + +You can also search for datasets intersecting a polygon. This is useful, for +example, when looking for optical/infrared/radio counterparts of a gravitational +wave (GW) event. GW spatial probability maps (e.g. via LIGO/Virgo skymaps) can be +converted into confidence contours at a given probability level, resulting in a +counterclockwise polygon suitable for ADQL spatial queries. + +The following example uses a polygon constructed for the GW170817 event and +returns matching datasets ordered by ``t_min``. + +.. doctest-remote-data:: + + >>> query = """SELECT t_min, snr, abmaglim, dataproduct_type as type, dp_id + ... FROM ivoa.ObsCore + ... WHERE INTERSECTS( + ... s_region, + ... POLYGON('J2000', + ... 196.8311,-23.5212, 196.7432,-23.3586, 196.6553,-23.1962, + ... 196.4795,-23.0339, 196.3916,-22.8719, 196.3037,-22.7100, + ... 196.2158,-22.5484, 196.1279,-22.3869, 196.0400,-22.2257, + ... 195.9521,-22.0646, 195.8643,-21.9037, 195.7764,-21.7429, + ... 195.7764,-21.5824, 195.6885,-21.4220, 195.6006,-21.2618, + ... 195.5127,-21.1018, 195.4248,-20.9420, 195.3369,-20.7823, + ... 195.3369,-20.6228, 195.2490,-20.4634, 195.1611,-20.3043, + ... 195.1611,-20.1452, 195.0732,-19.9864, 194.9854,-19.8277, + ... 194.8975,-19.6692, 194.8975,-19.5108, 194.8096,-19.3526, + ... 194.7217,-19.1945, 194.6338,-19.0366, 194.6338,-18.8788, + ... 194.5459,-18.7212, 194.4580,-18.5637, 194.4580,-18.4064, + ... 194.3701,-18.2492, 194.4580,-18.0922, 194.4580,-17.9353, + ... 194.6338,-18.0137, 194.8096,-18.1707, 194.9854,-18.3278, + ... 195.0732,-18.4851, 195.1611,-18.6425, 195.2490,-18.8000, + ... 195.3369,-18.9577, 195.4248,-19.1155, 195.5127,-19.2735, + ... 195.6006,-19.4317, 195.6885,-19.5900, 195.8643,-19.7484, + ... 195.9521,-19.9070, 196.1279,-20.0658, 196.2158,-20.2247, + ... 196.3916,-20.3838, 196.4795,-20.5431, 196.5674,-20.7025, + ... 196.6553,-20.8621, 196.7432,-21.0219, 196.8311,-21.1818, + ... 196.9189,-21.3419, 196.9189,-21.5022, 197.0068,-21.6626, + ... 197.0947,-21.8233, 197.1826,-21.9841, 197.2705,-22.1451, + ... 197.3584,-22.3063, 197.4463,-22.4676, 197.5342,-22.6292, + ... 197.6221,-22.7909, 197.7100,-22.9529, 197.7979,-23.1150, + ... 197.7979,-23.2773, 197.8857,-23.4399, 197.9736,-23.6026, + ... 196.9189,-23.6026 + ... ))=1 + ... ORDER BY t_min ASC + ... """ + >>> result = eso.query_tap(query=query) + >>> result +
+ t_min snr abmaglim type dp_id + d mag + float64 float64 float64 object object + -------------- ------- -------- ------------ --------------------------- + 52769.18223732 41.5 -- spectrum ADP.2020-08-04T16:44:55.149 + 53070.29890813 79.4 -- spectrum ADP.2016-09-20T08:08:13.622 + 53071.17154411 86.0 -- spectrum ADP.2016-09-20T08:08:12.820 + 53373.36714808 170.9 -- spectrum ADP.2016-09-21T07:05:31.370 + 54514.36343016 74.2 -- spectrum ADP.2020-06-28T16:34:31.221 + 54514.36347455 14.3 -- spectrum ADP.2020-06-28T16:34:31.235 + ... ... ... ... ... + 59355.20396096 6.9 -- spectrum ADP.2021-06-08T11:54:53.878 + 59759.00758508 -- -- visibility ADP.2025-12-18T13:15:42.554 + +Spatial joins +------------- + +Are you interested in finding images in different bands of the same sky region, +for photometric studies? The following example shows how to compose a spatial +join to find HAWKI source tables: + +* within 10 degrees from the Galactic plane, +* taken in the J and H bands (selected by wavelength constraints), +* where the J and H footprints overlap, and +* ensuring that the overlap covers at least 80% of the J-band footprint area. + +.. doctest-remote-data:: + + >>> query = """SELECT J.* FROM + ... (SELECT * FROM ivoa.ObsCore + ... WHERE dataproduct_subtype = 'srctbl' + ... AND obs_collection = 'HAWKI' + ... AND gal_lat < 10 AND gal_lat > -10 + ... AND em_min < 1.265E-6 AND em_max > 1.265E-6) J, + ... + ... (SELECT * FROM ivoa.ObsCore + ... WHERE dataproduct_subtype = 'srctbl' + ... AND obs_collection = 'HAWKI' + ... AND gal_lat < 10 AND gal_lat > -10 + ... AND em_min < 1.66E-6 AND em_max > 1.66E-6) H + ... + ... WHERE INTERSECTS(J.s_region, H.s_region)=1 + ... AND ESO_INTERSECTION(J.s_region, H.s_region) > 0.8 * AREA(J.s_region) + ... """ + >>> result = eso.query_tap(query=query) + >>> result +
+ abmaglim access_estsize access_format ... t_resolution t_xel target_name + float64 int64 object ... float64 int64 object + -------- -------------- ------------------------------------------ ... ------------ ----- ----------- + 23.029 2255 application/x-votable+xml;content=datalink ... 3580.209504 -- Lupus3-East + 23.054 2004 application/x-votable+xml;content=datalink ... 3580.209504 -- Lupus3-East + 23.035 1990 application/x-votable+xml;content=datalink ... 3580.94736 -- Lupus3-East + 23.01 2217 application/x-votable+xml;content=datalink ... 3580.94736 -- Lupus3-East + ... ... ... ... ... ... ... + 25.468 3081 application/x-votable+xml;content=datalink ... 280.788768 -- V U Car + 25.408 3093 application/x-votable+xml;content=datalink ... 280.788768 -- V U Car + +Notice the ``J.*`` in the ``SELECT`` clause: this returns only the metadata for +the J-band datasets. You can repeat the query selecting ``H.*`` to retrieve the +H-band metadata. This is useful, for example, when visualising results in tools +such as Aladin using different colours for different bands. Alternatively, you +can return both sets in one query (``SELECT *``) or select specific columns from +each side, e.g. ``SELECT J.dp_id, H.dp_id, ...``. + + +Query by wavelengths +-------------------- + +The IVOA ObsCore standard stores wavelengths in meters. You can rescale them in +the ``SELECT`` clause to display other units (e.g. nanometers), but you should +use meters when setting constraints on the ``em_min`` and ``em_max`` columns. +This applies even when the underlying data are naturally described in frequency +(e.g. radio observations), enabling uniform queries across observatories. + +.. doctest-remote-data:: + + >>> query = """SELECT obs_collection AS collection, + ... dataproduct_type AS type, + ... dataproduct_subtype AS subtype, + ... em_min*1E9 AS min_wavel_nm, + ... em_max*1E9 AS max_wavel_nm, + ... em_res_power + ... FROM ivoa.ObsCore + ... WHERE target_name = 'a370' + ... AND em_res_power < 3000""" + >>> result = eso.query_tap(query=query) + >>> result +
+ collection type subtype min_wavel_nm max_wavel_nm em_res_power + object object object float64 float64 float64 + ---------- ------------ -------- ------------------ ------------ ------------ + 092.A-0472 image 1981.9999999999998 2307.0 6.5985 + HAWKI measurements srctbl 1181.0 1335.0 8.1688 + HAWKI image pawprint 1181.0 1335.0 8.1688 + HAWKI image tile 1181.0 1335.0 8.1688 + ... ... ... ... ... ... + HAWKI image tile 1181.0 1335.0 8.1688 + +Query ESO Archive Stats +======================= + +The ESO TAP service can also be used to retrieve high-level statistics about the +contents of the ESO Science Archive. +The example below queries the total number of entries in the +``ivoa.ObsCore`` table, corresponding to the total number of data products +currently exposed through the ESO TAP service. + +.. doctest-remote-data:: + + >>> query = "SELECT COUNT(*) FROM ivoa.ObsCore" + >>> result = eso.query_tap(query) + >>> result +
+ COUNT_ALL + int32 + --------- + 4731624 + +More detailed statistics can be obtained by grouping on selected metadata +columns. The following example groups all data products by +``dataproduct_type`` and returns both the number of products in each category +and the total estimated archive volume, computed from the +``access_estsize`` column. The results are ordered by decreasing total data +volume. + +.. doctest-remote-data:: + + >>> query = """SELECT dataproduct_type, + ... COUNT(*) NumProducts, + ... SUM(access_estsize)/1000000000. TB + ... FROM ivoa.ObsCore + ... GROUP BY dataproduct_type + ... ORDER BY 3 DESC + ... """ + >>> result = eso.query_tap(query) + >>> result +
+ dataproduct_type numproducts tb + object int32 float64 + ---------------- ----------- ------------ + cube 690736 98.60936855 + image 969281 82.775606369 + measurements 675811 50.197075129 + spectrum 2377000 5.434112666 + visibility 18796 0.254378068 + +These are just some of the many possible ways to query for ESO Science Archive statistics +using the TAP service. For more examples on the available queries, +refer to the `TAP service `_. Download Data ============= -To download the data returned by the query, you can use the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes a list of data product IDs (``dp_id``) and downloads the corresponding files from the ESO archive. +As with the other functionality, to download the data returned by the above data queries (i.e. not the stats queries), you can use the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes a list of data product IDs (``dp_id``) and downloads the corresponding files from the ESO archive. .. doctest-remote-data:: >>> eso.retrieve_data(table["dp_id"]) From fedb8ff43e302ec5eb12f4f44cdf01b7d259981b Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Fri, 13 Feb 2026 20:34:33 +0100 Subject: [PATCH 16/18] Doc updates after AMI comments --- docs/eso/eso.rst | 21 +- docs/eso/eso_cat_query.rst | 4 + docs/eso/{eso_apex.rst => eso_obs_apex.rst} | 77 +++--- ...{eso_download.rst => eso_obs_download.rst} | 9 +- ...eader_info.rst => eso_obs_header_info.rst} | 10 +- docs/eso/eso_obs_raw_general.rst | 243 ++++++++++++++++++ ...trument.rst => eso_obs_raw_instrument.rst} | 13 +- .../{eso_reduced.rst => eso_obs_reduced.rst} | 7 +- docs/eso/eso_raw_general.rst | 170 ------------ docs/eso/eso_troubleshooting.rst | 19 +- 10 files changed, 340 insertions(+), 233 deletions(-) create mode 100644 docs/eso/eso_cat_query.rst rename docs/eso/{eso_apex.rst => eso_obs_apex.rst} (75%) rename docs/eso/{eso_download.rst => eso_obs_download.rst} (91%) rename docs/eso/{eso_header_info.rst => eso_obs_header_info.rst} (88%) create mode 100644 docs/eso/eso_obs_raw_general.rst rename docs/eso/{eso_raw_instrument.rst => eso_obs_raw_instrument.rst} (93%) rename docs/eso/{eso_reduced.rst => eso_obs_reduced.rst} (99%) delete mode 100644 docs/eso/eso_raw_general.rst diff --git a/docs/eso/eso.rst b/docs/eso/eso.rst index 510ec876d9..002ce5c9ce 100644 --- a/docs/eso/eso.rst +++ b/docs/eso/eso.rst @@ -9,17 +9,17 @@ Quick Start .. note:: - Check if this is a version of ``astroquery.eso`` (0.4.13 or later) that includes TAP support (see warning below). + Check if this is a version of ``astroquery.eso`` (0.4.12 or later) that includes TAP support (see warning below). .. doctest-skip:: >>> import astroquery - >>> astroquery.__version__ >= '0.4.13' + >>> astroquery.__version__ >= '0.4.12' True This quick start example shows how to use the ``astroquery.eso`` interface to query the ESO Science Archive for both raw and reduced observations. -The we can perform cone searches for `ESPRESSO `_ +Here, we perform a cone search for `ESPRESSO `_ spectra towards the star `HD 37903 `_, and how to download the corresponding data products. @@ -53,16 +53,17 @@ Getting Started =============== .. toctree:: - :maxdepth: 2 + :maxdepth: 1 eso_login - eso_raw_general - eso_raw_instrument - eso_reduced - eso_apex - eso_tap + eso_obs_raw_general + eso_obs_raw_instrument + eso_obs_reduced + eso_obs_apex + eso_obs_header_info + eso_cat_query eso_cone_search - eso_header_info + eso_tap eso_download eso_troubleshooting diff --git a/docs/eso/eso_cat_query.rst b/docs/eso/eso_cat_query.rst new file mode 100644 index 0000000000..ebaa8e83f2 --- /dev/null +++ b/docs/eso/eso_cat_query.rst @@ -0,0 +1,4 @@ +*************************** +Catalogues - Query for Data +*************************** + diff --git a/docs/eso/eso_apex.rst b/docs/eso/eso_obs_apex.rst similarity index 75% rename from docs/eso/eso_apex.rst rename to docs/eso/eso_obs_apex.rst index 59bf0b4e31..3c553acdc6 100644 --- a/docs/eso/eso_apex.rst +++ b/docs/eso/eso_obs_apex.rst @@ -1,7 +1,7 @@ -******************* -Query for APEX Data -******************* +********************************** +Observations - Query for APEX Data +********************************** APEX observations can be discovered via the standard archive query interfaces for reduced and raw data: @@ -25,40 +25,45 @@ Look products. As an example, we use APEX observations from the ALCOHOLS survey first locate the reduced survey products, then use an instrument-specific query to identify the corresponding raw datasets, and finally retrieve the associated Quick Look bundles. This indirection is sometimes necessary because Quick Look -queries may require the APEX proposal ID (distinct from the ESO programme ID). -If the APEX proposal ID is known (e.g. for proprietary data), Quick Look -products can be queried directly. +queries may require the APEX project ID (distinct from the ESO proposal - aka +programme - ID). If the APEX project ID is known (e.g. for proprietary data), +Quick Look products can be queried directly. Query Reduced APEX data ======================= -We first query for the reduced data from the **ALCOHOLS** survey, and retrieve the (ESO) proposal ID. +We first query for the reduced data from the **ALCOHOLS** survey, and retrieve +the ESO proposal ID. .. doctest-remote-data:: >>> # query the ESO archive for the ALCOHOLS survey >>> table_reduced = eso.query_surveys("ALCOHOLS") - >>> # extract unique proposal IDs from the query result - >>> proposal_id = list(set(table_reduced['proposal_id'])) + >>> # extract unique ESO run IDs from the query result + >>> # (the survey table actually contains ESO run IDs) + >>> run_ids = list(set(table_reduced["proposal_id"])) - >>> # check if we have a single proposal ID or multiple - >>> if len(proposal_id) == 1: - >>> proposal_id = proposal_id[0] - >>> else: - >>> print("Warning: Multiple proposal IDs found...") + >>> # check if we have a single run ID or multiple + >>> if len(run_ids) == 1: + ... run_id = run_ids[0] + ... else: + ... print("Warning: Multiple proposal IDs found...") + ... run_id = run_ids[0] - >>> # Extract the first part of the proposal ID before any parentheses (i.e. the run ID) - >>> def proposal_remove_run(proposal_id): - ... program = ".".join(proposal_id.split('.',2)[:2]) - ... program = ".".join(program.split('(')[:1]) - ... return program + >>> # Extract the proposal (aka programme) ID from the run ID + >>> def proposal_remove_run(run_id): + ... proposal_id = ".".join(run_id.split(".", 2)[:2]) + ... proposal_id = ".".join(proposal_id.split("(")[:1]) + ... return proposal_id - >>> proposal_id = proposal_remove_run(proposal_id) + >>> proposal_id = proposal_remove_run(run_id) >>> print(f"Proposal ID: {proposal_id}") Proposal ID: 094.C-0935 -Note that multiple values of ``proposal_id`` may be returned, which would require minor changes to the above example to loop through the projects. +Note that multiple values of ``proposal_id`` may be returned, +which would require minor changes to the above example to loop through all +relevant IDs. Available Query Constraints =========================== @@ -87,7 +92,6 @@ As always, it is good practice to check the available columns to search in the i wobcycle float s wobthrow float deg wobused char - Number of records present in the table ist.APEX: 913029 [astroquery.eso.core] @@ -95,14 +99,15 @@ As always, it is good practice to check the available columns to search in the i Query Raw APEX data =================== -We now query for raw data from the APEX instrument, using the proposal ID we retrieved from the previous query. +We now query for raw data from the APEX instrument, using the ESO proposal ID +retrieved from the previous query. .. doctest-remote-data:: - >>> # query the APEX instrument for data related to the proposal ID + >>> # query the APEX instrument for data related to the ESO proposal ID >>> table_raw = eso.query_instrument("APEX", column_filters={"prog_id": f"like '{proposal_id}%'"}) - >>> # extract unique project IDs from the raw data query + >>> # extract unique APEX project IDs from the raw data query >>> project_id = list(set(table_raw["project_id"])) >>> # assuming we only have one project ID @@ -110,10 +115,17 @@ We now query for raw data from the APEX instrument, using the proposal ID we ret >>> print(f"Project ID: {project_id}") Project ID: E-094.C-0935A-2014 -In this case, we know there is only **one** APEX proposal ID, but if there were multiple IDs, we would need to loop through them. +In this case, we know there is only **one** APEX project ID, but if there were +multiple IDs, we would need to loop through them. .. tip:: - In the :meth:`~astroquery.eso.EsoClass.query_surveys` query, the ``"proposal_id"`` column refers to the **ESO programme ID**. Whereas, in :meth:`~astroquery.eso.EsoClass.query_instrument`, the ``"prog_id"`` column refers to the **ESO programme ID** and ``"project_id"`` column refers to the **APEX proposal ID** - this is the value used to identify APEX Quick Look products (see below). + In the :meth:`~astroquery.eso.EsoClass.query_surveys` query, the + ``"proposal_id"`` column refers to the ESO program/run ID. Whereas, in + :meth:`~astroquery.eso.EsoClass.query_instrument`, it is the ``"prog_id"`` + column that refers to the ESO program/run ID (the proposal ID can be + extracted from the run ID), and ``"project_id"`` refers to the APEX project + ID. This is the value used to identify APEX Quick Look products (see + below). Query APEX Quick Look products ============================== @@ -139,12 +151,13 @@ We can check the available columns to search in the query. project_id char quicklook_id char release_date char timestamp - Number of records present in the table ist.apex_quicklooks: 282296 [astroquery.eso.core] -And now, use :meth:`~astroquery.eso.EsoClass.query_apex_quicklooks` to query for the APEX Quick Look products using the APEX proposal ID (``project_id``) we retrieved from the previous query. +And now, use :meth:`~astroquery.eso.EsoClass.query_apex_quicklooks` to query +for the APEX Quick Look products using the APEX project ID +(``project_id``) we retrieved from the previous query. .. doctest-remote-data:: @@ -161,7 +174,11 @@ And now, use :meth:`~astroquery.eso.EsoClass.query_apex_quicklooks` to query for 6389 https://dataportal.eso.org/dataPortal/file/E-094.C-0935A.2015AUG22.TAR APEXHET Heterodyne ESO ... The APEX Large CO Heterodyne Outflow Legacy Supercam survey of Orion (ALCOHOLS) Normal E-094.C-0935A-2014 E-094.C-0935A.2015AUG22.TAR 2015-04-25T18:41:53.900Z -As can be seen from the output above, there is one APEX Quick Look product available per UT date, per APEX proposal ID. Also note that the APEX Quick Look products are available in ``.tar`` format (e.g. ``E-094.C-0935A.2014DEC10.TAR``), which can be downloaded and extracted (see below). +As can be seen from the output above, there is one APEX Quick Look product +available per UT date, per APEX project ID. Also note that the APEX Quick Look +products are available in ``.tar`` format (e.g. +``E-094.C-0935A.2014DEC10.TAR``), which can be downloaded and extracted (see +below). Download APEX Quick Look products ================================= diff --git a/docs/eso/eso_download.rst b/docs/eso/eso_obs_download.rst similarity index 91% rename from docs/eso/eso_download.rst rename to docs/eso/eso_obs_download.rst index 29ebb0a10f..6f13e28de2 100644 --- a/docs/eso/eso_download.rst +++ b/docs/eso/eso_obs_download.rst @@ -1,7 +1,7 @@ -************* -Download Data -************* +**************************** +Observations - Download Data +**************************** Assuming you have already performed a query (e.g., using :meth:`~astroquery.eso.EsoClass.query_surveys`, :meth:`~astroquery.eso.EsoClass.query_instrument`, or :meth:`~astroquery.eso.EsoClass.query_main`) and have a resulting table of data products, you can download the actual datasets using the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes as input a list of data product IDs (``dp_id``) obtained from the query results. @@ -31,7 +31,8 @@ The file names returned in ``data_files`` point to the decompressed datasets By default, if a requested dataset is already found, it is not downloaded again from the archive. To force the retrieval of data that are present in the destination directory, use ``continuation=True`` in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. -When downloading datasets, you can optionally retrieve associated calibration files by using the ``with_calib`` argument. This allows you to obtain either raw or processed calibrations in addition to the science files. +When downloading datasets, you can optionally retrieve associated calibration files by using the ``with_calib`` argument. +This makes use of the ESO CalSelector service (see the `CalSelector information page `_). Available options: diff --git a/docs/eso/eso_header_info.rst b/docs/eso/eso_obs_header_info.rst similarity index 88% rename from docs/eso/eso_header_info.rst rename to docs/eso/eso_obs_header_info.rst index 47589c7931..813725e381 100644 --- a/docs/eso/eso_header_info.rst +++ b/docs/eso/eso_obs_header_info.rst @@ -1,7 +1,7 @@ -****************** -Header Information -****************** +********************************* +Observations - Header Information +********************************* Only a small subset of the keywords present in the data products can be obtained with the :meth:`~astroquery.eso.EsoClass.query_instrument`, :meth:`~astroquery.eso.EsoClass.query_main`, :meth:`~astroquery.eso.EsoClass.query_surveys`, or :meth:`~astroquery.eso.EsoClass.query_tap` methods. @@ -36,3 +36,7 @@ archive using a dedicated helper method. MIDI.2008-01-01T09:00:10.000 True 16 ... MIDI.2008-01-01T09:00:10.000.fits As shown above, for each data product ID (``DP.ID``; note that this is equivalent to ``dp_id`` in ``table``), the full primary header (336 columns in our case) of the archive FITS file is collected. In the above table ``table_headers``, there are as many rows as there are entries in the ``table['dp_id']`` column. + +.. note:: + + At present, astroquery returns only the primary header; the rest of the FITS header is not accessible through astroquery yet. Support for returning the entire header is planned for a future version. diff --git a/docs/eso/eso_obs_raw_general.rst b/docs/eso/eso_obs_raw_general.rst new file mode 100644 index 0000000000..9ecaa66fda --- /dev/null +++ b/docs/eso/eso_obs_raw_general.rst @@ -0,0 +1,243 @@ + +******************************************* +Observations - Query for Raw Data (Generic) +******************************************* + +.. tip:: + The `astroquery.eso` module provides several ways to search for data in + the ESO Science Archive for raw data. This section focuses on the + **generic query interface** for raw data + :meth:`~astroquery.eso.EsoClass.query_main`, while in + :doc:`the instrument-specific raw-data query section ` + we describe **instrument-specific queries** using + :meth:`~astroquery.eso.EsoClass.query_instrument`. + +You may want to query the ESO Archive **without targeting a specific instrument**. This is exactly what the :meth:`~astroquery.eso.EsoClass.query_main` method is designed for. It allows access to the **global raw data table**, which combines metadata across all instruments. +Internally, this method queries the ``dbo.raw`` table via ESO's `TAP service `_ (or using the :meth:`~astroquery.eso.EsoClass.query_tap` function), which you could also access directly using ADQL with a simple statement like: ``SELECT * FROM dbo.raw``. + +Available Query Constraints +=========================== + +We start by inspecting the available columns that can be queried by :meth:`~astroquery.eso.EsoClass.query_main` with the ``help=True`` keyword. +This will return a list of all columns available, along with their data types, units, and any additional metadata such as ``xtype`` information. + +The output includes column names, data types, units, and, where applicable, +``xtype`` information to indicate more specific column content. The ``xtype`` +attribute defines how a string-valued column should be interpreted during +querying. For example, a column with datatype ``char`` may represent a standard +string (``xtype = null``), a timestamp (``xtype = timestamp``), or a sky region +footprint usable in ADQL spatial queries (``xtype = adql:REGION``), with this +interpretation reflected in the ``xtype`` field. + +.. doctest-remote-data:: + + >>> eso.query_main(help=True) + INFO: + Columns present in the table dbo.raw: + column_name datatype xtype unit + ------------------- -------- ----------- ------ + access_estsize long kbyte + access_url char + datalink_url char + date_obs char + dec double deg + dec_pnt double deg + ... + tpl_start char + Number of records present in the table dbo.raw: + 36404227 + [astroquery.eso.core] + +Query with Constraints (All Instruments) +======================================== + +To perform a genuinely generic raw-data query, you can combine a cone search +with additional constraints in ``column_filters``. The example below retrieves +all data products from any instrument within 30 arcsec of ``SN2013am``, with a +time constraint specified down to the second. + +.. doctest-skip:: + + >>> from astropy.coordinates import SkyCoord + >>> import astropy.units as u + + >>> coords = SkyCoord.from_name("SN2013am") # The Prawn Nebula + >>> ra = coords.ra.value + >>> dec = coords.dec.value + >>> r = (30 * u.arcsec).to(u.deg).value + + >>> table = eso.query_main( + ... cone_ra=ra, + ... cone_dec=dec, + ... cone_radius=r, + ... column_filters={ + ... "exp_start": "between '2013-04-12' and '2013-04-12T04:52:06'" + ... }, + ... columns=["object", "ra", "dec", "date_obs", "instrument", "prog_id"], + ... ) + >>> table +
+ object ra dec date_obs prog_id instrument + deg deg + object float64 float64 object object object + -------- --------- -------- ------------------------ ------------- ---------- + SN2013AM 169.737 13.05858 2013-04-12T04:12:05.4817 188.D-3003(S) SOFI + SN2013AM 169.737 13.05858 2013-04-12T04:22:05.6191 188.D-3003(S) SOFI + SN2013AM 169.73717 13.06367 2013-04-12T02:40:17.0744 188.D-3003(S) SOFI + SKY 169.73717 13.06922 2013-04-12T02:47:39.0284 188.D-3003(S) SOFI + SKY 169.73717 13.06367 2013-04-12T02:48:19.3074 188.D-3003(S) SOFI + SKY 169.737 13.0672 2013-04-12T02:49:02.2986 188.D-3003(S) SOFI + SN2013AM 169.737 13.05887 2013-04-12T02:51:34.0298 188.D-3003(S) SOFI + SN2013AM 169.737 13.06029 2013-04-12T03:09:58.1859 188.D-3003(S) SOFI + SN2013AM 169.737 13.06029 2013-04-12T03:15:59.6815 188.D-3003(S) SOFI + SN2013AM 169.737 13.05682 2013-04-12T03:34:23.8352 188.D-3003(S) SOFI + SN2013AM 169.737 13.05887 2013-04-12T03:41:45.3991 188.D-3003(S) SOFI + SN2013AM 169.73717 13.06367 2013-04-12T02:31:32.9840 188.D-3003(S) SOFI + SN2013AM 169.73717 13.06367 2013-04-12T02:34:32.7246 188.D-3003(S) SOFI + +Query with Constraints (Specific Instrument) +============================================ + +If needed, you can still narrow a generic query to a single instrument by +passing ``instruments=...`` to :meth:`~astroquery.eso.EsoClass.query_main`. +For example, to retrieve only ``MIDI`` data products within some time range: + +.. doctest-remote-data:: + + >>> eso.ROW_LIMIT = -1 # Return all results without truncation + >>> table = eso.query_main( + ... instruments="midi", + ... column_filters={ + ... "exp_start": "between '2008-01-01' and '2009-05-12'"} + ... ) + >>> print(len(table)) + 58810 + >>> table.colnames + ['object', 'ra', 'dec', 'dp_id', 'date_obs', 'prog_id', + 'access_estsize', 'access_url', 'datalink_url', ... 'tpl_start'] + >>> table[["object", "ra", "dec", "date_obs", "prog_id"]] +
+ object ra dec date_obs prog_id + deg deg + object float64 float64 object object + -------------- ------------- ------------- ----------------------- ------------ + FLAT -596.52323555 -596.52323555 2008-09-28T06:49:15.000 60.A-9224(A) + FLAT -596.52323555 -596.52323555 2008-09-28T06:54:23.000 60.A-9224(A) + FLAT -596.52323555 -596.52323555 2008-09-28T06:59:48.000 60.A-9224(A) + FLAT -596.52323555 -596.52323555 2008-09-28T07:02:29.000 60.A-9224(A) + ... ... ... ... ... + WAVE,SPECTEMPL -596.52323555 -596.52323555 2009-01-20T10:21:43.000 60.A-9224(A) + WAVE,SPECTEMPL -596.52323555 -596.52323555 2009-01-20T10:22:36.000 60.A-9224(A) + OTHER -596.52323555 -596.52323555 2009-01-20T10:25:35.128 60.A-9224(A) + WAVE,SPECTEMPL -596.52323555 -596.52323555 2009-01-20T10:26:48.000 60.A-9224(A) + WAVE,SPECTEMPL -596.52323555 -596.52323555 2009-01-20T10:29:37.000 60.A-9224(A) + + +Suppose that we have a large query; it may be useful to limit the columns +returned in the results table. This can be done using the ``columns`` argument. +For example, to return only the target name, right ascension, declination, +observation date, and program ID: + +.. doctest-remote-data:: + + >>> table = eso.query_main( + ... instruments="midi", + ... column_filters={ + ... "exp_start": "between '2008-01-01' and '2009-05-12'"}, + ... columns=["object", "ra", "dec", "date_obs", "prog_id"] + ... ) + >>> table +
+ object ra dec date_obs prog_id + deg deg + object float64 float64 object object + ----------------------- ------------- ------------- ----------------------- ------------ + SEARCH,OBJECT,DISPERSED 0.49041805 -6.01429 2008-10-27T03:27:44.000 60.A-9224(A) + PHOTOMETRY,OBJECT 0.49041805 -6.01429 2008-10-27T03:36:04.000 60.A-9224(A) + COARSE,OBJECT 0.49041805 -6.01429 2008-10-27T03:45:32.000 60.A-9224(A) + BIAS -596.52323555 -596.52323555 2008-10-27T06:22:12.388 60.A-9224(A) + ... ... ... ... ... + FLAT -596.52323555 -596.52323555 2009-03-27T09:28:52.000 60.A-9224(A) + OTHER -596.52323555 -596.52323555 2009-03-27T09:33:31.378 60.A-9224(A) + FLAT -596.52323555 -596.52323555 2009-03-27T09:34:46.000 60.A-9224(A) + FLAT -596.52323555 -596.52323555 2009-03-27T09:43:46.000 60.A-9224(A) + FLAT -596.52323555 -596.52323555 2009-03-27T09:47:37.000 60.A-9224(A) + + +Query with Constraints (Multiple Instruments) +============================================= + +We can also query for raw data from multiple instruments in a single call to +:meth:`~astroquery.eso.EsoClass.query_main`. For example, to retrieve raw data +from the interferometric instruments ``MIDI``, ``GRAVITY``, and ``PIONIER`` within 10 arcsec of +``ETA CAR``: + +.. doctest-remote-data:: + + >>> coords = SkyCoord.from_name("ETA CAR") # Eta Carinae + >>> ra = coords.ra.value + >>> dec = coords.dec.value + >>> r = (10 * u.arcsec).to(u.deg).value + + >>> table = eso.query_main( + ... cone_ra=ra, + ... cone_dec=dec, + ... cone_radius=r, + ... instruments=["midi", "gravity", "pionier"], + ... column_filters={ + ... columns=["instrument", "object", "ra", "dec", "date_obs", "prog_id"] + ... ) + >>> print(len(table)) + 622 + >>> table[["object", "ra", "dec", "date_obs", "prog_id"]] +
+ object ra dec date_obs prog_id instrument + deg deg + object float64 float64 object object object + ------------ ------------ --------- ------------------------ ------------- ---------- + ETA CARINAE 161.2603 -59.68441 2016-02-25T05:14:21 60.A-9102(A) GRAVITY + ETA CARINAE 161.2603 -59.68441 2016-02-25T06:56:58 60.A-9102(A) GRAVITY + ETA CARINAE 161.2603 -59.68441 2016-02-25T09:36:24 60.A-9102(A) GRAVITY + ETA_CAR 161.26028194 -59.68442 2017-02-20T08:38:01 098.D-0488(A) GRAVITY + ... ... ... ... ... ... + KAPPA,OBJECT 161.26358805 -59.68441 2015-12-07T07:28:26.1927 60.A-9209(A) PIONIER + KAPPA,OBJECT 161.26358805 -59.68441 2015-12-07T07:29:14.3074 60.A-9209(A) PIONIER + KAPPA,OBJECT 161.26358805 -59.68441 2015-12-07T07:29:38.4820 60.A-9209(A) PIONIER + ETA_CAR 161.26358805 -59.68441 2015-12-07T07:48:38.6025 60.A-9209(A) PIONIER + ETA_CAR 161.26358805 -59.68441 2015-12-07T07:52:06.9548 60.A-9209(A) PIONIER + +.. tip:: + + Use :meth:`~astroquery.eso.EsoClass.query_main` when you want to search + **across all instruments**. Use + :meth:`~astroquery.eso.EsoClass.query_instrument` when you want a more + **refined, instrument-specific search** (e.g. instrument modes, + configurations, or ambient conditions). + + .. doctest-remote-data:: + + >>> column_filters = { + ... "dp_cat": "SCIENCE", # Science data only + ... "ins_opt1_name": "HIGH_SENS", # High sensitivity mode + ... "night_flag": "night", # Nighttime observations only + ... "moon_illu": "< 0", # No moon (below horizon) + ... "lst": "between 0 and 6" # Local sidereal time early in the night + ... } + >>> table = eso.query_instrument("midi", column_filters=column_filters) + +Download Data +============= + +To download the data returned by the query, you can use the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes a list of data product IDs (``dp_id``) and downloads the corresponding files from the ESO archive. + +.. doctest-remote-data:: + >>> eso.retrieve_data(table["dp_id"]) + +The ``data_files`` list points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. + +.. doctest-skip:: + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") + +For raw data, you can also retrieve associated calibration files by +passing ``with_calib="raw"`` (or ``with_calib="processed"``). See +:doc:`eso_download` for examples and the related CalSelector terminology. diff --git a/docs/eso/eso_raw_instrument.rst b/docs/eso/eso_obs_raw_instrument.rst similarity index 93% rename from docs/eso/eso_raw_instrument.rst rename to docs/eso/eso_obs_raw_instrument.rst index 1f07782d33..95c27b7edf 100644 --- a/docs/eso/eso_raw_instrument.rst +++ b/docs/eso/eso_obs_raw_instrument.rst @@ -1,10 +1,10 @@ -**************************************** -Query for Raw Data (Instrument-Specific) -**************************************** +******************************************************* +Observations - Query for Raw Data (Instrument-Specific) +******************************************************* .. note:: - The `astroquery.eso` module provides several ways to search for raw data in the ESO Science Archive. This section focuses on the **instrument-specific query interface** for raw data :meth:`~astroquery.eso.EsoClass.query_instrument`, while in another section we describe **generic queries** using :meth:`~astroquery.eso.EsoClass.query_main`. + The `astroquery.eso` module provides several ways to search for raw data in the ESO Science Archive. This section focuses on the **instrument-specific query interface** for raw data :meth:`~astroquery.eso.EsoClass.query_instrument`, while in :doc:`the generic raw-data query section ` we describe **generic queries** using :meth:`~astroquery.eso.EsoClass.query_main`. In many cases, you will want to query the ESO Archive **for data from a specific instrument**. This is exactly what the :meth:`~astroquery.eso.EsoClass.query_instrument` method is designed for. It allows you to search instrument-specific tables, which expose metadata fields and filters unique to each instrument. Internally, this method queries the corresponding instrument table (e.g., ``ist.muse``) via ESO's `TAP service `_. This approach is ideal when you need precise control over your query, such as filtering by instrument configuration, mode, or observational setup. @@ -73,7 +73,6 @@ interpretation reflected in the ``xtype`` field. s_region char adql:REGION ... utc float s - Number of records present in the table ist.midi: 437577 [astroquery.eso.core] @@ -144,3 +143,7 @@ The ``data_files`` list points to the decompressed dataset filenames that have b .. doctest-skip:: >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") + +For raw data, you can also retrieve associated calibration files by +passing ``with_calib="raw"`` (or ``with_calib="processed"``). See +:doc:`eso_download` for examples and the related CalSelector terminology. \ No newline at end of file diff --git a/docs/eso/eso_reduced.rst b/docs/eso/eso_obs_reduced.rst similarity index 99% rename from docs/eso/eso_reduced.rst rename to docs/eso/eso_obs_reduced.rst index 69788f3b3e..a0e912e47e 100644 --- a/docs/eso/eso_reduced.rst +++ b/docs/eso/eso_obs_reduced.rst @@ -1,7 +1,7 @@ -********************** -Query for Reduced Data -********************** +************************************* +Observations - Query for Reduced Data +************************************* In addition to raw observational files, the ESO Science Archive provides access to a wide range of **processed (reduced) data products**. These include science-ready images, spectra, and datacubes that have been validated by ESO (through the `ESO Phase 3 `_ process). @@ -48,7 +48,6 @@ As before, list the possible columns in :meth:`~astroquery.eso.EsoClass.query_su calib_level int ... target_name char - Number of records present in the table ivoa.ObsCore: 4559928 [astroquery.eso.core] diff --git a/docs/eso/eso_raw_general.rst b/docs/eso/eso_raw_general.rst deleted file mode 100644 index 1163163bf3..0000000000 --- a/docs/eso/eso_raw_general.rst +++ /dev/null @@ -1,170 +0,0 @@ - -**************************** -Query for Raw Data (Generic) -**************************** - -.. tip:: - The `astroquery.eso` module provides several ways to search for data in the ESO Science Archive for raw data. This section focuses on the **generic query interface** for raw data :meth:`~astroquery.eso.EsoClass.query_main`, while in another section we describe **instrument-specific queries** using :meth:`~astroquery.eso.EsoClass.query_instrument`. Functionally, :meth:`~astroquery.eso.EsoClass.query_main` works the same way as :meth:`~astroquery.eso.EsoClass.query_instrument`, except you don’t need to specify an instrument table name directly. You can still apply column filters, control the columns returned, and limit result counts. - -You may want to query the ESO Archive **without targeting a specific instrument**. This is exactly what the :meth:`~astroquery.eso.EsoClass.query_main` method is designed for. It allows access to the **global raw data table**, which combines metadata across all instruments. Internally, this method queries the ``dbo.raw`` table via ESO's `TAP service `_ (or using the :meth:`~astroquery.eso.EsoClass.query_tap` function), which you could also access directly using ADQL with a simple statement like: - -.. doctest-skip:: - SELECT * FROM dbo.raw - -Available Instruments -===================== - -To begin retrieving raw data from the ESO Science Archive, you may first want to know the relevant instrument(s) for your search. Each instrument has its own dedicated query table accessible through the archive’s programmatic `TAP `_ interface. - -The list of all supported instruments can be retrieved using the :meth:`~astroquery.eso.EsoClass.list_instruments` method: - -.. doctest-remote-data:: - - >>> from astroquery.eso import Eso - >>> eso = Eso() - >>> eso.list_instruments() - ['alpaca', 'amber', 'apex', 'crires', 'efosc', 'eris', - 'espresso', 'fiat', 'fors1', 'fors2', 'giraffe', 'gravity', - 'harps', 'hawki', 'isaac', 'kmos', 'matisse', 'midi', - 'muse', 'naco', 'nirps', 'omegacam', 'pionier', 'sinfoni', - 'sofi', 'sphere', 'uves', 'vimos', 'vircam', 'visir', - 'wlgsu', 'xshooter'] - -This list corresponds to the instruments currently available for programmatic raw data queries in the ESO archive. - -The list is dynamically generated by querying the archive’s internal TAP service. It can also be reproduced directly by executing the following ADQL query on the `ESO TAP interface `_ or using the :meth:`~astroquery.eso.EsoClass.query_tap` function: - -.. doctest-skip:: - - SELECT table_name - FROM TAP_SCHEMA.tables - WHERE schema_name = 'ist' - ORDER BY table_name - -Once you have identified the instrument of interest, you can proceed with constructing your query and retrieving raw data products. - -Available Query Constraints -=========================== - -We start by inspecting the available columns that can be queried by :meth:`~astroquery.eso.EsoClass.query_main` with the ``help=True`` keyword. This will return a list of all columns available, along with their data types, units, and any additional metadata such as ``xtype`` information. - -The output includes column names, data types, units, and, where applicable, -``xtype`` information to indicate more specific column content. The ``xtype`` -attribute defines how a string-valued column should be interpreted during -querying. For example, a column with datatype ``char`` may represent a standard -string (``xtype = null``), a timestamp (``xtype = timestamp``), or a sky region -footprint usable in ADQL spatial queries (``xtype = adql:REGION``), with this -interpretation reflected in the ``xtype`` field. - -.. doctest-remote-data:: - - >>> eso.query_main(help=True) - INFO: - Columns present in the table dbo.raw: - column_name datatype xtype unit - ------------------- -------- ----------- ------ - access_estsize long kbyte - access_url char - datalink_url char - date_obs char - dec double deg - dec_pnt double deg - ... - tpl_start char - - Number of records present in the table dbo.raw: - 34821254 - [astroquery.eso.core] - -Query with Constraints (Specific Instrument) -============================================ - -Now that we know which of the columns are available for queries, we can retrieve all-sky images from the ``APICAM`` instrument for a single night (i.e. 2019-04-26): - -.. doctest-remote-data:: - - >>> eso.ROW_LIMIT = -1 # Return all results without truncation - >>> table = eso.query_main( - ... column_filters={ - ... "instrument": "APICAM", - ... "exp_start": "between '2019-04-26' and '2019-04-27'"} - ... ) - >>> print(len(table)) - 215 - >>> table.colnames - ['object', 'ra', 'dec', 'dp_id', 'date_obs', 'prog_id', - 'access_estsize', 'access_url', 'datalink_url', ... 'tpl_start'] - >>> table[["object", "ra", "dec", "date_obs", "prog_id"]] -
- object ra dec date_obs prog_id - deg deg - object float64 float64 object object - ------- ------------ ------------ ----------------------- ------------ - ALL SKY 145.29212694 -24.53624194 2019-04-26T00:08:49.000 60.A-9008(A) - ALL SKY 145.92251305 -24.53560305 2019-04-26T00:11:20.000 60.A-9008(A) - ALL SKY 146.55707 -24.53497111 2019-04-26T00:13:52.000 60.A-9008(A) - ... - ALL SKY 143.56345694 -24.53804388 2019-04-26T23:57:59.000 60.A-9008(A) - - -Suppose that we have a large query, it may be useful to limit the columns returned in the results table. -This can be done using the ``columns`` argument. For example, to return only the target name, right ascension, declination, observation date, and program ID: - -.. doctest-remote-data:: - - >>> table = eso.query_main( - ... column_filters={ - ... "instrument": "APICAM", - ... "exp_start": "between '2019-04-26' and '2019-04-27'"}, - ... columns=["object", "ra", "dec", "date_obs", "prog_id"] - ... ) - >>> table -
- object ra dec date_obs prog_id - deg deg - object float64 float64 object object - ------- ------------ ------------ ----------------------- ------------ - ALL SKY 145.29212694 -24.53624194 2019-04-26T00:08:49.000 60.A-9008(A) - ALL SKY 145.92251305 -24.53560305 2019-04-26T00:11:20.000 60.A-9008(A) - ALL SKY 146.55707 -24.53497111 2019-04-26T00:13:52.000 60.A-9008(A) - ... - ALL SKY 143.56345694 -24.53804388 2019-04-26T23:57:59.000 60.A-9008(A) - -Query with Constraints (All Instruments) -======================================== - -Alternatively, if you want to retrieve all available raw data products for all instruments observed on a specfic day. - -.. doctest-remote-data:: - - >>> table = eso.query_main(column_filters={ - ... "exp_start": "between '2019-04-26' and '2019-04-27'" - ... }) - -.. tip:: - - Use :meth:`~astroquery.eso.EsoClass.query_main` when you want to search **across all instruments**. Use :meth:`~astroquery.eso.EsoClass.query_instrument` when you want a more **refined, instrument-specific search** (e.g. instrument modes, configurations, or ambient conditions). - - .. doctest-remote-data:: - - >>> column_filters = { - ... "dp_cat": "SCIENCE", # Science data only - ... "ins_opt1_name": "HIGH_SENS", # High sensitivity mode - ... "night_flag": "night", # Nighttime observations only - ... "moon_illu": "< 0", # No moon (below horizon) - ... "lst": "between 0 and 6" # Local sidereal time early in the night - ... } - >>> table = eso.query_instrument("midi", column_filters=column_filters) - -Download Data -============= - -To download the data returned by the query, you can use the :meth:`~astroquery.eso.EsoClass.retrieve_data` method. This method takes a list of data product IDs (``dp_id``) and downloads the corresponding files from the ESO archive. - -.. doctest-remote-data:: - >>> eso.retrieve_data(table["dp_id"]) - -The ``data_files`` list points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. - -.. doctest-skip:: - >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") diff --git a/docs/eso/eso_troubleshooting.rst b/docs/eso/eso_troubleshooting.rst index 7a54f98320..5ea8dcdcbd 100644 --- a/docs/eso/eso_troubleshooting.rst +++ b/docs/eso/eso_troubleshooting.rst @@ -31,15 +31,20 @@ Filtering between two dates # ❌ Invalid (WDB-specific fields, not recognized by TAP) column_filters = { - "stime": "2024-01-01", - "etime": "2024-12-31" + "stime": "2024-01-01 12:00:00", + "etime": "2024-01-03 12:00:00" } .. doctest-skip:: # ✅ Correct (TAP-compliant syntax using 'exp_start') column_filters = { - "exp_start": "between '2024-01-01' and '2024-12-31'" + "exp_start": "between '2024-01-01 12:00:00' and '2024-01-03 12:00:00'" + } + + # OR + column_filters = { + "exp_start": "between '2024-01-01T12:00:00' and '2024-01-03T12:00:00'" } Filtering with only a start date @@ -49,14 +54,14 @@ Filtering with only a start date # ❌ Invalid column_filters = { - "stime": "2024-01-01" + "stime": "2024-01-01 12:00:00" } .. doctest-skip:: # ✅ Correct column_filters = { - "exp_start": "> '2024-01-01'" + "exp_start": "> '2024-01-01 12:00:00'" } Filtering with only an end date @@ -66,12 +71,12 @@ Filtering with only an end date # ❌ Invalid column_filters = { - "etime": "2024-12-31" + "etime": "2024-12-31 12:00:00" } .. doctest-skip:: # ✅ Correct column_filters = { - "exp_start": "< '2024-12-31'" + "exp_start": "< '2024-12-31 12:00:00'" } From a1ef15f5362d2cf273a739dcb5021a656087b4db Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Tue, 24 Feb 2026 15:26:32 +0100 Subject: [PATCH 17/18] Updates - alberto 2nd review --- docs/eso/eso.rst | 74 ++++- docs/eso/eso_cat_cone_search.rst | 198 +++++++++++++ docs/eso/eso_cat_query.rst | 153 +++++++++- docs/eso/eso_cat_tap.rst | 265 ++++++++++++++++++ ...one_search.rst => eso_obs_cone_search.rst} | 8 +- docs/eso/{eso_tap.rst => eso_obs_tap.rst} | 40 +-- docs/index.rst | 1 + 7 files changed, 698 insertions(+), 41 deletions(-) create mode 100644 docs/eso/eso_cat_cone_search.rst create mode 100644 docs/eso/eso_cat_tap.rst rename docs/eso/{eso_cone_search.rst => eso_obs_cone_search.rst} (97%) rename docs/eso/{eso_tap.rst => eso_obs_tap.rst} (97%) diff --git a/docs/eso/eso.rst b/docs/eso/eso.rst index 002ce5c9ce..451f10f98b 100644 --- a/docs/eso/eso.rst +++ b/docs/eso/eso.rst @@ -9,7 +9,8 @@ Quick Start .. note:: - Check if this is a version of ``astroquery.eso`` (0.4.12 or later) that includes TAP support (see warning below). + Check if this is a version of ``astroquery.eso`` (0.4.12 or later) that includes TAP support + (see warning below). .. doctest-skip:: @@ -18,7 +19,11 @@ Quick Start True This quick start example shows how to use the ``astroquery.eso`` interface to query the ESO -Science Archive for both raw and reduced observations. +Science Archive for both raw and reduced observations, as well as catalogue data. + +Observations +------------ + Here, we perform a cone search for `ESPRESSO `_ spectra towards the star `HD 37903 `_, and how to download the corresponding data products. @@ -41,6 +46,18 @@ and how to download the corresponding data products. >>> eso.retrieve_data(table_raw["dp_id"]) >>> eso.retrieve_data(table_reduced["dp_id"]) +Catalogues +------------ + +Catalogue queries operate on published survey tables and return column-based +results. For example, +to query the `KiDS DR4 `_ +catalogue and inspect a small subset of rows: + +.. doctest-skip:: + + >>> table_cat = eso.query_catalogue("KiDS_DR4_1_ugriZYJHKs_cat_fits", ROW_LIMIT=5) + By default, queries are limited to returning a maximum of 1000 rows. This limit can be modified by setting the ``ROW_LIMIT`` attribute. To disable truncation and return all matching results (up to TAP limit of 15,000,000), set: @@ -52,19 +69,48 @@ and return all matching results (up to TAP limit of 15,000,000), set: Getting Started =============== +This guide walks through logging in, running your first archive queries, inspecting +metadata, downloading products, and querying catalogues. + +Authentication +-------------- + .. toctree:: - :maxdepth: 1 + :maxdepth: 2 eso_login + +Observation Queries +------------------- + +.. toctree:: + :maxdepth: 2 + + eso_obs_cone_search eso_obs_raw_general eso_obs_raw_instrument eso_obs_reduced + eso_obs_tap eso_obs_apex eso_obs_header_info + eso_obs_download + +Catalogue Queries +----------------- + +.. toctree:: + :maxdepth: 2 + + eso_cat_cone_search eso_cat_query - eso_cone_search - eso_tap - eso_download + eso_cat_tap + +Help +---- + +.. toctree:: + :maxdepth: 2 + eso_troubleshooting .. warning:: @@ -81,14 +127,14 @@ Getting Started In TAP, ``column_filters`` accepts ADQL expressions. For example: .. doctest-skip:: - - column_filters = { - "some_int_column": "< 5", - "some_float_column_2": ">= 1.23", - "some_char_column": "like '%John%'", - "some_generic_column": "in ('mango', 'apple', 'kiwi')", - "other_generic_column": "between '2024-01-01' and '2024-12-31'" - } + + >>> column_filters = { + ... "some_int_column": "< 5", + ... "some_float_column_2": ">= 1.23", + ... "some_char_column": "like '%John%'", + ... "some_generic_column": "in ('mango', 'apple', 'kiwi')", + ... "other_generic_column": "between '2024-01-01' and '2024-12-31'" + ... } Please review your queries carefully and update them accordingly to ensure compatibility with the new astroquery versions. See section :ref:`column-filters-fix` for more information and examples. diff --git a/docs/eso/eso_cat_cone_search.rst b/docs/eso/eso_cat_cone_search.rst new file mode 100644 index 0000000000..4c05efbc2a --- /dev/null +++ b/docs/eso/eso_cat_cone_search.rst @@ -0,0 +1,198 @@ +************************ +Catalogues - Cone Search +************************ + +In the simplest case, catalogue queries can be constrained by sky +position (cone search). For catalogue tables, this is currently done by +building an ADQL cone-search expression and passing it through +``column_filters`` in ``eso.query_catalogue``. + +.. note:: + The ``cone_ra``, ``cone_dec``, and ``cone_radius`` arguments are not + currently implemented in ``eso.query_catalogue`` (as they are with e.g. ``eso.query_main``). + For catalogue cone searches, use a custom ADQL spatial filter as shown + below. + + +Identify Coordinate Columns +=========================== + +Before building a cone search, inspect the catalogue schema with +``help=True`` and identify the right ascension and declination columns from +their UCD values: + +- ``pos.eq.ra;meta.main`` for right ascension +- ``pos.eq.dec;meta.main`` for declination + +For example: + +.. doctest-skip:: + + >>> eso.query_catalogue(catalogue='KiDS_DR4_1_ugriZYJHKs_cat_fits', help=True) + ... + RAJ2000 DOUBLE deg pos.eq.ra;meta.main + DECJ2000 DOUBLE deg pos.eq.dec;meta.main + ... + +In this case the column names are ``RAJ2000`` and ``DECJ2000``. +However, different catalogues may use different coordinate column names (e.g. +``RA2000``/``DEC2000`` or ``RA_Spec``/``Dec_Spec``), so checking the UCD is the +most robust way to identify the correct columns. + +Query with Cone Filter +=========================== + +Catalogue cone searches are implemented using the ADQL spatial function +``CONTAINS(POINT, CIRCLE)``. Since ``eso.query_catalogue`` builds its +``WHERE`` clause from the ``column_filters`` dictionary, the spatial +constraint must be provided as a key with value ``1`` (corresponding to +``CONTAINS(...) = 1`` in ADQL). + +Under the hood, ``eso.query_catalogue`` submits an ADQL query via TAP. +For example, a cone search on the KiDS DR4 catalogue corresponds to an +ADQL statement of the form: + +.. code-block:: sql + + SELECT * + FROM KiDS_DR4_1_ugriZYJHKs_cat_fits + WHERE CONTAINS(POINT('', RAJ2000, DECJ2000), CIRCLE('', , , ))=1 + +Here: + +- ``POINT('', RAJ2000, DECJ2000)`` defines the source position + using the catalogue RA and Dec columns. +- ``CIRCLE('', , , )`` defines the + search region in degrees. +- ``CONTAINS`` evaluates to ``1`` when the source lies within + the specified region. + +When using ``column_filters``, the dictionary *key* is inserted as the +left-hand side of the ``WHERE`` clause, and the dictionary *value* +becomes the right-hand side. Therefore, providing a key like +``CONTAINS(...)`` with value ``1`` yields ``CONTAINS(...) = 1`` in ADQL. + +The helper function below constructs this automatically. + +Helper Function +--------------- + +Define target of interest and search radius: + +.. doctest-skip:: + + >>> from astroquery.eso import Eso + >>> from astropy.coordinates import SkyCoord + >>> import astropy.units as u + >>> eso = Eso() + + >>> coords = SkyCoord.from_name("NGC1097") + >>> radius = 3 * u.arcmin + +Helper function to construct ADQL cone search predicate: + +.. doctest-skip:: + + >>> def make_cone_filter( + ... ra, + ... dec, + ... radius, + ... ra_name="RAJ2000", + ... dec_name="DECJ2000", + ... ): + ... """ + ... Construct an ADQL cone-search predicate for catalogue queries. + ... + ... Parameters + ... ---------- + ... ra : astropy.units.Quantity + ... Right ascension of cone centre. + ... dec : astropy.units.Quantity + ... Declination of cone centre. + ... radius : astropy.units.Quantity + ... Search radius. + ... ra_name : str, optional + ... Name of the RA column in the catalogue + ... (default: 'RAJ2000'). + ... dec_name : str, optional + ... Name of the Dec column in the catalogue + ... (default: 'DECJ2000'). + ... + ... Returns + ... ------- + ... dict + ... Dictionary suitable for ``column_filters`` of the form + ... ``{predicate: 1}``, corresponding to + ... ``CONTAINS(...) = 1`` in ADQL. + ... """ + ... + ... ra_deg = ra.to_value(u.deg) + ... dec_deg = dec.to_value(u.deg) + ... rad_deg = radius.to_value(u.deg) + ... + ... predicate = ( + ... f"CONTAINS(" + ... f"POINT('', {ra_name}, {dec_name}), " + ... f"CIRCLE('', {ra_deg}, {dec_deg}, {rad_deg})" + ... f")" + ... ) + ... + ... return {predicate: 1} + +Run search with cone filter: + +.. doctest-skip:: + + >>> column_filters = make_cone_filter( + ... ra=coords.ra, + ... dec=coords.dec, + ... radius=radius, + ... ) + + >>> table = eso.query_catalogue( + ... catalogue="KiDS_DR4_1_ugriZYJHKs_cat_fits", + ... column_filters=column_filters, + ... ) + >>> table +
+ Level ALPHA_J2000 A_IMAGE A_WORLD ... Z_B_MAX Z_B_MIN Z_ML + count deg pixel deg ... + float32 float64 float64 float32 ... float64 float64 float64 + ---------- ----------- ------------------ ------------ ... ------------------ ------------------- ------------------- + 0.02452336 41.624103 3.264482 0.0001940674 ... 0.76 0.69 0.73 + 0.02450416 41.632288 1.199719 7.132157e-05 ... 1.35 0.52 1.87 + 0.02449745 41.634118 1.714497 0.0001019072 ... 1.03 0.41000000000000003 7.0 + 0.02455906 41.62684 1.2852190000000001 7.640447e-05 ... 1.62 0.98 1.65 + ... ... ... ... ... ... ... ... + 0.02458552 41.62298 1.037333 6.16569e-05 ... 1.6400000000000001 0.63 3.66 + 0.02453066 41.625326 1.191271 7.081894e-05 ... 1.38 0.53 0.24000000000000002 + 0.02451551 41.624227 1.582422 9.406044e-05 ... 0.63 0.43000000000000005 0.55 + 0.02454042 41.618984 2.15449 0.0001280547 ... 0.6799999999999999 0.17 0.11 + +Combining with Additional Column Filters +---------------------------------------- + +The cone filter returns a standard ``column_filters`` dictionary. +Additional constraints (e.g. magnitude limits) can be added using +normal dictionary updates. + +.. doctest:: + + >>> # Add a magnitude constraint + >>> column_filters.update({"MAG_AUTO": "<22"}) + + >>> table = eso.query_catalogue( + ... catalogue="KiDS_DR4_1_ugriZYJHKs_cat_fits", + ... column_filters=column_filters, + ... ) +
+ Level ALPHA_J2000 A_IMAGE A_WORLD Agaper ... Ypos Z_B Z_B_MAX Z_B_MIN Z_ML + count deg pixel deg arcsec ... pixel + float32 float64 float64 float32 float32 ... float32 float64 float64 float64 float64 + ---------- ----------- ------------------ ------------ ------- ... -------- ------- ------- ------------------- ------------------ + 0.02452336 41.624103 3.264482 0.0001940674 0.99 ... 9077.715 0.73 0.76 0.69 0.73 + 0.02450728 41.622675 3.695437 0.0002196531 1.06 ... 9153.268 0.51 0.54 0.48000000000000004 0.51 + 0.02450339 41.63336 2.607809 0.000155017 0.9 ... 9207.265 0.96 1.35 0.8200000000000001 1.6400000000000001 + 0.02452449 41.62697 2.9889900000000003 0.0001776759 0.95 ... 9217.702 0.72 0.75 0.6699999999999999 0.72 + 0.02452196 41.626775 2.787715 0.0001656992 0.92 ... 9311.3 0.56 0.59 0.53 0.56 + 0.0245021 41.629953 5.531547 0.0003288784 1.38 ... 9338.087 0.7 0.74 0.6599999999999999 0.7 \ No newline at end of file diff --git a/docs/eso/eso_cat_query.rst b/docs/eso/eso_cat_query.rst index ebaa8e83f2..d71698465d 100644 --- a/docs/eso/eso_cat_query.rst +++ b/docs/eso/eso_cat_query.rst @@ -1,4 +1,151 @@ -*************************** -Catalogues - Query for Data -*************************** +************************************* +Catalogues - Query for Catalogue Data +************************************* +In addition to raw and reduced observations, the ESO archive provides access to +science catalogues. This section shows how to +list the available catalogues and query one of them. + +Available Catalogues +==================== + +You can list the available catalogues with :meth:`~astroquery.eso.EsoClass.list_catalogues`. By default, +the ``all_versions=False`` option returns only the latest version of each +catalogue. + +.. doctest-skip:: + + >>> from astroquery.eso import Eso + >>> eso = Eso() + + >>> eso.list_catalogues(all_versions=False) + ['AMBRE_V1', + 'ATLASGAL_V1', + 'COSMOS2015_Laigle_v1_1b_latestV7_fits_V1', + 'EREBOS_cat_fits_V1', + 'EREBOS_RV_cat_fits_V1', + ... + 'KiDS_DR3_0_ugri_src_fits_V2', + 'KiDS_DR3_1_ugri_shear_fits_V1', + 'KiDS_DR4_1_ugriZYJHKs_cat_fits', + ... + 'vmc_dr7_yjks_back_V1', + 'vmc_dr7_yjks_varCat_V3'] + + >>> print(len(eso.list_catalogues(all_versions=False))) + 86 + +To include every available version of every catalogue, set +``all_versions=True``: + +.. doctest-skip:: + + >>> eso.list_catalogues(all_versions=True) + ['AMBRE_V1', + 'atlas_er3_ugriz_catMetaData_fits_V2', + 'ATLASGAL_V1', + 'COSMOS2015_Laigle_v1_1b_latestV7_fits_V1', + 'EREBOS_cat_fits_V1', + 'EREBOS_RV_cat_fits_V1', + ... + 'KiDS_DR3_0_ugri_src_fits_V2', + 'KiDS_DR3_1_ugri_shear_fits_V1', + 'KiDS_DR4_1_ugriZYJHKs_cat_fits', + ... + 'vmc_dr7_yjks_back_V1', + 'vmc_dr7_yjks_varCat_V3'] + + >>> print(len(eso.list_catalogues(all_versions=True))) + 129 + +Available Query Constraints +=========================== + +Before querying a specific catalogue, inspect its schema with +``eso.query_catalogue(..., help=True)``. This prints the available columns and +the total number of records in that table -- in this case the ``'KiDS_DR4_1_ugriZYJHKs_cat_fits'`` +table. To see more infomation on this specific version of the Kilo-Degree Survey (KiDS) catalogue, see the +`release documentation `_. + +.. doctest-skip:: + + >>> eso.query_catalogue(catalogue='KiDS_DR4_1_ugriZYJHKs_cat_fits', help=True) + INFO: + Columns present in the table safcat.KiDS_DR4_1_ugriZYJHKs_cat_fits: + column_name datatype unit ucd + ------------------- -------- ---------------- ------------------------------------- + ID CHAR meta.id;meta.main + KIDS_TILE CHAR meta.id + THELI_NAME CHAR meta.id + SeqNr INTEGER meta.id + SLID INTEGER meta.id + SID INTEGER meta.id + FLUX_AUTO REAL count phot.flux;em.opt.R + FLUXERR_AUTO REAL count stat.error;phot.flux;em.opt.R + MAG_AUTO REAL mag phot.mag;em.opt.R + ... + MASK INTEGER meta.code + COLOUR_GAAP_u_g REAL mag phot.color.reddFree;em.opt.U;em.opt.B + COLOUR_GAAP_g_r REAL mag phot.color.reddFree;em.opt.B;em.opt.R + COLOUR_GAAP_r_i REAL mag phot.color.reddFree;em.opt.R;em.opt.I + COLOUR_GAAP_i_Z REAL mag phot.color.reddFree;em.opt.I + COLOUR_GAAP_Z_Y REAL mag phot.color.reddFree;em.opt.I;em.IR.J + COLOUR_GAAP_Y_J REAL mag phot.color.reddFree;em.IR.J + COLOUR_GAAP_J_H REAL mag phot.color.reddFree;em.IR.J;em.IR.H + COLOUR_GAAP_H_Ks REAL mag phot.color.reddFree;em.IR.H;em.IR.K + Number of records present in the table safcat.KiDS_DR4_1_ugriZYJHKs_cat_fits: + 100350804 + [astroquery.eso.core] + +Query with Constraints +====================== + +The table can be queried using the :meth:`~astroquery.eso.EsoClass.query_catalogue` method. +Note, however, that catalogue tables can be very large, so it is often useful to start with a +small row limit, which can be set with e.g ``eso.ROW_LIMIT = 5`` or by passing +e.g. ``ROW_LIMIT=5`` to the query method: + +.. doctest-skip:: + + >>> table = eso.query_catalogue(catalogue='KiDS_DR4_1_ugriZYJHKs_cat_fits', ROW_LIMIT=5) + >>> table + WARNING: MaxResultsWarning: Results truncated to 5. To retrieve all the records set to None the ROW_LIMIT attribute [astroquery.eso.core] +
+ Level ALPHA_J2000 A_IMAGE A_WORLD ... Z_B_MAX Z_B_MIN Z_ML + count deg pixel deg ... + float32 float64 float64 float32 ... float64 float64 float64 + ---------- ----------- ------------------ ------------ ... ------- ------------------ ------------------ + 0.03316526 5.079035 1.023542 6.084906e-05 ... 0.97 0.6699999999999999 0.7799999999999999 + 0.03316413 5.127818 2.087884 0.0001240919 ... 0.92 0.7899999999999999 0.8300000000000001 + 0.03320557 5.171088 2.6607149999999997 0.00015816 ... 0.89 0.7899999999999999 0.8600000000000001 + 0.02512704 5.912251 201.661499 0.0119862 ... 0.04 0.01 0.01 + 0.04201349 6.206854 3.289606 0.0001955687 ... 0.35 0.23 0.27 + +A larger number of rows can be returned by setting a higher row limit, or by disabling truncation +with ``eso.ROW_LIMIT = -1``. Be aware that the TAP service has a maximum limit of 15,000,000 rows +per query, so setting ``eso.ROW_LIMIT = -1`` will return all matching results up to that TAP limit. + +You can also combine selected columns with ADQL filters in ``column_filters``. +For example, to retrieve bright sources with r-band magnitude of ``MAG_AUTO < 10``, and only return a subset of +the available columns, you can do: + +.. doctest-skip:: + + >>> table = eso.query_catalogue( + ... catalogue='KiDS_DR4_1_ugriZYJHKs_cat_fits', + ... columns=["ID", "RAJ2000", "DECJ2000", "KIDS_TILE", "MAG_AUTO", "MAGERR_AUTO"], + ... column_filters={"MAG_AUTO": "<10"} + ... ) + >>> table +
+ ID RAJ2000 DECJ2000 ... MAG_AUTO MAGERR_AUTO + deg deg ... mag mag + object float64 float64 ... float32 float32 + ----------------------------- ------------------ ------------------- ... -------- ------------ + KiDSDR4 J092035.791+001110.92 140.14913 0.186369 ... 9.263525 3.264208e-05 + KiDSDR4 J110654.466+015628.34 166.726944 1.941208 ... 9.844576 4.391666e-05 + KiDSDR4 J225722.767-293856.66 344.34486300000003 -29.649074 ... 9.725844 0.0001094358 + ... + KiDSDR4 J223130.214-322011.16 337.875892 -32.336436 ... 9.379268 2.736172e-05 + KiDSDR4 J223131.492-322022.97 337.881217 -32.339715 ... 9.846291 3.40929e-05 + KiDSDR4 J032045.699-263559.19 50.190415 -26.599775 ... 9.857556 3.780224e-05 \ No newline at end of file diff --git a/docs/eso/eso_cat_tap.rst b/docs/eso/eso_cat_tap.rst new file mode 100644 index 0000000000..8a23f007eb --- /dev/null +++ b/docs/eso/eso_cat_tap.rst @@ -0,0 +1,265 @@ +***************************** +Catalogues - Query with TAP +***************************** + +The ESO TAP service can also be used to query catalogue tables directly with +free ADQL, using :meth:`~astroquery.eso.EsoClass.query_tap`. By default, +``query_tap(query)`` targets the observations TAP service (equivalent to +``which_tap="tap_obs"``). To query catalogue content, pass +``which_tap="tap_cat"`` -- i.e. ``eso.query_tap(query, which_tap="tap_cat")``. + +Basic Usage +=========== + +.. doctest-skip:: + + >>> from astroquery.eso import Eso + >>> eso = Eso() + >>> query = "SELECT table_name FROM TAP_SCHEMA.tables" + >>> table = eso.query_tap(query, which_tap="tap_cat") + +The examples below focus on common catalogue TAP workflows: schema discovery, +spatial filtering, metadata-driven joins, and light-curve retrieval. + +Schema Discovery Examples +========================= + +Inspect Columns in a Specific Catalogue Table +--------------------------------------------- + +Use this to list columns and data types for a specific catalogue table. +``TAP_SCHEMA.columns`` contains column names, units, UCDs, and additional +metadata for all published ESO tables. + +.. doctest-skip:: + + >>> query = """ + ... SELECT column_name, datatype, unit, ucd + ... FROM TAP_SCHEMA.columns + ... WHERE table_name='COSMOS2015_Laigle_v1_1b_latestV7_fits_V1' + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +Find the Main RA/Dec Columns via UCD +------------------------------------ + +Use this to identify which columns carry the main sky coordinates in a table. +Here, the UCD values mark the primary ``RA`` and ``Dec`` fields -- in this case +``TRANSIENT_RAJ2000`` and ``TRANSIENT_DECJ2000``. + +.. doctest-skip:: + + >>> query = """ + ... SELECT column_name + ... FROM TAP_SCHEMA.columns + ... WHERE table_name='PESSTO_TRAN_CAT_fits_V2' + ... AND (ucd = 'pos.eq.ra;meta.main' OR ucd = 'pos.eq.dec;meta.main') + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +List VMC Tables with Coordinate and Source-ID Columns +----------------------------------------------------- + +Use this to list coordinate and source-ID columns for all tables in the ``VMC`` +collection, ordered by publication date. + +.. doctest-skip:: + + >>> query = """ + ... SELECT publication_date, T.table_name, column_name, ucd + ... FROM TAP_SCHEMA.columns AS C, TAP_SCHEMA.tables AS T + ... WHERE T.table_name = C.table_name + ... AND cat_id IN ( + ... SELECT cat_id + ... FROM TAP_SCHEMA.tables + ... WHERE collection='VMC' + ... ) + ... AND ( + ... ucd = 'pos.eq.ra;meta.main' + ... OR ucd = 'pos.eq.dec;meta.main' + ... OR ucd = 'meta.id;meta.main' + ... ) + ... ORDER BY publication_date DESC, 2, 4 + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +List Coordinate and Source-ID Columns Across All Collections +------------------------------------------------------------ + +Use this to discover coordinate and source-ID columns across all published +catalogues, ordered by publication date and collection. + +.. doctest-skip:: + + >>> query = """ + ... SELECT publication_date, collection, T.table_name, column_name, ucd + ... FROM TAP_SCHEMA.columns AS C, TAP_SCHEMA.tables AS T + ... WHERE T.table_name = C.table_name + ... AND ( + ... ucd = 'pos.eq.ra;meta.main' + ... OR ucd = 'pos.eq.dec;meta.main' + ... OR ucd = 'meta.id;meta.main' + ... ) + ... ORDER BY publication_date DESC, 2, 3, 5 + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +Spatial Query Example +===================== + +Cone Search for PESSTO Transients Around ESO 154-10 +--------------------------------------------------- + +Search the PESSTO master catalogue for transients in a circle of ``0.04 deg`` +(``2.4 arcmin``) around galaxy ``ESO 154-10`` (ICRS position +``41.2863, -55.7406``). + +.. doctest-skip:: + + >>> query = """ + ... SELECT host_id, transient_id, transient_classification, + ... transient_raj2000, transient_decj2000 + ... FROM pessto_tran_cat_fits_V2 + ... WHERE CONTAINS( + ... POINT('', transient_raj2000, transient_decj2000), + ... CIRCLE('', 41.2863, -55.7406, 0.04) + ... ) = 1 + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +The target coordinates can also be resolved online using a name resolver and then included in +the query, as e.g.: + +.. doctest-skip:: + + >>> import astropy.units as u + >>> from astropy.coordinates import SkyCoord + + >>> target = SkyCoord.from_name("ESO 154-10") + >>> ra_deg = target.icrs.ra.to_value(u.deg) + >>> dec_deg = target.icrs.dec.to_value(u.deg) + >>> radius_deg = (2.4 * u.arcmin).to_value(u.deg) + + >>> query = f""" + ... SELECT host_id, transient_id, transient_classification, + ... transient_raj2000, transient_decj2000 + ... FROM pessto_tran_cat_fits_V2 + ... WHERE CONTAINS( + ... POINT('ICRS', transient_raj2000, transient_decj2000), + ... CIRCLE('ICRS', {ra_deg}, {dec_deg}, {radius_deg}) + ... ) = 1 + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +Metadata Join Example +===================== + +Discover Join Keys for VVV Tables +--------------------------------- + +Use this to discover how ``VVV_CAT_V2`` can be joined with related VVV tables. +The resulting table shows join pairs as: + +- ``from_table`` via ``from_column`` +- ``target_table`` via ``target_column`` + +.. doctest-skip:: + + >>> query = """ + ... SELECT k.from_table, kc.from_column, k.target_table, kc.target_column + ... FROM TAP_SCHEMA.columns AS c, TAP_SCHEMA.keys AS k, TAP_SCHEMA.key_columns AS kc + ... WHERE c.table_name='VVV_CAT_V2' + ... AND ( + ... (k.from_table = c.table_name AND kc.from_column = c.column_name) + ... OR + ... (k.target_table = c.table_name AND kc.target_column = c.column_name) + ... ) + ... AND k.key_id = kc.key_id + ... ORDER BY table_name, column_name, 1, 3, 2 + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +For example, after identifying join keys (in this case the ``VVV_MPHOT_Ks_V2`` and ``VVV_VAR_V2`` tables), +you can build a light-curve (see below). + +Light-Curve Query Examples +========================== + +Get a VVV Source Light Curve and Variability Metrics +---------------------------------------------------- + +Retrieve multi-epoch photometry and variability metrics for a known VVV source. + +.. doctest-skip:: + + >>> query = """ + ... SELECT TOP 10 VVV.IAUNAME, VVV.SOURCEID, VVV.RA2000, VVV.DEC2000, + ... M.MJD, M.KSMAG, M.KSERR, + ... V.KSMEANMAG, V.KSAMPL, KSPROBVAR + ... FROM VVV_CAT_V2 AS VVV + ... JOIN VVV_MPHOT_Ks_V2 AS M ON VVV.SOURCEID = M.SOURCEID + ... JOIN VVV_VAR_V2 AS V ON VVV.SOURCEID = V.SOURCEID + ... WHERE VVV.IAUNAME='VVV J135433.73-594836.43' + ... ORDER BY 1, 5 + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +In this query: + +- ``VVV`` is the VVV master catalogue alias. +- ``M`` is the multi-epoch photometry table alias. +- ``V`` is the variability table alias. + + +Get PESSTO Light-Curve Magnitudes in a Sky Region +------------------------------------------------- + +Retrieve selected magnitudes from the PESSTO light-curve catalogue for +transients within ``0.05 deg`` (``3 arcmin``) around ESO 154-10. + +.. doctest-skip:: + + >>> query = """ + ... SELECT host_id, transient_id, transient_classification, + ... lc.B_VEGA_MAG, lc.B_VEGA_MAGERR, + ... lc.R_AB_MAG, lc.R_AB_MAGERR, + ... lc.R_VEGA_MAG, lc.R_VEGA_MAGERR + ... FROM safcat.PESSTO_TRAN_CAT_V3 AS master + ... JOIN safcat.PESSTO_MPHOT_V3 AS lc ON lc.SOURCE_ID = master.TRANSIENT_ID + ... WHERE CONTAINS( + ... POINT('', transient_raj2000, transient_decj2000), + ... CIRCLE('', 41.2863, -55.7406, 0.05) + ... ) = 1 + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") + +The light-curve table does not provide coordinates directly, so it is joined to +the PESSTO master catalogue for spatial filtering. To retrieve all available +magnitude columns instead of only ``B`` and ``R``, use ``lc.*`` in the +``SELECT`` clause. + +This can also be done using a name resolver for the target coordinates, as previously shown above. + +.. doctest-skip:: + + >>> import astropy.units as u + >>> from astropy.coordinates import SkyCoord + + >>> target = SkyCoord.from_name("ESO 154-10") + >>> ra_deg = target.icrs.ra.to_value(u.deg) + >>> dec_deg = target.icrs.dec.to_value(u.deg) + >>> radius_deg = (3 * u.arcmin).to_value(u.deg) + + >>> query = f""" + ... SELECT host_id, transient_id, transient_classification, + ... lc.B_VEGA_MAG, lc.B_VEGA_MAGERR, + ... lc.R_AB_MAG, lc.R_AB_MAGERR, + ... lc.R_VEGA_MAG, lc.R_VEGA_MAGERR + ... FROM safcat.PESSTO_TRAN_CAT_V3 AS master + ... JOIN safcat.PESSTO_MPHOT_V3 AS lc ON lc.SOURCE_ID = master.TRANSIENT_ID + ... WHERE CONTAINS( + ... POINT('', transient_raj2000, transient_decj2000), + ... CIRCLE('', {ra_deg}, {dec_deg}, {radius_deg}) + ... ) = 1 + ... """ + >>> table = eso.query_tap(query, which_tap="tap_cat") \ No newline at end of file diff --git a/docs/eso/eso_cone_search.rst b/docs/eso/eso_obs_cone_search.rst similarity index 97% rename from docs/eso/eso_cone_search.rst rename to docs/eso/eso_obs_cone_search.rst index dfed119d50..d42dcd3981 100644 --- a/docs/eso/eso_cone_search.rst +++ b/docs/eso/eso_obs_cone_search.rst @@ -1,7 +1,7 @@ -****************** -Simple Cone Search -****************** +************************** +Observations - Cone Search +************************** In the simplest case, archive queries can be performed using a positional (cone) search, which returns all data products within a given angular radius @@ -94,4 +94,4 @@ To download the data returned by the query, you can use the :meth:`~astroquery.e The ``data_files`` list points to the decompressed dataset filenames that have been locally downloaded. The default location of the decompressed datasets can be adjusted by providing a ``destination`` keyword in the call to :meth:`~astroquery.eso.EsoClass.retrieve_data`. .. doctest-skip:: - >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") + >>> data_files = eso.retrieve_data(table["dp_id"], destination="./eso_data/") \ No newline at end of file diff --git a/docs/eso/eso_tap.rst b/docs/eso/eso_obs_tap.rst similarity index 97% rename from docs/eso/eso_tap.rst rename to docs/eso/eso_obs_tap.rst index 4e14767550..2ec7b23402 100644 --- a/docs/eso/eso_tap.rst +++ b/docs/eso/eso_obs_tap.rst @@ -41,7 +41,7 @@ The following example demonstrates how to query the ``dbo.raw`` table for raw da ... FROM dbo.raw ... AND instrument = 'MUSE' ... """ - >>> result = eso.query_tap(query) + >>> table = eso.query_tap(query) Query for Raw Data (Instrument-Specific) ======================================== @@ -63,7 +63,7 @@ an average seeing (FWHM) below 1.0 arcsec, and an exposure time greater than ... AND exptime > 100 ... AND lst between 0 and 6 ... """ - >>> result = eso.query_tap(query) + >>> table = eso.query_tap(query) Query for Reduced Data Products =============================== @@ -99,8 +99,8 @@ scales smaller than 0.2 arcsec: ... AND multi_ob = 'M' ... AND s_pixel_scale < 0.2 ... """ - >>> result = eso.query_tap(query) - >>> result + >>> table = eso.query_tap(query) + >>> table
obs_collection calib_level multi_ob filter s_pixel_scale instrument_name arcsec @@ -139,8 +139,8 @@ column, which represents the sky footprint of each dataset. .. doctest-remote-data:: - >>> result = eso.query_tap(query=query) - >>> print("Num matching datasets: %d" % (len(result))) + >>> table = eso.query_tap(query=query) + >>> print("Num matching datasets: %d" % (len(table))) Num matching datasets: 219 .. note:: @@ -180,8 +180,8 @@ Here we use ``CONTAINS`` and restrict the results to images and cubes. ... AND dataproduct_type IN ('image', 'cube') ... ORDER BY t_min ASC ... """ - >>> result = eso.query_tap(query=query) - >>> result + >>> table = eso.query_tap(query=query) + >>> table
t_min abmaglim ... obs_release_date d mag ... @@ -212,8 +212,8 @@ circle around NGC 253 and restrict to images and cubes. ... AND dataproduct_type IN ('image', 'cube') ... ORDER BY t_min ASC ... """ - >>> result = eso.query_tap(query=query) - >>> result + >>> table = eso.query_tap(query=query) + >>> table
t_min s_fov type dp_id obs_release_date d deg @@ -277,8 +277,8 @@ returns matching datasets ordered by ``t_min``. ... ))=1 ... ORDER BY t_min ASC ... """ - >>> result = eso.query_tap(query=query) - >>> result + >>> table = eso.query_tap(query=query) + >>> table
t_min snr abmaglim type dp_id d mag @@ -324,8 +324,8 @@ join to find HAWKI source tables: ... WHERE INTERSECTS(J.s_region, H.s_region)=1 ... AND ESO_INTERSECTION(J.s_region, H.s_region) > 0.8 * AREA(J.s_region) ... """ - >>> result = eso.query_tap(query=query) - >>> result + >>> table = eso.query_tap(query=query) + >>> table
abmaglim access_estsize access_format ... t_resolution t_xel target_name float64 int64 object ... float64 int64 object @@ -366,8 +366,8 @@ This applies even when the underlying data are naturally described in frequency ... FROM ivoa.ObsCore ... WHERE target_name = 'a370' ... AND em_res_power < 3000""" - >>> result = eso.query_tap(query=query) - >>> result + >>> table = eso.query_tap(query=query) + >>> table
collection type subtype min_wavel_nm max_wavel_nm em_res_power object object object float64 float64 float64 @@ -391,8 +391,8 @@ currently exposed through the ESO TAP service. .. doctest-remote-data:: >>> query = "SELECT COUNT(*) FROM ivoa.ObsCore" - >>> result = eso.query_tap(query) - >>> result + >>> table = eso.query_tap(query) + >>> table
COUNT_ALL int32 @@ -415,8 +415,8 @@ volume. ... GROUP BY dataproduct_type ... ORDER BY 3 DESC ... """ - >>> result = eso.query_tap(query) - >>> result + >>> table = eso.query_tap(query) + >>> table
dataproduct_type numproducts tb object int32 float64 diff --git a/docs/index.rst b/docs/index.rst index 64c3d35f66..074d19e8ca 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -339,6 +339,7 @@ for each source) :maxdepth: 1 alfalfa/alfalfa.rst + eso/eso.rst exoplanet_orbit_database/exoplanet_orbit_database.rst gama/gama.rst ipac/irsa/irsa_dust/irsa_dust.rst From ec57bf2703271be96536e1fd57b60924062a301d Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Thu, 26 Feb 2026 18:03:59 +0100 Subject: [PATCH 18/18] minor updates --- docs/eso/eso.rst | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/eso/eso.rst b/docs/eso/eso.rst index 451f10f98b..92060126e7 100644 --- a/docs/eso/eso.rst +++ b/docs/eso/eso.rst @@ -10,13 +10,7 @@ Quick Start .. note:: Check if this is a version of ``astroquery.eso`` (0.4.12 or later) that includes TAP support - (see warning below). - - .. doctest-skip:: - - >>> import astroquery - >>> astroquery.__version__ >= '0.4.12' - True + (see **⚠️ Backward Compatibility Notice ⚠️** below). This quick start example shows how to use the ``astroquery.eso`` interface to query the ESO Science Archive for both raw and reduced observations, as well as catalogue data. @@ -26,7 +20,7 @@ Observations Here, we perform a cone search for `ESPRESSO `_ spectra towards the star `HD 37903 `_, -and how to download the corresponding data products. +and show how to download the corresponding data products. .. doctest-skip::