From 78b3e8d4e55ee8411d771ffa4f628f8e4b7f6bc7 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 12 Sep 2025 09:42:38 +0200 Subject: [PATCH 1/5] Setup auto sync fork for main branch --- .github/workflows/sync_fork.yml | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/sync_fork.yml diff --git a/.github/workflows/sync_fork.yml b/.github/workflows/sync_fork.yml new file mode 100644 index 0000000000..18abb115e6 --- /dev/null +++ b/.github/workflows/sync_fork.yml @@ -0,0 +1,45 @@ +name: Sync Fork +run-name: Sync Fork +on: + schedule: + - cron: '58 23 * * *' # run every day - two minutes to midnight + workflow_dispatch: # to enable manual runs of the workflow + +jobs: + Get-Timestamp: + runs-on: ubuntu-latest + steps: + - run: date + + Sync-With-Upstream: + runs-on: ubuntu-latest + steps: + - run: echo "The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "This job is now running on a ${{ runner.os }} server hosted by GitHub" + - run: echo "Running on branch ${{ github.ref }}, repository ${{ github.repository }}." + - name: Check out repository code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - run: echo "The ${{ github.repository }} repository has been cloned to the runner." + - name: List files in the repository + run: | + ls ${{ github.workspace }} + - name: Sync repository with upstream + run: | + cd ${{ github.workspace }} + git config --global user.email "jcarmona@eso.org" + git config --global user.name "Nightly Sync" + git remote add upstream https://github.com/astropy/astroquery.git + git remote -v + git fetch upstream main + echo "--- upstream log: " + git log upstream/main --oneline -10 + echo "--- current branch log before merge: " + git log --oneline -10 + git merge upstream/main + echo "--- current branch log after merge: " + git log --oneline -10 + echo "--- push force with lease" + git push --force-with-lease + - run: echo "The job finished with status ${{ job.status }}." From f03b75f72a9986192cbda54cc042115c680a6df2 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 11 Sep 2025 16:19:21 +0200 Subject: [PATCH 2/5] Setup custom CI/CD for ESO submodule --- .github/workflows/ci_devtests.yml | 2 ++ .github/workflows/ci_tests.yml | 2 ++ setup.cfg | 4 +++- tox.ini | 6 +++--- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_devtests.yml b/.github/workflows/ci_devtests.yml index cfb68e4b1c..509fc3e7ea 100644 --- a/.github/workflows/ci_devtests.yml +++ b/.github/workflows/ci_devtests.yml @@ -5,11 +5,13 @@ on: push: branches: - main + - develop tags: - '*' pull_request: branches: - main + - develop schedule: # run every Monday at 5am UTC - cron: '0 5 * * 1' diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index d0db0f48cc..0135d90b1e 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -5,11 +5,13 @@ on: push: branches: - main + - develop tags: - '*' pull_request: branches: - main + - develop schedule: # run every Monday at 5am UTC - cron: '0 5 * * 1' diff --git a/setup.cfg b/setup.cfg index ecdc423640..9d128fac73 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,9 @@ show-response = 1 [tool:pytest] minversion = 7.4 norecursedirs = build docs/_build astroquery/irsa astroquery/nasa_exoplanet_archive astroquery/ned astroquery/ibe astroquery/irsa_dust astroquery/cds astroquery/sha astroquery/dace -testpaths = astroquery docs +testpaths = + docs/eso + astroquery/eso doctest_plus = enabled astropy_header = true text_file_format = rst diff --git a/tox.ini b/tox.ini index 5c19109f91..896c0f8341 100644 --- a/tox.ini +++ b/tox.ini @@ -68,12 +68,12 @@ commands = devdeps: pip install -U --pre --no-deps --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy python -m pip freeze - !cov: pytest --pyargs astroquery {toxinidir}/docs {env:PYTEST_ARGS} {posargs} - cov: pytest --pyargs astroquery {toxinidir}/docs --cov astroquery --cov-config={toxinidir}/setup.cfg {env:PYTEST_ARGS} {posargs} + !cov: pytest --pyargs astroquery.eso {toxinidir}/docs/eso {env:PYTEST_ARGS} {posargs} + cov: pytest --pyargs astroquery.eso {toxinidir}/docs/eso --cov astroquery.eso --cov-config={toxinidir}/setup.cfg {env:PYTEST_ARGS} {posargs} # For remote tests, we re-run the failures to filter out at least some of the flaky ones. # We use a second pytest run with --last-failed as opposed to --rerun in order to rerun the # failed ones at the end rather than right away. - online: pytest --pyargs astroquery {toxinidir}/docs {env:PYTEST_ARGS_2} {posargs} + online: pytest --pyargs astroquery.eso {toxinidir}/docs/eso {env:PYTEST_ARGS_2} {posargs} cov: coverage xml -o {toxinidir}/coverage.xml pip_pre = From 4d1cbf4d266a093e98b447153d12c42502990091 Mon Sep 17 00:00:00 2001 From: Ashley Barnes <30494539+ashleythomasbarnes@users.noreply.github.com> Date: Wed, 14 Jan 2026 21:51:36 +0100 Subject: [PATCH 3/5] Add query_ancillary method for Phase 3 product files Introduces the query_ancillary method to EsoClass for querying Phase 3 ancillary product files from the ESO archive. Adds the phase3_product_files_table constant, a helper _normalize_product_ids, and a new test Jupyter notebook demonstrating the new functionality. --- astroquery/eso/core.py | 150 +++++++++- .../eso/testing/test_query_ancillary.ipynb | 281 ++++++++++++++++++ astroquery/eso/utils.py | 30 ++ 3 files changed, 460 insertions(+), 1 deletion(-) create mode 100644 astroquery/eso/testing/test_query_ancillary.ipynb diff --git a/astroquery/eso/core.py b/astroquery/eso/core.py index 30c473b60d..e4c91efe2e 100644 --- a/astroquery/eso/core.py +++ b/astroquery/eso/core.py @@ -37,7 +37,7 @@ from ..query import QueryWithLogin from ..utils import schema from .utils import _UserParams, raise_if_coords_not_valid, _reorder_columns, \ - _raise_if_has_deprecated_keys, _build_adql_string, \ + _raise_if_has_deprecated_keys, _build_adql_string, _normalize_product_ids, \ DEFAULT_LEAD_COLS_PHASE3, DEFAULT_LEAD_COLS_RAW @@ -69,6 +69,7 @@ def _expired(self) -> bool: class _EsoNames: raw_table = "dbo.raw" phase3_table = "ivoa.ObsCore" + phase3_product_files_table = "phase3v2.product_files" raw_instruments_column = "instrument" phase3_surveys_column = "obs_collection" @@ -677,6 +678,153 @@ def query_instrument( t = _reorder_columns(t, DEFAULT_LEAD_COLS_RAW) return t + def query_ancillary( + self, + dp_id=None, *, + help: bool = False, + columns: Union[List, str] = None, + column_filters: Optional[dict] = None, + maxrec: int = None, + **kwargs, + ) -> Union[Table, int, str, None]: + """ + Query Phase 3 ancillary product files contained in the ESO archive. + + Parameters + ---------- + dp_id : str or list-like or Column, optional + Phase 3 product identifier(s) to query. These map to ``product_id`` + in the ``phase3v2.product_files`` table. When ``help`` is ``True``, + this parameter is optional. + help : bool, optional + If ``True``, prints all the parameters accepted in ``column_filters`` + and ``columns``. Default is ``False``. + columns : str or list of str, optional + Name of the columns the query should return. If specified as a string, + it should be a comma-separated list of column names. + column_filters : dict or None, optional + Constraints applied to the query in ADQL syntax, + e.g., ``{"quality": "like '%SCIENCE%'"}``. + Default is ``None``. + maxrec : int or None, optional + Overrides the configured row limit for this query only. + **kwargs + Additional optional parameters forwarded to the query. + + Returns + ------- + astropy.table.Table, str, int, or None + - By default, returns an :class:`~astropy.table.Table` containing records + based on the specified columns and constraints. Returns ``None`` if no results. + - When ``count_only`` is ``True``, returns an ``int`` representing the + record count for the specified filters. + - When ``get_query_payload`` is ``True``, returns the query string that + would be issued to the TAP service given the specified arguments. + """ + table_name = _EsoNames.phase3_product_files_table + if help: + self.list_column(table_name) + return + + dp_ids = _normalize_product_ids(dp_id) + if not dp_ids: + raise ValueError("dp_id must be specified when help=False.") + + column_filters = dict(column_filters) if column_filters else {} + + def _parse_product_id_filter(value): + if isinstance(value, (list, tuple, set)): + return _normalize_product_ids(value) + if not isinstance(value, str): + return _normalize_product_ids([value]) + + val = value.strip() + if not val: + return [] + + lower = val.lower() + if lower.startswith("in "): + rest = val[3:].strip() + if not (rest.startswith("(") and rest.endswith(")")): + return None + inner = rest[1:-1] + items = [item.strip() for item in inner.split(",")] + cleaned = [] + for item in items: + if len(item) >= 2 and item[0] == item[-1] == "'": + item = item[1:-1] + item = item.strip() + if item: + cleaned.append(item) + return cleaned + if lower.startswith("="): + item = val[1:].strip() + if len(item) >= 2 and item[0] == item[-1] == "'": + item = item[1:-1] + return [item] if item else [] + if lower.startswith(("like ", "not like ", "between ", "not between ", + "<=", ">=", "!=", "<", ">")): + return None + + if len(val) >= 2 and val[0] == val[-1] == "'": + val = val[1:-1] + return [val] + + product_id_key = next((k for k in column_filters if k.lower() == "product_id"), None) + if product_id_key: + filter_values = _parse_product_id_filter(column_filters[product_id_key]) + if filter_values is None: + raise ValueError( + "column_filters for 'product_id' must use '=' or 'in' when dp_id is provided." + ) + filter_values = _normalize_product_ids(filter_values) + if not filter_values: + raise ValueError("column_filters for 'product_id' is empty.") + filter_set = set(filter_values) + merged = [val for val in dp_ids if val in filter_set] + if not merged: + raise ValueError( + "column_filters for 'product_id' conflicts with dp_id values." + ) + dp_ids = merged + column_filters.pop(product_id_key, None) + + allowed_kwargs = { + "top", "count_only", "get_query_payload", "authenticated", + "order_by", "order_by_desc", + } + unknown_kwargs = set(kwargs) - allowed_kwargs + if unknown_kwargs: + unknown_str = ", ".join(sorted(unknown_kwargs)) + raise TypeError(f"Unexpected keyword argument(s): {unknown_str}") + + row_limit = None + if maxrec is not None: + row_limit = self.ROW_LIMIT + self.ROW_LIMIT = maxrec + + try: + user_params = _UserParams(table_name=table_name, + column_name="product_id", + allowed_values=dp_ids, + cone_ra=None, + cone_dec=None, + cone_radius=None, + columns=columns, + column_filters=column_filters, + top=kwargs.get("top"), + count_only=kwargs.get("count_only", False), + get_query_payload=kwargs.get("get_query_payload", False), + print_help=False, + authenticated=kwargs.get("authenticated", False), + order_by=kwargs.get("order_by", ''), + order_by_desc=kwargs.get("order_by_desc", True), + ) + return self._query_on_allowed_values(user_params) + finally: + if row_limit is not None: + self.ROW_LIMIT = row_limit + def get_headers(self, product_ids, *, cache=True): """ Get the headers associated to a list of data product IDs diff --git a/astroquery/eso/testing/test_query_ancillary.ipynb b/astroquery/eso/testing/test_query_ancillary.ipynb new file mode 100644 index 0000000000..322c88fc66 --- /dev/null +++ b/astroquery/eso/testing/test_query_ancillary.ipynb @@ -0,0 +1,281 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "9ad322a1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.4.12.dev10525\n", + "/opt/anaconda3/envs/astroquery_testing/lib/python3.9/site-packages/astroquery/__init__.py\n" + ] + } + ], + "source": [ + "import astroquery\n", + "print(astroquery.__version__) # Print the version of astroquery\n", + "print(astroquery.__file__) # Print the file path of astroquery\n", + "from astroquery.eso import Eso # Import the ESO module\n", + "\n", + "eso = Eso()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "476180a0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO: \n", + "Columns present in the table phase3v2.product_files:\n", + " column_name datatype xtype unit\n", + "----------------- -------- ----- -----\n", + " access_estsize long kbyte\n", + " access_url char \n", + " archive_id char \n", + " eso_category char \n", + " extension char \n", + " internal_file_id long \n", + "original_filename char \n", + " product_id char \n", + "\n", + "Number of records present in the table phase3v2.product_files:\n", + "15209363\n", + " [astroquery.eso.core]\n" + ] + } + ], + "source": [ + "eso.query_ancillary(help=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "d98cd474", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Coordinates of Sgr A*: \n" + ] + } + ], + "source": [ + "from astropy.coordinates import SkyCoord # import the SkyCoord class from the astropy.coordinates module\n", + "import astropy.units as u # import the astropy.units module\n", + "\n", + "target = \"Sgr A*\" # set the target to Sgr A*\n", + "coords = SkyCoord.from_name(target) # create a SkyCoord object from the name of the source \n", + "radius = 3 *u.arcmin # set the radius of the search to 3 arcminutes\n", + "\n", + "print(f\"Coordinates of Sgr A*: {coords}\") # print the coordinates of Sgr A*\n", + "\n", + "table_reduced = eso.query_surveys(column_filters={\"instrument_name\": \"HAWKI\"},\n", + " cone_ra=coords.ra.value, \n", + " cone_dec=coords.dec.value, \n", + " cone_radius=radius.to(\"deg\").value) # query the ESO archive for HAWKI data" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "ce33704d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<MaskedColumn name='dp_id' dtype='object' description='ESO-specific field not present in the standard ObsCore. The original ESO identifier of the main science file. ObsCore uses obs_publisher_did, which returns the full IVO Identifier. This field can be used to join with other ESO (non-VO) tables, like provenance.' length=105>\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
ADP.2021-04-30T12:05:58.657
ADP.2021-04-30T12:05:58.661
ADP.2021-04-30T12:05:58.763
ADP.2021-04-30T12:05:58.795
ADP.2021-04-30T12:05:58.889
ADP.2021-04-30T12:05:58.745
ADP.2021-04-30T12:05:58.747
ADP.2021-04-30T12:05:58.893
ADP.2021-04-30T12:05:59.035
ADP.2021-04-30T12:05:59.249
ADP.2021-04-30T12:05:58.797
ADP.2021-04-30T12:05:58.807
...
ADP.2021-04-30T12:05:59.265
ADP.2021-04-30T12:05:59.363
ADP.2021-04-30T12:05:59.383
ADP.2021-04-30T12:05:59.413
ADP.2021-04-30T12:05:59.423
ADP.2021-12-16T10:45:25.800
ADP.2021-12-16T10:45:25.828
ADP.2021-12-16T10:45:25.872
ADP.2021-12-16T10:45:25.918
ADP.2021-12-16T10:45:25.962
ADP.2021-12-16T10:45:25.805
ADP.2021-12-16T10:45:25.939
" + ], + "text/plain": [ + "\n", + "ADP.2021-04-30T12:05:58.657\n", + "ADP.2021-04-30T12:05:58.661\n", + "ADP.2021-04-30T12:05:58.763\n", + "ADP.2021-04-30T12:05:58.795\n", + "ADP.2021-04-30T12:05:58.889\n", + "ADP.2021-04-30T12:05:58.745\n", + "ADP.2021-04-30T12:05:58.747\n", + "ADP.2021-04-30T12:05:58.893\n", + "ADP.2021-04-30T12:05:59.035\n", + "ADP.2021-04-30T12:05:59.249\n", + "ADP.2021-04-30T12:05:58.797\n", + "ADP.2021-04-30T12:05:58.807\n", + " ...\n", + "ADP.2021-04-30T12:05:59.265\n", + "ADP.2021-04-30T12:05:59.363\n", + "ADP.2021-04-30T12:05:59.383\n", + "ADP.2021-04-30T12:05:59.413\n", + "ADP.2021-04-30T12:05:59.423\n", + "ADP.2021-12-16T10:45:25.800\n", + "ADP.2021-12-16T10:45:25.828\n", + "ADP.2021-12-16T10:45:25.872\n", + "ADP.2021-12-16T10:45:25.918\n", + "ADP.2021-12-16T10:45:25.962\n", + "ADP.2021-12-16T10:45:25.805\n", + "ADP.2021-12-16T10:45:25.939" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table_reduced[\"dp_id\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "361b7809", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Table length=2\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
access_estsizeaccess_urlarchive_ideso_categoryextensioninternal_file_idoriginal_filenameproduct_id
kbyte
int64objectobjectobjectobjectint64objectobject
46088https://dataportal.eso.org/dataPortal/file/ADP.2021-04-30T12:05:58.657ADP.2021-04-30T12:05:58.657SCIENCE.IMAGEFITS24843754F1_chip1_J.fitsADP.2021-04-30T12:05:58.657
46088https://dataportal.eso.org/dataPortal/file/ADP.2021-04-30T12:05:58.658ADP.2021-04-30T12:05:58.658ANCILLARY.RMSMAPFITS24843755F1_chip1_J_error_map.fitsADP.2021-04-30T12:05:58.657
" + ], + "text/plain": [ + "\n", + "access_estsize ... product_id \n", + " kbyte ... \n", + " int64 ... object \n", + "-------------- ... ---------------------------\n", + " 46088 ... ADP.2021-04-30T12:05:58.657\n", + " 46088 ... ADP.2021-04-30T12:05:58.657" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table = eso.query_ancillary(table_reduced[\"dp_id\"][0])\n", + "table" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "dc0ba64f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Table length=10\n", + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
access_estsizeaccess_urlarchive_ideso_categoryextensioninternal_file_idoriginal_filenameproduct_id
kbyte
int64objectobjectobjectobjectint64objectobject
46088https://dataportal.eso.org/dataPortal/file/ADP.2021-04-30T12:05:58.657ADP.2021-04-30T12:05:58.657SCIENCE.IMAGEFITS24843754F1_chip1_J.fitsADP.2021-04-30T12:05:58.657
46088https://dataportal.eso.org/dataPortal/file/ADP.2021-04-30T12:05:58.658ADP.2021-04-30T12:05:58.658ANCILLARY.RMSMAPFITS24843755F1_chip1_J_error_map.fitsADP.2021-04-30T12:05:58.657
46088https://dataportal.eso.org/dataPortal/file/ADP.2021-04-30T12:05:58.661ADP.2021-04-30T12:05:58.661SCIENCE.IMAGEFITS24843758F1_chip3_J.fitsADP.2021-04-30T12:05:58.661
46088https://dataportal.eso.org/dataPortal/file/ADP.2021-04-30T12:05:58.662ADP.2021-04-30T12:05:58.662ANCILLARY.RMSMAPFITS24843759F1_chip3_J_error_map.fitsADP.2021-04-30T12:05:58.661
46088https://dataportal.eso.org/dataPortal/file/ADP.2021-04-30T12:05:58.763ADP.2021-04-30T12:05:58.763SCIENCE.IMAGEFITS24843860F3_chip3_J.fitsADP.2021-04-30T12:05:58.763
46088https://dataportal.eso.org/dataPortal/file/ADP.2021-04-30T12:05:58.764ADP.2021-04-30T12:05:58.764ANCILLARY.RMSMAPFITS24843861F3_chip3_J_error_map.fitsADP.2021-04-30T12:05:58.763
46088https://dataportal.eso.org/dataPortal/file/ADP.2021-04-30T12:05:58.795ADP.2021-04-30T12:05:58.795SCIENCE.IMAGEFITS24843892F8_chip2_J.fitsADP.2021-04-30T12:05:58.795
46088https://dataportal.eso.org/dataPortal/file/ADP.2021-04-30T12:05:58.796ADP.2021-04-30T12:05:58.796ANCILLARY.RMSMAPFITS24843893F8_chip2_J_error_map.fitsADP.2021-04-30T12:05:58.795
46088https://dataportal.eso.org/dataPortal/file/ADP.2021-04-30T12:05:58.889ADP.2021-04-30T12:05:58.889SCIENCE.IMAGEFITS24843986F1_chip1_H.fitsADP.2021-04-30T12:05:58.889
46088https://dataportal.eso.org/dataPortal/file/ADP.2021-04-30T12:05:58.890ADP.2021-04-30T12:05:58.890ANCILLARY.RMSMAPFITS24843987F1_chip1_H_error_map.fitsADP.2021-04-30T12:05:58.889
" + ], + "text/plain": [ + "\n", + "access_estsize ... product_id \n", + " kbyte ... \n", + " int64 ... object \n", + "-------------- ... ---------------------------\n", + " 46088 ... ADP.2021-04-30T12:05:58.657\n", + " 46088 ... ADP.2021-04-30T12:05:58.657\n", + " 46088 ... ADP.2021-04-30T12:05:58.661\n", + " 46088 ... ADP.2021-04-30T12:05:58.661\n", + " 46088 ... ADP.2021-04-30T12:05:58.763\n", + " 46088 ... ADP.2021-04-30T12:05:58.763\n", + " 46088 ... ADP.2021-04-30T12:05:58.795\n", + " 46088 ... ADP.2021-04-30T12:05:58.795\n", + " 46088 ... ADP.2021-04-30T12:05:58.889\n", + " 46088 ... ADP.2021-04-30T12:05:58.889" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table = eso.query_ancillary(table_reduced[\"dp_id\"][0:5])\n", + "table" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "astroquery_testing", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.25" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/astroquery/eso/utils.py b/astroquery/eso/utils.py index e4f910e279..931b2f1129 100644 --- a/astroquery/eso/utils.py +++ b/astroquery/eso/utils.py @@ -3,6 +3,8 @@ """ from dataclasses import dataclass from typing import Dict, List, Optional, Union + +import numpy as np from astropy.table import Table DEFAULT_LEAD_COLS_RAW = ['object', 'ra', 'dec', 'dp_id', 'date_obs', 'prog_id'] @@ -39,6 +41,34 @@ def _split_str_as_list_of_str(column_str: str): return column_list +def _normalize_product_ids(dp_id) -> List[str]: + if dp_id is None: + return [] + + if isinstance(dp_id, (str, bytes)): + raw_values = [dp_id] + else: + try: + raw_values = list(dp_id) + except TypeError: + raw_values = [dp_id] + + normalized = [] + seen = set() + for val in raw_values: + if val is None or np.ma.is_masked(val): + continue + if isinstance(val, bytes): + val = val.decode() + val = str(val).strip() + if not val or val in seen: + continue + seen.add(val) + normalized.append(val) + + return normalized + + def _raise_if_has_deprecated_keys(filters: Optional[Dict[str, str]]) -> bool: if not filters: return From e6d74eaf71df472cf96c88d066940b964e7dda30 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Fri, 12 Sep 2025 09:42:38 +0200 Subject: [PATCH 4/5] Setup auto sync fork for main branch --- .github/workflows/sync_fork.yml | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/sync_fork.yml diff --git a/.github/workflows/sync_fork.yml b/.github/workflows/sync_fork.yml new file mode 100644 index 0000000000..18abb115e6 --- /dev/null +++ b/.github/workflows/sync_fork.yml @@ -0,0 +1,45 @@ +name: Sync Fork +run-name: Sync Fork +on: + schedule: + - cron: '58 23 * * *' # run every day - two minutes to midnight + workflow_dispatch: # to enable manual runs of the workflow + +jobs: + Get-Timestamp: + runs-on: ubuntu-latest + steps: + - run: date + + Sync-With-Upstream: + runs-on: ubuntu-latest + steps: + - run: echo "The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "This job is now running on a ${{ runner.os }} server hosted by GitHub" + - run: echo "Running on branch ${{ github.ref }}, repository ${{ github.repository }}." + - name: Check out repository code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - run: echo "The ${{ github.repository }} repository has been cloned to the runner." + - name: List files in the repository + run: | + ls ${{ github.workspace }} + - name: Sync repository with upstream + run: | + cd ${{ github.workspace }} + git config --global user.email "jcarmona@eso.org" + git config --global user.name "Nightly Sync" + git remote add upstream https://github.com/astropy/astroquery.git + git remote -v + git fetch upstream main + echo "--- upstream log: " + git log upstream/main --oneline -10 + echo "--- current branch log before merge: " + git log --oneline -10 + git merge upstream/main + echo "--- current branch log after merge: " + git log --oneline -10 + echo "--- push force with lease" + git push --force-with-lease + - run: echo "The job finished with status ${{ job.status }}." From 2345b66c3be1883e8beabf221782dcbaa091e5f5 Mon Sep 17 00:00:00 2001 From: "Juan M. Carmona Loaiza" Date: Thu, 11 Sep 2025 16:19:21 +0200 Subject: [PATCH 5/5] Setup custom CI/CD for ESO submodule --- .github/workflows/ci_devtests.yml | 2 ++ .github/workflows/ci_tests.yml | 2 ++ setup.cfg | 4 +++- tox.ini | 6 +++--- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_devtests.yml b/.github/workflows/ci_devtests.yml index e0f73c1310..7f0b61b132 100644 --- a/.github/workflows/ci_devtests.yml +++ b/.github/workflows/ci_devtests.yml @@ -5,11 +5,13 @@ on: push: branches: - main + - develop tags: - '*' pull_request: branches: - main + - develop schedule: # run every Monday at 5am UTC - cron: '0 5 * * 1' diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 9786f85fcf..278b38491e 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -5,11 +5,13 @@ on: push: branches: - main + - develop tags: - '*' pull_request: branches: - main + - develop schedule: # run every Monday at 5am UTC - cron: '0 5 * * 1' diff --git a/setup.cfg b/setup.cfg index d51f8c8b1a..351e654fbf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,9 @@ show-response = 1 [tool:pytest] minversion = 7.4 norecursedirs = build docs/_build astroquery/irsa astroquery/nasa_exoplanet_archive astroquery/ned astroquery/ibe astroquery/irsa_dust astroquery/cds astroquery/sha astroquery/dace -testpaths = astroquery docs +testpaths = + docs/eso + astroquery/eso doctest_plus = enabled astropy_header = true text_file_format = rst diff --git a/tox.ini b/tox.ini index 53a67b9341..1f7fcceef4 100644 --- a/tox.ini +++ b/tox.ini @@ -66,12 +66,12 @@ commands = devdeps: pip install -U --pre --no-deps --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy python -m pip freeze - !cov: pytest --pyargs astroquery {toxinidir}/docs {env:PYTEST_ARGS} {posargs} - cov: pytest --pyargs astroquery {toxinidir}/docs --cov astroquery --cov-config={toxinidir}/setup.cfg {env:PYTEST_ARGS} {posargs} + !cov: pytest --pyargs astroquery.eso {toxinidir}/docs/eso {env:PYTEST_ARGS} {posargs} + cov: pytest --pyargs astroquery.eso {toxinidir}/docs/eso --cov astroquery.eso --cov-config={toxinidir}/setup.cfg {env:PYTEST_ARGS} {posargs} # For remote tests, we re-run the failures to filter out at least some of the flaky ones. # We use a second pytest run with --last-failed as opposed to --rerun in order to rerun the # failed ones at the end rather than right away. - online: pytest --pyargs astroquery {toxinidir}/docs {env:PYTEST_ARGS_2} {posargs} + online: pytest --pyargs astroquery.eso {toxinidir}/docs/eso {env:PYTEST_ARGS_2} {posargs} cov: coverage xml -o {toxinidir}/coverage.xml pip_pre =