From 1d0aba95518df608f6250c7bead141057ce230bd Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Thu, 19 Mar 2026 11:13:45 -0400 Subject: [PATCH 01/27] Initial commit for read_product function --- astroquery/mast/observations.py | 38 +++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 2ae5fc9c6b..75808a5a6a 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -13,6 +13,7 @@ from urllib.parse import quote import numpy as np +import boto3 from requests import HTTPError @@ -1176,6 +1177,43 @@ def _parse_result(self, responses, *, verbose=False): # Used by the async_to_sy return self._portal_api_connection._parse_result(responses, verbose) + def read_product(s3_uri:str, + read_as: ReadAs="auto", + encoding: str = "utf-8", + boto3_session: Optional[boto3.session.Session] = None, + s3_client=None, + extra_kwargs: Optional[dict] = None + ): + + bucket, key = Observations._parse_result(s3_uri) + + if s3_client is None: + session = boto3_session + s3_client = session.client("s3") + + get_kwargs = {"Bucket": bucket, "Key": key} + if extra_kwargs: + get_kwargs.update(extra_kwargs) + + obj = s3_client.get_object(**get_kwargs) + body = obj["Body"].read() + + if read_as == "auto": + lowered = key.lower() + + if lowered.endswith(".fits"): + read_as = "fits" + + if lowered.endswith(".asdf"): + read_as = "asdf" + + if lowered.endswith(".parquet"): + read_as = "parquet" + + raise ValueError(f"Unsupported read_as: {read_as}") + + + @class_or_instance def service_request_async(self, service, params, *, pagesize=None, page=None, **kwargs): """ From 376cc1f09672817beda495d3d8383b6edd68e7e6 Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Thu, 19 Mar 2026 11:42:03 -0400 Subject: [PATCH 02/27] adding returns for fits and asdf product types --- astroquery/mast/observations.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 75808a5a6a..7c6e49d5e8 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -18,6 +18,7 @@ from requests import HTTPError import astropy.units as u +from astropy.io import fits, asdf import astropy.coordinates as coord from botocore.exceptions import ClientError, BotoCoreError @@ -1198,21 +1199,19 @@ def read_product(s3_uri:str, obj = s3_client.get_object(**get_kwargs) body = obj["Body"].read() + # Attempt to automatically load the file based on known extensions if read_as == "auto": lowered = key.lower() if lowered.endswith(".fits"): read_as = "fits" + return fits.open(body) if lowered.endswith(".asdf"): read_as = "asdf" + return asdf.open(body) - if lowered.endswith(".parquet"): - read_as = "parquet" - raise ValueError(f"Unsupported read_as: {read_as}") - - @class_or_instance def service_request_async(self, service, params, *, pagesize=None, page=None, **kwargs): From 4a7fcca44a57401be9b05bb47a291d9352c4260a Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Fri, 20 Mar 2026 13:22:05 -0400 Subject: [PATCH 03/27] Updating read_product function --- astroquery/mast/observations.py | 56 +++++++++++---------------------- 1 file changed, 19 insertions(+), 37 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 7c6e49d5e8..26fc048f73 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -10,7 +10,8 @@ import warnings import time import os -from urllib.parse import quote +from typing import Optional +from urllib.parse import quote, urlparse import numpy as np import boto3 @@ -18,7 +19,7 @@ from requests import HTTPError import astropy.units as u -from astropy.io import fits, asdf +from astropy.io import fits import astropy.coordinates as coord from botocore.exceptions import ClientError, BotoCoreError @@ -1147,6 +1148,22 @@ def get_unique_product_list(self, observations, *, batch_size=500): if len(unique_products) < len(products): log.info("To return all products, use `Observations.get_product_list`") return unique_products + +def read_product(s3_uri: str, read_as="auto"): + + # NOTE: How stand alone does this function need to be, do we want it to handle query critera and retreiving the S3 URI or is the assumption that is done previous to invoking this function? + if read_as == "auto": + if s3_uri.endswith(".fits"): + return fits.open(s3_uri, fsspec_kwargs={"anon": True}) + + elif s3_uri.endswith(".asdf"): + fs = s3fs.S3FileSystem(anon=True) + + with fs.open(s3_uri, 'rb') as s3_file: + af = asdf.open(s3_file, s3_uri) + + af.close() + @async_to_sync @@ -1178,41 +1195,6 @@ def _parse_result(self, responses, *, verbose=False): # Used by the async_to_sy return self._portal_api_connection._parse_result(responses, verbose) - def read_product(s3_uri:str, - read_as: ReadAs="auto", - encoding: str = "utf-8", - boto3_session: Optional[boto3.session.Session] = None, - s3_client=None, - extra_kwargs: Optional[dict] = None - ): - - bucket, key = Observations._parse_result(s3_uri) - - if s3_client is None: - session = boto3_session - s3_client = session.client("s3") - - get_kwargs = {"Bucket": bucket, "Key": key} - if extra_kwargs: - get_kwargs.update(extra_kwargs) - - obj = s3_client.get_object(**get_kwargs) - body = obj["Body"].read() - - # Attempt to automatically load the file based on known extensions - if read_as == "auto": - lowered = key.lower() - - if lowered.endswith(".fits"): - read_as = "fits" - return fits.open(body) - - if lowered.endswith(".asdf"): - read_as = "asdf" - return asdf.open(body) - - raise ValueError(f"Unsupported read_as: {read_as}") - @class_or_instance def service_request_async(self, service, params, *, pagesize=None, page=None, **kwargs): """ From 3e3b56a1a37089e533c9ec8e320c0bebac060f91 Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Fri, 20 Mar 2026 13:41:03 -0400 Subject: [PATCH 04/27] Updating for cleaner asdf handling --- astroquery/mast/observations.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 26fc048f73..5d56611b06 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -10,11 +10,11 @@ import warnings import time import os -from typing import Optional -from urllib.parse import quote, urlparse +from urllib.parse import quote import numpy as np -import boto3 +import s3fs +import asdf from requests import HTTPError @@ -1153,16 +1153,18 @@ def read_product(s3_uri: str, read_as="auto"): # NOTE: How stand alone does this function need to be, do we want it to handle query critera and retreiving the S3 URI or is the assumption that is done previous to invoking this function? if read_as == "auto": + + # Block to load fits file to memory if s3_uri.endswith(".fits"): return fits.open(s3_uri, fsspec_kwargs={"anon": True}) + # block to load asdf file to memory elif s3_uri.endswith(".asdf"): fs = s3fs.S3FileSystem(anon=True) with fs.open(s3_uri, 'rb') as s3_file: - af = asdf.open(s3_file, s3_uri) - - af.close() + af = asdf.open(s3_file) + return af From 7d8b427820ac1f7ea70a8ccd72d7da3f28d8f264 Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Fri, 20 Mar 2026 14:02:51 -0400 Subject: [PATCH 05/27] Updating and adding log on failure --- astroquery/mast/observations.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 5d56611b06..2a756ccf93 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -1149,24 +1149,21 @@ def get_unique_product_list(self, observations, *, batch_size=500): log.info("To return all products, use `Observations.get_product_list`") return unique_products -def read_product(s3_uri: str, read_as="auto"): +def read_product(s3_uri, read_as="auto"): # NOTE: How stand alone does this function need to be, do we want it to handle query critera and retreiving the S3 URI or is the assumption that is done previous to invoking this function? if read_as == "auto": - - # Block to load fits file to memory if s3_uri.endswith(".fits"): return fits.open(s3_uri, fsspec_kwargs={"anon": True}) - # block to load asdf file to memory + # This needs roman-datamodels installed to work elif s3_uri.endswith(".asdf"): fs = s3fs.S3FileSystem(anon=True) - with fs.open(s3_uri, 'rb') as s3_file: af = asdf.open(s3_file) return af - - + else: + log.info(f"Unsupported extension type") @async_to_sync class MastClass(MastQueryWithLogin): From d45c0f72841b7e56b5ff0f5f6dfbd175808b75ec Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Tue, 24 Mar 2026 13:20:56 -0400 Subject: [PATCH 06/27] Adding in description --- astroquery/mast/observations.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 2a756ccf93..6089e2ea22 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -1149,9 +1149,25 @@ def get_unique_product_list(self, observations, *, batch_size=500): log.info("To return all products, use `Observations.get_product_list`") return unique_products -def read_product(s3_uri, read_as="auto"): +def read_product(s3_uri: str, read_as="auto"): - # NOTE: How stand alone does this function need to be, do we want it to handle query critera and retreiving the S3 URI or is the assumption that is done previous to invoking this function? + """ + Read a product from S3 to memory. + + Parameters + ---------- + s3_uri: str + S3 URI to the product in open bucket. + + read_as: str, optional + How to read the file. Currently only .fits and .asdf is supported by "auto". + + Returns + ------- + object + FITS or ASDF object. + + """ if read_as == "auto": if s3_uri.endswith(".fits"): return fits.open(s3_uri, fsspec_kwargs={"anon": True}) From beb09b5d8ac259c1b00a4650831c4faaff780159 Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Tue, 24 Mar 2026 13:23:10 -0400 Subject: [PATCH 07/27] Updating comments --- astroquery/mast/observations.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 6089e2ea22..f0c0a21e8b 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -1152,7 +1152,7 @@ def get_unique_product_list(self, observations, *, batch_size=500): def read_product(s3_uri: str, read_as="auto"): """ - Read a product from S3 to memory. + Read a product from Open S3 bucket to memory. Parameters ---------- @@ -1168,11 +1168,14 @@ def read_product(s3_uri: str, read_as="auto"): FITS or ASDF object. """ + if read_as == "auto": + # Read logic for FITS if s3_uri.endswith(".fits"): return fits.open(s3_uri, fsspec_kwargs={"anon": True}) - # This needs roman-datamodels installed to work + # Read logic for ASDF + # NOTE: Since asdf files are changing so rapidly this may need addition packages to run (e.x. roman-datamodels) elif s3_uri.endswith(".asdf"): fs = s3fs.S3FileSystem(anon=True) with fs.open(s3_uri, 'rb') as s3_file: From 1dc505eaa901707a2660277005e70c3c1598e1d0 Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Mon, 30 Mar 2026 10:35:04 -0400 Subject: [PATCH 08/27] and exception logic --- astroquery/mast/observations.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index f0c0a21e8b..89beaae960 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -1172,15 +1172,22 @@ def read_product(s3_uri: str, read_as="auto"): if read_as == "auto": # Read logic for FITS if s3_uri.endswith(".fits"): - return fits.open(s3_uri, fsspec_kwargs={"anon": True}) + try: + return fits.open(s3_uri, fsspec_kwargs={"anon": True}) + except Exception as e: + log.exception(f"Failed to open FITS File: {s3_uri} {e}") # Read logic for ASDF # NOTE: Since asdf files are changing so rapidly this may need addition packages to run (e.x. roman-datamodels) elif s3_uri.endswith(".asdf"): - fs = s3fs.S3FileSystem(anon=True) - with fs.open(s3_uri, 'rb') as s3_file: - af = asdf.open(s3_file) - return af + try: + fs = s3fs.S3FileSystem(anon=True) + with fs.open(s3_uri, 'rb') as s3_file: + af = asdf.open(s3_file) + return af + except Exception as e: + log.exception(f"Failed to open ASD File: {s3_uri} {e}") + else: log.info(f"Unsupported extension type") From 587ccd692f52f134a100e60bf80296d3a6c6f360 Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Mon, 6 Apr 2026 09:00:53 -0400 Subject: [PATCH 09/27] Added in comments on package requirements for read_product functions --- astroquery/mast/observations.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 89beaae960..bc7af5d4d4 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -1148,9 +1148,14 @@ def get_unique_product_list(self, observations, *, batch_size=500): if len(unique_products) < len(products): log.info("To return all products, use `Observations.get_product_list`") return unique_products - -def read_product(s3_uri: str, read_as="auto"): +# Function requires following packages +# [Required] gwcs (which would also instasdf-astropy) - To obtain the general asdf schemas to correctly parse the asdf metadata. Should handle most non-roman schemas +# [Required] s3fs - for actually connecting to s3 and retreiving the files +# [Required] lz4 - to handle the asdf file compression +# [Optional] roman-datamodels: Required for specific roman asdf schemas (i.e. Roman SOC and PIT) +# [Optional Future]: Most likely some HLSP schema package +def read_product(s3_uri: str, read_as="auto"): """ Read a product from Open S3 bucket to memory. @@ -1168,7 +1173,7 @@ def read_product(s3_uri: str, read_as="auto"): FITS or ASDF object. """ - + if read_as == "auto": # Read logic for FITS if s3_uri.endswith(".fits"): @@ -1178,7 +1183,6 @@ def read_product(s3_uri: str, read_as="auto"): log.exception(f"Failed to open FITS File: {s3_uri} {e}") # Read logic for ASDF - # NOTE: Since asdf files are changing so rapidly this may need addition packages to run (e.x. roman-datamodels) elif s3_uri.endswith(".asdf"): try: fs = s3fs.S3FileSystem(anon=True) @@ -1187,10 +1191,9 @@ def read_product(s3_uri: str, read_as="auto"): return af except Exception as e: log.exception(f"Failed to open ASD File: {s3_uri} {e}") - else: log.info(f"Unsupported extension type") - + @async_to_sync class MastClass(MastQueryWithLogin): """ From 1bdb8a5e7bbc19c8a374188c4b2503a97ea35d0a Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Mon, 6 Apr 2026 09:03:38 -0400 Subject: [PATCH 10/27] Updated imports to reflect only required packages --- astroquery/mast/observations.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index bc7af5d4d4..2f9f938300 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -13,8 +13,9 @@ from urllib.parse import quote import numpy as np +import gwcs import s3fs -import asdf +import lz4 from requests import HTTPError @@ -1193,7 +1194,7 @@ def read_product(s3_uri: str, read_as="auto"): log.exception(f"Failed to open ASD File: {s3_uri} {e}") else: log.info(f"Unsupported extension type") - + @async_to_sync class MastClass(MastQueryWithLogin): """ From 80238f643e85468bdb54542d4d9822c889faf9f6 Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Mon, 6 Apr 2026 09:05:38 -0400 Subject: [PATCH 11/27] Updating description --- astroquery/mast/observations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 2f9f938300..01189818b3 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -1158,7 +1158,7 @@ def get_unique_product_list(self, observations, *, batch_size=500): # [Optional Future]: Most likely some HLSP schema package def read_product(s3_uri: str, read_as="auto"): """ - Read a product from Open S3 bucket to memory. + Read a product from Open S3 bucket to memory. Currently can handle FITS and ASDF product types with appropriate package installation. Parameters ---------- From f883b8d4e8b0b6adc21bb710a84dba9c9a3dd389 Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Thu, 9 Apr 2026 10:09:17 -0400 Subject: [PATCH 12/27] minor changes --- astroquery/mast/observations.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 01189818b3..934cf9b22a 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -14,6 +14,7 @@ import numpy as np import gwcs +import asdf import s3fs import lz4 @@ -1151,22 +1152,25 @@ def get_unique_product_list(self, observations, *, batch_size=500): return unique_products # Function requires following packages -# [Required] gwcs (which would also instasdf-astropy) - To obtain the general asdf schemas to correctly parse the asdf metadata. Should handle most non-roman schemas +# [Required] gwcs (which would also inst asdf-astropy) - To obtain the general asdf schemas to correctly parse the asdf metadata. Should handle most non-roman schemas # [Required] s3fs - for actually connecting to s3 and retreiving the files # [Required] lz4 - to handle the asdf file compression # [Optional] roman-datamodels: Required for specific roman asdf schemas (i.e. Roman SOC and PIT) # [Optional Future]: Most likely some HLSP schema package -def read_product(s3_uri: str, read_as="auto"): +def read_product(uri, read_as="auto", ignore_unrecognized=False): """ Read a product from Open S3 bucket to memory. Currently can handle FITS and ASDF product types with appropriate package installation. Parameters ---------- - s3_uri: str - S3 URI to the product in open bucket. + uri: str + URI to the product in open bucket. read_as: str, optional - How to read the file. Currently only .fits and .asdf is supported by "auto". + How to read the file. Currently only .fits and .asdf is supported by "auto". Defaults to "auto". + + ignore_unrecognized: bool + Tells asdf.open() to include or ignore warnings from unrecognized asdf tags. Defaults to False Returns ------- @@ -1177,21 +1181,21 @@ def read_product(s3_uri: str, read_as="auto"): if read_as == "auto": # Read logic for FITS - if s3_uri.endswith(".fits"): + if uri.endswith(".fits"): try: - return fits.open(s3_uri, fsspec_kwargs={"anon": True}) + return fits.open(uri, fsspec_kwargs={"anon": True}) except Exception as e: - log.exception(f"Failed to open FITS File: {s3_uri} {e}") + log.exception(f"Failed to open FITS File: {uri} {e}") # Read logic for ASDF - elif s3_uri.endswith(".asdf"): + elif uri.endswith(".asdf"): try: fs = s3fs.S3FileSystem(anon=True) - with fs.open(s3_uri, 'rb') as s3_file: - af = asdf.open(s3_file) + with fs.open(uri, 'rb') as s3_file: + af = asdf.open(s3_file, ignore_unrecognized_tag=ignore_unrecognized) return af except Exception as e: - log.exception(f"Failed to open ASD File: {s3_uri} {e}") + log.exception(f"Failed to open ASD File: {uri} {e}") else: log.info(f"Unsupported extension type") From 66b4a6c9a3e4982105608493591fafab5cd65509 Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Thu, 9 Apr 2026 10:14:23 -0400 Subject: [PATCH 13/27] gwcs and lz4 packages do not need to be imported, they just need to be in the users environment --- astroquery/mast/observations.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 934cf9b22a..dfd7313f1c 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -13,10 +13,8 @@ from urllib.parse import quote import numpy as np -import gwcs import asdf import s3fs -import lz4 from requests import HTTPError From 831baf45ccb4345522c8256c253448d9e99bbd5a Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Thu, 16 Apr 2026 10:02:33 -0400 Subject: [PATCH 14/27] Updating uri variable name to product_path be more general, as well as adding notes --- astroquery/mast/observations.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index dfd7313f1c..f06487ee8f 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -1151,11 +1151,13 @@ def get_unique_product_list(self, observations, *, batch_size=500): # Function requires following packages # [Required] gwcs (which would also inst asdf-astropy) - To obtain the general asdf schemas to correctly parse the asdf metadata. Should handle most non-roman schemas +# NOTE: It looks like gwc is a dev dependency in asdf-astropy # [Required] s3fs - for actually connecting to s3 and retreiving the files # [Required] lz4 - to handle the asdf file compression # [Optional] roman-datamodels: Required for specific roman asdf schemas (i.e. Roman SOC and PIT) -# [Optional Future]: Most likely some HLSP schema package -def read_product(uri, read_as="auto", ignore_unrecognized=False): +# [Optional Future]: Most likely some HLSP schema package +# TODO: Need to inlcude way to parse if it is a MAST on prem URL and handle the streaming of that +def read_product(product_path, read_as="auto", ignore_unrecognized=False): """ Read a product from Open S3 bucket to memory. Currently can handle FITS and ASDF product types with appropriate package installation. @@ -1179,21 +1181,23 @@ def read_product(uri, read_as="auto", ignore_unrecognized=False): if read_as == "auto": # Read logic for FITS - if uri.endswith(".fits"): + if product_path.endswith(".fits"): try: - return fits.open(uri, fsspec_kwargs={"anon": True}) + return fits.open(product_path, fsspec_kwargs={"anon": True}) + except Exception as e: - log.exception(f"Failed to open FITS File: {uri} {e}") + log.exception(f"Failed to open FITS File: {product_path} {e}") # Read logic for ASDF - elif uri.endswith(".asdf"): + elif product_path.endswith(".asdf"): try: fs = s3fs.S3FileSystem(anon=True) - with fs.open(uri, 'rb') as s3_file: - af = asdf.open(s3_file, ignore_unrecognized_tag=ignore_unrecognized) + with fs.open(product_path, 'rb') as asdf_file: + af = asdf.open(asdf_file, ignore_unrecognized_tag=ignore_unrecognized) + return af except Exception as e: - log.exception(f"Failed to open ASD File: {uri} {e}") + log.exception(f"Failed to open ASD File: {product_path} {e}") else: log.info(f"Unsupported extension type") From 50c0ef335baf31d42717a6747706a85b5b20ee6d Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Thu, 16 Apr 2026 10:03:08 -0400 Subject: [PATCH 15/27] Updating description --- astroquery/mast/observations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index f06487ee8f..2f6d1a5923 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -1163,7 +1163,7 @@ def read_product(product_path, read_as="auto", ignore_unrecognized=False): Parameters ---------- - uri: str + product_path: str URI to the product in open bucket. read_as: str, optional From a8cb877c82896dab1338530cd54116628fd8caac Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Fri, 17 Apr 2026 11:36:48 -0400 Subject: [PATCH 16/27] Updating import logic for read_product functions --- astroquery/mast/observations.py | 67 ++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 2f6d1a5923..c061cea4ad 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -13,8 +13,6 @@ from urllib.parse import quote import numpy as np -import asdf -import s3fs from requests import HTTPError @@ -38,6 +36,29 @@ __all__ = ['Observations', 'ObservationsClass', 'MastClass', 'Mast'] +from importlib.metadata import PackageNotFoundError, version + +asdf_module_req= ["asdf", "s3fs"] +asdf_package_req = ["lz4", "gwcs"] +asdf_missing = [] + +for module in asdf_module_req: + try: + # Is there a better way to do this + globals()[module] = __import__(module) + + except ModuleNotFoundError: + asdf_missing.append(module) + log.debug(f"Module Not Found: {module} please pip install to stream and open asdf data products") + +for package in asdf_package_req: + try: + version(package) + + except PackageNotFoundError: + asdf_missing.append(package) + log.debug(f"Package Not Found: {package} please pip install to stream and open asdf data products") + @async_to_sync class ObservationsClass(MastQueryWithLogin): @@ -1149,13 +1170,7 @@ def get_unique_product_list(self, observations, *, batch_size=500): log.info("To return all products, use `Observations.get_product_list`") return unique_products -# Function requires following packages -# [Required] gwcs (which would also inst asdf-astropy) - To obtain the general asdf schemas to correctly parse the asdf metadata. Should handle most non-roman schemas -# NOTE: It looks like gwc is a dev dependency in asdf-astropy -# [Required] s3fs - for actually connecting to s3 and retreiving the files -# [Required] lz4 - to handle the asdf file compression -# [Optional] roman-datamodels: Required for specific roman asdf schemas (i.e. Roman SOC and PIT) -# [Optional Future]: Most likely some HLSP schema package + # TODO: Need to inlcude way to parse if it is a MAST on prem URL and handle the streaming of that def read_product(product_path, read_as="auto", ignore_unrecognized=False): """ @@ -1182,24 +1197,30 @@ def read_product(product_path, read_as="auto", ignore_unrecognized=False): if read_as == "auto": # Read logic for FITS if product_path.endswith(".fits"): - try: + try: return fits.open(product_path, fsspec_kwargs={"anon": True}) - - except Exception as e: - log.exception(f"Failed to open FITS File: {product_path} {e}") - + + except Exception as e: + log.exception(f"Failed to open FITS File: {product_path} {e}") + + # Read logic for ASDF elif product_path.endswith(".asdf"): - try: - fs = s3fs.S3FileSystem(anon=True) - with fs.open(product_path, 'rb') as asdf_file: - af = asdf.open(asdf_file, ignore_unrecognized_tag=ignore_unrecognized) - - return af - except Exception as e: - log.exception(f"Failed to open ASD File: {product_path} {e}") + + if asdf_missing: + log.debug(f"Missing Required Packaged for ASDF product type handeling: {asdf_missing}") + + else: + try: + fs = s3fs.S3FileSystem(anon=True) + with fs.open(product_path, 'rb') as s3_file: + af = asdf.open(s3_file, ignore_unrecognized_tag=ignore_unrecognized) + return af + + except Exception as e: + log.exception(f"Failed to open ASD File: {product_path} {e}") else: - log.info(f"Unsupported extension type") + print(f"Unsupported extension type") @async_to_sync class MastClass(MastQueryWithLogin): From 18fd33cdb7426cd8ac70d75bf11f3923c76be858 Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Fri, 17 Apr 2026 11:40:38 -0400 Subject: [PATCH 17/27] minor format change --- astroquery/mast/observations.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 7ffdd0aad8..ac44d4bb6f 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -59,6 +59,7 @@ except ModuleNotFoundError: asdf_missing.append(module) log.debug(f"Module Not Found: {module} please pip install to stream and open asdf data products") + pass for package in asdf_package_req: try: @@ -67,6 +68,7 @@ except PackageNotFoundError: asdf_missing.append(package) log.debug(f"Package Not Found: {package} please pip install to stream and open asdf data products") + pass @async_to_sync From 4f99c05993055b3b42c7da14585191fd83be1fcd Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Fri, 17 Apr 2026 11:47:14 -0400 Subject: [PATCH 18/27] Formatting Fixes --- astroquery/mast/observations.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index ac44d4bb6f..b92fcb2c54 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -47,7 +47,7 @@ from importlib.metadata import PackageNotFoundError, version -asdf_module_req= ["asdf", "s3fs"] +asdf_module_req = ["asdf", "s3fs"] asdf_package_req = ["lz4", "gwcs"] asdf_missing = [] @@ -1233,16 +1233,14 @@ def get_unique_product_list(self, observations, *, batch_size=500): # TODO: Need to inlcude way to parse if it is a MAST on prem URL and handle the streaming of that def read_product(product_path, read_as="auto", ignore_unrecognized=False): """ - Read a product from Open S3 bucket to memory. Currently can handle FITS and ASDF product types with appropriate package installation. + Read a product from Open S3 bucket to memory. Currently can handle FITS and ASDF product types. Parameters ---------- product_path: str URI to the product in open bucket. - read_as: str, optional How to read the file. Currently only .fits and .asdf is supported by "auto". Defaults to "auto". - ignore_unrecognized: bool Tells asdf.open() to include or ignore warnings from unrecognized asdf tags. Defaults to False @@ -1250,32 +1248,26 @@ def read_product(product_path, read_as="auto", ignore_unrecognized=False): ------- object FITS or ASDF object. - """ - if read_as == "auto": # Read logic for FITS if product_path.endswith(".fits"): try: return fits.open(product_path, fsspec_kwargs={"anon": True}) - except Exception as e: log.exception(f"Failed to open FITS File: {product_path} {e}") # Read logic for ASDF elif product_path.endswith(".asdf"): - if asdf_missing: log.debug(f"Missing Required Packaged for ASDF product type handeling: {asdf_missing}") - else: try: fs = s3fs.S3FileSystem(anon=True) with fs.open(product_path, 'rb') as s3_file: af = asdf.open(s3_file, ignore_unrecognized_tag=ignore_unrecognized) return af - except Exception as e: log.exception(f"Failed to open ASD File: {product_path} {e}") else: From e6ad337bc2c35056aab27c2ac81f7b22da37bd7c Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Fri, 17 Apr 2026 12:05:44 -0400 Subject: [PATCH 19/27] Changes import logic and checking to be cleaner --- astroquery/mast/observations.py | 55 +++++++++++++-------------------- 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index b92fcb2c54..040d64d5d7 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -11,6 +11,7 @@ import time import os from urllib.parse import quote +from importlib.metadata import version import numpy as np import astropy.units as u @@ -45,31 +46,12 @@ '`~astroquery.mast.ObservationsClass.enable_cloud_dataset` method.' ) -from importlib.metadata import PackageNotFoundError, version - -asdf_module_req = ["asdf", "s3fs"] -asdf_package_req = ["lz4", "gwcs"] -asdf_missing = [] - -for module in asdf_module_req: - try: - # Is there a better way to do this - globals()[module] = __import__(module) - - except ModuleNotFoundError: - asdf_missing.append(module) - log.debug(f"Module Not Found: {module} please pip install to stream and open asdf data products") - pass - -for package in asdf_package_req: - try: - version(package) - - except PackageNotFoundError: - asdf_missing.append(package) - log.debug(f"Package Not Found: {package} please pip install to stream and open asdf data products") - pass - +asdf_modules= ["asdf", "s3fs", "lz4", "gwcs"] +try: + import asdf + import s3fs +except ImportError: + pass @async_to_sync class ObservationsClass(MastQueryWithLogin): @@ -1260,16 +1242,21 @@ def read_product(product_path, read_as="auto", ignore_unrecognized=False): # Read logic for ASDF elif product_path.endswith(".asdf"): - if asdf_missing: - log.debug(f"Missing Required Packaged for ASDF product type handeling: {asdf_missing}") - else: + # Check all required modules are available + for module in asdf_modules: try: - fs = s3fs.S3FileSystem(anon=True) - with fs.open(product_path, 'rb') as s3_file: - af = asdf.open(s3_file, ignore_unrecognized_tag=ignore_unrecognized) - return af - except Exception as e: - log.exception(f"Failed to open ASD File: {product_path} {e}") + version(module) + except: + log.debug(f"Missing Required Module: {module}") + return + + try: + fs = s3fs.S3FileSystem(anon=True) + with fs.open(product_path, 'rb') as s3_file: + af = asdf.open(s3_file, ignore_unrecognized_tag=ignore_unrecognized) + return af + except Exception as e: + log.exception(f"Failed to open ASD File: {product_path} {e}") else: print(f"Unsupported extension type") From 29d6ad4fa2e0a813fc16e633ac2b4cf8fad88e86 Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Fri, 17 Apr 2026 12:11:05 -0400 Subject: [PATCH 20/27] Formatting --- astroquery/mast/observations.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 040d64d5d7..7d0f33bae7 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -46,13 +46,14 @@ '`~astroquery.mast.ObservationsClass.enable_cloud_dataset` method.' ) -asdf_modules= ["asdf", "s3fs", "lz4", "gwcs"] +asdf_modules = ["asdf", "s3fs", "lz4", "gwcs"] try: import asdf import s3fs except ImportError: pass + @async_to_sync class ObservationsClass(MastQueryWithLogin): """ @@ -1232,21 +1233,19 @@ def read_product(product_path, read_as="auto", ignore_unrecognized=False): FITS or ASDF object. """ if read_as == "auto": - # Read logic for FITS if product_path.endswith(".fits"): - try: + try: return fits.open(product_path, fsspec_kwargs={"anon": True}) - except Exception as e: + except Exception as e: log.exception(f"Failed to open FITS File: {product_path} {e}") - # Read logic for ASDF elif product_path.endswith(".asdf"): # Check all required modules are available for module in asdf_modules: try: version(module) - except: + except ModuleNotFoundError: log.debug(f"Missing Required Module: {module}") return @@ -1258,7 +1257,8 @@ def read_product(product_path, read_as="auto", ignore_unrecognized=False): except Exception as e: log.exception(f"Failed to open ASD File: {product_path} {e}") else: - print(f"Unsupported extension type") + print("Unsupported extension type") + @async_to_sync class MastClass(MastQueryWithLogin): From 548414dbd5ab1f11be45d413dc6a353e4a0531ff Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Fri, 17 Apr 2026 12:14:25 -0400 Subject: [PATCH 21/27] Formatting fix --- astroquery/mast/observations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 7d0f33bae7..d84b4a3481 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -1237,7 +1237,7 @@ def read_product(product_path, read_as="auto", ignore_unrecognized=False): try: return fits.open(product_path, fsspec_kwargs={"anon": True}) except Exception as e: - log.exception(f"Failed to open FITS File: {product_path} {e}") + log.exception(f"Failed to open FITS File: {product_path} {e}") # Read logic for ASDF elif product_path.endswith(".asdf"): From cfeed724af531c93f9372d2b22d79b20d73c54df Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Fri, 17 Apr 2026 12:37:09 -0400 Subject: [PATCH 22/27] Moving into observations class --- astroquery/mast/observations.py | 86 ++++++++++++++++----------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index d84b4a3481..ea82449c53 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -1213,51 +1213,51 @@ def get_unique_product_list(self, observations, *, batch_size=500): return unique_products -# TODO: Need to inlcude way to parse if it is a MAST on prem URL and handle the streaming of that -def read_product(product_path, read_as="auto", ignore_unrecognized=False): - """ - Read a product from Open S3 bucket to memory. Currently can handle FITS and ASDF product types. - - Parameters - ---------- - product_path: str - URI to the product in open bucket. - read_as: str, optional - How to read the file. Currently only .fits and .asdf is supported by "auto". Defaults to "auto". - ignore_unrecognized: bool - Tells asdf.open() to include or ignore warnings from unrecognized asdf tags. Defaults to False - - Returns - ------- - object - FITS or ASDF object. - """ - if read_as == "auto": - if product_path.endswith(".fits"): - try: - return fits.open(product_path, fsspec_kwargs={"anon": True}) - except Exception as e: - log.exception(f"Failed to open FITS File: {product_path} {e}") - - # Read logic for ASDF - elif product_path.endswith(".asdf"): - # Check all required modules are available - for module in asdf_modules: + # TODO: Need to inlcude way to parse if it is a MAST on prem URL and handle the streaming of that + def read_product(self, product_path, read_as="auto", ignore_unrecognized=False): + """ + Read a product from Open S3 bucket to memory. Currently can handle FITS and ASDF product types. + + Parameters + ---------- + product_path: str + URI to the product in open bucket. + read_as: str, optional + How to read the file. Currently only .fits and .asdf is supported by "auto". Defaults to "auto". + ignore_unrecognized: bool + Tells asdf.open() to include or ignore warnings from unrecognized asdf tags. Defaults to False + + Returns + ------- + object + FITS or ASDF object. + """ + if read_as == "auto": + if product_path.endswith(".fits"): try: - version(module) - except ModuleNotFoundError: - log.debug(f"Missing Required Module: {module}") - return + return fits.open(product_path, fsspec_kwargs={"anon": True}) + except Exception as e: + log.exception(f"Failed to open FITS File: {product_path} {e}") + + # Read logic for ASDF + elif product_path.endswith(".asdf"): + # Check all required modules are available + for module in asdf_modules: + try: + version(module) + except ModuleNotFoundError: + log.debug(f"Missing Required Module: {module}") + return - try: - fs = s3fs.S3FileSystem(anon=True) - with fs.open(product_path, 'rb') as s3_file: - af = asdf.open(s3_file, ignore_unrecognized_tag=ignore_unrecognized) - return af - except Exception as e: - log.exception(f"Failed to open ASD File: {product_path} {e}") - else: - print("Unsupported extension type") + try: + fs = s3fs.S3FileSystem(anon=True) + with fs.open(product_path, 'rb') as s3_file: + af = asdf.open(s3_file, ignore_unrecognized_tag=ignore_unrecognized) + return af + except Exception as e: + log.exception(f"Failed to open ASD File: {product_path} {e}") + else: + print("Unsupported extension type") @async_to_sync From 909a45a1ad895048c40b254d990aff8aa5d69b77 Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Fri, 17 Apr 2026 12:38:31 -0400 Subject: [PATCH 23/27] Formatting --- astroquery/mast/observations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index ea82449c53..090df74a3d 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -1212,7 +1212,6 @@ def get_unique_product_list(self, observations, *, batch_size=500): log.info("To return all products, use `Observations.get_product_list`") return unique_products - # TODO: Need to inlcude way to parse if it is a MAST on prem URL and handle the streaming of that def read_product(self, product_path, read_as="auto", ignore_unrecognized=False): """ From a2556cc6a65a2c03264605e80de0c3a9f791c041 Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Fri, 17 Apr 2026 15:07:15 -0400 Subject: [PATCH 24/27] Updating CHANGES.rst to reflect updates to read_product --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 536e238f0a..911a1451c2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -147,6 +147,8 @@ mast - The cloud dataset in ``Observations`` is now enabled by default if the ``boto3`` and ``botocore`` packages are installed. This default can be overridden by setting the ``enable_cloud_dataset`` configuration option to False. [#3534] +- Adding in ability to read FITS and ASDF dataproducts to memory from s3:// using ``Observations.read_product()`` function. [#3561] + jplspec ^^^^^^^ From 91f322f99214505f06de1256a1cc2acfe906d2de Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Fri, 17 Apr 2026 15:07:48 -0400 Subject: [PATCH 25/27] Adding fsspec to imports --- astroquery/mast/observations.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 090df74a3d..554a9f12e8 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -46,7 +46,7 @@ '`~astroquery.mast.ObservationsClass.enable_cloud_dataset` method.' ) -asdf_modules = ["asdf", "s3fs", "lz4", "gwcs"] +asdf_modules = ["asdf", "s3fs", "fsspec", "lz4", "gwcs"] try: import asdf import s3fs @@ -1233,6 +1233,13 @@ def read_product(self, product_path, read_as="auto", ignore_unrecognized=False): """ if read_as == "auto": if product_path.endswith(".fits"): + fits_packages = ['s3fs', 'fsspec'] + for package in fits_packages: + try: + version(package) + except ModuleNotFoundError: + log.debug(f"Missing Required Module: {module}") + return try: return fits.open(product_path, fsspec_kwargs={"anon": True}) except Exception as e: From 8dd81d26f228f89d0f442a4129d51a48be559c88 Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Fri, 17 Apr 2026 15:15:29 -0400 Subject: [PATCH 26/27] Formatting and typo --- astroquery/mast/observations.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 554a9f12e8..57031b5b8c 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -46,7 +46,9 @@ '`~astroquery.mast.ObservationsClass.enable_cloud_dataset` method.' ) -asdf_modules = ["asdf", "s3fs", "fsspec", "lz4", "gwcs"] +asdf_packages = ["asdf", "s3fs", "fsspec", "lz4", "gwcs"] +fits_packages = ['s3fs', 'fsspec'] + try: import asdf import s3fs @@ -1233,12 +1235,11 @@ def read_product(self, product_path, read_as="auto", ignore_unrecognized=False): """ if read_as == "auto": if product_path.endswith(".fits"): - fits_packages = ['s3fs', 'fsspec'] for package in fits_packages: try: version(package) except ModuleNotFoundError: - log.debug(f"Missing Required Module: {module}") + log.debug(f"Missing Required Package: {package}") return try: return fits.open(product_path, fsspec_kwargs={"anon": True}) @@ -1248,11 +1249,11 @@ def read_product(self, product_path, read_as="auto", ignore_unrecognized=False): # Read logic for ASDF elif product_path.endswith(".asdf"): # Check all required modules are available - for module in asdf_modules: + for package in asdf_packages: try: - version(module) + version(package) except ModuleNotFoundError: - log.debug(f"Missing Required Module: {module}") + log.debug(f"Missing Required Package: {package}") return try: From 12ab349be8d8e3cb13139f14f9ddcf0a4cff783e Mon Sep 17 00:00:00 2001 From: Alex Reedy Date: Wed, 22 Apr 2026 16:36:19 -0400 Subject: [PATCH 27/27] removing print --- astroquery/mast/observations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 57031b5b8c..8183d03697 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -1264,7 +1264,8 @@ def read_product(self, product_path, read_as="auto", ignore_unrecognized=False): except Exception as e: log.exception(f"Failed to open ASD File: {product_path} {e}") else: - print("Unsupported extension type") + log.error("Unsupported extension type") + return @async_to_sync