From 9adebff4255b5db1744aa887273be54a6836877a Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Thu, 2 Apr 2026 11:51:15 -0700 Subject: [PATCH 01/35] Introduce new ReducedDatum models Moves almost all field from the existing model an abstract model from which all models inherit. This means existing ReducedDatums remain unchanged, but 3 new tables are created for the concrete types. A datamigration can be run to convert from the old model to the new (future commit). --- ...trument_reduceddatum_telescope_and_more.py | 101 +++++++++++ tom_dataproducts/models.py | 161 +++++++++++------- 2 files changed, 196 insertions(+), 66 deletions(-) create mode 100644 tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py diff --git a/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py b/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py new file mode 100644 index 000000000..cc8fb8e0c --- /dev/null +++ b/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py @@ -0,0 +1,101 @@ +# Generated by Django 5.2.12 on 2026-04-02 18:46 + +import django.db.models.deletion +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tom_alerts', '0007_alter_alertstreammessage_message_id'), + ('tom_dataproducts', '0014_alter_reduceddatum_timestamp'), + ('tom_targets', '0030_alter_basetarget_slope'), + ] + + operations = [ + migrations.AddField( + model_name='reduceddatum', + name='instrument', + field=models.CharField(blank=True, default='', max_length=255), + ), + migrations.AddField( + model_name='reduceddatum', + name='telescope', + field=models.CharField(blank=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='reduceddatum', + name='value', + field=models.JSONField(default=dict, verbose_name='extra data'), + ), + migrations.CreateModel( + name='AstrometryReducedDatum', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.DateTimeField(db_index=True, default=django.utils.timezone.now)), + ('value', models.JSONField(default=dict, verbose_name='extra data')), + ('telescope', models.CharField(blank=True, default='', max_length=255)), + ('instrument', models.CharField(blank=True, default='', max_length=255)), + ('source_name', models.CharField(blank=True, default='', max_length=100)), + ('ra', models.FloatField()), + ('dec', models.FloatField()), + ('ra_error', models.FloatField(blank=True, default=None, null=True)), + ('dec_error', models.FloatField(blank=True, default=None, null=True)), + ('ra_error_units', models.CharField(blank=True, default='', max_length=32)), + ('dec_error_units', models.CharField(blank=True, default='', max_length=32)), + ('data_product', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='tom_dataproducts.dataproduct')), + ('message', models.ManyToManyField(blank=True, to='tom_alerts.alertstreammessage')), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tom_targets.basetarget')), + ], + options={ + 'constraints': [models.UniqueConstraint(fields=('target', 'timestamp', 'telescope', 'instrument'), name='unique_astrometry')], + }, + ), + migrations.CreateModel( + name='PhotometryReducedDatum', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.DateTimeField(db_index=True, default=django.utils.timezone.now)), + ('value', models.JSONField(default=dict, verbose_name='extra data')), + ('telescope', models.CharField(blank=True, default='', max_length=255)), + ('instrument', models.CharField(blank=True, default='', max_length=255)), + ('source_name', models.CharField(blank=True, default='', max_length=100)), + ('discovery', models.BooleanField(default=False)), + ('brightness', models.FloatField()), + ('brightness_error', models.FloatField()), + ('unit', models.CharField(blank=True, default='', max_length=32)), + ('bandpass', models.CharField(max_length=32)), + ('exposure_time', models.FloatField()), + ('data_product', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='tom_dataproducts.dataproduct')), + ('message', models.ManyToManyField(blank=True, to='tom_alerts.alertstreammessage')), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tom_targets.basetarget')), + ], + options={ + 'constraints': [models.UniqueConstraint(fields=('target', 'bandpass', 'timestamp'), name='unique_photometry')], + }, + ), + migrations.CreateModel( + name='SpectroscopyReducedDatum', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.DateTimeField(db_index=True, default=django.utils.timezone.now)), + ('value', models.JSONField(default=dict, verbose_name='extra data')), + ('telescope', models.CharField(blank=True, default='', max_length=255)), + ('instrument', models.CharField(blank=True, default='', max_length=255)), + ('source_name', models.CharField(blank=True, default='', max_length=100)), + ('setup', models.CharField(blank=True, default='', max_length=2000)), + ('exposure_time', models.FloatField()), + ('flux', models.TextField(blank=True, default='')), + ('flux_unit', models.TextField(blank=True, default='')), + ('error', models.TextField(blank=True, default='')), + ('wavelength', models.TextField(blank=True, default='')), + ('data_product', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='tom_dataproducts.dataproduct')), + ('message', models.ManyToManyField(blank=True, to='tom_alerts.alertstreammessage')), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tom_targets.basetarget')), + ], + options={ + 'constraints': [models.UniqueConstraint(fields=('target', 'timestamp', 'telescope', 'instrument'), name='unique_spectroscopy')], + }, + ), + ] diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index cbde50216..150cc7362 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -1,20 +1,20 @@ import logging import os import tempfile +from importlib import import_module from astropy.io import fits from django.conf import settings +from django.core.exceptions import ValidationError from django.core.files import File from django.db import models -from django.utils import timezone, text -from django.core.exceptions import ValidationError +from django.utils import text, timezone from fits2image.conversions import fits_to_jpg from PIL import Image -from importlib import import_module -from tom_targets.base_models import BaseTarget from tom_alerts.models import AlertStreamMessage from tom_observations.models import ObservationRecord +from tom_targets.base_models import BaseTarget logger = logging.getLogger(__name__) @@ -311,75 +311,79 @@ def create_thumbnail(self, width=None, height=None): return -class ReducedDatum(models.Model): +class ReducedDatumCommon(models.Model): """ - Class representing a datum in a TOM. + Abstract base class for all reduced datum models. A ``ReducedDatum`` generally refers to a single piece of data--e.g., a spectrum, or a photometry point. It is associated with a target, and optionally with the data product it came from. An example of a ``ReducedDatum`` - without an associated data product would be photometry ingested from a broker. + without an associated data product would be photometry ingested from a broker. There are + concrete implementations of Photometry, Spectroscopy and Astronmetry ReducedDatum models. :param target: The ``Target`` with which this object is associated. :param data_product: The ``DataProduct`` with which this object is optionally associated. - :param data_type: The type of data this datum represents. Default choices are the default values found in - DATA_PRODUCT_TYPES in settings.py. - :type data_type: str - - :param source_name: The original source of this datum. The current major use of this field is to track the broker a - datum came from, but can be used for other sources. - :type source_name: str - - :param source_location: A reference to the location that this datum was originally sourced from. The current major - use of this field is the URL path to the alert that this datum came from. - :type source_name: str - :param timestamp: The timestamp of this datum. :type timestamp: datetime - :param value: The value of the datum. This is a dict, intended to store data with a variety of - scopes. As an example, a photometry value might contain the following: + :param value: Freeform data. This is a dict, intended to store extra data with a variety of + scopes. As an example, one might want to store the originating survey: - :: + :: { - 'magnitude': 18.5, - 'error': .5 + 'survey': 'lsst', } - but could also contain a filter, a telescope, an instrument, and/or a unit: - - :: - - { - 'magnitude': 18.5, - 'error': .5, - 'filter': 'r', - 'telescope': 'ELP.domeA.1m0a', - 'instrument': 'fa07', - } :type value: dict + :param source_name: The original source of this datum. The current major use of this field is to track the broker a + datum came from, but can be used for other sources. + :type source_name: str + :param message: Set of ``AlertStreamMessage`` objects this object is associated with. :type message: ManyRelatedManager object """ target = models.ForeignKey(BaseTarget, null=False, on_delete=models.CASCADE) - data_product = models.ForeignKey(DataProduct, null=True, blank=True, on_delete=models.CASCADE) - data_type = models.CharField( - max_length=100, - default='' + data_product = models.ForeignKey( + DataProduct, null=True, blank=True, on_delete=models.CASCADE + ) + timestamp = models.DateTimeField( + null=False, blank=False, default=timezone.now, db_index=True ) - source_name = models.CharField(max_length=100, default='', blank=True) - source_location = models.CharField(max_length=200, default='', blank=True) - timestamp = models.DateTimeField(null=False, blank=False, default=timezone.now, db_index=True) - value = models.JSONField(null=False, blank=False) + value = models.JSONField( + null=False, blank=False, default=dict, verbose_name="extra data" + ) + telescope = models.CharField(max_length=255, blank=True, default="") + instrument = models.CharField(max_length=255, blank=True, default="") + source_name = models.CharField(max_length=100, default="", blank=True) message = models.ManyToManyField(AlertStreamMessage, blank=True) class Meta: - get_latest_by = ('timestamp',) + abstract = True + + +class ReducedDatum(ReducedDatumCommon): + """ + Class representing a generic datum in a TOM that isn't represented by any of the existing data types. + + :param data_type: The type of data this datum represents. Default choices are the default values found in + DATA_PRODUCT_TYPES in settings.py. + :type data_type: str + + :param source_location: A reference to the location that this datum was originally sourced from. The current major + use of this field is the URL path to the alert that this datum came from. + :type source_location: str + """ + + data_type = models.CharField(max_length=100, default="") + source_location = models.CharField(max_length=200, default="", blank=True) + + class Meta: + get_latest_by = ("timestamp",) def save(self, *args, **kwargs): # Validate data_type based on options in settings.py or default types: (type, display) @@ -387,34 +391,59 @@ def save(self, *args, **kwargs): if self.data_type and self.data_type == dp_type: break else: - raise ValidationError('Not a valid DataProduct type.') + raise ValidationError("Not a valid DataProduct type.") # because we have a custom way of validating the uniqueness of the ReducedDatum, # we need to call full_clean() here to invoke our validate_unique() method. self.full_clean() return super().save() - def validate_unique(self, *args, **kwargs): - """ - Validates that the ReducedDatum is unique. Because the `value` field is a JSONField, it is not possible to rely - on standard validation. - Do nothing if the uniqueness test passes. Otherwise, raise a ValidationError. +class PhotometryReducedDatum(ReducedDatumCommon): + discovery = models.BooleanField(default=False) + brightness = models.FloatField() + brightness_error = models.FloatField() + unit = models.CharField(max_length=32, blank=True, default="") + bandpass = models.CharField(max_length=32) + exposure_time = models.FloatField() - see https://docs.djangoproject.com/en/5.0/ref/models/instances/#validating-objects - """ - super().validate_unique(*args, **kwargs) + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["target", "bandpass", "timestamp"], name="unique_photometry" + ) + ] - # Check if the Reduced Datum exists in the database - try: - existing_reduced_datum = ReducedDatum.objects.get(target=self.target, - data_type=self.data_type, - timestamp=self.timestamp, - value=self.value) - if existing_reduced_datum and existing_reduced_datum.id != self.id: # not the same object - # found ReducedDatum with the same values. Don't save this duplicate ReducedDatum. - raise ValidationError(f'ReducedDatum already exists: {self.data_type} data with value of {self.value} ' - f'found for {self.target} at {self.timestamp}') - except ReducedDatum.DoesNotExist: - # this means that our check for uniqueness passed: so do not raise ValidationError - pass + +class SpectroscopyReducedDatum(ReducedDatumCommon): + setup = models.CharField(max_length=2000, blank=True, default="") + exposure_time = models.FloatField() + flux = models.TextField(blank=True, default="") + flux_unit = models.TextField(blank=True, default="") + error = models.TextField(blank=True, default="") + wavelength = models.TextField(blank=True, default="") + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["target", "timestamp", "telescope", "instrument"], + name="unique_spectroscopy", + ) + ] + + +class AstrometryReducedDatum(ReducedDatumCommon): + ra = models.FloatField() + dec = models.FloatField() + ra_error = models.FloatField(null=True, blank=True, default=None) + dec_error = models.FloatField(null=True, blank=True, default=None) + ra_error_units = models.CharField(max_length=32, blank=True, default="") + dec_error_units = models.CharField(max_length=32, blank=True, default="") + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["target", "timestamp", "telescope", "instrument"], + name="unique_astrometry", + ) + ] From ecda66e49d8b51dda977a234977918c25d6c0991 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Thu, 2 Apr 2026 15:30:11 -0700 Subject: [PATCH 02/35] Introduce Flux and FloatArray fields for spectra Two custom field types that use TextField as storage, but se/deserialize as arrays. Flux being an array of 2-tuple flux/wavelength pairs, and FloatArray as a single array of floats. --- tom_dataproducts/fields.py | 78 ++++++++++++ ...trument_reduceddatum_telescope_and_more.py | 15 +-- tom_dataproducts/models.py | 14 +- tom_dataproducts/tests/tests.py | 120 +++++++++++------- 4 files changed, 166 insertions(+), 61 deletions(-) create mode 100644 tom_dataproducts/fields.py diff --git a/tom_dataproducts/fields.py b/tom_dataproducts/fields.py new file mode 100644 index 000000000..a0fa52013 --- /dev/null +++ b/tom_dataproducts/fields.py @@ -0,0 +1,78 @@ +import ast + +from django.db import models + + +class FloatArrayField(models.Field): + """ + Stores a list of floats as a TextField. + + Python: [1.23, 4.567, 3.423e-19] + Database: "[1.23, 4.567, 3.423e-19]" + """ + + def get_internal_type(self): + return 'TextField' + + def from_db_value(self, value, expression, connection): + if value is None: + return value + return self._parse(value) + + def to_python(self, value): + if value is None or isinstance(value, list): + return value + return self._parse(value) + + def get_prep_value(self, value): + if value is None: + return value + if isinstance(value, str): + return value + return repr([float(x) for x in value]) + + def value_to_string(self, obj): + return self.get_prep_value(self.value_from_object(obj)) + + @staticmethod + def _parse(value): + if not value: + return [] + return [float(x) for x in ast.literal_eval(value)] + + +class FluxField(models.Field): + """ + Stores a list of (x, y) float 2-tuples as a TextField. + Useful for storing spectral data with wavelength/flux pairs. + """ + + def get_internal_type(self): + return 'TextField' + + def from_db_value(self, value, expression, connection): + if value is None: + return value + return self._parse(value) + + def to_python(self, value): + if value is None or isinstance(value, list): + return value + return self._parse(value) + + def get_prep_value(self, value): + if value is None: + return value + if isinstance(value, str): + return value + return repr([(float(x), float(y)) for x, y in value]) + + def value_to_string(self, obj): + return self.get_prep_value(self.value_from_object(obj)) + + @staticmethod + def _parse(value): + if not value: + return [] + parsed = ast.literal_eval(value) + return [(float(x), float(y)) for x, y in parsed] diff --git a/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py b/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py index cc8fb8e0c..b0c77ecce 100644 --- a/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py +++ b/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py @@ -1,7 +1,8 @@ -# Generated by Django 5.2.12 on 2026-04-02 18:46 +# Generated by Django 5.2.12 on 2026-04-02 21:53 import django.db.models.deletion import django.utils.timezone +import tom_dataproducts.fields from django.db import migrations, models @@ -61,12 +62,11 @@ class Migration(migrations.Migration): ('telescope', models.CharField(blank=True, default='', max_length=255)), ('instrument', models.CharField(blank=True, default='', max_length=255)), ('source_name', models.CharField(blank=True, default='', max_length=100)), - ('discovery', models.BooleanField(default=False)), - ('brightness', models.FloatField()), - ('brightness_error', models.FloatField()), + ('brightness', models.FloatField(blank=True, null=True)), + ('brightness_error', models.FloatField(blank=True, null=True)), ('unit', models.CharField(blank=True, default='', max_length=32)), ('bandpass', models.CharField(max_length=32)), - ('exposure_time', models.FloatField()), + ('exposure_time', models.FloatField(blank=True, null=True)), ('data_product', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='tom_dataproducts.dataproduct')), ('message', models.ManyToManyField(blank=True, to='tom_alerts.alertstreammessage')), ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tom_targets.basetarget')), @@ -86,10 +86,9 @@ class Migration(migrations.Migration): ('source_name', models.CharField(blank=True, default='', max_length=100)), ('setup', models.CharField(blank=True, default='', max_length=2000)), ('exposure_time', models.FloatField()), - ('flux', models.TextField(blank=True, default='')), + ('flux', tom_dataproducts.fields.FluxField(blank=True, default=list)), + ('error', tom_dataproducts.fields.FloatArrayField(blank=True, default=list)), ('flux_unit', models.TextField(blank=True, default='')), - ('error', models.TextField(blank=True, default='')), - ('wavelength', models.TextField(blank=True, default='')), ('data_product', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='tom_dataproducts.dataproduct')), ('message', models.ManyToManyField(blank=True, to='tom_alerts.alertstreammessage')), ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tom_targets.basetarget')), diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index 150cc7362..5742e1b8e 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -16,6 +16,8 @@ from tom_observations.models import ObservationRecord from tom_targets.base_models import BaseTarget +from .fields import FluxField, FloatArrayField + logger = logging.getLogger(__name__) try: @@ -400,12 +402,11 @@ def save(self, *args, **kwargs): class PhotometryReducedDatum(ReducedDatumCommon): - discovery = models.BooleanField(default=False) - brightness = models.FloatField() - brightness_error = models.FloatField() + brightness = models.FloatField(blank=True, null=True) + brightness_error = models.FloatField(blank=True, null=True) unit = models.CharField(max_length=32, blank=True, default="") bandpass = models.CharField(max_length=32) - exposure_time = models.FloatField() + exposure_time = models.FloatField(blank=True, null=True) class Meta: constraints = [ @@ -418,10 +419,9 @@ class Meta: class SpectroscopyReducedDatum(ReducedDatumCommon): setup = models.CharField(max_length=2000, blank=True, default="") exposure_time = models.FloatField() - flux = models.TextField(blank=True, default="") + flux = FluxField(blank=True, default=list) + error = FloatArrayField(blank=True, default=list) flux_unit = models.TextField(blank=True, default="") - error = models.TextField(blank=True, default="") - wavelength = models.TextField(blank=True, default="") class Meta: constraints = [ diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index 3d5b6e312..05bc5cf09 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -1,34 +1,43 @@ -import datetime -from http import HTTPStatus import os import tempfile -import responses +from datetime import date, time +from http import HTTPStatus +from unittest.mock import patch +import numpy as np +import responses from astropy import units from astropy.io import fits from astropy.table import Table -from datetime import date, time -from django.test import TestCase, override_settings from django.conf import settings from django.contrib.auth.models import Group, User -from django.core.exceptions import ValidationError from django.core.files.uploadedfile import SimpleUploadedFile +from django.db import IntegrityError +from django.test import TestCase, override_settings from django.urls import reverse -from django.utils import timezone, text +from django.utils import text, timezone from guardian.shortcuts import assign_perm -import numpy as np from specutils import Spectrum1D -from unittest.mock import patch from tom_dataproducts.exceptions import InvalidFileFormatException from tom_dataproducts.forms import DataProductUploadForm -from tom_dataproducts.models import DataProduct, is_fits_image_file, ReducedDatum, data_product_path +from tom_dataproducts.models import ( + DataProduct, + PhotometryReducedDatum, + ReducedDatum, + SpectroscopyReducedDatum, + data_product_path, + is_fits_image_file, +) from tom_dataproducts.processors.data_serializers import SpectrumSerializer from tom_dataproducts.processors.photometry_processor import PhotometryProcessor from tom_dataproducts.processors.spectroscopy_processor import SpectroscopyProcessor from tom_dataproducts.utils import create_image_dataproduct +from tom_observations.tests.factories import ( + ObservingRecordFactory, + SiderealTargetFactory, +) from tom_observations.tests.utils import FakeRoboticFacility -from tom_observations.tests.factories import SiderealTargetFactory, ObservingRecordFactory def mock_fits2image(file1, file2, width, height): @@ -538,6 +547,46 @@ def test_create_thumbnail(self, mock_is_fits_image_file): self.assertIn(expected, logs.output) +class TestCustomFields(TestCase): + def test_create_spectra_with_flux_data(self): + flux = [ + (7.427265572723272e-16, 3399.697753906248), + (7.796862575906174e-16, 3401.383788108824), + ] + rd = SpectroscopyReducedDatum.objects.create( + target=SiderealTargetFactory.create(), + timestamp=timezone.now(), + exposure_time=1000.0, + flux=flux, + flux_unit="Å", + ) + rd.refresh_from_db() # ensure we round trip to the database + self.assertEqual(flux, rd.flux) + + def test_create_spectra_with_error_data(self): + error = [0.0001, 0.0002] + rd = SpectroscopyReducedDatum.objects.create( + target=SiderealTargetFactory.create(), + timestamp=timezone.now(), + exposure_time=1000.0, + flux=[(1.0, 2.0), (3.0, 4.0)], + error=error, + flux_unit="Å", + ) + rd.refresh_from_db() + self.assertEqual(error, rd.error) + + def test_create_spectra_bad_flux(self): + with self.assertRaises(TypeError): + SpectroscopyReducedDatum.objects.create( + target=SiderealTargetFactory.create(), + timestamp=timezone.now(), + exposure_time=1000.0, + flux=[1.0, 2.0, 3.0, 4.0], # oops, not tuples. + flux_unit="Å", + ) + + class TestReducedDatumModel(TestCase): def setUp(self): # set up a ReducedDatum instance to test against @@ -577,43 +626,22 @@ def test_create_reduced_datum(self): self.assertEqual(2, ReducedDatum.objects.count()) def test_create_reduced_datum_duplicate(self): - """Test that we cannot add a second ReducedDatum with the same target, data_type, - timestamp, and value dict""" - # in this case ALL fields are the same as the self.existing_reduced_datum - with self.assertRaises(ValidationError): - ReducedDatum.objects.create( - target=self.target, - data_type=self.data_type, - source_name=self.source_name, - timestamp=self.timestamp, - value=self.existing_reduced_datum_value) - - # in this case only the target, data_type and value fields - # are the same as the self.existing_reduced_datum - # so an exception should NOT be raised - try: - ReducedDatum.objects.create( - target=self.target, - data_type=self.data_type, - source_name='new_source_name', - timestamp=(self.timestamp - datetime.timedelta(days=1)), # different timestamp - value=self.existing_reduced_datum_value) - except ValidationError: - self.fail("ValidationError raised when it should not have been (timestamps differ)") - - # by NOT raising ValidationError, this shows that - # ReducedDatum.objects.bulk_create() bypasses the ReducedDatum.save() - # method which validated uniqueness!! - # (this is a duplicate ReducedDatum that we are trying to add here - unsaved_reduced_datum = ReducedDatum( + """Test that we cannot add a second PhotometryReducedDatum with the same target, + timestamp, and bandpass""" + PhotometryReducedDatum.objects.create( target=self.target, - data_type=self.data_type, - source_name=self.source_name, timestamp=self.timestamp, - value=self.existing_reduced_datum_value) - # does bulk_create bypass the ReducedDatum.save() method which validated uniqueness? - # (this is a duplicate ReducedDatum that we are trying to add here - ReducedDatum.objects.bulk_create([unsaved_reduced_datum]) + brightness=1.0, + bandpass="r" + ) + + with self.assertRaises(IntegrityError): + PhotometryReducedDatum.objects.create( + target=self.target, + timestamp=self.timestamp, + brightness=2.0, + bandpass="r" + ) @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], From 65980f2da83ef6bf596cd20c46c861b186f6e612 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 8 Apr 2026 15:07:02 -0700 Subject: [PATCH 03/35] Exposure time optional field value can now be blank --- ...tum_instrument_reduceddatum_telescope_and_more.py | 12 ++++++------ tom_dataproducts/models.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py b/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py index b0c77ecce..554e8df82 100644 --- a/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py +++ b/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 5.2.12 on 2026-04-02 21:53 +# Generated by Django 5.2.12 on 2026-04-08 22:20 import django.db.models.deletion import django.utils.timezone @@ -28,14 +28,14 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='reduceddatum', name='value', - field=models.JSONField(default=dict, verbose_name='extra data'), + field=models.JSONField(blank=True, default=dict, verbose_name='extra data'), ), migrations.CreateModel( name='AstrometryReducedDatum', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('timestamp', models.DateTimeField(db_index=True, default=django.utils.timezone.now)), - ('value', models.JSONField(default=dict, verbose_name='extra data')), + ('value', models.JSONField(blank=True, default=dict, verbose_name='extra data')), ('telescope', models.CharField(blank=True, default='', max_length=255)), ('instrument', models.CharField(blank=True, default='', max_length=255)), ('source_name', models.CharField(blank=True, default='', max_length=100)), @@ -58,7 +58,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('timestamp', models.DateTimeField(db_index=True, default=django.utils.timezone.now)), - ('value', models.JSONField(default=dict, verbose_name='extra data')), + ('value', models.JSONField(blank=True, default=dict, verbose_name='extra data')), ('telescope', models.CharField(blank=True, default='', max_length=255)), ('instrument', models.CharField(blank=True, default='', max_length=255)), ('source_name', models.CharField(blank=True, default='', max_length=100)), @@ -80,12 +80,12 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('timestamp', models.DateTimeField(db_index=True, default=django.utils.timezone.now)), - ('value', models.JSONField(default=dict, verbose_name='extra data')), + ('value', models.JSONField(blank=True, default=dict, verbose_name='extra data')), ('telescope', models.CharField(blank=True, default='', max_length=255)), ('instrument', models.CharField(blank=True, default='', max_length=255)), ('source_name', models.CharField(blank=True, default='', max_length=100)), ('setup', models.CharField(blank=True, default='', max_length=2000)), - ('exposure_time', models.FloatField()), + ('exposure_time', models.FloatField(blank=True, null=True)), ('flux', tom_dataproducts.fields.FluxField(blank=True, default=list)), ('error', tom_dataproducts.fields.FloatArrayField(blank=True, default=list)), ('flux_unit', models.TextField(blank=True, default='')), diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index 5742e1b8e..17c4d88fc 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -357,7 +357,7 @@ class ReducedDatumCommon(models.Model): null=False, blank=False, default=timezone.now, db_index=True ) value = models.JSONField( - null=False, blank=False, default=dict, verbose_name="extra data" + null=False, blank=True, default=dict, verbose_name="extra data" ) telescope = models.CharField(max_length=255, blank=True, default="") instrument = models.CharField(max_length=255, blank=True, default="") @@ -418,7 +418,7 @@ class Meta: class SpectroscopyReducedDatum(ReducedDatumCommon): setup = models.CharField(max_length=2000, blank=True, default="") - exposure_time = models.FloatField() + exposure_time = models.FloatField(blank=True, null=True) flux = FluxField(blank=True, default=list) error = FloatArrayField(blank=True, default=list) flux_unit = models.TextField(blank=True, default="") From 5861979f77f584b95cce6a326f69329af1a98a7f Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 8 Apr 2026 16:28:43 -0700 Subject: [PATCH 04/35] Create ReducedDatum subclasses via API --- tom_dataproducts/models.py | 138 ++++++++++++++++++++++++++++- tom_dataproducts/serializers.py | 4 +- tom_dataproducts/tests/test_api.py | 81 ++++++++++++++++- 3 files changed, 217 insertions(+), 6 deletions(-) diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index 17c4d88fc..54c454b26 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -16,7 +16,7 @@ from tom_observations.models import ObservationRecord from tom_targets.base_models import BaseTarget -from .fields import FluxField, FloatArrayField +from .fields import FloatArrayField, FluxField logger = logging.getLogger(__name__) @@ -447,3 +447,139 @@ class Meta: name="unique_astrometry", ) ] + + +def _pop_find_field(possible_fields: set, value: dict): + """ + Helper function to find a field in a dict given a set of possible field names. + Pops the value of the first found field, or returns None if no field is found. + """ + for field in possible_fields: + if field in value: + return value.pop(field) + + return None + + +def _extract_extra_fields(data: dict, model: type[models.Model]) -> dict: + """ + Helper function to extract fields from the value dict that are not part of the model. + Pops the extra fields from the value dict and returns them as a new dict to be saved + in the models `value` field. + """ + model_fields = set(f.name for f in model._meta.get_fields()) + extra_fields = {} + for field in list(data.keys()): + if field not in model_fields: + extra_fields[field] = data.pop(field) + + return extra_fields + + +def _build_photometry_reduced_datum(data: dict) -> PhotometryReducedDatum: + BRIGHTNESS_FIELDS = {"brightness", "magnitude", "mag"} + BRIGHTNESS_ERROR_FIELDS = { + "error", + "brightness_error", + "magnitude_error", + "mag_err", + } + BANDPASS_FIELDS = {"bandpass", "filter", "band", "f"} + + brightness = _pop_find_field(BRIGHTNESS_FIELDS, data) + brightness_error = _pop_find_field(BRIGHTNESS_ERROR_FIELDS, data) + bandpass = _pop_find_field(BANDPASS_FIELDS, data) or "" + + extra_fields = _extract_extra_fields(data, PhotometryReducedDatum) + + return PhotometryReducedDatum( + brightness=brightness, + brightness_error=brightness_error, + bandpass=bandpass, + value=extra_fields, + **data, + ) + + +def _build_spectroscopy_reduced_datum(data: dict) -> SpectroscopyReducedDatum: + FLUX_FIELDS = {"flux", "f"} + ERROR_FIELDS = {"error", "err", "flux_error", "f_error"} + FLUX_UNIT_FIELDS = {"flux_unit", "f_unit", "flux_units", "f_units"} + + flux = _pop_find_field(FLUX_FIELDS, data) + error = _pop_find_field(ERROR_FIELDS, data) + flux_unit = _pop_find_field(FLUX_UNIT_FIELDS, data) or "" + + extra_fields = _extract_extra_fields(data, SpectroscopyReducedDatum) + + return SpectroscopyReducedDatum( + flux=flux, + error=error, + flux_unit=flux_unit, + value=extra_fields, + **data, + ) + + +def _build_astrometry_reduced_datum(data: dict) -> AstrometryReducedDatum: + RA_FIELDS = {"ra", "right_ascension", "ra_deg"} + DEC_FIELDS = {"dec", "declination", "dec_deg"} + RA_ERROR_FIELDS = {"ra_error", "right_ascension_error", "ra_err"} + DEC_ERROR_FIELDS = {"dec_error", "declination_error", "dec_err"} + RA_ERROR_UNIT_FIELDS = {"ra_error_units", "ra_err_units"} + DEC_ERROR_UNIT_FIELDS = {"dec_error_units", "dec_err_units"} + + ra = _pop_find_field(RA_FIELDS, data) + dec = _pop_find_field(DEC_FIELDS, data) + ra_error = _pop_find_field(RA_ERROR_FIELDS, data) + dec_error = _pop_find_field(DEC_ERROR_FIELDS, data) + ra_error_units = _pop_find_field(RA_ERROR_UNIT_FIELDS, data) or "" + dec_error_units = _pop_find_field(DEC_ERROR_UNIT_FIELDS, data) or "" + + extra_fields = _extract_extra_fields(data, AstrometryReducedDatum) + + return AstrometryReducedDatum( + ra=ra, + dec=dec, + ra_error=ra_error, + dec_error=dec_error, + ra_error_units=ra_error_units, + dec_error_units=dec_error_units, + value=extra_fields, + **data, + ) + + +def _build_generic_reduced_datum(data: dict, data_type: str) -> ReducedDatum: + extra_fields = _extract_extra_fields(data, ReducedDatum) + + return ReducedDatum(value=extra_fields, data_type=data_type, **data) + + +def try_parse_reduced_datum( + data: dict, +) -> ( + ReducedDatum + | PhotometryReducedDatum + | SpectroscopyReducedDatum + | AstrometryReducedDatum +): + """ + Accepts unstructured data and attempts to create the correct ReducedDatum sublcass. + If the heuristics fail, returns a generic ReducedDatum. + """ + if data.get("value") and isinstance(data["value"], dict): + # `value` is the existing free-form field. Pull those values to the top of the dict + # so we can use them to determine which reduced datum type to create + value = data.pop("value") + data = {**data, **value} + + match data_type := data.pop("data_type", "").lower(): + case "photometry": + return _build_photometry_reduced_datum(data) + case "spectroscopy": + return _build_spectroscopy_reduced_datum(data) + case "astrometry": + return _build_astrometry_reduced_datum(data) + case _: + return _build_generic_reduced_datum(data, data_type) diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py index 7c1d92c0f..58e697b43 100644 --- a/tom_dataproducts/serializers.py +++ b/tom_dataproducts/serializers.py @@ -4,7 +4,7 @@ from rest_framework import serializers from tom_common.serializers import GroupSerializer -from tom_dataproducts.models import DataProductGroup, DataProduct, ReducedDatum +from tom_dataproducts.models import DataProductGroup, DataProduct, ReducedDatum, try_parse_reduced_datum from tom_observations.models import ObservationRecord from tom_observations.serializers import ObservationRecordFilteredPrimaryKeyRelatedField from tom_targets.models import Target @@ -38,7 +38,7 @@ def create(self, validated_data): """ groups = validated_data.pop('groups', []) - rd = ReducedDatum(**validated_data) + rd = try_parse_reduced_datum(validated_data) rd.full_clean() rd.save() diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py index a8ad6dc69..6338847f8 100644 --- a/tom_dataproducts/tests/test_api.py +++ b/tom_dataproducts/tests/test_api.py @@ -1,12 +1,18 @@ from django.contrib.auth.models import Group, User +from django.core.exceptions import ValidationError from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse -from django.core.exceptions import ValidationError from guardian.shortcuts import assign_perm from rest_framework import status from rest_framework.test import APITestCase -from tom_dataproducts.models import DataProduct, ReducedDatum +from tom_dataproducts.models import ( + AstrometryReducedDatum, + DataProduct, + PhotometryReducedDatum, + ReducedDatum, + SpectroscopyReducedDatum, +) from tom_observations.tests.factories import ObservingRecordFactory from tom_targets.tests.factories import SiderealTargetFactory @@ -139,7 +145,7 @@ def test_upload_same_reduced_datum_twice(self): self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') self.rd_data['value'] = {'magnitude': 15.582, 'filter': 'B', 'error': 0.005} self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') - rd_queryset = ReducedDatum.objects.all() + rd_queryset = PhotometryReducedDatum.objects.all() self.assertEqual(rd_queryset.count(), 2) def test_upload_reduced_datum_no_sharing_location(self): @@ -195,3 +201,72 @@ def test_reduced_datum_filter(self): response3 = self.client.get(reverse('api:reduceddatums-list'), QUERY_STRING='source_name=thin_air') self.assertEqual(response3.data['count'], 0) self.assertEqual(response3.data['results'], []) + + def test_upload_reduced_photometry_datum(self): + payload = { + "data_product": "", + "data_type": "photometry", + "value": {"magnitude": 15.582, "filter": "r", "error": 0.005}, + "target": self.st.id, + "timestamp": "2012-02-12T01:40:47Z", + } + response = self.client.post( + reverse("api:reduceddatums-list"), payload, format="json" + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(PhotometryReducedDatum.objects.count(), 1) + rd = PhotometryReducedDatum.objects.first() + self.assertEqual(rd.brightness, payload["value"]["magnitude"]) + self.assertEqual(rd.target.id, payload["target"]) + + def test_upload_spectroscopy_datum(self): + payload = { + "data_product": "", + "data_type": "spectroscopy", + "value": {"flux": "[(123.4, 4.321)]", "error": "[0.005]", "flux_unit": "s"}, + "target": self.st.id, + "timestamp": "2012-02-12T01:40:47Z", + } + response = self.client.post( + reverse("api:reduceddatums-list"), payload, format="json" + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(SpectroscopyReducedDatum.objects.count(), 1) + rd = SpectroscopyReducedDatum.objects.first() + self.assertEqual(rd.flux, [(123.4, 4.321)]) + self.assertEqual(rd.target.id, payload["target"]) + + def test_upload_astrometry_reduced_datum(self): + payload = { + "data_product": "", + "data_type": "astrometry", + "value": {"ra": 11.2, "dec": 30.0, "error": 0.005}, + "target": self.st.id, + "timestamp": "2012-02-12T01:40:47Z", + } + response = self.client.post( + reverse("api:reduceddatums-list"), payload, format="json" + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(AstrometryReducedDatum.objects.count(), 1) + rd = AstrometryReducedDatum.objects.first() + self.assertEqual(rd.ra, 11.2) + self.assertEqual(rd.target.id, payload["target"]) + + def test_upload_generic_reduced_datum(self): + payload = { + "data_product": "", + "data_type": "image_file", # can be any custom data type + "value": {"ra": 11.2, "dec": 30.0, "foobar": 0.005}, + "target": self.st.id, + "timestamp": "2012-02-12T01:40:47Z", + } + response = self.client.post( + reverse("api:reduceddatums-list"), payload, format="json" + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ReducedDatum.objects.count(), 1) + rd = ReducedDatum.objects.first() + self.assertEqual(rd.value["ra"], 11.2) + self.assertEqual(rd.value["foobar"], 0.005) + self.assertEqual(rd.target.id, payload["target"]) From f325611ee981afb9381cbb8ca1370bd988777a13 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Thu, 9 Apr 2026 11:27:16 -0700 Subject: [PATCH 05/35] Refactor data product upload processor and view --- tom_dataproducts/data_processor.py | 66 +++++++++++++----------------- tom_dataproducts/models.py | 8 ++++ tom_dataproducts/tests/test_api.py | 4 +- tom_dataproducts/views.py | 17 +++++--- 4 files changed, 51 insertions(+), 44 deletions(-) diff --git a/tom_dataproducts/data_processor.py b/tom_dataproducts/data_processor.py index 1bf7eaac5..36aa6f266 100644 --- a/tom_dataproducts/data_processor.py +++ b/tom_dataproducts/data_processor.py @@ -1,11 +1,10 @@ -import json import logging import mimetypes from django.conf import settings from importlib import import_module -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import try_parse_reduced_datum from tom_targets.sharing import continuous_share_data logger = logging.getLogger(__name__) @@ -26,8 +25,8 @@ def run_data_processor(dp, dp_type_override=None): type from the `dp` object is used. :type dp_type_override: str, optional - :returns: QuerySet of `ReducedDatum` objects created by the `run_data_processor` call - :rtype: `QuerySet` of `ReducedDatum` + :returns: List of typed ReducedDatum objects created by the `run_data_processor` call + :rtype: list """ data_type = dp_type_override or dp.data_product_type try: @@ -43,39 +42,35 @@ def run_data_processor(dp, dp_type_override=None): raise ImportError('Could not import {}. Did you provide the correct path?'.format(processor_class)) data_processor = clazz() - # data returned by process_data is a list of 3-tuples: (timestamp, datum, source) + # 1. data returned by process_data is a list of 3-tuples: (timestamp, datum, source) data = data_processor.process_data(dp) data_type = data_processor.data_type_override() or data_type - # Add only the new (non-duplicate) ReducedDatum objects to the database - - # 1. For quick O(1) lookup, create a hash table of existing ReducedDatum objects - - # Extract exising ReducedDatums for this target, and create a hash table (dict) - # (We make the reduced_dataum.value JSONField dict hashable by converting it to a json string). - # This is so we can do O(1) lookups below as we check for duplicate data. - existing_reduced_datum_values = {json.dumps(rd.value, sort_keys=True, skipkeys=True): 1 - for rd in ReducedDatum.objects.filter(target=dp.target)} - - # 2. Create the list of new ReducedDatum objects (ready for bulk_create) + # 2. Build typed ReducedDatum instances from the raw data. + # try_parse_reduced_datum inspects the data_type and field names to determine the + # correct concrete subclass (photometry, spectroscopy, astrometry, or generic). new_reduced_datums = [] - skipped_data = [] for datum in data: - # Check if the value is already in the ReducedDatum table - # (via lookup in the hash table created above for this purpose) - if json.dumps(datum[1], sort_keys=True, skipkeys=True) in existing_reduced_datum_values: - skipped_data.append(datum) - else: - new_reduced_datums.append( - ReducedDatum(target=dp.target, data_product=dp, data_type=data_type, - timestamp=datum[0], value=datum[1], source_name=datum[2])) - - # prior to checking for duplicates, we created the (yet-to-be-inserted) ReducedDatum list like this: - # reduced_datums = [ReducedDatum(target=dp.target, data_product=dp, data_type=data_type, - # timestamp=datum[0], value=datum[1], source_name=datum[2]) for datum in data] - - # 3. Finally, insert the new ReducedDatum objects into the database - reduced_datums = ReducedDatum.objects.bulk_create(new_reduced_datums) + instance = try_parse_reduced_datum({ + 'target': dp.target, + 'data_product': dp, + 'timestamp': datum[0], + 'source_name': datum[2], + 'data_type': data_type, + **datum[1], + }) + new_reduced_datums.append(instance) + + # 3. bulk_create requires a uniform model type, so group instances by their concrete class. + # Then create them + by_type: dict[type, list] = {} + for instance in new_reduced_datums: + by_type.setdefault(type(instance), []).append(instance) + + reduced_datums = [] + for model_class, instances in by_type.items(): + # ignore_conflicts uses DB level cosntraints for deduplication + reduced_datums.extend(model_class.objects.bulk_create(instances, ignore_conflicts=True)) # 4. Trigger any sharing you may have set to occur when new data comes in # Encapsulate this in a try/catch so sharing failure doesn't prevent dataproduct ingestion @@ -85,12 +80,9 @@ def run_data_processor(dp, dp_type_override=None): logger.warning(f"Failed to share new dataproduct {dp.product_id}: {repr(e)}") # log what happened - if skipped_data: - logger.warning(f'{len(skipped_data)} of {len(data)} skipped as duplicates') - logger.info(f'{len(new_reduced_datums)} of {len(data)} new ReducedDatums ' - f'added for DataProduct: {dp.product_id}') + logger.info(f'{len(reduced_datums)} of {len(data)} new ReducedDatums added for DataProduct: {dp.product_id}') - return ReducedDatum.objects.filter(data_product=dp) + return reduced_datums class DataProcessor(): diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index 54c454b26..b52dc2760 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -449,6 +449,14 @@ class Meta: ] +REDUCED_DATUM_MODELS = ( + ReducedDatum, + PhotometryReducedDatum, + SpectroscopyReducedDatum, + AstrometryReducedDatum, +) + + def _pop_find_field(possible_fields: set, value: dict): """ Helper function to find a field in a dict given a set of possible field names. diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py index 6338847f8..5ee61dfdd 100644 --- a/tom_dataproducts/tests/test_api.py +++ b/tom_dataproducts/tests/test_api.py @@ -45,7 +45,7 @@ def test_data_product_upload_for_target(self): response = self.client.post(reverse('api:dataproducts-list'), self.dp_data, format='multipart') self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual(DataProduct.objects.count(), 1) - self.assertEqual(ReducedDatum.objects.count(), 3) + self.assertEqual(PhotometryReducedDatum.objects.count(), 3) dp = DataProduct.objects.get(pk=response.data['id']) self.assertEqual(dp.target_id, self.st.id) @@ -62,7 +62,7 @@ def test_data_product_upload_for_observation(self): self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual(DataProduct.objects.count(), 1) - self.assertEqual(ReducedDatum.objects.count(), 3) + self.assertEqual(PhotometryReducedDatum.objects.count(), 3) dp = DataProduct.objects.get(pk=response.data['id']) self.assertEqual(dp.target_id, self.st.id) self.assertEqual(dp.observation_record_id, self.obsr.id) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index d5787ea73..cc9961698 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -23,7 +23,7 @@ from tom_common.hooks import run_hook from tom_common.hints import add_hint from tom_common.mixins import Raise403PermissionRequiredMixin -from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum +from tom_dataproducts.models import DataProduct, DataProductGroup, REDUCED_DATUM_MODELS from tom_dataproducts.exceptions import InvalidFileFormatException from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm, DataShareForm from tom_dataproducts.filters import DataProductFilter @@ -39,6 +39,11 @@ logger.setLevel(logging.DEBUG) +def _delete_reduced_datums_for_product(dp): + for model in REDUCED_DATUM_MODELS: + model.objects.filter(data_product=dp).delete() + + class DataProductSaveView(LoginRequiredMixin, View): """ View that handles saving a ``DataProduct`` generated by an observation. Requires authentication. @@ -213,17 +218,19 @@ def form_valid(self, form): for group in form.cleaned_data['groups']: assign_perm('tom_dataproducts.view_dataproduct', group, dp) assign_perm('tom_dataproducts.delete_dataproduct', group, dp) - assign_perm('tom_dataproducts.view_reduceddatum', group, reduced_data) + for datum in reduced_data: + perm = f'tom_dataproducts.view_{type(datum).__name__.lower()}' + assign_perm(perm, group, datum) successful_uploads.append(str(dp)) except InvalidFileFormatException as iffe: - ReducedDatum.objects.filter(data_product=dp).delete() + _delete_reduced_datums_for_product(dp) dp.delete() messages.error( self.request, f'File format invalid for file {str(dp)} -- error was {iffe}' ) except Exception as e: - ReducedDatum.objects.filter(data_product=dp).delete() + _delete_reduced_datums_for_product(dp) dp.delete() messages.error(self.request, f'There was a problem processing your file: {str(dp)} -- Error: {e}') if successful_uploads: @@ -288,7 +295,7 @@ def form_valid(self, form): data_product = self.get_object() # Delete associated ReducedDatum objects - ReducedDatum.objects.filter(data_product=data_product).delete() + _delete_reduced_datums_for_product(data_product) # Delete the file reference. data_product.data.delete() From 1269b6a0dbab25f576e5fa7c977d976b0bf2281e Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 15 Apr 2026 14:37:32 -0700 Subject: [PATCH 06/35] Small changes to rd models: photometry has limit, source_location moved to base --- ...duceddatum_instrument_reduceddatum_telescope_and_more.py | 6 +++++- tom_dataproducts/models.py | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py b/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py index 554e8df82..b595db0be 100644 --- a/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py +++ b/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 5.2.12 on 2026-04-08 22:20 +# Generated by Django 5.2.13 on 2026-04-15 21:36 import django.db.models.deletion import django.utils.timezone @@ -39,6 +39,7 @@ class Migration(migrations.Migration): ('telescope', models.CharField(blank=True, default='', max_length=255)), ('instrument', models.CharField(blank=True, default='', max_length=255)), ('source_name', models.CharField(blank=True, default='', max_length=100)), + ('source_location', models.CharField(blank=True, default='', max_length=200)), ('ra', models.FloatField()), ('dec', models.FloatField()), ('ra_error', models.FloatField(blank=True, default=None, null=True)), @@ -62,8 +63,10 @@ class Migration(migrations.Migration): ('telescope', models.CharField(blank=True, default='', max_length=255)), ('instrument', models.CharField(blank=True, default='', max_length=255)), ('source_name', models.CharField(blank=True, default='', max_length=100)), + ('source_location', models.CharField(blank=True, default='', max_length=200)), ('brightness', models.FloatField(blank=True, null=True)), ('brightness_error', models.FloatField(blank=True, null=True)), + ('limit', models.FloatField(blank=True, null=True)), ('unit', models.CharField(blank=True, default='', max_length=32)), ('bandpass', models.CharField(max_length=32)), ('exposure_time', models.FloatField(blank=True, null=True)), @@ -84,6 +87,7 @@ class Migration(migrations.Migration): ('telescope', models.CharField(blank=True, default='', max_length=255)), ('instrument', models.CharField(blank=True, default='', max_length=255)), ('source_name', models.CharField(blank=True, default='', max_length=100)), + ('source_location', models.CharField(blank=True, default='', max_length=200)), ('setup', models.CharField(blank=True, default='', max_length=2000)), ('exposure_time', models.FloatField(blank=True, null=True)), ('flux', tom_dataproducts.fields.FluxField(blank=True, default=list)), diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index b52dc2760..6654e366c 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -362,6 +362,7 @@ class ReducedDatumCommon(models.Model): telescope = models.CharField(max_length=255, blank=True, default="") instrument = models.CharField(max_length=255, blank=True, default="") source_name = models.CharField(max_length=100, default="", blank=True) + source_location = models.CharField(max_length=200, default="", blank=True) message = models.ManyToManyField(AlertStreamMessage, blank=True) class Meta: @@ -382,7 +383,6 @@ class ReducedDatum(ReducedDatumCommon): """ data_type = models.CharField(max_length=100, default="") - source_location = models.CharField(max_length=200, default="", blank=True) class Meta: get_latest_by = ("timestamp",) @@ -404,6 +404,7 @@ def save(self, *args, **kwargs): class PhotometryReducedDatum(ReducedDatumCommon): brightness = models.FloatField(blank=True, null=True) brightness_error = models.FloatField(blank=True, null=True) + limit = models.FloatField(blank=True, null=True) unit = models.CharField(max_length=32, blank=True, default="") bandpass = models.CharField(max_length=32) exposure_time = models.FloatField(blank=True, null=True) From 240d9b9d80aa539a9aba74c2edb27cd05539e048 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 15 Apr 2026 14:49:22 -0700 Subject: [PATCH 07/35] Migrate alerce broker to use PhotometryReducedDatum --- tom_alerts/brokers/alerce.py | 44 +++++++++++-------------- tom_alerts/tests/brokers/test_alerce.py | 4 +-- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/tom_alerts/brokers/alerce.py b/tom_alerts/brokers/alerce.py index 41a49e923..b9a4568da 100644 --- a/tom_alerts/brokers/alerce.py +++ b/tom_alerts/brokers/alerce.py @@ -10,7 +10,7 @@ from tom_alerts.alerts import GenericAlert, GenericBroker, GenericQueryForm from tom_targets.models import Target -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum logger = logging.getLogger(__name__) @@ -573,35 +573,31 @@ def process_reduced_data(self, target, alert=None): for detection in lightcurve['detections']: mjd = Time(detection['mjd'], format='mjd', scale='utc') - value = { - 'filter': FILTERS[detection['fid']], - 'magnitude': detection['magpsf'], - 'error': detection['sigmapsf'], - 'telescope': 'ZTF', - } - ReducedDatum.objects.get_or_create( + PhotometryReducedDatum.objects.get_or_create( + target=target, + bandpass=FILTERS[detection['fid']], timestamp=mjd.to_datetime(TimezoneInfo()), - value=value, - source_name=self.name, - source_location=oid, - data_type='photometry', - target=target + defaults={ + 'brightness': detection['magpsf'], + 'brightness_error': detection['sigmapsf'], + 'telescope': 'ZTF', + 'source_name': self.name, + 'source_location': oid, + } ) for non_detection in lightcurve['non_detections']: mjd = Time(non_detection['mjd'], format='mjd', scale='utc') - value = { - 'filter': FILTERS[non_detection['fid']], - 'limit': non_detection['diffmaglim'], - 'telescope': 'ZTF', - } - ReducedDatum.objects.get_or_create( + PhotometryReducedDatum.objects.get_or_create( + target=target, + bandpass=FILTERS[non_detection['fid']], timestamp=mjd.to_datetime(TimezoneInfo()), - value=value, - source_name=self.name, - source_location=oid, - data_type='photometry', - target=target + defaults={ + 'limit': non_detection['diffmaglim'], + 'telescope': 'ZTF', + 'source_name': self.name, + 'source_location': oid, + } ) def to_target(self, alert): diff --git a/tom_alerts/tests/brokers/test_alerce.py b/tom_alerts/tests/brokers/test_alerce.py index 8e3187050..041482474 100644 --- a/tom_alerts/tests/brokers/test_alerce.py +++ b/tom_alerts/tests/brokers/test_alerce.py @@ -7,7 +7,7 @@ from faker import Faker from tom_alerts.brokers.alerce import ALeRCEBroker, ALeRCEQueryForm -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum from tom_targets.models import Target from tom_targets.tests.factories import SiderealTargetFactory @@ -401,7 +401,7 @@ def test_process_reduced_datum(self, mock_fetch_lightcurve): mock_fetch_lightcurve.return_value = test_data target = SiderealTargetFactory() ALeRCEBroker().process_reduced_data(target) - self.assertEqual(ReducedDatum.objects.count(), 2) + self.assertEqual(PhotometryReducedDatum.objects.count(), 2) def test_to_generic_alert(self): """Test to_generic_alert broker method.""" From ab0f2414d495f7d2af8da1cd695004f1d67a9ffb Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 15 Apr 2026 14:58:34 -0700 Subject: [PATCH 08/35] Migrate gaia broker to PhotometryReducedDatum --- tom_alerts/brokers/gaia.py | 23 +++++++++++------------ tom_alerts/tests/brokers/test_gaia.py | 13 ++++++------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/tom_alerts/brokers/gaia.py b/tom_alerts/brokers/gaia.py index e1385fabc..e4d902a03 100644 --- a/tom_alerts/brokers/gaia.py +++ b/tom_alerts/brokers/gaia.py @@ -12,7 +12,7 @@ from django import forms from tom_alerts.alerts import GenericAlert, GenericBroker, GenericQueryForm -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum BASE_BROKER_URL = 'http://gsaweb.ast.cam.ac.uk' @@ -181,17 +181,16 @@ def process_reduced_data(self, target, alert=None): jd = Time(float(phot_data[1]), format='jd', scale='utc') jd.to_datetime(timezone=TimezoneInfo()) - value = { - 'magnitude': float(phot_data[2]), - 'filter': 'G' - } - - rd, _ = ReducedDatum.objects.get_or_create( + rd, _ = PhotometryReducedDatum.objects.get_or_create( + target=target, + bandpass='G', timestamp=jd.to_datetime(timezone=TimezoneInfo()), - value=value, - source_name=self.name, - source_location=alert_url, - data_type='photometry', - target=target) + defaults={ + 'brightness': float(phot_data[2]), + 'source_name': self.name, + 'source_location': alert_url, + + } + ) return diff --git a/tom_alerts/tests/brokers/test_gaia.py b/tom_alerts/tests/brokers/test_gaia.py index 803032306..7c208b0cc 100644 --- a/tom_alerts/tests/brokers/test_gaia.py +++ b/tom_alerts/tests/brokers/test_gaia.py @@ -9,7 +9,7 @@ from tom_alerts.brokers.gaia import GaiaQueryForm from tom_alerts.brokers.gaia import GaiaBroker from tom_targets.models import Target -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum @override_settings(TOM_ALERT_CLASSES=['tom_alerts.brokers.gaia.GaiaBroker']) @@ -102,13 +102,12 @@ def setUp(self): "rvs": 'false'} ] self.test_target = Target.objects.create(name=self.alert_list[0]['name']) - ReducedDatum.objects.create( + PhotometryReducedDatum.objects.create( source_name='Gaia', source_location=111111, target=self.test_target, - data_type='photometry', timestamp=timezone.now(), - value=12345.6789 + brightness=12345.6789 ) @mock.patch('tom_alerts.brokers.gaia.requests.get') @@ -141,7 +140,7 @@ def test_process_reduced_data_with_alert(self, mock_requests_get): GaiaBroker().process_reduced_data(self.test_target, alert=self.alert_list[0]) - reduced_data = ReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') + reduced_data = PhotometryReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') self.assertGreater(reduced_data.count(), 1) self.assertEqual(reduced_data.count(), 3) # one from setUp and two from this test @@ -158,7 +157,7 @@ def test_process_reduced_data_without_alert(self, mock_fetch_alerts, mock_reques GaiaBroker().process_reduced_data(self.test_target) - reduced_data = ReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') + reduced_data = PhotometryReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') self.assertGreater(reduced_data.count(), 1) self.assertEqual(reduced_data.count(), 3) # one from setUp and two from this test @@ -183,6 +182,6 @@ def test_rewrite_process_reduced_data_with_alert(self): except ValidationError as e: self.fail(f'This test should have created two UNIQUE ReducedDatum objects, but {e}') - reduced_data = ReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') + reduced_data = PhotometryReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') self.assertGreater(reduced_data.count(), 1) self.assertEqual(reduced_data.count(), 3) # one from setUp and two from this test From 7fb69fac0afce15ddaa9fd8c2b656fd9e7da25d2 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 15 Apr 2026 15:17:23 -0700 Subject: [PATCH 09/35] Migrate hermes to use PhotometryReducedDatum --- tom_dataproducts/alertstreams/hermes.py | 28 ++++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index d3cd8b2c7..8bf88bd8c 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -9,7 +9,7 @@ from tom_alerts.models import AlertStreamMessage from tom_targets.models import Target, TargetList -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum import requests @@ -330,15 +330,23 @@ def hermes_alert_handler(alert, metadata): except ValueError: continue - datum = { - 'target': target, - 'data_type': 'photometry', - 'source_name': alert_as_dict['topic'], - 'source_location': 'Hermes via HOP', # TODO Add message URL here once message ID's exist - 'timestamp': obs_date, - 'value': get_hermes_phot_value(row) - } - new_rd, created = ReducedDatum.objects.get_or_create(**datum) + value = get_hermes_phot_value(row) + new_rd, created = PhotometryReducedDatum.objects.get_or_create( + target=target, + timestamp=obs_date, + bandpass=value.get('filter', ''), + defaults={ + 'brightness': value.get('magnitude', None), + 'brightness_error': value.get('error', None), + 'unit': value.get('unit', None), + 'limit': value.get('limit', None), + 'source_name': alert_as_dict['topic'], + 'source_location': 'Hermes via HOP', # TODO Add message URL here once message ID's exist + 'timestamp': obs_date, + 'telescope': value.get('telescope', ''), + 'instrument': value.get('instrument', ''), + } + ) if created: hermes_alert.save() new_rd.message.add(hermes_alert) From a75fb8ecf161fd039d6a5bc3e129abd9eb924ad8 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 15 Apr 2026 16:11:36 -0700 Subject: [PATCH 10/35] get_photometry_data (data list for target) refactor to use PhotometryReducedDatum --- .../templatetags/dataproduct_extras.py | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index a7085eb09..7f709c229 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -18,7 +18,7 @@ import numpy as np from tom_dataproducts.forms import DataProductUploadForm, DataShareForm -from tom_dataproducts.models import DataProduct, ReducedDatum +from tom_dataproducts.models import DataProduct, PhotometryReducedDatum, ReducedDatum from tom_dataproducts.processors.data_serializers import SpectrumSerializer from tom_dataproducts.single_target_data_service.single_target_data_service import get_service_classes, \ get_service_class @@ -164,38 +164,31 @@ def get_photometry_data(context, target, target_share=False): """ Displays a table of the all photometric points for a target. """ - photometry = ReducedDatum.objects.filter(data_type='photometry', target=target).order_by('-timestamp') + photometry = PhotometryReducedDatum.objects.filter(target=target).order_by('-timestamp') if not settings.TARGET_PERMISSIONS_ONLY: photometry = get_objects_for_user( context["request"].user, - "tom_dataproducts.view_reduceddatum", + "tom_dataproducts.view_photometryreduceddatum", klass=photometry, ) - # Possibilities for reduced_datums from ZTF/MARS: - # reduced_datum.value: {'error': 0.0929680392146111, 'filter': 'r', 'magnitude': 18.2364940643311} - # reduced_datum.value: {'limit': 20.1023998260498, 'filter': 'g'} - - # for limit magnitudes, set the value of the limit key to True and - # the value of the magnitude key to the limit so the template and - # treat magnitudes as such and prepend a '>' to the limit magnitudes - # see recent_photometry.html + # Non detections have limit set and brightness null, detections are the reverse. data = [] for reduced_datum in photometry: rd_data = {'id': reduced_datum.pk, 'timestamp': reduced_datum.timestamp, 'source': reduced_datum.source_name, - 'filter': reduced_datum.value.get('filter', ''), - 'telescope': reduced_datum.value.get('telescope', ''), - 'error': reduced_datum.value.get('error', reduced_datum.value.get('magnitude_error', '')) + 'filter': reduced_datum.bandpass, + 'telescope': reduced_datum.telescope, } - - if 'limit' in reduced_datum.value.keys(): - rd_data['magnitude'] = reduced_datum.value['limit'] + if reduced_datum.limit is not None: + rd_data['magnitude'] = reduced_datum.limit rd_data['limit'] = True + rd_data['error'] = '' else: - rd_data['magnitude'] = reduced_datum.value['magnitude'] + rd_data['magnitude'] = reduced_datum.brightness rd_data['limit'] = False + rd_data['error'] = reduced_datum.brightness_error or '' data.append(rd_data) initial = {'submitter': context['request'].user, From ef66ebb2a7995a3d995dbd13999f56c1bb93f940 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 15 Apr 2026 16:34:30 -0700 Subject: [PATCH 11/35] Update photometry plot to use PhotometryDataProduct --- .../templatetags/dataproduct_extras.py | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 7f709c229..b4c10178b 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -1,4 +1,5 @@ import logging +from collections import defaultdict from urllib.parse import urlencode from django import template @@ -241,28 +242,20 @@ def photometry_for_target(context, target, width=700, height=600, background=Non 'i': 'black' } - try: - photometry_data_type = settings.DATA_PRODUCT_TYPES['photometry'][0] - except (AttributeError, KeyError): - photometry_data_type = 'photometry' - photometry_data = {} + photometry_data = defaultdict(lambda: {'time': [], 'magnitude': [], 'error': [], 'limit': []}) if settings.TARGET_PERMISSIONS_ONLY: - datums = ReducedDatum.objects.filter(target=target, data_type=photometry_data_type) + datums = PhotometryReducedDatum.objects.filter(target=target) else: datums = get_objects_for_user(context['request'].user, - 'tom_dataproducts.view_reduceddatum', - klass=ReducedDatum.objects.filter( - target=target, - data_type=photometry_data_type)) + 'tom_dataproducts.view_photometryreduceddatum', + klass=PhotometryReducedDatum.objects.filter(target=target)) for datum in datums: - if (isinstance(datum.value.get('magnitude', 0), float) and isinstance(datum.value.get('error', 0), float)) \ - or isinstance(datum.value.get('limit', 0), float): - photometry_data.setdefault(datum.value['filter'], {}) - photometry_data[datum.value['filter']].setdefault('time', []).append(datum.timestamp) - photometry_data[datum.value['filter']].setdefault('magnitude', []).append(datum.value.get('magnitude')) - photometry_data[datum.value['filter']].setdefault('error', []).append(datum.value.get('error')) - photometry_data[datum.value['filter']].setdefault('limit', []).append(datum.value.get('limit')) + if datum.brightness is not None or datum.limit is not None: + photometry_data[datum.bandpass]['time'].append(datum.timestamp) + photometry_data[datum.bandpass]['magnitude'].append(datum.brightness) + photometry_data[datum.bandpass]['error'].append(datum.brightness_error) + photometry_data[datum.bandpass]['limit'].append(datum.limit) plot_data = [] all_ydata = [] From d410d6fbc43d1a2a92358253a5c6b19f6609d535 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 15 Apr 2026 16:40:38 -0700 Subject: [PATCH 12/35] Update latest photometry widget --- tom_dataproducts/templatetags/dataproduct_extras.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index b4c10178b..28013a1fd 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -135,7 +135,7 @@ def recent_photometry(target, limit=1): """ Displays a table of the most recent photometric points for a target. """ - photometry = ReducedDatum.objects.filter(data_type='photometry', target=target).order_by('-timestamp')[:limit] + photometry = PhotometryReducedDatum.objects.filter(target=target).order_by('-timestamp')[:limit] # Possibilities for reduced_datums from ZTF/MARS: # reduced_datum.value: {'error': 0.0929680392146111, 'filter': 'r', 'magnitude': 18.2364940643311} @@ -148,11 +148,11 @@ def recent_photometry(target, limit=1): data = [] for reduced_datum in photometry: rd_data = {'timestamp': reduced_datum.timestamp} - if 'limit' in reduced_datum.value.keys(): - rd_data['magnitude'] = reduced_datum.value['limit'] + if reduced_datum.limit is not None: + rd_data['magnitude'] = reduced_datum.limit rd_data['limit'] = True else: - rd_data['magnitude'] = reduced_datum.value['magnitude'] + rd_data['magnitude'] = reduced_datum.brightness rd_data['limit'] = False data.append(rd_data) From 3a1405622a923d1e3393dfaeb1fd9c80dea06ec1 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Thu, 16 Apr 2026 14:44:36 -0700 Subject: [PATCH 13/35] Refactor spectrum to use seperate flux/wl fields --- tom_dataproducts/fields.py | 86 ++++++------------- ...trument_reduceddatum_telescope_and_more.py | 5 +- tom_dataproducts/models.py | 12 ++- tom_dataproducts/tests/test_api.py | 4 +- tom_dataproducts/tests/tests.py | 20 +++-- 5 files changed, 50 insertions(+), 77 deletions(-) diff --git a/tom_dataproducts/fields.py b/tom_dataproducts/fields.py index a0fa52013..6806013ea 100644 --- a/tom_dataproducts/fields.py +++ b/tom_dataproducts/fields.py @@ -1,78 +1,44 @@ -import ast - +from django.core.exceptions import ValidationError from django.db import models -class FloatArrayField(models.Field): +class FloatArrayField(models.JSONField): """ - Stores a list of floats as a TextField. - - Python: [1.23, 4.567, 3.423e-19] - Database: "[1.23, 4.567, 3.423e-19]" + Stores a list of floats as a JSON array. + All values are coerced to float on assignment and validated. """ - def get_internal_type(self): - return 'TextField' - - def from_db_value(self, value, expression, connection): - if value is None: - return value - return self._parse(value) - - def to_python(self, value): - if value is None or isinstance(value, list): - return value - return self._parse(value) - - def get_prep_value(self, value): - if value is None: - return value - if isinstance(value, str): - return value - return repr([float(x) for x in value]) - - def value_to_string(self, obj): - return self.get_prep_value(self.value_from_object(obj)) - @staticmethod - def _parse(value): - if not value: - return [] - return [float(x) for x in ast.literal_eval(value)] - - -class FluxField(models.Field): - """ - Stores a list of (x, y) float 2-tuples as a TextField. - Useful for storing spectral data with wavelength/flux pairs. - """ - - def get_internal_type(self): - return 'TextField' + def _coerce_to_floats(value): + if not isinstance(value, list): + raise ValidationError("FloatArrayField value must be a list.") + result = [] + for i, item in enumerate(value): + try: + result.append(float(item)) + except (TypeError, ValueError): + raise ValidationError( + f"Element at index {i} cannot be converted to float: {item!r}" + ) + return result def from_db_value(self, value, expression, connection): + value = super().from_db_value(value, expression, connection) if value is None: return value - return self._parse(value) + return [float(x) for x in value] def to_python(self, value): - if value is None or isinstance(value, list): + value = super().to_python(value) + if value is None or value == []: return value - return self._parse(value) + return self._coerce_to_floats(value) def get_prep_value(self, value): if value is None: - return value - if isinstance(value, str): - return value - return repr([(float(x), float(y)) for x, y in value]) - - def value_to_string(self, obj): - return self.get_prep_value(self.value_from_object(obj)) + return super().get_prep_value(value) + return super().get_prep_value(self._coerce_to_floats(value)) - @staticmethod - def _parse(value): - if not value: - return [] - parsed = ast.literal_eval(value) - return [(float(x), float(y)) for x, y in parsed] + def validate(self, value, model_instance): + super().validate(value, model_instance) + self._coerce_to_floats(value) diff --git a/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py b/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py index b595db0be..8b27e2201 100644 --- a/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py +++ b/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 5.2.13 on 2026-04-15 21:36 +# Generated by Django 5.2.13 on 2026-04-16 21:38 import django.db.models.deletion import django.utils.timezone @@ -90,7 +90,8 @@ class Migration(migrations.Migration): ('source_location', models.CharField(blank=True, default='', max_length=200)), ('setup', models.CharField(blank=True, default='', max_length=2000)), ('exposure_time', models.FloatField(blank=True, null=True)), - ('flux', tom_dataproducts.fields.FluxField(blank=True, default=list)), + ('wavelength', tom_dataproducts.fields.FloatArrayField(blank=True, default=list)), + ('flux', tom_dataproducts.fields.FloatArrayField(blank=True, default=list)), ('error', tom_dataproducts.fields.FloatArrayField(blank=True, default=list)), ('flux_unit', models.TextField(blank=True, default='')), ('data_product', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='tom_dataproducts.dataproduct')), diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index 6654e366c..a9011bc5b 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -16,7 +16,7 @@ from tom_observations.models import ObservationRecord from tom_targets.base_models import BaseTarget -from .fields import FloatArrayField, FluxField +from .fields import FloatArrayField logger = logging.getLogger(__name__) @@ -420,7 +420,8 @@ class Meta: class SpectroscopyReducedDatum(ReducedDatumCommon): setup = models.CharField(max_length=2000, blank=True, default="") exposure_time = models.FloatField(blank=True, null=True) - flux = FluxField(blank=True, default=list) + wavelength = FloatArrayField(blank=True, default=list) + flux = FloatArrayField(blank=True, default=list) error = FloatArrayField(blank=True, default=list) flux_unit = models.TextField(blank=True, default="") @@ -512,16 +513,19 @@ def _build_photometry_reduced_datum(data: dict) -> PhotometryReducedDatum: def _build_spectroscopy_reduced_datum(data: dict) -> SpectroscopyReducedDatum: FLUX_FIELDS = {"flux", "f"} + WAVELENGTH_FIELDS = {"wavelength", "wave", "wl"} ERROR_FIELDS = {"error", "err", "flux_error", "f_error"} FLUX_UNIT_FIELDS = {"flux_unit", "f_unit", "flux_units", "f_units"} - flux = _pop_find_field(FLUX_FIELDS, data) - error = _pop_find_field(ERROR_FIELDS, data) + flux = _pop_find_field(FLUX_FIELDS, data) or [] + wavelength = _pop_find_field(WAVELENGTH_FIELDS, data) or [] + error = _pop_find_field(ERROR_FIELDS, data) or [] flux_unit = _pop_find_field(FLUX_UNIT_FIELDS, data) or "" extra_fields = _extract_extra_fields(data, SpectroscopyReducedDatum) return SpectroscopyReducedDatum( + wavelength=wavelength, flux=flux, error=error, flux_unit=flux_unit, diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py index 5ee61dfdd..be313485d 100644 --- a/tom_dataproducts/tests/test_api.py +++ b/tom_dataproducts/tests/test_api.py @@ -223,7 +223,7 @@ def test_upload_spectroscopy_datum(self): payload = { "data_product": "", "data_type": "spectroscopy", - "value": {"flux": "[(123.4, 4.321)]", "error": "[0.005]", "flux_unit": "s"}, + "value": {"flux": [123.4, 4.321], "wavelength": [150, 151], "error": [0.005], "flux_unit": "s"}, "target": self.st.id, "timestamp": "2012-02-12T01:40:47Z", } @@ -233,7 +233,7 @@ def test_upload_spectroscopy_datum(self): self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual(SpectroscopyReducedDatum.objects.count(), 1) rd = SpectroscopyReducedDatum.objects.first() - self.assertEqual(rd.flux, [(123.4, 4.321)]) + self.assertEqual(rd.flux, [123.4, 4.321]) self.assertEqual(rd.target.id, payload["target"]) def test_upload_astrometry_reduced_datum(self): diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index e7d836aba..ee27183bd 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -12,6 +12,7 @@ from django.conf import settings from django.contrib.auth.models import Group, User from django.core.files.uploadedfile import SimpleUploadedFile +from django.core.exceptions import ValidationError from django.db import IntegrityError from django.test import TestCase, override_settings from django.urls import reverse @@ -549,27 +550,27 @@ def test_create_thumbnail(self, mock_is_fits_image_file): class TestCustomFields(TestCase): def test_create_spectra_with_flux_data(self): - flux = [ - (7.427265572723272e-16, 3399.697753906248), - (7.796862575906174e-16, 3401.383788108824), - ] + flux = [7.427265572723272, 3399.697753906248] + wavelength = [3399.697753906248, 3401.383788108824] rd = SpectroscopyReducedDatum.objects.create( target=SiderealTargetFactory.create(), timestamp=timezone.now(), exposure_time=1000.0, flux=flux, + wavelength=wavelength, flux_unit="Å", ) rd.refresh_from_db() # ensure we round trip to the database self.assertEqual(flux, rd.flux) def test_create_spectra_with_error_data(self): - error = [0.0001, 0.0002] + error = [0.0001, 0.0002, 0.0003, 0.0004] rd = SpectroscopyReducedDatum.objects.create( target=SiderealTargetFactory.create(), timestamp=timezone.now(), exposure_time=1000.0, - flux=[(1.0, 2.0), (3.0, 4.0)], + flux=[1.0, 2.0, 3.0, 4.0], + wavelength=[1, 2, 3, 4], error=error, flux_unit="Å", ) @@ -577,13 +578,14 @@ def test_create_spectra_with_error_data(self): self.assertEqual(error, rd.error) def test_create_spectra_bad_flux(self): - with self.assertRaises(TypeError): + with self.assertRaises(ValidationError): SpectroscopyReducedDatum.objects.create( target=SiderealTargetFactory.create(), timestamp=timezone.now(), exposure_time=1000.0, - flux=[1.0, 2.0, 3.0, 4.0], # oops, not tuples. - flux_unit="Å", + flux=[1.0, 2.0, 3.0, "asd"], + wavelength=[1, 2, 3, 4], + flux_unit="cm^2", ) From 52163d18b27e2a0206a5da1387183b54618fdde5 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Thu, 16 Apr 2026 15:48:25 -0700 Subject: [PATCH 14/35] Update sharing code paths to use concrete classes --- tom_dataproducts/alertstreams/hermes.py | 72 +++++--------- tom_dataproducts/serializers.py | 31 +++++- tom_dataproducts/sharing.py | 99 +++++++++---------- .../templatetags/dataproduct_extras.py | 15 ++- tom_dataproducts/tests/test_sharing.py | 91 +++++++---------- tom_dataproducts/tests/tests.py | 27 ++--- tom_targets/sharing.py | 6 +- tom_targets/tests/tests.py | 53 +++++----- tom_targets/views.py | 8 +- 9 files changed, 196 insertions(+), 206 deletions(-) diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index 8bf88bd8c..55fe6ef86 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -9,7 +9,7 @@ from tom_alerts.models import AlertStreamMessage from tom_targets.models import Target, TargetList -from tom_dataproducts.models import PhotometryReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum, SpectroscopyReducedDatum import requests @@ -74,56 +74,34 @@ def get_hermes_photometry(self, datum): phot_table_row = { 'target_name': datum.target.name, 'date_obs': datum.timestamp.isoformat(), - 'telescope': datum.value.get('telescope'), - 'instrument': datum.value.get('instrument'), - 'bandpass': datum.value.get('filter', ''), + 'telescope': datum.telescope, + 'instrument': datum.instrument, + 'bandpass': datum.bandpass, } - brightness_unit = convert_astropy_brightness_to_hermes(datum.value.get('unit')) + brightness_unit = convert_astropy_brightness_to_hermes(datum.unit) if brightness_unit: phot_table_row['brightness_unit'] = brightness_unit - if datum.value.get('magnitude', None): - phot_table_row['brightness'] = datum.value['magnitude'] + if datum.brightness is not None: + phot_table_row['brightness'] = datum.brightness else: - phot_table_row['limiting_brightness'] = datum.value.get('limit', None) - error_value = datum.value.get('error', datum.value.get('magnitude_error', None)) - if error_value is not None: - phot_table_row['brightness_error'] = error_value + phot_table_row['limiting_brightness'] = datum.limit + if datum.brightness_error is not None: + phot_table_row['brightness_error'] = datum.brightness_error return phot_table_row def get_hermes_spectroscopy(self, datum): - """Build a row for a Hermes Spectroscopy Table using a TOM Spectroscopy datum - The datum is assumed to have is json value be of the form {1: {flux: 1, wavelength:200}, 2: {},...} - Or the form {'flux': [1,2,3,...], 'wavelength': [1,2,3,...]} - """ - flux_list = [] - flux_error_list = [] - wavelength_list = [] - if 'flux' in datum.value and 'wavelength' in datum.value: - flux_list = datum.value['flux'] - wavelength_list = datum.value['wavelength'] - flux_error_list = datum.value.get('flux_error', datum.value.get('error', [])) - else: - for entry in datum.value.values(): - if 'flux' in entry: - flux_list.append(entry['flux']) - if 'wavelength' in entry: - wavelength_list.append(entry['wavelength']) - if 'error' in entry: - flux_error_list.append(entry['error']) - if 'flux_error' in entry: - flux_error_list.append(entry['flux_error']) + """Build a row for a Hermes Spectroscopy Table using a SpectroscopyReducedDatum.""" if self.validate: - if len(flux_list) != len(wavelength_list): + if len(datum.flux) != len(datum.wavelength): msg = f"Spectroscopy Datum {datum.id} has mismatched flux and wavelength values" logger.error(msg) raise HermesMessageException(msg) - elif len(flux_list) == 0 or len(wavelength_list) == 0: - msg = f"Spectroscopy Datum {datum.id} has spectrum data in unknown format." - msg += "Please implement a custom HermesDatumConverter to support your data format." + elif len(datum.flux) == 0 or len(datum.wavelength) == 0: + msg = f"Spectroscopy Datum {datum.id} has no spectral data." logger.error(msg) raise HermesMessageException(msg) - if flux_error_list and len(flux_error_list) != len(flux_list): + if datum.error and len(datum.error) != len(datum.flux): msg = f"Spectroscopy Datum {datum.id} must have the same number of flux and flux error datapoints" logger.error(msg) raise HermesMessageException(msg) @@ -131,17 +109,15 @@ def get_hermes_spectroscopy(self, datum): spectroscopy_table_row = { 'target_name': datum.target.name, 'date_obs': datum.timestamp.isoformat(), - 'telescope': datum.value.get('telescope'), - 'instrument': datum.value.get('instrument'), - 'reducer': datum.value.get('reducer'), - 'observer': datum.value.get('observer'), - 'flux': flux_list, - 'wavelength': wavelength_list, - 'flux_units': datum.value.get('flux_units'), + 'telescope': datum.telescope, + 'instrument': datum.instrument, + 'flux': datum.flux, + 'wavelength': datum.wavelength, + 'flux_units': datum.flux_unit, 'wavelength_units': convert_astropy_wavelength_to_hermes(datum.value.get('wavelength_units')), } - if flux_error_list: - spectroscopy_table_row['flux_error'] = flux_error_list + if datum.error: + spectroscopy_table_row['flux_error'] = datum.error return spectroscopy_table_row @@ -249,9 +225,9 @@ def create_hermes_alert(message_info, datums, targets=Target.objects.none(), **k for datum in datums: if datum.target.name not in hermes_target_dict: hermes_target_dict[datum.target.name] = hermes_data_converter.get_hermes_target(datum.target) - if datum.data_type == 'photometry': + if isinstance(datum, PhotometryReducedDatum): hermes_photometry_data.append(hermes_data_converter.get_hermes_photometry(datum)) - elif datum.data_type == 'spectroscopy': + elif isinstance(datum, SpectroscopyReducedDatum): hermes_spectroscopy_data.append(hermes_data_converter.get_hermes_spectroscopy(datum)) # Now go through the targets queryset and ensure we have all of them in the table diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py index 58e697b43..2cf774f4b 100644 --- a/tom_dataproducts/serializers.py +++ b/tom_dataproducts/serializers.py @@ -4,13 +4,42 @@ from rest_framework import serializers from tom_common.serializers import GroupSerializer -from tom_dataproducts.models import DataProductGroup, DataProduct, ReducedDatum, try_parse_reduced_datum +from tom_dataproducts.models import DataProductGroup, DataProduct, ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum, try_parse_reduced_datum from tom_observations.models import ObservationRecord from tom_observations.serializers import ObservationRecordFilteredPrimaryKeyRelatedField from tom_targets.models import Target from tom_targets.fields import TargetFilteredPrimaryKeyRelatedField +class PhotometryReducedDatumSerializer(serializers.ModelSerializer): + """Serializes a PhotometryReducedDatum into the legacy ReducedDatum wire format + """ + data_type = serializers.SerializerMethodField() + value = serializers.SerializerMethodField() + data_product = serializers.SerializerMethodField() + + class Meta: + model = PhotometryReducedDatum + fields = ('data_product', 'data_type', 'source_name', 'source_location', 'timestamp', 'value', 'target') + + def get_data_type(self, obj): + return 'photometry' + + def get_value(self, obj): + return { + 'brightness': obj.brightness, + 'brightness_error': obj.brightness_error, + 'bandpass': obj.bandpass, + 'unit': obj.unit, + 'telescope': obj.telescope, + 'instrument': obj.instrument, + } + + def get_data_product(self, obj): + return None + + class DataProductGroupSerializer(serializers.ModelSerializer): class Meta: model = DataProductGroup diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index 58ce87088..2506ffd23 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -13,9 +13,9 @@ from tom_targets.models import Target -from tom_dataproducts.models import DataProduct, ReducedDatum +from tom_dataproducts.models import DataProduct, PhotometryReducedDatum from tom_dataproducts.alertstreams.hermes import publish_to_hermes, BuildHermesMessage, get_hermes_topics -from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer +from tom_dataproducts.serializers import DataProductSerializer, PhotometryReducedDatumSerializer def share_target_list_with_hermes(share_destination, form_data, selected_targets=None, include_all_data=False): @@ -33,9 +33,9 @@ def share_target_list_with_hermes(share_destination, form_data, selected_targets title_name = f"{target_list.name} target list" targets = Target.objects.filter(id__in=selected_targets) if include_all_data: - reduced_datums = ReducedDatum.objects.filter(target__id__in=selected_targets, data_type='photometry') + reduced_datums = PhotometryReducedDatum.objects.filter(target__id__in=selected_targets) else: - reduced_datums = ReducedDatum.objects.none() + reduced_datums = PhotometryReducedDatum.objects.none() return _share_with_hermes(share_destination, form_data, title_name, reduced_datums, targets) @@ -50,29 +50,27 @@ def share_data_with_hermes(share_destination, form_data, product_id=None, target :return: json response for the sharing """ # Query relevant Reduced Datums Queryset - accepted_data_types = ['photometry', 'spectroscopy'] if product_id: product = DataProduct.objects.get(pk=product_id) target = product.target - reduced_datums = ReducedDatum.objects.filter(data_product=product) + reduced_datums = PhotometryReducedDatum.objects.filter(data_product=product) elif selected_data: - reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) + reduced_datums = PhotometryReducedDatum.objects.filter(pk__in=selected_data) target = reduced_datums[0].target elif target_id: target = Target.objects.get(pk=target_id) - reduced_datums = ReducedDatum.objects.none() + reduced_datums = PhotometryReducedDatum.objects.none() else: - reduced_datums = ReducedDatum.objects.none() + reduced_datums = PhotometryReducedDatum.objects.none() target = Target.objects.none() title_name = target.name if target else '' - reduced_datums.filter(data_type__in=accepted_data_types) return _share_with_hermes( share_destination, form_data, title_name, reduced_datums, targets=Target.objects.filter(pk=target.pk) ) def _share_with_hermes(share_destination, form_data, title_name, - reduced_datums=ReducedDatum.objects.none(), + reduced_datums=PhotometryReducedDatum.objects.none(), targets=Target.objects.none()): """ Helper method to serialize and share data with hermes @@ -127,7 +125,7 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id dataproducts_url = destination_tom_base_url + 'api/dataproducts/' targets_url = destination_tom_base_url + 'api/targets/' reduced_datums_url = destination_tom_base_url + 'api/reduceddatums/' - reduced_datums = ReducedDatum.objects.none() + reduced_datums = PhotometryReducedDatum.objects.none() # If a DataProduct is provided, share that DataProduct if product_id: @@ -151,7 +149,7 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id elif selected_data or target_id: # If ReducedDatums are provided, share those ReducedDatums if selected_data: - reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) + reduced_datums = PhotometryReducedDatum.objects.filter(pk__in=selected_data) targets = set(reduced_datum.target for reduced_datum in reduced_datums) target_dict = {} for target in targets: @@ -166,7 +164,7 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id # If Target is provided, share all ReducedDatums for that Target # (Will not create New Target in Destination TOM) target = Target.objects.get(pk=target_id) - reduced_datums = ReducedDatum.objects.filter(target=target) + reduced_datums = PhotometryReducedDatum.objects.filter(target=target) destination_target_id, _ = get_destination_target(target, targets_url, headers, auth) if destination_target_id is None: return {'message': 'ERROR: No matching target found.'} @@ -179,9 +177,8 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id return {'message': 'ERROR: No valid data to share.'} for datum in reduced_datums: if target_dict[datum.target.name]: - serialized_data = ReducedDatumSerializer(datum).data + serialized_data = PhotometryReducedDatumSerializer(datum).data serialized_data['target'] = target_dict[datum.target.name] - serialized_data['data_product'] = '' if not serialized_data['source_name']: serialized_data['source_name'] = settings.TOM_NAME serialized_data['source_location'] = f"ReducedDatum shared from " \ @@ -313,35 +310,25 @@ def sharing_feedback_handler(response, request): return -def process_spectro_data_for_download(serialized_datum): - """ Turns a serialized spectrograph datum into a list of serialized datums with the - spectrograph info expanded one piece per line +def process_spectro_data_for_download(datum): + """ Turns a SpectroscopyReducedDatum into a list of row dicts with the spectrum + expanded to one row per wavelength/flux point. """ download_datums = [] - spectra_data = serialized_datum.pop('value') - if ('flux' in spectra_data and isinstance(spectra_data['flux'], list) - and 'wavelength' in spectra_data and isinstance(spectra_data['wavelength'], list) - and len(spectra_data['flux']) == len(spectra_data['wavelength'])): - datum_to_copy = serialized_datum.copy() - # If its a data dict with certain array or dict fields, then first copy the scalar fields over - for key, value in spectra_data.items(): - if not isinstance(value, (list, dict)) and key not in datum_to_copy: - datum_to_copy[key] = value - # And then iterate over the expected array fields to build output rows - for i, flux in enumerate(spectra_data['flux']): - expanded_datum = datum_to_copy.copy() - expanded_datum['flux'] = flux - expanded_datum['wavelength'] = spectra_data['wavelength'][i] - if 'flux_error' in spectra_data and isinstance(spectra_data['flux_error'], list): - expanded_datum['flux_error'] = spectra_data['flux_error'][i] - download_datums.append(expanded_datum) - else: - for entry in spectra_data.values(): - if isinstance(entry, dict): - expanded_datum = serialized_datum.copy() - # If its an "array" of dicts, just expand each dict into the output - expanded_datum.update(entry) - download_datums.append(expanded_datum) + if datum.flux and datum.wavelength and len(datum.flux) == len(datum.wavelength): + scalar_fields = { + 'timestamp': datum.timestamp, + 'telescope': datum.telescope, + 'instrument': datum.instrument, + 'setup': datum.setup, + 'flux_unit': datum.flux_unit, + 'source_name': datum.source_name, + } + for i, (wavelength, flux) in enumerate(zip(datum.wavelength, datum.flux)): + row = {**scalar_fields, 'wavelength': wavelength, 'flux': flux} + if datum.error and i < len(datum.error): + row['flux_error'] = datum.error[i] + download_datums.append(row) return download_datums @@ -354,18 +341,24 @@ def download_data(form_data, selected_data): :param selected_data: ReducucedDatums selected via the checkboxes in the DataShareForm :return: CSV photometry or spectroscopy table as a StreamingHttpResponse """ - reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) - serialized_data = [ReducedDatumSerializer(rd).data for rd in reduced_datums] + # TODO: selected_data can only contain photometry PKs as of now. the share-box checkboxes are + # only rendered in photometry_datalist_for_target.html. + # There is no spectroscopy share UI. + phot_datums = PhotometryReducedDatum.objects.filter(pk__in=selected_data) data_to_save = [] sort_fields = ['timestamp'] - for datum in serialized_data: - if datum.get('data_type') == 'photometry': - datum.update(datum.pop('value')) - data_to_save.append(datum) - elif datum.get('data_type') == 'spectroscopy': - sort_fields = ['timestamp', 'wavelength'] - # Attempt to expand the photometry table stored in the .value into multiple entries in serialized data - data_to_save.extend(process_spectro_data_for_download(datum)) + for datum in phot_datums: + data_to_save.append({ + 'timestamp': datum.timestamp, + 'telescope': datum.telescope, + 'instrument': datum.instrument, + 'bandpass': datum.bandpass, + 'brightness': datum.brightness, + 'brightness_error': datum.brightness_error, + 'limit': datum.limit, + 'unit': datum.unit, + 'source_name': datum.source_name, + }) table = Table(data_to_save) if form_data.get('share_message'): table.meta['comments'] = [form_data['share_message']] diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 28013a1fd..47aa7f433 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -19,8 +19,7 @@ import numpy as np from tom_dataproducts.forms import DataProductUploadForm, DataShareForm -from tom_dataproducts.models import DataProduct, PhotometryReducedDatum, ReducedDatum -from tom_dataproducts.processors.data_serializers import SpectrumSerializer +from tom_dataproducts.models import DataProduct, PhotometryReducedDatum, SpectroscopyReducedDatum from tom_dataproducts.single_target_data_service.single_target_data_service import get_service_classes, \ get_service_class from tom_observations.models import ObservationRecord @@ -376,16 +375,16 @@ def spectroscopy_for_target(context, target, dataproduct=None): plot_data = [] if settings.TARGET_PERMISSIONS_ONLY: - datums = ReducedDatum.objects.filter(data_product__in=spectral_dataproducts) + datums = SpectroscopyReducedDatum.objects.filter(data_product__in=spectral_dataproducts) else: datums = get_objects_for_user(context['request'].user, - 'tom_dataproducts.view_reduceddatum', - klass=ReducedDatum.objects.filter(data_product__in=spectral_dataproducts)) + 'tom_dataproducts.view_spectroscopyreduceddatum', + klass=SpectroscopyReducedDatum.objects.filter( + data_product__in=spectral_dataproducts)) for datum in datums: - deserialized = SpectrumSerializer().deserialize(datum.value) plot_data.append(go.Scatter( - x=deserialized.wavelength.value, - y=deserialized.flux.value, + x=datum.wavelength, + y=datum.flux, name=datetime.strftime(datum.timestamp, '%Y%m%d-%H:%M:%s') )) diff --git a/tom_dataproducts/tests/test_sharing.py b/tom_dataproducts/tests/test_sharing.py index 8aac8bb30..2e9144c80 100644 --- a/tom_dataproducts/tests/test_sharing.py +++ b/tom_dataproducts/tests/test_sharing.py @@ -1,7 +1,7 @@ from django.test import TestCase, override_settings from tom_dataproducts.alertstreams.hermes import create_hermes_alert, BuildHermesMessage, HermesMessageException -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum, SpectroscopyReducedDatum from tom_observations.tests.utils import FakeRoboticFacility from tom_observations.tests.factories import SiderealTargetFactory, ObservingRecordFactory from django.contrib.auth.models import User @@ -28,47 +28,43 @@ def setUp(self): parameters={} ) self.user = User.objects.create_user(username='test', email='test@example.com') - self.rd1 = ReducedDatum.objects.create( + self.rd1 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 18.5, 'error': .5, 'filter': 'V', 'telescope': 'tst'} + brightness=18.5, + brightness_error=0.5, + bandpass='V', + telescope='tst', ) - self.rd2 = ReducedDatum.objects.create( + self.rd2 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 19.5, 'error': .5, 'filter': 'B', 'telescope': 'tst'} + brightness=19.5, + brightness_error=0.5, + bandpass='B', + telescope='tst', ) - self.rd3 = ReducedDatum.objects.create( + self.rd3 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 17.5, 'error': .5, 'filter': 'R', 'telescope': 'tst'} + brightness=17.5, + brightness_error=0.5, + bandpass='R', + telescope='tst', ) - self.rd4 = ReducedDatum.objects.create( + self.rd4 = SpectroscopyReducedDatum.objects.create( target=self.target, - data_type='spectroscopy', - value={ - 'flux': [1, 2, 3], - 'wavelength': [6000, 5999, 5998], - 'error': [0.11, 0.22, 0.33], - 'telescope': 'SpectraTelescope' - } + flux=[1, 2, 3], + wavelength=[6000, 5999, 5998], + error=[0.11, 0.22, 0.33], + telescope='SpectraTelescope', ) - self.rd5 = ReducedDatum.objects.create( + self.rd5 = SpectroscopyReducedDatum.objects.create( target=self.target, - data_type='spectroscopy', - value={ - '1': {'flux': 20, 'wavelength': 3000}, - '2': {'flux': 21, 'wavelength': 3001}, - '3': {'flux': 22, 'wavelength': 3002}, - } + flux=[20, 21, 22], + wavelength=[3000, 3001, 3002], ) - self.bad_rd = ReducedDatum.objects.create( + self.bad_rd = SpectroscopyReducedDatum.objects.create( target=self.target, - data_type='spectroscopy', - value={ - 'myflux': [1, 2, 3], - 'wavelength_function': 'lambda_xyz' - } + flux=[1, 2, 3], + wavelength=[6000, 5999], # mismatched length — triggers HermesMessageException ) self.message_info = BuildHermesMessage( title='Test Title', @@ -100,38 +96,30 @@ def _check_alert(self, alert, message_info, datums, targets): photometry_count = 0 spectroscopy_count = 0 self.assertEqual(photometry_datums + spectroscopy_datums, len(datums)) - # These should line up for datum in datums: - if datum.data_type == 'photometry': + if isinstance(datum, PhotometryReducedDatum): hermes_datum = alert['data']['photometry'][photometry_count] self.assertEqual(hermes_datum['target_name'], datum.target.name) self.assertEqual(hermes_datum['date_obs'], datum.timestamp.isoformat()) - self.assertEqual(hermes_datum['telescope'], datum.value.get('telescope')) - self.assertEqual(hermes_datum['brightness'], datum.value.get('magnitude')) - self.assertEqual(hermes_datum['brightness_error'], datum.value.get('error')) - self.assertEqual(hermes_datum['bandpass'], datum.value.get('filter')) + self.assertEqual(hermes_datum['telescope'], datum.telescope) + self.assertEqual(hermes_datum['brightness'], datum.brightness) + self.assertEqual(hermes_datum['brightness_error'], datum.brightness_error) + self.assertEqual(hermes_datum['bandpass'], datum.bandpass) photometry_count += 1 - elif datum.data_type == 'spectroscopy': + elif isinstance(datum, SpectroscopyReducedDatum): hermes_datum = alert['data']['spectroscopy'][spectroscopy_count] self.assertEqual(hermes_datum['target_name'], datum.target.name) self.assertEqual(hermes_datum['date_obs'], datum.timestamp.isoformat()) - if 'flux' in datum.value and 'wavelength' in datum.value: - self.assertEqual(hermes_datum['flux'], datum.value.get('flux')) - self.assertEqual(hermes_datum['wavelength'], datum.value.get('wavelength')) - if 'error' in datum.value: - self.assertEqual(hermes_datum['flux_error'], datum.value.get('error')) - else: - for i, entry in enumerate(datum.value.values()): - if 'flux' in entry: - self.assertEqual(hermes_datum['flux'][i], entry['flux']) - self.assertEqual(hermes_datum['wavelength'][i], entry['wavelength']) + self.assertEqual(hermes_datum['flux'], datum.flux) + self.assertEqual(hermes_datum['wavelength'], datum.wavelength) + if datum.error: + self.assertEqual(hermes_datum['flux_error'], datum.error) spectroscopy_count += 1 def test_convert_to_hermes_format(self): datums = [self.rd1, self.rd2, self.rd3] targets = [self.target] alert = create_hermes_alert(self.message_info, datums, targets) - # Now check the alerts formatting is correct self._check_alert(alert, self.message_info, datums, targets) def test_convert_to_hermes_format_extra_target(self): @@ -139,32 +127,27 @@ def test_convert_to_hermes_format_extra_target(self): datums = [self.rd1, self.rd2, self.rd3] targets = [target2, self.target] alert = create_hermes_alert(self.message_info, datums, targets) - # Now check the alerts formatting is correct self._check_alert(alert, self.message_info, datums, targets) def test_convert_to_hermes_format_only_targets(self): target2 = SiderealTargetFactory.create() targets = [target2, self.target] alert = create_hermes_alert(self.message_info, [], targets) - # Now check the alerts formatting is correct self._check_alert(alert, self.message_info, [], targets) def test_convert_to_hermes_format_only_datums(self): datums = [self.rd1, self.rd2, self.rd3] alert = create_hermes_alert(self.message_info, datums, []) - # Now check the alerts formatting is correct self._check_alert(alert, self.message_info, datums, [self.target]) def test_convert_to_hermes_format_spectro_datums(self): datums = [self.rd4, self.rd5] alert = create_hermes_alert(self.message_info, datums, []) - # Now check the alerts formatting is correct self._check_alert(alert, self.message_info, datums, [self.target]) def test_convert_to_hermes_format_mixed_datums(self): datums = [self.rd1, self.rd2, self.rd3, self.rd4, self.rd5] alert = create_hermes_alert(self.message_info, datums, []) - # Now check the alerts formatting is correct self._check_alert(alert, self.message_info, datums, [self.target]) def test_convert_to_hermes_format_bad_spectro_datum_fails(self): diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index ee27183bd..1a0f0f0c4 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -669,20 +669,23 @@ def setUp(self): assign_perm('tom_targets.view_target', self.user, self.target) self.client.force_login(self.user) - self.rd1 = ReducedDatum.objects.create( + self.rd1 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 18.5, 'error': .5, 'filter': 'V'} + brightness=18.5, + brightness_error=0.5, + bandpass='V', ) - self.rd2 = ReducedDatum.objects.create( + self.rd2 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 19.5, 'error': .5, 'filter': 'B'} + brightness=19.5, + brightness_error=0.5, + bandpass='B', ) - self.rd3 = ReducedDatum.objects.create( + self.rd3 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 17.5, 'error': .5, 'filter': 'R'} + brightness=17.5, + brightness_error=0.5, + bandpass='R', ) @responses.activate @@ -779,7 +782,7 @@ def test_share_reduced_datums_no_valid_responses(self): 'share_destination': [share_destination], 'share_title': ['Updated data for thingy.'], 'share_message': ['test_message'], - 'share-box': [1, 2] + 'share-box': [self.rd1.pk, self.rd2.pk] }, follow=True ) @@ -897,7 +900,7 @@ def test_share_reduced_datums_valid_responses(self): 'share_destination': [share_destination], 'share_title': ['Updated data for thingy.'], 'share_message': ['test_message'], - 'share-box': [1, 2] + 'share-box': [self.rd1.pk, self.rd2.pk] }, follow=True ) @@ -929,7 +932,7 @@ def test_share_reduced_datums_invalid_responses(self): 'share_destination': [share_destination], 'share_title': ['Updated data for thingy.'], 'share_message': ['test_message'], - 'share-box': [1, 2] + 'share-box': [self.rd1.pk, self.rd2.pk] } # Check 500 error responses.add( diff --git a/tom_targets/sharing.py b/tom_targets/sharing.py index ebeba11f9..815ea5bde 100644 --- a/tom_targets/sharing.py +++ b/tom_targets/sharing.py @@ -7,7 +7,7 @@ from tom_targets.models import PersistentShare from tom_dataproducts.sharing import (check_for_share_safe_datums, share_data_with_tom, get_destination_target, sharing_feedback_converter) -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum from tom_dataproducts.alertstreams.hermes import publish_to_hermes, BuildHermesMessage @@ -22,7 +22,7 @@ def share_target_and_all_data(share_destination, target): hermes_topic = share_destination.split(':')[1] destination = share_destination.split(':')[0] filtered_reduced_datums = check_for_share_safe_datums( - destination, ReducedDatum.objects.filter(target=target), topic=hermes_topic) + destination, PhotometryReducedDatum.objects.filter(target=target), topic=hermes_topic) sharing = getattr(settings, "DATA_SHARING", {}) tom_name = f"{getattr(settings, 'TOM_NAME', 'TOM Toolkit')}" message = BuildHermesMessage(title=f"Setting up continuous sharing for {target.name} from " @@ -56,7 +56,7 @@ def continuous_share_data(target, reduced_datums): hermes_topic = share_destination.split(':')[1] destination = share_destination.split(':')[0] filtered_reduced_datums = check_for_share_safe_datums( - destination, ReducedDatum.objects.filter(pk__in=reduced_datum_pks), topic=hermes_topic) + destination, PhotometryReducedDatum.objects.filter(pk__in=reduced_datum_pks), topic=hermes_topic) sharing = getattr(settings, "DATA_SHARING", {}) tom_name = f"{getattr(settings, 'TOM_NAME', 'TOM Toolkit')}" message = BuildHermesMessage(title=f"Updated data for {target.name} from " diff --git a/tom_targets/tests/tests.py b/tom_targets/tests/tests.py index 522b09b49..c0f4222ec 100644 --- a/tom_targets/tests/tests.py +++ b/tom_targets/tests/tests.py @@ -21,7 +21,7 @@ from tom_targets.base_models import BaseTarget from tom_targets.templatetags.targets_extras import target_table_headers, target_table_row from tom_targets.permissions import targets_for_user -from tom_dataproducts.models import ReducedDatum, DataProduct +from tom_dataproducts.models import PhotometryReducedDatum, ReducedDatum, DataProduct from tom_observations.models import ObservationRecord from guardian.shortcuts import assign_perm, get_perms @@ -1466,20 +1466,23 @@ def setUp(self): self.user = User.objects.create_user(username='test', email='test@example.com') assign_perm('tom_targets.view_target', self.user, self.target) self.client.force_login(self.user) - self.rd1 = ReducedDatum.objects.create( + self.rd1 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 18.5, 'error': .5, 'filter': 'V'} + brightness=18.5, + brightness_error=0.5, + bandpass='V', ) - self.rd2 = ReducedDatum.objects.create( + self.rd2 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 19.5, 'error': .5, 'filter': 'B'} + brightness=19.5, + brightness_error=0.5, + bandpass='B', ) - self.rd3 = ReducedDatum.objects.create( + self.rd3 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 17.5, 'error': .5, 'filter': 'R'} + brightness=17.5, + brightness_error=0.5, + bandpass='R', ) @responses.activate @@ -1616,7 +1619,7 @@ def test_share_reduceddatums_target_valid_responses(self): 'submitter': ['test_submitter'], 'target': self.target.id, 'share_destination': [share_destination], - 'share-box': [1, 2] + 'share-box': [self.rd1.pk, self.rd2.pk] }, follow=True ) @@ -1646,25 +1649,29 @@ def setUp(self): self.user = User.objects.create_user(username='test', email='test@example.com') assign_perm('tom_targets.view_target', self.user, self.target) self.client.force_login(self.user) - self.rd1 = ReducedDatum.objects.create( + self.rd1 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 18.5, 'error': .5, 'filter': 'V'} + brightness=18.5, + brightness_error=0.5, + bandpass='V', ) - self.rd2 = ReducedDatum.objects.create( + self.rd2 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 19.5, 'error': .5, 'filter': 'B'} + brightness=19.5, + brightness_error=0.5, + bandpass='B', ) - self.rd3 = ReducedDatum.objects.create( + self.rd3 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 17.5, 'error': .5, 'filter': 'R'} + brightness=17.5, + brightness_error=0.5, + bandpass='R', ) - self.rd4 = ReducedDatum.objects.create( + self.rd4 = PhotometryReducedDatum.objects.create( target=self.target2, - data_type='photometry', - value={'magnitude': 17.5, 'error': .5, 'filter': 'R'} + brightness=17.5, + brightness_error=0.5, + bandpass='R', ) @responses.activate diff --git a/tom_targets/views.py b/tom_targets/views.py index 18c0b2024..b256725f1 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -48,7 +48,7 @@ from tom_targets.merge import target_merge from tom_dataproducts.sharing import (share_data_with_hermes, share_data_with_tom, sharing_feedback_handler, share_target_list_with_hermes) -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum from tom_targets.groups import ( add_all_to_grouping, add_selected_to_grouping, remove_all_from_grouping, remove_selected_from_grouping, move_all_to_grouping, move_selected_to_grouping @@ -554,7 +554,7 @@ def post(self, request, *args, **kwargs): message=request.POST.get('share_message', ''), authors=sharing['hermes'].get('DEFAULT_AUTHORS') ) - reduced_datums = ReducedDatum.objects.filter(pk__in=request.POST.getlist('share-box', [])) + reduced_datums = PhotometryReducedDatum.objects.filter(pk__in=request.POST.getlist('share-box', [])) preload_key = preload_to_hermes(hermes_message, reduced_datums, [target]) load_url = sharing['hermes']['BASE_URL'] + f'submit-message?id={preload_key}' return HttpResponseRedirect(load_url) @@ -923,9 +923,9 @@ def post(self, request, *args, **kwargs): ) targets = Target.objects.filter(pk__in=request.POST.getlist('selected-target', [])) if request.POST.get('dataSwitch', '') == 'on': - reduced_datums = ReducedDatum.objects.filter(target__in=targets, data_type='photometry') + reduced_datums = PhotometryReducedDatum.objects.filter(target__in=targets) else: - reduced_datums = ReducedDatum.objects.none() + reduced_datums = PhotometryReducedDatum.objects.none() preload_key = preload_to_hermes(hermes_message, reduced_datums, targets) load_url = sharing['hermes']['BASE_URL'] + f'submit-message?id={preload_key}' return HttpResponseRedirect(load_url) From 04964fd071082d469662baed0e23004957c079b1 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Thu, 16 Apr 2026 16:36:42 -0700 Subject: [PATCH 15/35] Update api views to use concrete data types, preserving json format --- tom_dataproducts/api_views.py | 59 ++++++++++++++++- tom_dataproducts/serializers.py | 81 +++++++++++++++-------- tom_dataproducts/sharing.py | 4 +- tom_dataproducts/tests/test_api.py | 101 ++++++++++++++++++++++++++--- 4 files changed, 205 insertions(+), 40 deletions(-) diff --git a/tom_dataproducts/api_views.py b/tom_dataproducts/api_views.py index ff2df3729..15f2176d1 100644 --- a/tom_dataproducts/api_views.py +++ b/tom_dataproducts/api_views.py @@ -11,11 +11,21 @@ from tom_common.hooks import run_hook from tom_dataproducts.data_processor import run_data_processor from tom_dataproducts.filters import DataProductFilter, ReducedDatumFilter -from tom_dataproducts.models import DataProduct, ReducedDatum +from tom_dataproducts.models import (DataProduct, ReducedDatum, PhotometryReducedDatum, + SpectroscopyReducedDatum, AstrometryReducedDatum, + REDUCED_DATUM_MODELS) from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer from tom_targets.models import Target +# Maps the data_type query param to the concrete model that holds those rows. +_DATA_TYPE_MODEL_MAP = { + 'photometry': PhotometryReducedDatum, + 'spectroscopy': SpectroscopyReducedDatum, + 'astrometry': AstrometryReducedDatum, +} + + class DataProductViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet, PermissionListMixin): """ Viewset for DataProduct objects. Supports list, create, and delete. @@ -50,7 +60,8 @@ def create(self, request, *args, **kwargs): assign_perm('tom_dataproducts.delete_dataproduct', group, dp) assign_perm('tom_dataproducts.view_reduceddatum', group, reduced_data) except Exception: - ReducedDatum.objects.filter(data_product=dp).delete() + for model in REDUCED_DATUM_MODELS: + model.objects.filter(data_product=dp).delete() dp.delete() return Response({'Data processing error': '''There was an error in processing your DataProduct into \ individual ReducedDatum objects.'''}, @@ -77,6 +88,9 @@ class ReducedDatumViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, G Viewset for ReducedDatum objects. Supports list, create, and delete. To view supported query parameters, please use the OPTIONS endpoint, which can be accessed through the web UI. + + The list endpoint queries all concrete ReducedDatum and returns them in the legacy json format. + TODO: Deprecate the legacy format and have seperate enpoints for each type? """ queryset = ReducedDatum.objects.all() serializer_class = ReducedDatumSerializer @@ -85,6 +99,47 @@ class ReducedDatumViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, G permission_required = 'tom_dataproducts.view_reduceddatum' parser_classes = [FormParser, JSONParser] + def _base_queryset_for_model(self, model): + qs = model.objects.all() + if settings.TARGET_PERMISSIONS_ONLY: + qs = qs.filter( + target__in=get_objects_for_user(self.request.user, f'{Target._meta.app_label}.view_target') + ) + return qs + + def list(self, request, *args, **kwargs): + params = request.query_params + requested_data_type = params.get('data_type', '').lower() + + # Determine which models to query + if requested_data_type in _DATA_TYPE_MODEL_MAP: + # A typed data_type + models_to_query = [_DATA_TYPE_MODEL_MAP[requested_data_type]] + elif requested_data_type: + # An unmapped data_type is a generic ReducedDatum + models_to_query = [ReducedDatum] + else: + models_to_query = REDUCED_DATUM_MODELS + + # Strip data_type before passing to ReducedDatumFilter it doesn't exist on concrete types + filter_params = {k: v for k, v in params.items() if k != 'data_type'} + + all_instances = [] + for model in models_to_query: + qs = self._base_queryset_for_model(model) + if model is ReducedDatum and requested_data_type: + qs = qs.filter(data_type=requested_data_type) + qs = ReducedDatumFilter(data=filter_params, queryset=qs).qs + all_instances.extend(list(qs)) + + all_instances.sort(key=lambda x: x.timestamp, reverse=True) + + page = self.paginate_queryset(all_instances) + if page is not None: + return self.get_paginated_response(self.get_serializer(page, many=True).data) + + return Response(self.get_serializer(all_instances, many=True).data) + def create(self, request, *args, **kwargs): response = super().create(request, *args, **kwargs) diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py index 2cf774f4b..87e32fbf9 100644 --- a/tom_dataproducts/serializers.py +++ b/tom_dataproducts/serializers.py @@ -6,40 +6,13 @@ from tom_common.serializers import GroupSerializer from tom_dataproducts.models import DataProductGroup, DataProduct, ReducedDatum from tom_dataproducts.models import PhotometryReducedDatum, try_parse_reduced_datum +from tom_dataproducts.models import SpectroscopyReducedDatum, AstrometryReducedDatum from tom_observations.models import ObservationRecord from tom_observations.serializers import ObservationRecordFilteredPrimaryKeyRelatedField from tom_targets.models import Target from tom_targets.fields import TargetFilteredPrimaryKeyRelatedField -class PhotometryReducedDatumSerializer(serializers.ModelSerializer): - """Serializes a PhotometryReducedDatum into the legacy ReducedDatum wire format - """ - data_type = serializers.SerializerMethodField() - value = serializers.SerializerMethodField() - data_product = serializers.SerializerMethodField() - - class Meta: - model = PhotometryReducedDatum - fields = ('data_product', 'data_type', 'source_name', 'source_location', 'timestamp', 'value', 'target') - - def get_data_type(self, obj): - return 'photometry' - - def get_value(self, obj): - return { - 'brightness': obj.brightness, - 'brightness_error': obj.brightness_error, - 'bandpass': obj.bandpass, - 'unit': obj.unit, - 'telescope': obj.telescope, - 'instrument': obj.instrument, - } - - def get_data_product(self, obj): - return None - - class DataProductGroupSerializer(serializers.ModelSerializer): class Meta: model = DataProductGroup @@ -61,6 +34,58 @@ class Meta: 'target' ) + def to_representation(self, instance): + if isinstance(instance, (PhotometryReducedDatum, SpectroscopyReducedDatum, AstrometryReducedDatum)): + return { + 'data_product': None, + 'data_type': self._get_data_type(instance), + 'source_name': instance.source_name, + 'source_location': instance.source_location, + 'timestamp': self.fields['timestamp'].to_representation(instance.timestamp), + 'value': self._get_typed_value(instance), + 'target': instance.target_id, + } + return super().to_representation(instance) + + def _get_data_type(self, instance): + if isinstance(instance, PhotometryReducedDatum): + return 'photometry' + if isinstance(instance, SpectroscopyReducedDatum): + return 'spectroscopy' + if isinstance(instance, AstrometryReducedDatum): + return 'astrometry' + + def _get_typed_value(self, instance): + if isinstance(instance, PhotometryReducedDatum): + return { + 'brightness': instance.brightness, + 'brightness_error': instance.brightness_error, + 'bandpass': instance.bandpass, + 'unit': instance.unit, + 'telescope': instance.telescope, + 'instrument': instance.instrument, + } + if isinstance(instance, SpectroscopyReducedDatum): + return { + 'flux': instance.flux, + 'wavelength': instance.wavelength, + 'error': instance.error, + 'flux_unit': instance.flux_unit, + 'telescope': instance.telescope, + 'instrument': instance.instrument, + } + if isinstance(instance, AstrometryReducedDatum): + return { + 'ra': instance.ra, + 'dec': instance.dec, + 'ra_error': instance.ra_error, + 'dec_error': instance.dec_error, + 'ra_error_units': instance.ra_error_units, + 'dec_error_units': instance.dec_error_units, + 'telescope': instance.telescope, + 'instrument': instance.instrument, + } + def create(self, validated_data): """DRF requires explicitly handling writeable nested serializers, here we pop the groups data and save it using its serializer. diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index 2506ffd23..47d9914a0 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -15,7 +15,7 @@ from tom_dataproducts.models import DataProduct, PhotometryReducedDatum from tom_dataproducts.alertstreams.hermes import publish_to_hermes, BuildHermesMessage, get_hermes_topics -from tom_dataproducts.serializers import DataProductSerializer, PhotometryReducedDatumSerializer +from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer def share_target_list_with_hermes(share_destination, form_data, selected_targets=None, include_all_data=False): @@ -177,7 +177,7 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id return {'message': 'ERROR: No valid data to share.'} for datum in reduced_datums: if target_dict[datum.target.name]: - serialized_data = PhotometryReducedDatumSerializer(datum).data + serialized_data = ReducedDatumSerializer(datum).data serialized_data['target'] = target_dict[datum.target.name] if not serialized_data['source_name']: serialized_data['source_name'] = settings.TOM_NAME diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py index be313485d..bac7e868c 100644 --- a/tom_dataproducts/tests/test_api.py +++ b/tom_dataproducts/tests/test_api.py @@ -176,26 +176,28 @@ def test_reduced_datum_list(self): self.assertContains(response, rd.data_type, status_code=status.HTTP_200_OK) def test_reduced_datum_filter(self): - rd1 = ReducedDatum.objects.create( + rd1 = PhotometryReducedDatum.objects.create( target=self.st, - data_type='photometry', source_name='TOM Toolkit', - value={'magnitude': 15.582, 'filter': 'r', 'error': 0.005}, + brightness=15.582, + brightness_error=0.005, + bandpass='r', ) - rd2 = ReducedDatum.objects.create( + rd2 = SpectroscopyReducedDatum.objects.create( target=self.st, - data_type='spectroscopy', source_name='TOM Toolkit', - value={'wavelength': 150, 'flux': 12, 'error': 0.005}, + wavelength=[150.0], + flux=[12.0], + error=[0.005] ) # test filter for one object response = self.client.get(reverse('api:reduceddatums-list'), QUERY_STRING='data_type=photometry') - self.assertContains(response, rd1.data_type, status_code=status.HTTP_200_OK, count=1) + self.assertContains(response, rd1.brightness, status_code=status.HTTP_200_OK, count=1) # test filter for both objects response2 = self.client.get(reverse('api:reduceddatums-list'), QUERY_STRING=f'target_name={self.st.name}') - self.assertContains(response2, rd2.data_type, status_code=status.HTTP_200_OK, count=2) + self.assertContains(response2, rd2.flux, status_code=status.HTTP_200_OK, count=2) # test filter for no objects response3 = self.client.get(reverse('api:reduceddatums-list'), QUERY_STRING='source_name=thin_air') @@ -270,3 +272,86 @@ def test_upload_generic_reduced_datum(self): self.assertEqual(rd.value["ra"], 11.2) self.assertEqual(rd.value["foobar"], 0.005) self.assertEqual(rd.target.id, payload["target"]) + + def test_list_includes_all_typed_models(self): + """GET /api/reduceddatums/ returns rows from all concrete model tables.""" + PhotometryReducedDatum.objects.create(target=self.st, brightness=15.0, bandpass='r') + SpectroscopyReducedDatum.objects.create(target=self.st, flux=[1.0], wavelength=[6000.0]) + AstrometryReducedDatum.objects.create(target=self.st, ra=10.0, dec=20.0) + + response = self.client.get(reverse('api:reduceddatums-list')) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['count'], 3) + + def test_photometry_representation(self): + """PhotometryReducedDatum is serialized to the legacy wire format.""" + PhotometryReducedDatum.objects.create( + target=self.st, brightness=15.582, brightness_error=0.005, + bandpass='r', unit='AB', telescope='tst' + ) + + result = self.client.get(reverse('api:reduceddatums-list')).data['results'][0] + self.assertEqual(result['data_type'], 'photometry') + self.assertIsNone(result['data_product']) + self.assertEqual(result['value']['brightness'], 15.582) + self.assertEqual(result['value']['brightness_error'], 0.005) + self.assertEqual(result['value']['bandpass'], 'r') + self.assertEqual(result['value']['telescope'], 'tst') + + def test_spectroscopy_representation(self): + """SpectroscopyReducedDatum is serialized to the legacy wire format.""" + SpectroscopyReducedDatum.objects.create( + target=self.st, flux=[1.0, 2.0], wavelength=[6000.0, 6001.0], + error=[0.1, 0.1], flux_unit='erg/cm2/s/A' + ) + + result = self.client.get(reverse('api:reduceddatums-list')).data['results'][0] + self.assertEqual(result['data_type'], 'spectroscopy') + self.assertIsNone(result['data_product']) + self.assertEqual(result['value']['flux'], [1.0, 2.0]) + self.assertEqual(result['value']['wavelength'], [6000.0, 6001.0]) + self.assertEqual(result['value']['error'], [0.1, 0.1]) + self.assertEqual(result['value']['flux_unit'], 'erg/cm2/s/A') + + def test_astrometry_representation(self): + """AstrometryReducedDatum is serialized to the legacy wire format.""" + AstrometryReducedDatum.objects.create( + target=self.st, ra=10.5, dec=20.3, ra_error=0.001, dec_error=0.002 + ) + + result = self.client.get(reverse('api:reduceddatums-list')).data['results'][0] + self.assertEqual(result['data_type'], 'astrometry') + self.assertIsNone(result['data_product']) + self.assertEqual(result['value']['ra'], 10.5) + self.assertEqual(result['value']['dec'], 20.3) + self.assertEqual(result['value']['ra_error'], 0.001) + self.assertEqual(result['value']['dec_error'], 0.002) + + def test_data_type_filter_routes_to_correct_model(self): + """?data_type= returns only rows from the matching concrete model table.""" + PhotometryReducedDatum.objects.create(target=self.st, brightness=15.0, bandpass='r') + SpectroscopyReducedDatum.objects.create(target=self.st, flux=[1.0], wavelength=[6000.0]) + + response = self.client.get(reverse('api:reduceddatums-list'), QUERY_STRING='data_type=photometry') + self.assertEqual(response.data['count'], 1) + self.assertEqual(response.data['results'][0]['data_type'], 'photometry') + + response = self.client.get(reverse('api:reduceddatums-list'), QUERY_STRING='data_type=spectroscopy') + self.assertEqual(response.data['count'], 1) + self.assertEqual(response.data['results'][0]['data_type'], 'spectroscopy') + + def test_round_trip_post_then_get(self): + """Data POSTed in legacy wire format is stored as a typed model and returned in the same format.""" + payload = { + 'data_type': 'photometry', + 'value': {'magnitude': 15.582, 'filter': 'r', 'error': 0.005}, + 'target': self.st.id, + 'timestamp': '2012-02-12T01:40:47Z', + } + self.client.post(reverse('api:reduceddatums-list'), payload, format='json') + self.assertEqual(PhotometryReducedDatum.objects.count(), 1) + + result = self.client.get(reverse('api:reduceddatums-list')).data['results'][0] + self.assertEqual(result['data_type'], 'photometry') + self.assertEqual(result['value']['brightness'], 15.582) + self.assertEqual(result['value']['bandpass'], 'r') From 2eaa5c3f18e38b7e528cd18384779680fc840249 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 22 Apr 2026 14:46:41 -0700 Subject: [PATCH 16/35] Update target merge logic for ReducedDatums --- tom_targets/merge.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tom_targets/merge.py b/tom_targets/merge.py index 11831cbbc..1ffcf1f68 100644 --- a/tom_targets/merge.py +++ b/tom_targets/merge.py @@ -1,7 +1,7 @@ from django.contrib import messages from django.core.exceptions import ValidationError from tom_targets.models import TargetName, TargetExtra -from tom_dataproducts.models import ReducedDatum, DataProduct +from tom_dataproducts.models import DataProduct, REDUCED_DATUM_MODELS from tom_observations.models import ObservationRecord @@ -61,15 +61,15 @@ def target_merge(primary_target, secondary_target): dataproduct.save() # take secondary_target reduceddatums and save them as primary_target reduceddatums - st_reduceddatums = ReducedDatum.objects.filter(target=secondary_target) - for reduceddatum in st_reduceddatums: - reduceddatum.target = primary_target - try: - reduceddatum.validate_unique() - except ValidationError: - reduceddatum.delete() # delete what would become a duplicate reduceddatum - else: - reduceddatum.save() + for model in REDUCED_DATUM_MODELS: + for reduceddatum in model.objects.filter(target=secondary_target): + reduceddatum.target = primary_target + try: + reduceddatum.validate_unique() + except ValidationError: + reduceddatum.delete() # delete what would become a duplicate reducedatum + else: + reduceddatum.save() # take secondary target extras without repeated keys and save them as primary target extras pt_targetextra_keys = list(TargetExtra.objects.filter(target=primary_target).values_list("key", flat=True)) From 9e1a93f7ba6bd3ac7c6e47dcaa9cafdde23a303c Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 22 Apr 2026 15:01:48 -0700 Subject: [PATCH 17/35] Missed some data in share_data_With_tom --- tom_dataproducts/sharing.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index 47d9914a0..6067adfcc 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -13,7 +13,7 @@ from tom_targets.models import Target -from tom_dataproducts.models import DataProduct, PhotometryReducedDatum +from tom_dataproducts.models import DataProduct, PhotometryReducedDatum, REDUCED_DATUM_MODELS from tom_dataproducts.alertstreams.hermes import publish_to_hermes, BuildHermesMessage, get_hermes_topics from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer @@ -164,7 +164,9 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id # If Target is provided, share all ReducedDatums for that Target # (Will not create New Target in Destination TOM) target = Target.objects.get(pk=target_id) - reduced_datums = PhotometryReducedDatum.objects.filter(target=target) + reduced_datums = [] + for model in REDUCED_DATUM_MODELS: + reduced_datums.extend(model.objects.filter(target=target)) destination_target_id, _ = get_destination_target(target, targets_url, headers, auth) if destination_target_id is None: return {'message': 'ERROR: No matching target found.'} From 9b675cd3887bb2c108cb6c5d9c305256fa349252 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 22 Apr 2026 15:40:27 -0700 Subject: [PATCH 18/35] Fire reduced datum post_save for all concrete types --- tom_targets/signals/handlers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tom_targets/signals/handlers.py b/tom_targets/signals/handlers.py index 8563d3b27..8ad4312a6 100644 --- a/tom_targets/signals/handlers.py +++ b/tom_targets/signals/handlers.py @@ -2,12 +2,11 @@ from django.db.models.signals import post_save, post_delete from guardian.utils import clean_orphan_obj_perms -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import REDUCED_DATUM_MODELS from tom_targets.sharing import continuous_share_data from tom_targets.models import Target -@receiver(post_save, sender=ReducedDatum) def cb_reduceddatum_post_save(sender, instance, *args, **kwargs): # When a new ReducedDatum is created or updated, check for any persistentshare instances on that target # and if they exist, attempt to share the new data @@ -15,6 +14,10 @@ def cb_reduceddatum_post_save(sender, instance, *args, **kwargs): continuous_share_data(target, reduced_datums=[instance]) +for model in REDUCED_DATUM_MODELS: + post_save.connect(cb_reduceddatum_post_save, sender=model) + + @receiver(post_delete, sender=Target) def cb_target_post_delete(sender, instance, *args, **kwargs): # When a Target is deleted, clean up orphaned permissions. From 1c99ac80ff24b0f45a3b5cf774e6b175f8f8a640 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 22 Apr 2026 16:01:15 -0700 Subject: [PATCH 19/35] Update updatereduceddatum command to use concrete types --- .../management/commands/updatereduceddata.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tom_dataproducts/management/commands/updatereduceddata.py b/tom_dataproducts/management/commands/updatereduceddata.py index dbaa6b521..34ae98099 100644 --- a/tom_dataproducts/management/commands/updatereduceddata.py +++ b/tom_dataproducts/management/commands/updatereduceddata.py @@ -4,7 +4,7 @@ from tom_alerts import alerts from tom_targets.models import Target -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import REDUCED_DATUM_MODELS class Command(BaseCommand): @@ -21,18 +21,28 @@ def handle(self, *args, **options): for broker in brokers: broker_classes[broker] = alerts.get_service_class(broker)() - target = None - sources = [s.source_name for s in ReducedDatum.objects.filter(source_name__in=broker_classes.keys()).distinct()] + all_source_names = set() + for model in REDUCED_DATUM_MODELS: + all_source_names.update( + model.objects.filter(source_name__in=broker_classes.keys()) + .values_list('source_name', flat=True).distinct() + ) + sources = list(all_source_names) + + all_target_ids = set() + for model in REDUCED_DATUM_MODELS: + all_target_ids.update( + model.objects.filter(source_name__in=sources) + .values_list('target', flat=True).distinct() + ) + if options['target_id']: try: targets = [Target.objects.get(pk=options['target_id'])] except ObjectDoesNotExist: raise Exception('Invalid target id provided') else: - targets = Target.objects.filter( - id__in=ReducedDatum.objects.filter( - source_name__in=sources - ).values_list('target').distinct()) + targets = Target.objects.filter(id__in=all_target_ids) failed_records = {} for target in targets: From 0abae154041ff28927e43729bfdc65f27b22b0d5 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 22 Apr 2026 16:49:21 -0700 Subject: [PATCH 20/35] add concrete reduced datums admins --- tom_dataproducts/admin.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tom_dataproducts/admin.py b/tom_dataproducts/admin.py index 2cdcb3f87..1be7d6b38 100644 --- a/tom_dataproducts/admin.py +++ b/tom_dataproducts/admin.py @@ -1,7 +1,12 @@ from django.contrib import admin -from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum +from tom_dataproducts.models import (DataProduct, DataProductGroup, ReducedDatum, + PhotometryReducedDatum, SpectroscopyReducedDatum, + AstrometryReducedDatum) admin.site.register(DataProduct) admin.site.register(DataProductGroup) admin.site.register(ReducedDatum) +admin.site.register(PhotometryReducedDatum) +admin.site.register(SpectroscopyReducedDatum) +admin.site.register(AstrometryReducedDatum) From 5b2638c32787dc502a741cfce1631deb71e4a278 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Thu, 23 Apr 2026 11:48:31 -0700 Subject: [PATCH 21/35] ReducedDatumViewset get return correct model --- tom_dataproducts/api_views.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tom_dataproducts/api_views.py b/tom_dataproducts/api_views.py index 15f2176d1..d875cb402 100644 --- a/tom_dataproducts/api_views.py +++ b/tom_dataproducts/api_views.py @@ -1,4 +1,5 @@ from django.conf import settings +from django.http import Http404 from django_filters import rest_framework as drf_filters from guardian.mixins import PermissionListMixin from guardian.shortcuts import assign_perm, get_objects_for_user @@ -99,6 +100,17 @@ class ReducedDatumViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, G permission_required = 'tom_dataproducts.view_reduceddatum' parser_classes = [FormParser, JSONParser] + def get_object(self): + pk = self.kwargs.get(self.lookup_field) + for model in REDUCED_DATUM_MODELS: + try: + obj = model.objects.get(pk=pk) + self.check_object_permissions(self.request, obj) + return obj + except model.DoesNotExist: + pass + raise Http404 + def _base_queryset_for_model(self, model): qs = model.objects.all() if settings.TARGET_PERMISSIONS_ONLY: From df1751b7a6c36954b5b78133ecff52d83a707701 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Thu, 23 Apr 2026 11:58:47 -0700 Subject: [PATCH 22/35] Update sparkline code for photometry reduced datum --- .../templatetags/dataproduct_extras.py | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 47aa7f433..b1999de20 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -135,11 +135,6 @@ def recent_photometry(target, limit=1): Displays a table of the most recent photometric points for a target. """ photometry = PhotometryReducedDatum.objects.filter(target=target).order_by('-timestamp')[:limit] - - # Possibilities for reduced_datums from ZTF/MARS: - # reduced_datum.value: {'error': 0.0929680392146111, 'filter': 'r', 'magnitude': 18.2364940643311} - # reduced_datum.value: {'limit': 20.1023998260498, 'filter': 'g'} - # for limit magnitudes, set the value of the limit key to True and # the value of the magnitude key to the limit so the template and # treat magnitudes as such and prepend a '>' to the limit magnitudes @@ -216,9 +211,6 @@ def photometry_for_target(context, target, width=700, height=600, background=Non """ Renders a photometric plot for a target. - This templatetag requires all ``ReducedDatum`` objects with a data_type of ``photometry`` to be structured with the - following keys in the JSON representation: magnitude, error, filter - :param width: Width of generated plot :type width: int @@ -456,29 +448,29 @@ def reduceddatum_sparkline(target, height, spacing=5, color_map=None, limit_y=Tr 'i': (0, 0, 0) } - vals = target.reduceddatum_set.filter( + vals = target.photometryreduceddatum_set.filter( timestamp__gte=datetime.utcnow() - timedelta(days=days) - ).values('value', 'timestamp') + ).values('brightness', 'limit', 'bandpass', 'timestamp') if len(vals) < 1: return {'sparkline': None} - vals = [v for v in vals if v['value']] + vals = [v for v in vals if v['brightness'] is not None or v['limit'] is not None] - min_mag = min([val['value']['magnitude'] for val in vals if val['value'].get('magnitude')]) - max_mag = max([val['value']['magnitude'] for val in vals if val['value'].get('magnitude')]) + min_mag = min([val['brightness'] for val in vals if val.get('brightness')]) + max_mag = max([val['brightness'] for val in vals if val.get('brightness')]) if not limit_y: # The following values are used if we want the graph's y range to extend to the values of non-detections - min_mag = min([min_mag, *[val['value']['limit'] for val in vals if val['value'].get('limit')]]) - max_mag = max([max_mag, *[val['value']['limit'] for val in vals if val['value'].get('limit')]]) + min_mag = min([min_mag, *[val['limit'] for val in vals if val.get('limit')]]) + max_mag = max([max_mag, *[val['limit'] for val in vals if val.get('limit')]]) - distinct_filters = set([val['value']['filter'] for val in vals]) + distinct_filters = set([val['bandpass'] for val in vals]) by_filter = {f: [(None, None)] * days for f in distinct_filters} for val in vals: day_index = (val['timestamp'].replace(tzinfo=timezone.utc) - timezone.now()).days - by_filter[val['value']['filter']][day_index] = (val['value'].get('magnitude'), val['value'].get('limit')) + by_filter[val['bandpass']][day_index] = (val.get('brightness'), val.get('limit')) val_range = max_mag - min_mag image_width = (spacing + 1) * (days - 1) From 10dd141a9ab3080810142f627e63cb4c5c445646 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Thu, 23 Apr 2026 13:23:13 -0700 Subject: [PATCH 23/35] Add data migration for reduceddatum to concrete types --- .../migrations/0016_auto_20260423_2012.py | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 tom_dataproducts/migrations/0016_auto_20260423_2012.py diff --git a/tom_dataproducts/migrations/0016_auto_20260423_2012.py b/tom_dataproducts/migrations/0016_auto_20260423_2012.py new file mode 100644 index 000000000..2441bd727 --- /dev/null +++ b/tom_dataproducts/migrations/0016_auto_20260423_2012.py @@ -0,0 +1,136 @@ +# Generated by Django 5.2.13 on 2026-04-23 20:12 + +from django.db import migrations + +BATCH_SIZE = 1000 + +BRIGHTNESS_FIELDS = ('brightness', 'magnitude', 'mag') +BRIGHTNESS_ERROR_FIELDS = ('brightness_error', 'error', 'magnitude_error', 'mag_error') +BANDPASS_FIELDS = ('bandpass', 'filter', 'band', 'f') + + +def _pop_first(d, keys, default=None): + for key in keys: + if key in d and d[key] is not None: + return d[key] + return default + + +def _run_batched(queryset, create_fn, TypedModel, ReducedDatum): + to_create = [] + pks_to_delete = [] + + for rd in queryset.iterator(chunk_size=BATCH_SIZE): + instance = create_fn(rd) + if instance is not None: + to_create.append(instance) + pks_to_delete.append(rd.pk) + + if len(to_create) >= BATCH_SIZE: + TypedModel.objects.bulk_create(to_create, ignore_conflicts=True) + ReducedDatum.objects.filter(pk__in=pks_to_delete).delete() + to_create = [] + pks_to_delete = [] + + if to_create: + TypedModel.objects.bulk_create(to_create, ignore_conflicts=True) + ReducedDatum.objects.filter(pk__in=pks_to_delete).delete() + + +def migrate_photometry(apps, schema_editor): + ReducedDatum = apps.get_model('tom_dataproducts', 'ReducedDatum') + PhotometryReducedDatum = apps.get_model('tom_dataproducts', 'PhotometryReducedDatum') + + def build(rd): + value = rd.value or {} + return PhotometryReducedDatum( + target_id=rd.target_id, + data_product_id=rd.data_product_id, + source_name=rd.source_name, + source_location=rd.source_location, + timestamp=rd.timestamp, + telescope=rd.telescope or value.get('telescope', ''), + instrument=rd.instrument or value.get('instrument', ''), + brightness=_pop_first(value, BRIGHTNESS_FIELDS), + brightness_error=_pop_first(value, BRIGHTNESS_ERROR_FIELDS), + limit=value.get('limit'), + unit=value.get('unit', ''), + bandpass=_pop_first(value, BANDPASS_FIELDS, default=''), + ) + + _run_batched( + ReducedDatum.objects.filter(data_type='photometry'), + build, PhotometryReducedDatum, ReducedDatum, + ) + + +def migrate_spectroscopy(apps, _schema_editor): + ReducedDatum = apps.get_model('tom_dataproducts', 'ReducedDatum') + SpectroscopyReducedDatum = apps.get_model('tom_dataproducts', 'SpectroscopyReducedDatum') + + def build(rd): + value = rd.value or {} + + flux = value.get('flux', []) + wavelength = value.get('wavelength', []) + error = value.get('error', value.get('flux_error', [])) + + return SpectroscopyReducedDatum( + target_id=rd.target_id, + data_product_id=rd.data_product_id, + source_name=rd.source_name, + source_location=rd.source_location, + timestamp=rd.timestamp, + telescope=rd.telescope or value.get('telescope', ''), + instrument=rd.instrument or value.get('instrument', ''), + flux=flux if isinstance(flux, list) else [], + wavelength=wavelength if isinstance(wavelength, list) else [], + error=error if isinstance(error, list) else [], + flux_unit=value.get('flux_units', value.get('flux_unit', '')), + ) + + _run_batched( + ReducedDatum.objects.filter(data_type='spectroscopy'), + build, SpectroscopyReducedDatum, ReducedDatum, + ) + + +def migrate_astrometry(apps, schema_editor): + ReducedDatum = apps.get_model('tom_dataproducts', 'ReducedDatum') + AstrometryReducedDatum = apps.get_model('tom_dataproducts', 'AstrometryReducedDatum') + + def build(rd): + value = rd.value or {} + return AstrometryReducedDatum( + target_id=rd.target_id, + data_product_id=rd.data_product_id, + source_name=rd.source_name, + source_location=rd.source_location, + timestamp=rd.timestamp, + telescope=rd.telescope or value.get('telescope', ''), + instrument=rd.instrument or value.get('instrument', ''), + ra=value.get('ra'), + dec=value.get('dec'), + ra_error=value.get('ra_error'), + dec_error=value.get('dec_error'), + ra_error_units=value.get('ra_error_units', ''), + dec_error_units=value.get('dec_error_units', ''), + ) + + _run_batched( + ReducedDatum.objects.filter(data_type='astrometry'), + build, AstrometryReducedDatum, ReducedDatum, + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ('tom_dataproducts', '0015_reduceddatum_instrument_reduceddatum_telescope_and_more'), + ] + + operations = [ + migrations.RunPython(migrate_photometry, migrations.RunPython.noop), + migrations.RunPython(migrate_spectroscopy, migrations.RunPython.noop), + migrations.RunPython(migrate_astrometry, migrations.RunPython.noop), + ] From 634551532e56d17e48a69abffbd307235ddd049c Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Thu, 23 Apr 2026 13:29:51 -0700 Subject: [PATCH 24/35] Convert data migration to management command --- .../commands/migrateReducedDatums.py | 140 ++++++++++++++++++ .../migrations/0016_auto_20260423_2012.py | 136 ----------------- 2 files changed, 140 insertions(+), 136 deletions(-) create mode 100644 tom_dataproducts/management/commands/migrateReducedDatums.py delete mode 100644 tom_dataproducts/migrations/0016_auto_20260423_2012.py diff --git a/tom_dataproducts/management/commands/migrateReducedDatums.py b/tom_dataproducts/management/commands/migrateReducedDatums.py new file mode 100644 index 000000000..d12ba9d29 --- /dev/null +++ b/tom_dataproducts/management/commands/migrateReducedDatums.py @@ -0,0 +1,140 @@ +from django.core.management.base import BaseCommand + +from tom_dataproducts.models import (ReducedDatum, PhotometryReducedDatum, + SpectroscopyReducedDatum, AstrometryReducedDatum) + +BATCH_SIZE = 1000 + +BRIGHTNESS_FIELDS = ('brightness', 'magnitude', 'mag') +BRIGHTNESS_ERROR_FIELDS = ('brightness_error', 'error', 'magnitude_error', 'mag_error') +BANDPASS_FIELDS = ('bandpass', 'filter', 'band', 'f') + + +def _pop_first(d, keys, default=None): + for key in keys: + if key in d and d[key] is not None: + return d[key] + return default + + +def _run_batched(queryset, build_fn, TypedModel, dry_run, stdout): + to_create = [] + pks_to_delete = [] + total_created = 0 + total_deleted = 0 + + for rd in queryset.iterator(chunk_size=BATCH_SIZE): + instance = build_fn(rd) + to_create.append(instance) + pks_to_delete.append(rd.pk) + + if len(to_create) >= BATCH_SIZE: + if not dry_run: + TypedModel.objects.bulk_create(to_create, ignore_conflicts=True) + ReducedDatum.objects.filter(pk__in=pks_to_delete).delete() + total_created += len(to_create) + total_deleted += len(pks_to_delete) + to_create = [] + pks_to_delete = [] + + if to_create: + if not dry_run: + TypedModel.objects.bulk_create(to_create, ignore_conflicts=True) + ReducedDatum.objects.filter(pk__in=pks_to_delete).delete() + total_created += len(to_create) + total_deleted += len(pks_to_delete) + + label = '[DRY RUN] Would migrate' if dry_run else 'Migrated' + stdout.write(f' {label} {total_created} {TypedModel.__name__} rows, deleted {total_deleted} ReducedDatum rows.') + + +def _build_photometry(rd): + value = rd.value or {} + return PhotometryReducedDatum( + target_id=rd.target_id, + data_product_id=rd.data_product_id, + source_name=rd.source_name, + source_location=rd.source_location, + timestamp=rd.timestamp, + telescope=rd.telescope or value.get('telescope', ''), + instrument=rd.instrument or value.get('instrument', ''), + brightness=_pop_first(value, BRIGHTNESS_FIELDS), + brightness_error=_pop_first(value, BRIGHTNESS_ERROR_FIELDS), + limit=value.get('limit'), + unit=value.get('unit', ''), + bandpass=_pop_first(value, BANDPASS_FIELDS, default=''), + ) + + +def _build_spectroscopy(rd): + value = rd.value or {} + return SpectroscopyReducedDatum( + target_id=rd.target_id, + data_product_id=rd.data_product_id, + source_name=rd.source_name, + source_location=rd.source_location, + timestamp=rd.timestamp, + telescope=rd.telescope or value.get('telescope', ''), + instrument=rd.instrument or value.get('instrument', ''), + flux=value.get('flux', []), + wavelength=value.get('wavelength', []), + error=value.get('error', value.get('flux_error', [])), + flux_unit=value.get('flux_units', value.get('flux_unit', '')), + ) + + +def _build_astrometry(rd): + value = rd.value or {} + return AstrometryReducedDatum( + target_id=rd.target_id, + data_product_id=rd.data_product_id, + source_name=rd.source_name, + source_location=rd.source_location, + timestamp=rd.timestamp, + telescope=rd.telescope or value.get('telescope', ''), + instrument=rd.instrument or value.get('instrument', ''), + ra=value.get('ra'), + dec=value.get('dec'), + ra_error=value.get('ra_error'), + dec_error=value.get('dec_error'), + ra_error_units=value.get('ra_error_units', ''), + dec_error_units=value.get('dec_error_units', ''), + ) + + +class Command(BaseCommand): + help = ( + 'Migrates generic ReducedDatum rows into their concrete typed models ' + '(PhotometryReducedDatum, SpectroscopyReducedDatum, AstrometryReducedDatum) ' + 'and deletes the originals. Run this once after deploying the reduceddatum refactor.' + ) + + def add_arguments(self, parser): + parser.add_argument( + '--dry-run', + action='store_true', + help='Report what would be migrated without writing any changes.', + ) + + def handle(self, *args, **options): + dry_run = options['dry_run'] + + if dry_run: + self.stdout.write('Dry run — no changes will be written.\n') + + steps = [ + ('photometry', ReducedDatum.objects.filter(data_type='photometry'), + _build_photometry, PhotometryReducedDatum), + ('spectroscopy', ReducedDatum.objects.filter(data_type='spectroscopy'), + _build_spectroscopy, SpectroscopyReducedDatum), + ('astrometry', ReducedDatum.objects.filter(data_type='astrometry'), + _build_astrometry, AstrometryReducedDatum), + ] + + for name, queryset, build_fn, TypedModel in steps: + count = queryset.count() + self.stdout.write(f'{name}: {count} rows to migrate.') + if count: + _run_batched(queryset, build_fn, TypedModel, dry_run, self.stdout) + + self.stdout.write(self.style.SUCCESS('Done.')) diff --git a/tom_dataproducts/migrations/0016_auto_20260423_2012.py b/tom_dataproducts/migrations/0016_auto_20260423_2012.py deleted file mode 100644 index 2441bd727..000000000 --- a/tom_dataproducts/migrations/0016_auto_20260423_2012.py +++ /dev/null @@ -1,136 +0,0 @@ -# Generated by Django 5.2.13 on 2026-04-23 20:12 - -from django.db import migrations - -BATCH_SIZE = 1000 - -BRIGHTNESS_FIELDS = ('brightness', 'magnitude', 'mag') -BRIGHTNESS_ERROR_FIELDS = ('brightness_error', 'error', 'magnitude_error', 'mag_error') -BANDPASS_FIELDS = ('bandpass', 'filter', 'band', 'f') - - -def _pop_first(d, keys, default=None): - for key in keys: - if key in d and d[key] is not None: - return d[key] - return default - - -def _run_batched(queryset, create_fn, TypedModel, ReducedDatum): - to_create = [] - pks_to_delete = [] - - for rd in queryset.iterator(chunk_size=BATCH_SIZE): - instance = create_fn(rd) - if instance is not None: - to_create.append(instance) - pks_to_delete.append(rd.pk) - - if len(to_create) >= BATCH_SIZE: - TypedModel.objects.bulk_create(to_create, ignore_conflicts=True) - ReducedDatum.objects.filter(pk__in=pks_to_delete).delete() - to_create = [] - pks_to_delete = [] - - if to_create: - TypedModel.objects.bulk_create(to_create, ignore_conflicts=True) - ReducedDatum.objects.filter(pk__in=pks_to_delete).delete() - - -def migrate_photometry(apps, schema_editor): - ReducedDatum = apps.get_model('tom_dataproducts', 'ReducedDatum') - PhotometryReducedDatum = apps.get_model('tom_dataproducts', 'PhotometryReducedDatum') - - def build(rd): - value = rd.value or {} - return PhotometryReducedDatum( - target_id=rd.target_id, - data_product_id=rd.data_product_id, - source_name=rd.source_name, - source_location=rd.source_location, - timestamp=rd.timestamp, - telescope=rd.telescope or value.get('telescope', ''), - instrument=rd.instrument or value.get('instrument', ''), - brightness=_pop_first(value, BRIGHTNESS_FIELDS), - brightness_error=_pop_first(value, BRIGHTNESS_ERROR_FIELDS), - limit=value.get('limit'), - unit=value.get('unit', ''), - bandpass=_pop_first(value, BANDPASS_FIELDS, default=''), - ) - - _run_batched( - ReducedDatum.objects.filter(data_type='photometry'), - build, PhotometryReducedDatum, ReducedDatum, - ) - - -def migrate_spectroscopy(apps, _schema_editor): - ReducedDatum = apps.get_model('tom_dataproducts', 'ReducedDatum') - SpectroscopyReducedDatum = apps.get_model('tom_dataproducts', 'SpectroscopyReducedDatum') - - def build(rd): - value = rd.value or {} - - flux = value.get('flux', []) - wavelength = value.get('wavelength', []) - error = value.get('error', value.get('flux_error', [])) - - return SpectroscopyReducedDatum( - target_id=rd.target_id, - data_product_id=rd.data_product_id, - source_name=rd.source_name, - source_location=rd.source_location, - timestamp=rd.timestamp, - telescope=rd.telescope or value.get('telescope', ''), - instrument=rd.instrument or value.get('instrument', ''), - flux=flux if isinstance(flux, list) else [], - wavelength=wavelength if isinstance(wavelength, list) else [], - error=error if isinstance(error, list) else [], - flux_unit=value.get('flux_units', value.get('flux_unit', '')), - ) - - _run_batched( - ReducedDatum.objects.filter(data_type='spectroscopy'), - build, SpectroscopyReducedDatum, ReducedDatum, - ) - - -def migrate_astrometry(apps, schema_editor): - ReducedDatum = apps.get_model('tom_dataproducts', 'ReducedDatum') - AstrometryReducedDatum = apps.get_model('tom_dataproducts', 'AstrometryReducedDatum') - - def build(rd): - value = rd.value or {} - return AstrometryReducedDatum( - target_id=rd.target_id, - data_product_id=rd.data_product_id, - source_name=rd.source_name, - source_location=rd.source_location, - timestamp=rd.timestamp, - telescope=rd.telescope or value.get('telescope', ''), - instrument=rd.instrument or value.get('instrument', ''), - ra=value.get('ra'), - dec=value.get('dec'), - ra_error=value.get('ra_error'), - dec_error=value.get('dec_error'), - ra_error_units=value.get('ra_error_units', ''), - dec_error_units=value.get('dec_error_units', ''), - ) - - _run_batched( - ReducedDatum.objects.filter(data_type='astrometry'), - build, AstrometryReducedDatum, ReducedDatum, - ) - - -class Migration(migrations.Migration): - - dependencies = [ - ('tom_dataproducts', '0015_reduceddatum_instrument_reduceddatum_telescope_and_more'), - ] - - operations = [ - migrations.RunPython(migrate_photometry, migrations.RunPython.noop), - migrations.RunPython(migrate_spectroscopy, migrations.RunPython.noop), - migrations.RunPython(migrate_astrometry, migrations.RunPython.noop), - ] From d88d33f464c1323e88081b60e7107dae0b751f80 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 29 Apr 2026 11:19:09 -0700 Subject: [PATCH 25/35] Update latex generation docs for new reduceddatums --- docs/common/latex_generation.rst | 36 +++++++++++++++----------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/docs/common/latex_generation.rst b/docs/common/latex_generation.rst index 929722d32..0d852a5c1 100644 --- a/docs/common/latex_generation.rst +++ b/docs/common/latex_generation.rst @@ -100,7 +100,7 @@ to access the object for which we’re generating data. from django import forms - from tom_dataproducts.models import ReducedDatum + from tom_dataproducts.models import PhotometryReducedDatum, SpectroscopyReducedDatum from tom_publications.latex import GenericLatexProcessor, GenericLatexForm from tom_targets.models import Target @@ -110,18 +110,17 @@ to access the object for which we’re generating data. form_class = TargetDataLatexForm def create_latex_table_data(self, cleaned_data): - target = Target.objects.get(pk=cleaned_data.get('model_pk')) - data = ReducedDatum.objects.filter(target=target, data_type=cleaned_data.get('data_type')) - - table_data = {} - if cleaned_data.get('data_type') == 'photometry': - for datum in data: - for key, value in json.loads(datum.value).items(): - table_data.setdefault(key, []).append(value) - elif cleaned_data.get('data_type') == 'spectroscopy': - ... - - return table_data + target = Target.objects.get(pk=cleaned_data.get('model_pk')) + table_data = {} + if cleaned_data.get('data_type') == 'photometry': + data = PhotometryReducedDatum.objects.filter(target=target) + for brightness, bandpass in data.values_list('brightness', 'bandpass'): + table_data.setdefault('brightness', []).append(brightness) + table_data.setdefault('bandpass', []).append(bandpass) + elif cleaned_data.get('data_type') == 'spectroscopy': + ... + + return table_data The above example only shows the photometric table generation, but spectroscopic can be left as an exercise to the reader. @@ -184,7 +183,7 @@ TOM. Here’s our final ``target_data_latex_processor.py``: from django import forms - from tom_dataproducts.models import ReducedDatum + from tom_dataproducts.models import PhotometryReducedDatum, SpectroscopyReducedDatum from tom_publications.latex import GenericLatexProcessor, GenericLatexForm from tom_targets.models import Target @@ -202,13 +201,12 @@ TOM. Here’s our final ``target_data_latex_processor.py``: def create_latex_table_data(self, cleaned_data): target = Target.objects.get(pk=cleaned_data.get('model_pk')) - data = ReducedDatum.objects.filter(target=target, data_type=cleaned_data.get('data_type')) - table_data = {} if cleaned_data.get('data_type') == 'photometry': - for datum in data: - for key, value in json.loads(datum.value).items(): - table_data.setdefault(key, []).append(value) + data = PhotometryReducedDatum.objects.filter(target=target) + for brightness, bandpass in data.values_list('brightness', 'bandpass'): + table_data.setdefault('brightness', []).append(brightness) + table_data.setdefault('bandpass', []).append(bandpass) elif cleaned_data.get('data_type') == 'spectroscopy': ... From 8f7bf334ac16384fa6d0a72f62f1d29e8837ce05 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 29 Apr 2026 11:26:58 -0700 Subject: [PATCH 26/35] Update permissions docs for new reduceddatums --- docs/common/permissions.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/common/permissions.rst b/docs/common/permissions.rst index a4dd46fef..663077bb4 100644 --- a/docs/common/permissions.rst +++ b/docs/common/permissions.rst @@ -139,3 +139,15 @@ The above code will allow all users in the groups that the example user belongs ``ReducedDatum``: * ``tom_dataproducts.view_reduceddatum`` + +``PhotometryReducedDatum``: + +* ``tom_dataproducts.view_photometryreduceddatum`` + +``SpectroscopyReducedDatum``: + +* ``tom_dataproducts.view_spectroscopyreduceddatum`` + +``AstrometryReducedDatum``: + +* ``tom_dataproducts.view_astrometryreduceddatum`` From 859f4a8a22ea2e0b37ff9172bdba0c17d243765c Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 29 Apr 2026 11:31:52 -0700 Subject: [PATCH 27/35] Update customize template tags for new reducedatums --- docs/customization/customize_template_tags.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/customization/customize_template_tags.rst b/docs/customization/customize_template_tags.rst index 557189f64..b9e88f04c 100644 --- a/docs/customization/customize_template_tags.rst +++ b/docs/customization/customize_template_tags.rst @@ -154,8 +154,8 @@ approach here: You can see that we’ll eventually be returning a dictionary, but first we need to add our logic. We’ll need to use the ``Target`` passed in to -get all ``ReducedDatum`` objects for that ``Target`` with a -``data_type`` of ``photometry``. Then we’ll need to order by +get all ``PhotometryReducedDatum`` objects for that ``Target`` +Then we’ll need to order by ``timestamp`` descending, and slice just the first few. Make sure to take note of the imports in this step! @@ -165,7 +165,7 @@ take note of the imports in this step! from django import template - from tom_dataproducts.models import ReducedDatum + from tom_dataproducts.models import PhotometryReducedDatum register = template.Library() @@ -173,8 +173,8 @@ take note of the imports in this step! @register.inclusion_tag('custom_code/partials/recent_photometry.html') def recent_photometry(target, num_points=1): - photometry = ReducedDatum.objects.filter(data_type='photometry').order_by('-timestamp')[:num_points] - return {'recent_photometry': [(datum.timestamp, json.loads(datum.value)['magnitude']) for datum in photometry]} + photometry = PhotometryReducedDatum.objects.order_by('-timestamp')[:num_points] + return {'recent_photometry': [(datum.timestamp, datum.brightness) for datum in photometry]} It’s only a couple of lines, but there’s a lot going on here. The first line does the aforemention database query and slices the first point of @@ -330,4 +330,4 @@ As far as this template tag goes, as of this tutorial, it’s now a part of the base TOM Toolkit, but all of the information here should provide you with the ability to write your own. -.. |image0| image:: /_static/customize_template_tags_doc/Templatetags.png \ No newline at end of file +.. |image0| image:: /_static/customize_template_tags_doc/Templatetags.png From d013b86f90572ed45dbfa211fd10eadaf9332e2f Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 29 Apr 2026 11:53:06 -0700 Subject: [PATCH 28/35] Add new reduceddatum models to architecture doc --- docs/introduction/tomarchitecture.rst | 42 +++++++++++++++++++-------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/docs/introduction/tomarchitecture.rst b/docs/introduction/tomarchitecture.rst index c109124d8..0bf49f940 100644 --- a/docs/introduction/tomarchitecture.rst +++ b/docs/introduction/tomarchitecture.rst @@ -52,7 +52,7 @@ they are left with a functioning but generic TOM. It is then up to the developer to implement the specific features that their science case requires. The toolkit tries to facilitate this as efficiently as possible and provides :doc:`documentation ` in areas of customization from :doc:`changing the HTML layout of a page ` -to :doc:`customizing an OCS facility and forms ` and even +to :doc:`customizing an OCS facility and forms ` and even :doc:`creating a new alert broker `. Django, and by extension the toolkit, rely heavily on object oriented @@ -95,7 +95,7 @@ each other. This means a TOM developer can easily change the layout and style of any page without modifying the underlying framework's code directly. Entire pages may be replaced, or only "blocks" within a template. -Compare these screenshots of the `standard target detail page <../../../_static/architecture/snex2layout.png>`_ and the +Compare these screenshots of the `standard target detail page <../../../_static/architecture/snex2layout.png>`_ and the `Global Supernova Project's target detail page <../../../_static/architecture/snex2layout.png>`_, the latter taking heavy advantage of template inheritance. @@ -266,28 +266,46 @@ ReducedDatum ------------ A ``ReducedDatum`` is a single point of data associated with a ``Target`` and optionally a -``DataProduct``. The single data point is typically a single point of photometry or an individual -spectrum. The ``ReducedDatum`` model has the following fields, in addition to its aforementioned +``DataProduct``. +There are three classes of ReducedDatum for the common data types: +``PhotometryReducedDatum``, ``SpectroscopyReducedDatum``, and ``AstrometryReducedDatum``. +The ``ReducedDatum`` is a general model meant to be flexible enough to allow for other data types as well. + +The ``ReducedDatum`` model has the following fields, in addition to its aforementioned foreign key relationships: - ``data_type`` is maintained on both the ``ReducedDatum`` and ``DataProduct`` for the case when data is brought in from another source, such as a broker - The ``source_name`` optionally refers to the original source of the data. The intent of this field was to track data ingested from brokers, but could potentially be used for other purposes. - ``source_location`` optionally gives a hard location to the source--for a broker, it would be a link to the original alert. - The ``timestamp`` time at which the datum was produced. -- ``value`` is a ``TextField`` that can take any series of data. As implemented, photometry is stored as JSON with keys for magnitude and error, but the ``TextField`` provides flexibility for additional photometry values on the datum. Spectroscopy is also stored as JSON, with keys for ``magnitude`` and ``flux``. - -Feedback and bug reporting -========================== +- ``value`` is a ``JSONField`` that can take any series of data. +- ``telescope`` and ``instrument`` are optional fields that can be used to track additional metadata. -We hope the TOM Toolkit is helpful to you and your project. If you have any -concerns about implementation details, or questions about your own needs, please -don't hesitate to `reach out `_. Issues and pull requests -are also welcome on the project's `GitHub page `_. +The ``PhotometryReducedDatum`` model has the following Photometry specific fields: +- ``brightness`` and ``brightness_error`` are float fields that track the magnitude and error, respectively. +- ``bandpass`` is a char field that tracks the bandpass/filter of the photometry. +- ``limit`` optional float field that tracks the limiting magnitude of the photometry +- ``unit`` optional char field that tracks the unit of the photometry +- ``exposure_time`` optional float field that tracks the exposure time of the photometry +The ``SpectroscopyReducedDatum`` model has the following Spectroscopy specific fields: +- ``wavelength``, ``flux`` and ``error`` are all FloatArrayFields that track the wavelength, flux, and error of the spectroscopy, respectively +- ``unit`` optional char field that tracks the unit of the spectroscopy +- ``setup`` optional text field for arbitrary metadata about the spectroscopic setup +- ``exposure_time`` optional float field that tracks the exposure time of the spectroscopy +The ``AstrometryReducedDatum`` model has the following Astrometry specific fields: +- ``ra``, ``dec``, ``ra_error`` and ``dec_error`` are all float fields for tracking coordinates and error. Errors are optional. +- ``ra_error_units`` and ``dec_error_units`` optional char fields that track the units of the errors +Feedback and bug reporting +========================== +We hope the TOM Toolkit is helpful to you and your project. If you have any +concerns about implementation details, or questions about your own needs, please +don't hesitate to `reach out `_. Issues and pull requests +are also welcome on the project's `GitHub page `_. From 58951b45c45ef93540a90c14170bc8a126d7910b Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 29 Apr 2026 14:30:14 -0700 Subject: [PATCH 29/35] Update customizing data processing docs for new reduceddatums --- docs/managing_data/customizing_data_processing.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/managing_data/customizing_data_processing.rst b/docs/managing_data/customizing_data_processing.rst index 86bcec121..3f728e74d 100644 --- a/docs/managing_data/customizing_data_processing.rst +++ b/docs/managing_data/customizing_data_processing.rst @@ -21,12 +21,12 @@ tom_dataproducts app in the TOM Toolkit: Let’s start with a quick overview of ``models.py``. The file contains the Django models for the dataproducts app–in our case, ``DataProduct`` -and ``ReducedDatum``. The ``DataProduct`` contains information about +, ``ReducedDatum`` and the three specialized reduced data product classes: +``PhotometryReducedDatum``, ``SpectroscopyReducedDatum`` and ``AstrometryReducedDatum``. +The ``DataProduct`` contains information about uploaded or saved ``DataProducts``, such as the file name, file path, -and what kind of file it is. The ``ReducedDatum`` contains individual +and what kind of file it is. The ``*ReducedDatum`` classes contain individual science data points that are taken from the ``DataProduct`` files. -Examples of ``ReducedDatum`` points would be individual photometry -points or individual spectra. Each ``DataProduct`` also has a ``data_product_type``. The ``data_product_type`` is simply a description of what the file is, more From 3fb80f9cfdfd78ef528ceb02e1cfb412cd936347 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 29 Apr 2026 14:33:15 -0700 Subject: [PATCH 30/35] Update plotting data docs for new reduceddatums --- docs/managing_data/plotting_data.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/managing_data/plotting_data.rst b/docs/managing_data/plotting_data.rst index 22bac5722..30300d2bf 100644 --- a/docs/managing_data/plotting_data.rst +++ b/docs/managing_data/plotting_data.rst @@ -128,7 +128,7 @@ Next, add the function body: # x axis: target names. y axis: datum count data = [go.Bar( x=[target.name for target in targets], - y=[target.reduceddatum_set.count() for target in targets] + y=[target.photometryreduceddatum_set.count() for target in targets] )] # Create the plot figure = offline.plot(go.Figure(data=data), output_type='div', show_link=False) From b573eaa2990bb1782139442668f200c8e2407dd8 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 29 Apr 2026 14:35:28 -0700 Subject: [PATCH 31/35] Update direct sharing docs for new reduceddatums --- docs/managing_data/tom_direct_sharing.rst | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/docs/managing_data/tom_direct_sharing.rst b/docs/managing_data/tom_direct_sharing.rst index bc0edb2ca..9ab67abfc 100644 --- a/docs/managing_data/tom_direct_sharing.rst +++ b/docs/managing_data/tom_direct_sharing.rst @@ -46,7 +46,7 @@ Receiving Shared Data: Reduced Datums: --------------- When your TOM receives a new ``ReducedDatum`` from another TOM it will be saved to your TOM's database with its source -set to the name of the TOM that submitted it. Currently, only Photometry data can be directly shared between +set to the name of the TOM that submitted it. Currently, only ``PhotometryReducedDatum`` can be directly shared between TOMS and a ``Target`` with a matching name or alias must exist in both TOMS for sharing to take place. Data Products: @@ -66,9 +66,3 @@ Target Lists: When your TOM receives a new ``TargetList`` from another TOM it will be saved to your TOM's database. If the targets in the ``TargetList`` are also shared, but already exist in the destination TOM, they will be added to the new ``TargetList``. - - - - - - From c8df07f286a6dccb465e3fbced44ea7e984c49f6 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 29 Apr 2026 14:39:23 -0700 Subject: [PATCH 32/35] Move source_location docstring to correct class --- tom_dataproducts/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index 7e1846dbe..3343159f9 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -368,6 +368,10 @@ class ReducedDatumCommon(models.Model): datum came from, but can be used for other sources. :type source_name: str + :param source_location: A reference to the location that this datum was originally sourced from. The current major + use of this field is the URL path to the alert that this datum came from. + :type source_location: str + :param message: Set of ``AlertStreamMessage`` objects this object is associated with. :type message: ManyRelatedManager object @@ -402,10 +406,6 @@ class ReducedDatum(ReducedDatumCommon): :param data_type: The type of data this datum represents. Default choices are the default values found in DATA_PRODUCT_TYPES in settings.py. :type data_type: str - - :param source_location: A reference to the location that this datum was originally sourced from. The current major - use of this field is the URL path to the alert that this datum came from. - :type source_location: str """ data_type = models.CharField(max_length=100, default="") From acd6e797a89aa1565cdf1d983dc4bc0959cc96d6 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 29 Apr 2026 14:42:32 -0700 Subject: [PATCH 33/35] Remove data product type check on save for generic reduceddatum --- tom_dataproducts/models.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index 3343159f9..ad58175e0 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -413,19 +413,6 @@ class ReducedDatum(ReducedDatumCommon): class Meta: get_latest_by = ("timestamp",) - def save(self, *args, **kwargs): - # Validate data_type based on options in settings.py or default types: (type, display) - for dp_type, _ in DATA_TYPE_CHOICES: - if self.data_type and self.data_type == dp_type: - break - else: - raise ValidationError("Not a valid DataProduct type.") - - # because we have a custom way of validating the uniqueness of the ReducedDatum, - # we need to call full_clean() here to invoke our validate_unique() method. - self.full_clean() - return super().save() - class PhotometryReducedDatum(ReducedDatumCommon): brightness = models.FloatField(blank=True, null=True) From d5ad27de4b4701bedd714136c631d74ae2f29eac Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 29 Apr 2026 14:47:37 -0700 Subject: [PATCH 34/35] Revert uniqueness check for generic reduceddatum Note that this does not scale, which is fine as long as we are only expecting to scale on the concrete types. --- tom_dataproducts/models.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index ad58175e0..371681deb 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -413,6 +413,38 @@ class ReducedDatum(ReducedDatumCommon): class Meta: get_latest_by = ("timestamp",) + def validate_unique(self, *args, **kwargs): + """ + Validates that the ReducedDatum is unique. Because the `value` field is a JSONField, it is not possible to rely + on standard validation. Also, We do not want to repeat identical data from two different sources. + + Do nothing if the uniqueness test passes. Otherwise, raise a ValidationError. + see https://docs.djangoproject.com/en/5.0/ref/models/instances/#validating-objects + """ + super().validate_unique(*args, **kwargs) + # Check if the Reduced Datum exists in the database + try: + existing_reduced_datum = ReducedDatum.objects.get( + target=self.target, + data_type=self.data_type, + timestamp=self.timestamp, + value=self.value, + ) + if ( + existing_reduced_datum and existing_reduced_datum.id != self.id + ): # not the same object + existing_source = existing_reduced_datum.__dict__.get( + "source_name", "Unknown Source" + ) + # found ReducedDatum with the same values. Don't save this duplicate ReducedDatum. + raise ValidationError( + f"ReducedDatum already exists: Identical {self.data_type} data " + f"found for {self.target} from {existing_source}." + ) + except ReducedDatum.DoesNotExist: + # this means that our check for uniqueness passed: so do not raise ValidationError + pass + class PhotometryReducedDatum(ReducedDatumCommon): brightness = models.FloatField(blank=True, null=True) From 86efd70a147c8105c9b6ed972a59ec40e0972b26 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Wed, 29 Apr 2026 15:28:11 -0700 Subject: [PATCH 35/35] Add note about v3 to migration script --- .../management/commands/migrateReducedDatums.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tom_dataproducts/management/commands/migrateReducedDatums.py b/tom_dataproducts/management/commands/migrateReducedDatums.py index d12ba9d29..744cd0a68 100644 --- a/tom_dataproducts/management/commands/migrateReducedDatums.py +++ b/tom_dataproducts/management/commands/migrateReducedDatums.py @@ -103,11 +103,12 @@ def _build_astrometry(rd): class Command(BaseCommand): - help = ( - 'Migrates generic ReducedDatum rows into their concrete typed models ' - '(PhotometryReducedDatum, SpectroscopyReducedDatum, AstrometryReducedDatum) ' - 'and deletes the originals. Run this once after deploying the reduceddatum refactor.' - ) + help = """ + Migrates generic ReducedDatum rows into their concrete typed models + (PhotometryReducedDatum, SpectroscopyReducedDatum, AstrometryReducedDatum) + and deletes the originals. Run this once after deploying the reduceddatum refactor. + Only necessary for TOMs that existed prior to v3. + """ def add_arguments(self, parser): parser.add_argument(