From c327cfa76a0c2a64d962cf0796c0ba2108c253c5 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Tue, 8 Jul 2025 16:26:33 -0700 Subject: [PATCH 1/7] HERD Changes internal file --- src/pynwb/file.py | 30 +++++- src/pynwb/io/file.py | 2 + src/pynwb/nwb-schema | 2 +- tests/unit/test_resources.py | 192 +++++++++++++++++++++++++++++++++++ 4 files changed, 224 insertions(+), 2 deletions(-) diff --git a/src/pynwb/file.py b/src/pynwb/file.py index c2fd317ca..d98c4604d 100644 --- a/src/pynwb/file.py +++ b/src/pynwb/file.py @@ -339,6 +339,7 @@ class NWBFile(MultiContainerInterface, HERDManager): {'name': 'keywords', 'type': 'array_data', 'doc': 'Terms to search over', 'default': None}, {'name': 'notes', 'type': str, 'doc': 'Notes about the experiment.', 'default': None}, + {'name': 'external_resources', 'child': True, 'required_name': 'external_resources'}, {'name': 'pharmacology', 'type': str, 'doc': 'Description of drugs used, including how and when they were administered. ' 'Anesthesia(s), painkiller(s), etc., plus dosage, concentration, etc.', 'default': None}, @@ -483,9 +484,13 @@ def __init__(self, **kwargs): 'icephys_experimental_conditions' ] args_to_set = popargs_to_dict(keys_to_set, kwargs) + args_to_set['internal_herd'] = popargs('external_resources', kwargs) kwargs['name'] = 'root' super().__init__(**kwargs) + self.reset_herd = False + self.external_herd = None + # add timezone to session_start_time if missing session_start_time = args_to_set['session_start_time'] if session_start_time.tzinfo is None: @@ -570,6 +575,29 @@ def all_children(self): stack.append(c) return ret + def link_resources(self, herd): + """ + This method is to set an external HERD file as the external resources for this file. + This will not persist on export. # TODO: This could change in the future with further development. + """ + self.external_herd = herd + self.reset_herd = True + + @property + def external_resources(self): + if self.reset_herd: + return self.external_herd + else: + return self.internal_herd + + @external_resources.setter + def external_resources(self, herd): + """ + This is here to set HERD for the file if the user did not do so using __init__. + """ + self.internal_herd = herd + self.internal_herd.parent = self + @property def objects(self): if self.__obj is None: @@ -1152,4 +1180,4 @@ def ElectrodeTable(name='electrodes', description='metadata about extracellular electrodes'): warn("The ElectrodeTable convenience function is deprecated. Please create a new instance of " "the ElectrodesTable class instead.", DeprecationWarning) - return ElectrodesTable() \ No newline at end of file + return ElectrodesTable() diff --git a/src/pynwb/io/file.py b/src/pynwb/io/file.py index d74c66be1..a3233a01d 100644 --- a/src/pynwb/io/file.py +++ b/src/pynwb/io/file.py @@ -110,6 +110,8 @@ def __init__(self, spec): self.map_spec('subject', general_spec.get_group('subject')) + self.map_spec('external_resources', general_spec.get_group('external_resources')) + device_spec = general_spec.get_group('devices') self.unmap(device_spec) self.map_spec('devices', device_spec.get_neurodata_type('Device')) diff --git a/src/pynwb/nwb-schema b/src/pynwb/nwb-schema index ade50ef33..39aaaec2b 160000 --- a/src/pynwb/nwb-schema +++ b/src/pynwb/nwb-schema @@ -1 +1 @@ -Subproject commit ade50ef33446beb3c7df4c6f1072ae0e821b5115 +Subproject commit 39aaaec2b199f7509c60da4a6287c5df68c96259 diff --git a/tests/unit/test_resources.py b/tests/unit/test_resources.py index 108a7fd84..85d9279f0 100644 --- a/tests/unit/test_resources.py +++ b/tests/unit/test_resources.py @@ -1,10 +1,27 @@ import warnings +from datetime import datetime +from uuid import uuid4 +import os +import numpy as np + +from dateutil import tz from pynwb.resources import HERD +from pynwb.file import Subject +from pynwb import NWBHDF5IO, NWBFile from pynwb.testing import TestCase class TestNWBContainer(TestCase): + def setUp(self): + self.path = "resources_file.nwb" + self.export_path = "export_file.nwb" + + def tearDown(self): + for path in [self.path, self.export_path]: + if os.path.isfile(path): + os.remove(path) + def test_constructor(self): """ Test constructor @@ -17,3 +34,178 @@ def test_constructor(self): ) er = HERD() self.assertIsInstance(er, HERD) + + def test_nwbfile_init_herd(self): + session_start_time = datetime(2018, 4, 25, 2, 30, 3, tzinfo=tz.gettz("US/Pacific")) + herd = HERD() + nwbfile = NWBFile( + session_description="A Person undergoing brain pokes.", + identifier=str(uuid4()), + session_start_time=session_start_time, + external_resources=herd + ) + self.assertTrue(isinstance(nwbfile.external_resources, HERD)) + + def test_nwbfile_set_herd(self): + session_start_time = datetime(2018, 4, 25, 2, 30, 3, tzinfo=tz.gettz("US/Pacific")) + herd = HERD() + nwbfile = NWBFile( + session_description="A Person undergoing brain pokes.", + identifier=str(uuid4()), + session_start_time=session_start_time, + ) + nwbfile.external_resources = herd + self.assertTrue(isinstance(nwbfile.external_resources, HERD)) + self.assertEqual(nwbfile.external_resources.parent, nwbfile) + + def test_resources_roundtrip(self): + session_start_time = datetime(2018, 4, 25, 2, 30, 3, tzinfo=tz.gettz("US/Pacific")) + + nwbfile = NWBFile( + session_description="A Person undergoing brain pokes.", + identifier=str(uuid4()), + session_start_time=session_start_time, + ) + subject = Subject( + subject_id="001", + age="26", + description="human 5", + species='Homo sapiens', + sex="M", + ) + + nwbfile.subject = subject + herd = HERD() + nwbfile.external_resources = herd + + nwbfile.external_resources.add_ref(container=nwbfile.subject, + key=nwbfile.subject.species, + entity_id="NCBI_TAXON:9606", + entity_uri='https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=9606') + + with NWBHDF5IO(self.path, "w") as io: + io.write(nwbfile) + + with NWBHDF5IO(self.path, "r") as io: + read_nwbfile = io.read() + self.assertEqual( + read_nwbfile.external_resources.keys[:], + np.array( + [[(b'Homo sapiens',)]], + dtype=[('key', 'O')] + ) + ) + + self.assertEqual( + read_nwbfile.external_resources.entities[:], + np.array( + [ + ('NCBI_TAXON:9606', + 'https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=9606') + ], + dtype=[('entity_id', 'O'), ('entity_uri', 'O')] + ) + ) + + self.assertEqual( + read_nwbfile.external_resources.objects[:], + np.array( + [ + (0, + subject.object_id, + 'Subject', + '', + '') + ], + dtype=[ + ('files_idx', ' Date: Tue, 8 Jul 2025 16:54:41 -0700 Subject: [PATCH 2/7] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 282156763..1154441f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # PyNWB Changelog +## PyNWB 3.1.0 (Upcoming) +### Enhancements and minor changes +- Added HERD to to `general` within the the `NWBFile`. (https://github.com/NeurodataWithoutBorders/pynwb/pull/2111) + ## PyNWB 3.1.0 (July 8, 2025) ### Breaking changes From 2c97c0d17ad5df5ed25c4ae17c20266a926b09b6 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Tue, 8 Jul 2025 17:11:49 -0700 Subject: [PATCH 3/7] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1154441f2..9c59cf953 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## PyNWB 3.1.0 (Upcoming) ### Enhancements and minor changes -- Added HERD to to `general` within the the `NWBFile`. (https://github.com/NeurodataWithoutBorders/pynwb/pull/2111) +- Added HERD to to `general` within the the `NWBFile`. @mavaylon1 [#2111](https://github.com/NeurodataWithoutBorders/pynwb/pull/2111) ## PyNWB 3.1.0 (July 8, 2025) From dc71149d5cc46a27339915951f4a24c8015121bc Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Thu, 19 Feb 2026 22:41:00 -0500 Subject: [PATCH 4/7] Fix external_resources docval and update nwb-schema submodule Move 'child' and 'required_name' keys from @docval to __nwbfields__ where they belong, fixing the import-time crash: "docval for external_resources: keys ['child', 'required_name'] are not supported by docval" Also update nwb-schema submodule to latest herd branch (with dev merge conflict resolved). Co-Authored-By: Claude Opus 4.6 --- src/pynwb/file.py | 5 ++++- src/pynwb/nwb-schema | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pynwb/file.py b/src/pynwb/file.py index d98c4604d..39570048b 100644 --- a/src/pynwb/file.py +++ b/src/pynwb/file.py @@ -9,6 +9,7 @@ from hdmf.common import DynamicTableRegion, DynamicTable from hdmf.container import HERDManager +from hdmf.common import HERD from hdmf.utils import docval, getargs, get_docval, popargs, popargs_to_dict, AllowPositional from . import register_class, CORE_NAMESPACE @@ -287,6 +288,7 @@ class NWBFile(MultiContainerInterface, HERDManager): {'name': 'trials', 'child': True, 'required_name': 'trials'}, {'name': 'units', 'child': True, 'required_name': 'units'}, {'name': 'subject', 'child': True, 'required_name': 'subject'}, + {'name': 'external_resources', 'child': True, 'required_name': 'external_resources'}, {'name': 'sweep_table', 'child': True, 'required_name': 'sweep_table'}, {'name': 'invalid_times', 'child': True, 'required_name': 'invalid_times'}, # icephys_filtering is temporary. /intracellular_ephys/filtering dataset will be deprecated @@ -339,7 +341,8 @@ class NWBFile(MultiContainerInterface, HERDManager): {'name': 'keywords', 'type': 'array_data', 'doc': 'Terms to search over', 'default': None}, {'name': 'notes', 'type': str, 'doc': 'Notes about the experiment.', 'default': None}, - {'name': 'external_resources', 'child': True, 'required_name': 'external_resources'}, + {'name': 'external_resources', 'type': HERD, + 'doc': 'the HERD external resources object for this NWBFile', 'default': None}, {'name': 'pharmacology', 'type': str, 'doc': 'Description of drugs used, including how and when they were administered. ' 'Anesthesia(s), painkiller(s), etc., plus dosage, concentration, etc.', 'default': None}, diff --git a/src/pynwb/nwb-schema b/src/pynwb/nwb-schema index 39aaaec2b..77642b99f 160000 --- a/src/pynwb/nwb-schema +++ b/src/pynwb/nwb-schema @@ -1 +1 @@ -Subproject commit 39aaaec2b199f7509c60da4a6287c5df68c96259 +Subproject commit 77642b99f3d1bd09b5a8c1b947c1e6965ecd308a From c97a8e392f4a52d770fc8ed0247fd43da97ab049 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Thu, 19 Feb 2026 22:49:45 -0500 Subject: [PATCH 5/7] Update nwb-schema submodule to include hdmf-common-schema 1.9.0 The nwb-schema herd branch now includes hdmf-common-schema 1.9.0, which moved HERD from hdmf-experimental to hdmf-common. This is required so the NWB core namespace can find the HERD type. Note: This also requires hdmf >= 4.3.2 (unreleased) which bundles hdmf-common-schema 1.9.0. CI should install hdmf from dev branch until 4.3.2 is released. Co-Authored-By: Claude Opus 4.6 --- src/pynwb/nwb-schema | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynwb/nwb-schema b/src/pynwb/nwb-schema index 77642b99f..d15dfb130 160000 --- a/src/pynwb/nwb-schema +++ b/src/pynwb/nwb-schema @@ -1 +1 @@ -Subproject commit 77642b99f3d1bd09b5a8c1b947c1e6965ecd308a +Subproject commit d15dfb130a280d9d4882ef340ed6e780055d852b From 2c3afeca6acfe2f9c804722df8748ca04aa56c88 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Thu, 19 Feb 2026 23:05:05 -0500 Subject: [PATCH 6/7] Pin hdmf>=5.0.0 (requires hdmf-common-schema 1.9.0 for HERD) hdmf 5.0.0 will bundle hdmf-common-schema 1.9.0, which moved HERD from the hdmf-experimental namespace to hdmf-common. This is required for the NWB core namespace to resolve the HERD type used in nwb.file.yaml. Co-Authored-By: Claude Opus 4.6 --- pyproject.toml | 2 +- requirements-min.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 24a6cb405..4fcb9f89c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ classifiers = [ ] dependencies = [ "h5py>=3.2.0", - "hdmf>=4.1.2,<5", + "hdmf>=5.0.0,<6", "numpy>=1.24.0", "pandas>=1.2.0", "python-dateutil>=2.8.2", diff --git a/requirements-min.txt b/requirements-min.txt index 61241e7c6..6bbe0d1ba 100644 --- a/requirements-min.txt +++ b/requirements-min.txt @@ -1,6 +1,6 @@ # minimum versions of package dependencies for installing PyNWB h5py==3.2.0 -hdmf==4.1.2 +hdmf==5.0.0 numpy==1.24.0 pandas==1.2.0 python-dateutil==2.8.2 diff --git a/requirements.txt b/requirements.txt index 79b6db693..be92e06f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # pinned dependencies to reproduce an entire development environment to use PyNWB h5py==3.12.1 -hdmf==4.1.2 +hdmf==5.0.0 numpy==2.1.1; python_version > "3.9" # numpy 2.1+ is not compatible with py3.9 numpy==2.0.2; python_version == "3.9" pandas==2.2.3 From eaae41b5b6d46e77cda6cd802f05849b9faf5055 Mon Sep 17 00:00:00 2001 From: rly <310197+rly@users.noreply.github.com> Date: Thu, 5 Mar 2026 22:08:58 -0800 Subject: [PATCH 7/7] Improve NWBFile external_resources HERD integration - Rename external_herd/internal_herd to _external_herd/_internal_herd - Use external_resources setter in __init__ to properly set parent - Remove reset_herd flag; check _external_herd is not None instead - Add docstrings to external_resources property, setter, and link_resources - Fix test assertions for HERD keys array shape (1-D not 2-D) Co-Authored-By: Claude Opus 4.6 --- src/pynwb/file.py | 41 +++++++++++++++++++++--------------- tests/unit/test_resources.py | 4 ++-- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/pynwb/file.py b/src/pynwb/file.py index 39570048b..ffa8ad1b0 100644 --- a/src/pynwb/file.py +++ b/src/pynwb/file.py @@ -487,12 +487,15 @@ def __init__(self, **kwargs): 'icephys_experimental_conditions' ] args_to_set = popargs_to_dict(keys_to_set, kwargs) - args_to_set['internal_herd'] = popargs('external_resources', kwargs) + external_resources = popargs('external_resources', kwargs) kwargs['name'] = 'root' super().__init__(**kwargs) - self.reset_herd = False - self.external_herd = None + self._external_herd = None + self._internal_herd = None + + if external_resources is not None: + self.external_resources = external_resources # add timezone to session_start_time if missing session_start_time = args_to_set['session_start_time'] @@ -579,27 +582,31 @@ def all_children(self): return ret def link_resources(self, herd): + """Link an external HERD object as the external resources for this file. + + The linked HERD will be returned by the ``external_resources`` property + but will not be written on export; the original internal HERD (if any) + is preserved in the exported file. """ - This method is to set an external HERD file as the external resources for this file. - This will not persist on export. # TODO: This could change in the future with further development. - """ - self.external_herd = herd - self.reset_herd = True + self._external_herd = herd @property def external_resources(self): - if self.reset_herd: - return self.external_herd - else: - return self.internal_herd + """Return the HERD external resources object for this NWBFile. + + If an external HERD has been linked via ``link_resources``, that object + is returned. Otherwise, the internal HERD set via ``__init__`` or the + setter is returned. + """ + if self._external_herd is not None: + return self._external_herd + return self._internal_herd @external_resources.setter def external_resources(self, herd): - """ - This is here to set HERD for the file if the user did not do so using __init__. - """ - self.internal_herd = herd - self.internal_herd.parent = self + """Set the internal HERD external resources object for this NWBFile.""" + self._internal_herd = herd + self._internal_herd.parent = self @property def objects(self): diff --git a/tests/unit/test_resources.py b/tests/unit/test_resources.py index 85d9279f0..ec07f24a4 100644 --- a/tests/unit/test_resources.py +++ b/tests/unit/test_resources.py @@ -91,7 +91,7 @@ def test_resources_roundtrip(self): self.assertEqual( read_nwbfile.external_resources.keys[:], np.array( - [[(b'Homo sapiens',)]], + [('Homo sapiens',)], dtype=[('key', 'O')] ) ) @@ -174,7 +174,7 @@ def test_link_resources(self): self.assertEqual( read_export_nwbfile.external_resources.keys[:], np.array( - [[(b'Homo sapiens',)]], + [('Homo sapiens',)], dtype=[('key', 'O')] ) )