From 46706b1f0a719101012ea6162c2213c98e188928 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Fri, 29 May 2026 14:06:48 -0700 Subject: [PATCH 1/6] fix: allow subject/procedures 400 responses, expose error messages, fix typo (#159) * fix: expose full contents of error messages * fix: allow 400 responses for subject/procedures * docs: typo --- .../prepare_before_acquisition.md | 48 ++++++++++--------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/docs/source/acquire_upload/prepare_before_acquisition.md b/docs/source/acquire_upload/prepare_before_acquisition.md index 710625d..e825a3e 100644 --- a/docs/source/acquire_upload/prepare_before_acquisition.md +++ b/docs/source/acquire_upload/prepare_before_acquisition.md @@ -21,7 +21,7 @@ The funding endpoint will be used during data upload to populate your data descr ```{raw} html
- + @@ -84,7 +84,7 @@ The funding endpoint will be used during data upload to populate your data descr fetch('https://aind-metadata-service/api/v2/funding/' + encodeURIComponent(projectName)) .then(response => { if (!response.ok) { - throw new Error('HTTP error! status: ' + response.status); + return response.text().then(text => { throw new Error(text || 'HTTP error! status: ' + response.status); }); } return response.json(); }) @@ -175,7 +175,7 @@ The investigators endpoint will be used during data upload to populate your data fetch('https://aind-metadata-service/api/v2/investigators/' + encodeURIComponent(projectName)) .then(response => { if (!response.ok) { - throw new Error('HTTP error! status: ' + response.status); + return response.text().then(text => { throw new Error(text || 'HTTP error! status: ' + response.status); }); } return response.json(); }) @@ -266,17 +266,19 @@ Subject metadata is populated by lab animal services (LAS) without your involvem fetch('https://aind-metadata-service/api/v2/subject/' + encodeURIComponent(subjectId)) .then(response => { - if (!response.ok) { - throw new Error('HTTP error! status: ' + response.status); + if (!response.ok && response.status !== 400) { + return response.text().then(text => { throw new Error(text || 'HTTP error! status: ' + response.status); }); } - return response.json(); + return response.json().then(data => ({ data, status: response.status })); }) - .then(response => { - const data = response.data || response; - resultDiv.style.backgroundColor = '#d4edda'; - resultDiv.style.border = '1px solid #28a745'; - resultDiv.innerHTML = 'Subject Information:
' + 
-                              JSON.stringify(data, null, 2) + '
'; + .then(({ data, status }) => { + const subject = data.data || data; + const isInvalid = status === 400; + resultDiv.style.backgroundColor = isInvalid ? '#fff3cd' : '#d4edda'; + resultDiv.style.border = isInvalid ? '1px solid #ffc107' : '1px solid #28a745'; + resultDiv.innerHTML = (isInvalid ? 'Warning: subject data failed schema validation:' : 'Subject Information:') + + '
' +
+                              JSON.stringify(subject, null, 2) + '
'; }) .catch(error => { resultDiv.style.backgroundColor = '#f8d7da'; @@ -395,7 +397,7 @@ Currently, only NSB procedures are automatically attached to data assets during ### Custom procedures -Custom [Procedures](https://aind-data-schema.readthedocs.io/en/latest/procedures.html) require you to generate a `procedures.json` file manually. Note that the `data-transfer-service` will **NOT** merge your procedures with any stored in NSB, you must pull the NSB procedures and manually merge them ahead of time, please reach out to Scientific Computing for help with this process. +Custom [Procedures](https://aind-data-schema.readthedocs.io/en/latest/procedures.html) require you to generate a `procedures.json` file manually. Please only provide metadata for procedures that are not stored by NSB. ### NSB procedures @@ -457,17 +459,19 @@ Standardized procedures that are performed by NSB (link?) are uploaded and acces fetch('https://aind-metadata-service/api/v2/procedures/' + encodeURIComponent(subjectId)) .then(response => { - if (!response.ok) { - throw new Error('HTTP error! status: ' + response.status); + if (!response.ok && response.status !== 400) { + return response.text().then(text => { throw new Error(text || 'HTTP error! status: ' + response.status); }); } - return response.json(); + return response.json().then(data => ({ data, status: response.status })); }) - .then(response => { - const data = response.data || response; - resultDiv.style.backgroundColor = '#d4edda'; - resultDiv.style.border = '1px solid #28a745'; - resultDiv.innerHTML = 'Procedures Information:
' + 
-                              JSON.stringify(data, null, 2) + '
'; + .then(({ data, status }) => { + const procedures = data.data || data; + const isInvalid = status === 400; + resultDiv.style.backgroundColor = isInvalid ? '#fff3cd' : '#d4edda'; + resultDiv.style.border = isInvalid ? '1px solid #ffc107' : '1px solid #28a745'; + resultDiv.innerHTML = (isInvalid ? 'Warning: procedures data failed schema validation:' : 'Procedures Information:') + + '
' +
+                              JSON.stringify(procedures, null, 2) + '
'; }) .catch(error => { resultDiv.style.backgroundColor = '#f8d7da'; From 3822dc51e44ca2e350b1bfd430b63d30b4a722af Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Mon, 1 Jun 2026 09:40:57 -0700 Subject: [PATCH 2/6] docs: draft QC diagram (#160) --- docs/source/diagrams/mid_level/QC.drawio | 115 ++++++++++++++++++ docs/source/diagrams/mid_level/QC.drawio.svg | 4 + .../source/explore_analyze/quality_control.md | 6 +- 3 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 docs/source/diagrams/mid_level/QC.drawio create mode 100644 docs/source/diagrams/mid_level/QC.drawio.svg diff --git a/docs/source/diagrams/mid_level/QC.drawio b/docs/source/diagrams/mid_level/QC.drawio new file mode 100644 index 0000000..23c37df --- /dev/null +++ b/docs/source/diagrams/mid_level/QC.drawio @@ -0,0 +1,115 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/source/diagrams/mid_level/QC.drawio.svg b/docs/source/diagrams/mid_level/QC.drawio.svg new file mode 100644 index 0000000..0330332 --- /dev/null +++ b/docs/source/diagrams/mid_level/QC.drawio.svg @@ -0,0 +1,4 @@ + + + +
Metadata
(DocDB)
Data Assets
(S3)
Static Data Stores
Static QC Portal
(data.allenneuraldynamics.org/qc)
Microsoft Entra
Editable QC Portal
(qc.allenneuraldynamics.org/view)
v
QC metadata
v
Reference media
OAuth
Data Consumers
User QCMetric
updates
Processes that transform data
User review
submission
 \ No newline at end of file diff --git a/docs/source/explore_analyze/quality_control.md b/docs/source/explore_analyze/quality_control.md index 06d6e94..87489c3 100644 --- a/docs/source/explore_analyze/quality_control.md +++ b/docs/source/explore_analyze/quality_control.md @@ -8,4 +8,8 @@ Please see the documentation on [QualityControl](https://aind-data-schema.readth ## QC Portal -Please see the [QC Portal](https://github.com/AllenNeuralDynamics/aind-qc-portal?tab=readme-ov-file) documentation for more information. \ No newline at end of file +![QC diagram](../diagrams/mid_level/QC.drawio.svg) + +The QC Portal is a web app that allows users to explore the quality control metadata for data assets and, in edit mode, modify the value and state of metrics to annotate assets as passing or failing QC. + +Please see the [QC Portal](https://github.com/AllenNeuralDynamics/aind-qc-portal?tab=readme-ov-file) documentation for more information. From 79c80c8e0ca9b671860c8f0e34f99e9cdbe071a3 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Mon, 1 Jun 2026 10:20:53 -0700 Subject: [PATCH 3/6] docs: rework the find data sections --- docs/source/explore_analyze/find_data.md | 25 ++++++++++++++++++++---- docs/source/explore_analyze/index.md | 14 ++++++------- docs/source/index.md | 8 ++++---- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/docs/source/explore_analyze/find_data.md b/docs/source/explore_analyze/find_data.md index fdb2d24..46871f5 100644 --- a/docs/source/explore_analyze/find_data.md +++ b/docs/source/explore_analyze/find_data.md @@ -1,6 +1,23 @@ # Find data -Raw assets uploaded from platforms at AIND are run through automated pipelines that produce derived assets. You can find these assets by performing a query on our metadata database using your project name and other fields unique to your experiment. **All analyses at AIND should begin with a query that returns a group of data assets, filtered by passing quality control**. +Raw assets uploaded from platforms at AIND are run through automated pipelines that produce derived assets. You can explore these assets through the [Data Portal](https://data.allenneuraldynamics.org/assets). + +The Data Portal exposes a range of views tailored to different slices of the data: + +- The [**Assets**](https://data.allenneuraldynamics.org/assets) view is your entry point into all data assets acquired in Neural Dynamics. +- [**Subject**](https://data.allenneuraldynamics.org/subject) views let you explore the history of an experimental subject from birth through surgical procedures, data acquisitions, perfusion, etc. Clicking into individual events pulls up the detailed metadata about that event as well as interactive viewers. +- [**Project**](https://data.allenneuraldynamics.org/project) views show you the assets associated with each `project_name`. Projects that are tracking curriculum metadata can easily view the status of all mice from this dashboard. + +There are also Platform dashboards for each of the major platforms in Neural Dynamics: + +| Platform | Dashboard | Internal Site | +| -- | -- | +| SmartSPIM | [Dashboard](https://data.allenneuraldynamics.org/smartspim) | [Internal Site](https://alleninstitute.sharepoint.com/sites/NeuralDynamics/SitePages/SmartSPIM-Platform.aspx) | +| Fiber Photometry | [Dashboard](https://data.allenneuraldynamics.org/fiber_photometry) | [Internal site](https://alleninstitute.sharepoint.com/sites/NeuralDynamics/SitePages/Fiber-Photometry-Platform.aspx) | +| Dynamic Foraging | [Dashboard](https://data.allenneuraldynamics.org/dynamic_foraging)| | +| VR Foraging | [Dashboard](https://data.allenneuraldynamics.org/vr_foraging) | | + +Analysis scripts should find assets using queries on our metadata database using your project name and other fields unique to your experiment. **All analyses at AIND should begin with a query that returns a group of data assets, filtered by passing quality control**. Some fields that are commonly used to filter assets: @@ -9,11 +26,11 @@ Some fields that are commonly used to filter assets: - `data_description.modalities.abbreviation` - `subject.subject_id` - `acquisition.acquisition_start_time` -- `quality_control.status` and `quality_control.metrics.status_history` +- `acquisition.acquisition_type`: this is the primary string that should differentiate acquisitions within the same project +- `quality_control.status` -You may also find it useful to tag your data with custom strings at the time of upload. These tags will make it easy to find cluster your data into different subsets. +You may also find it useful to tag your data with custom strings at the time of upload. These tags will make it easy to cluster your data into different subsets. -- `acquisition.acquisition_type`: this is the primary string that should differentiate acquisitions within the same project - `data_description.tags`: this is a list of strings you can use to cluster assets by things that aren't well represented in the metadata. ## Querying the metadata database diff --git a/docs/source/explore_analyze/index.md b/docs/source/explore_analyze/index.md index fbb4e37..bbd1183 100644 --- a/docs/source/explore_analyze/index.md +++ b/docs/source/explore_analyze/index.md @@ -4,19 +4,19 @@ orphan: true # Explore, QC & analyze +The [Data Portal](https://data.allenneuraldynamics.org/assets) should be your first stop for finding data assets. + ## I want to... [Quality control my processed data assets](quality_control.md) before starting analysis. -[Find data](find_data.md) based on its stored metadata. - -Learn [](co_best_practices.md) - -Learn about different approaches to [analyze data](analyze_data.md) in the cloud: +[Find and query data](find_data.md) based on its stored metadata. -[Automate my analysis](analyze_data.md#analysis-framework) using the Analysis Framework. +[Learn about different approaches to analyze data](analyze_data.md) in the cloud. -[Explore custom tools and apps](analyze_data.md#custom-tools) for annotation and data exploration. +- [Learn about Code Ocean](co_best_practices.md) best practices. +- [Automate my analysis](analyze_data.md#analysis-framework) using the Analysis Framework. +- [Explore custom tools and apps](analyze_data.md#custom-tools) for annotation and data exploration. [Find outreach events](outreach.md). diff --git a/docs/source/index.md b/docs/source/index.md index 15555e6..feb3ae3 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -35,7 +35,7 @@ Follow these links to request access to: [Data organization](policies_practices/data_organization.md), [data governance](policies_practices/data_governance.md), [software practices](policies_practices/software_practices.md), or [visualize how our software and systems interact](aind/diagrams.md). ```{toctree} -:maxdepth: 1 +:maxdepth: 2 :hidden: :caption: Acquire, upload & process @@ -48,7 +48,7 @@ acquire_upload/process_data ``` ```{toctree} -:maxdepth: 1 +:maxdepth: 2 :hidden: :caption: Explore, QC & analyze @@ -61,7 +61,7 @@ explore_analyze/outreach ``` ```{toctree} -:maxdepth: 1 +:maxdepth: 2 :hidden: :caption: Policies & practices @@ -76,7 +76,7 @@ policies_practices/docs ``` ```{toctree} -:maxdepth: 1 +:maxdepth: 2 :hidden: :caption: AIND Resources From d08d4a5ab8777c15095a9ff8f675f2d11459fa75 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Mon, 1 Jun 2026 10:26:21 -0700 Subject: [PATCH 4/6] docs: move data portal up --- docs/source/explore_analyze/find_data.md | 17 ----------------- docs/source/explore_analyze/index.md | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/docs/source/explore_analyze/find_data.md b/docs/source/explore_analyze/find_data.md index 46871f5..323bb8d 100644 --- a/docs/source/explore_analyze/find_data.md +++ b/docs/source/explore_analyze/find_data.md @@ -1,22 +1,5 @@ # Find data -Raw assets uploaded from platforms at AIND are run through automated pipelines that produce derived assets. You can explore these assets through the [Data Portal](https://data.allenneuraldynamics.org/assets). - -The Data Portal exposes a range of views tailored to different slices of the data: - -- The [**Assets**](https://data.allenneuraldynamics.org/assets) view is your entry point into all data assets acquired in Neural Dynamics. -- [**Subject**](https://data.allenneuraldynamics.org/subject) views let you explore the history of an experimental subject from birth through surgical procedures, data acquisitions, perfusion, etc. Clicking into individual events pulls up the detailed metadata about that event as well as interactive viewers. -- [**Project**](https://data.allenneuraldynamics.org/project) views show you the assets associated with each `project_name`. Projects that are tracking curriculum metadata can easily view the status of all mice from this dashboard. - -There are also Platform dashboards for each of the major platforms in Neural Dynamics: - -| Platform | Dashboard | Internal Site | -| -- | -- | -| SmartSPIM | [Dashboard](https://data.allenneuraldynamics.org/smartspim) | [Internal Site](https://alleninstitute.sharepoint.com/sites/NeuralDynamics/SitePages/SmartSPIM-Platform.aspx) | -| Fiber Photometry | [Dashboard](https://data.allenneuraldynamics.org/fiber_photometry) | [Internal site](https://alleninstitute.sharepoint.com/sites/NeuralDynamics/SitePages/Fiber-Photometry-Platform.aspx) | -| Dynamic Foraging | [Dashboard](https://data.allenneuraldynamics.org/dynamic_foraging)| | -| VR Foraging | [Dashboard](https://data.allenneuraldynamics.org/vr_foraging) | | - Analysis scripts should find assets using queries on our metadata database using your project name and other fields unique to your experiment. **All analyses at AIND should begin with a query that returns a group of data assets, filtered by passing quality control**. Some fields that are commonly used to filter assets: diff --git a/docs/source/explore_analyze/index.md b/docs/source/explore_analyze/index.md index bbd1183..ea98302 100644 --- a/docs/source/explore_analyze/index.md +++ b/docs/source/explore_analyze/index.md @@ -4,7 +4,22 @@ orphan: true # Explore, QC & analyze -The [Data Portal](https://data.allenneuraldynamics.org/assets) should be your first stop for finding data assets. +Raw assets uploaded from platforms at AIND are run through automated pipelines that produce derived assets. You can explore these assets through the [Data Portal](https://data.allenneuraldynamics.org/assets). + +The Data Portal exposes a range of views tailored to different slices of the data: + +- The [**Assets**](https://data.allenneuraldynamics.org/assets) view is your entry point into all data assets acquired in Neural Dynamics. +- [**Subject**](https://data.allenneuraldynamics.org/subject) views let you explore the history of an experimental subject from birth through surgical procedures, data acquisitions, perfusion, etc. Clicking into individual events pulls up the detailed metadata about that event as well as interactive viewers. +- [**Project**](https://data.allenneuraldynamics.org/project) views show you the assets associated with each `project_name`. Projects that are tracking curriculum metadata can easily view the status of all mice from this dashboard. + +There are also Platform dashboards for each of the major platforms in Neural Dynamics: + +| Platform | Dashboard | Internal Site | +| -- | -- | +| SmartSPIM | [Dashboard](https://data.allenneuraldynamics.org/smartspim) | [Internal Site](https://alleninstitute.sharepoint.com/sites/NeuralDynamics/SitePages/SmartSPIM-Platform.aspx) | +| Fiber Photometry | [Dashboard](https://data.allenneuraldynamics.org/fiber_photometry) | [Internal site](https://alleninstitute.sharepoint.com/sites/NeuralDynamics/SitePages/Fiber-Photometry-Platform.aspx) | +| Dynamic Foraging | [Dashboard](https://data.allenneuraldynamics.org/dynamic_foraging)| | +| VR Foraging | [Dashboard](https://data.allenneuraldynamics.org/vr_foraging) | | ## I want to... From 2be86d3a2cbe9f77058ced6911faf82e58db9c96 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Mon, 1 Jun 2026 10:26:45 -0700 Subject: [PATCH 5/6] docs: typo --- docs/source/explore_analyze/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/explore_analyze/index.md b/docs/source/explore_analyze/index.md index ea98302..f491413 100644 --- a/docs/source/explore_analyze/index.md +++ b/docs/source/explore_analyze/index.md @@ -15,7 +15,7 @@ The Data Portal exposes a range of views tailored to different slices of the dat There are also Platform dashboards for each of the major platforms in Neural Dynamics: | Platform | Dashboard | Internal Site | -| -- | -- | +| -- | -- | -- | | SmartSPIM | [Dashboard](https://data.allenneuraldynamics.org/smartspim) | [Internal Site](https://alleninstitute.sharepoint.com/sites/NeuralDynamics/SitePages/SmartSPIM-Platform.aspx) | | Fiber Photometry | [Dashboard](https://data.allenneuraldynamics.org/fiber_photometry) | [Internal site](https://alleninstitute.sharepoint.com/sites/NeuralDynamics/SitePages/Fiber-Photometry-Platform.aspx) | | Dynamic Foraging | [Dashboard](https://data.allenneuraldynamics.org/dynamic_foraging)| | From 31a3733d6c7314667995cd1ebce0c27d1a4007b3 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Mon, 1 Jun 2026 13:24:48 -0700 Subject: [PATCH 6/6] docs: generalize --- docs/source/explore_analyze/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/explore_analyze/index.md b/docs/source/explore_analyze/index.md index f491413..ab68aea 100644 --- a/docs/source/explore_analyze/index.md +++ b/docs/source/explore_analyze/index.md @@ -10,7 +10,7 @@ The Data Portal exposes a range of views tailored to different slices of the dat - The [**Assets**](https://data.allenneuraldynamics.org/assets) view is your entry point into all data assets acquired in Neural Dynamics. - [**Subject**](https://data.allenneuraldynamics.org/subject) views let you explore the history of an experimental subject from birth through surgical procedures, data acquisitions, perfusion, etc. Clicking into individual events pulls up the detailed metadata about that event as well as interactive viewers. -- [**Project**](https://data.allenneuraldynamics.org/project) views show you the assets associated with each `project_name`. Projects that are tracking curriculum metadata can easily view the status of all mice from this dashboard. +- [**Project**](https://data.allenneuraldynamics.org/project) views show data acquisitions for each subject within a project, and can be used to identify the modality or behavior curriculum stage for each acquisition. There are also Platform dashboards for each of the major platforms in Neural Dynamics: