Draft dataset loader Xenium cervical cancer with pathology-based ground truth cell type annotation

README.md

@@ -90,7 +90,7 @@ Format:
      labels: 'cell_labels', 'nucleus_labels', 'groundtruth_cell_labels'
      points: 'transcripts'
      shapes: 'cell_boundaries', 'nucleus_boundaries'
-     tables: 'metadata'
+     tables: 'table'
      coordinate_systems: 'global'
 
 </div>
@@ -148,7 +148,7 @@ Data structure:
 
 *tables*
 
-`metadata`: Metadata of spatial dataset.
+`table`: Metadata of spatial dataset.
 
 | Slot | Type | Description |
 |:---|:---|:---|

scripts/create_resources/xenium_gt_annotated_data.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# get the root of the directory
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+# ensure that the command below is run from the root of the repository
+cd "$REPO_ROOT"
+
+set -e
+
+cat > /tmp/params.yaml << HERE
+param_list:
+  - id: tenx_xenium_groundtruth/cervical_cancer
+    input: s3://hca-op-spatial/datasets/gt_annotated_data/Xenium_Prime_Cervical_Cancer_FFPE_Aligned.zarr
+    dataset_name: 10X Xenium - Cervical Cancer
+    dataset_url: https://www.10xgenomics.com/datasets/xenium-prime-ffpe-human-cervical-cancer
+    dataset_summary: Gene expression library for 5K Xenium Prime panel + 100 custom genes on cervical cancer sample
+    dataset_description: Xenium Prime 5K In Situ Gene Expression with Cell Segmentation data for human cervical cancer (FFPE) using the Xenium Prime 5K Human Pan Tissue and Pathways Panel plus 100 Custom Genes.
+    dataset_organism: homo_sapiens
+
+publish_dir: temp
+output_dataset: '\$id/dataset.zarr'
+output_state: '\$id/state.yaml'
+HERE
+
+# convert to zarr
+nextflow run . \
+  -main-script target/nextflow/datasets/loaders/tenx_xenium_groundtruth/main.nf \
+  -profile docker \
+  -resume \
+  -params-file /tmp/params.yaml
+
+# sync to s3
+# aws s3 sync --profile op \
+#   "resources_test/datasets/2023_10x_mouse_brain_xenium_rep1" \
+#   "s3://openproblems-data/resources_test/common/2023_10x_mouse_brain_xenium_rep1" \
+#   --delete --dryrun

src/api/file_common_ist.yaml

@@ -112,7 +112,7 @@ info:
             description: Geometry of the nucleus boundary
     tables:
       - type: anndata
-        name: "metadata"
+        name: table
         description: Metadata of spatial dataset
         required: true
         uns:

src/datasets/loaders/tenx_xenium_groundtruth/config.vsh.yaml

@@ -0,0 +1,70 @@
+name: tenx_xenium_groundtruth
+namespace: datasets/loaders
+
+argument_groups:
+  - name: Inputs
+    arguments:
+      - type: string
+        name: --input
+        required: true
+        description: A 10x xenium directory or zip file or download url or spatialData object
+      - type: string
+        name: --segmentation_id
+        required: true
+        description: The segmentation identifier
+        multiple: true
+  - name: Metadata
+    arguments:
+      - type: string
+        name: --dataset_id
+        description: "A unique identifier for the dataset"
+        required: true
+      - name: --dataset_name
+        type: string
+        description: Nicely formatted name.
+        required: true
+      - type: string
+        name: --dataset_url
+        description: Link to the original source of the dataset.
+        required: false
+      - name: --dataset_reference
+        type: string
+        description: Bibtex reference of the paper in which the dataset was published.
+        required: false
+      - name: --dataset_summary
+        type: string
+        description: Short description of the dataset.
+        required: true
+      - name: --dataset_description
+        type: string
+        description: Long description of the dataset.
+        required: true
+      - name: --dataset_organism
+        type: string
+        description: The organism of the sample in the dataset.
+        required: false
+  - name: Outputs
+    arguments:
+      - name: "--output"
+        __merge__: /src/api/file_common_ist.yaml
+        direction: output
+        required: true
+
+resources:
+  - type: python_script
+    path: script.py
+
+engines:
+  - type: docker
+    image: openproblems/base_python:1
+    setup:
+      - type: python
+        pypi:
+          - spatialdata-io
+  - type: native
+
+runners:
+  - type: executable
+  - type: nextflow
+    directives:
+      label: [midmem, midcpu, midtime]

src/datasets/loaders/tenx_xenium_groundtruth/script.py

@@ -0,0 +1,75 @@
+import spatialdata as sd
+import shutil
+import os
+
+## VIASH START
+par = {
+    "input": "resources/datasets/gt_annotated_data/Xenium_Prime_Cervical_Cancer_FFPE_Aligned.zarr",
+    "segmentation_id": [
+        "cell",
+        "nucleus",
+    ],
+    "dataset_id": "value",
+    "dataset_name": "value",
+    "dataset_url": "value",
+    "dataset_reference": "value",
+    "dataset_summary": "value",
+    "dataset_description": "value",
+    "dataset_organism": "value",
+    "output": "temp/datasets/10x_xenium/cervical_cancer/spatialData.zarr"
+}
+meta = {
+    "cpus": 1,
+}
+## VIASH END
+
+# read the data
+sdata = sd.read_zarr(
+    store=par["input"],
+    selection=None
+)
+
+print("Raw data input: ", sdata, flush=True)
+
+print("Add uns to table", flush=True)
+new_uns = {
+    "dataset_id": par["dataset_id"],
+    "dataset_name": par["dataset_name"],
+    "dataset_url": par["dataset_url"],
+    "dataset_reference": par["dataset_reference"],
+    "dataset_summary": par["dataset_summary"],
+    "dataset_description": par["dataset_description"],
+    "dataset_organism": par["dataset_organism"],
+    "segmentation_id": par["segmentation_id"],
+}
+for key, value in new_uns.items():
+    sdata.tables["table"].uns[key] = value
+
+# add ground truth cell labels
+## these annotations were derived by Caner Ercan
+sdata.tables["table"].obs["groundtruth_celltype"] = sdata.tables["table"].obs.pop("histoplus_cell_class")
+
+# rename Images
+## rename raw images to accomodate format
+sdata.images['image'] = sdata.images['morphology_focus']
+## rm morphology_focus
+_ = sdata.images.pop("morphology_focus")
+## rename hne image 
+sdata.images['he_image'] = sdata.images['hne_aligned']
+## rm hne_aligned
+_ = sdata.images.pop("hne_aligned")
+
+# rename Labels
+## add ground truth to cell labels
+## these annotations were derived by Caner Ercan
+sdata.Labels['groundtruth_cell_labels'] = sdata.tables['table'].obs.pop('histoplus_cell_class')
+
+print(f"Output: {sdata}", flush=True)
+
+print(f"Writing to '{par['output']}'", flush=True)
+if os.path.exists(par["output"]):
+    shutil.rmtree(par["output"])
+
+print(f"Output: {sdata}")
+
+sdata.write(par["output"])