Redesign STAC generation

.dockerignore

@@ -0,0 +1,6 @@
+.git
+.github
+examples
+notebooks
+**/.pixi
+**/.venv

bin/force-level2-wrapper.sh

@@ -183,7 +183,7 @@ elif [ "$dem" == "Copernicus_30m" ]; then
             export AWS_ENDPOINT_URL_S3='https://eodata.dataspace.copernicus.eu'
 	    echo "Environmental variables AWS_ENDPOINT_URL_S3 not defined. Using default: $AWS_ENDPOINT_URL_S3"
         fi
-        # for f5cmd:
+        # for s5cmd:
         export S3_ENDPOINT_URL=$AWS_ENDPOINT_URL_S3
 
         echo "Running s5cmd commands file"
@@ -239,43 +239,14 @@ rm -rf outputs/.parallel
 
 # create stac catalogue for output
 
-# CITEME_0x65.txt
-export citeme_path=$(cd outputs/l2-ard; ls CITEME*)
-cat /opt/apex-force-wrapper/etc/output-item-header.template | envsubst > outputs/l2-ard/$processing_name-l2-ard.json
-for continent_prj_path in $(cd outputs/l2-ard; ls */datacube-definition.prj); do
-    # europe
-    continent_dir=$(dirname $continent_prj_path)
-    for tile_dir in $(cd outputs/l2-ard; ls -d $continent_dir/X*_Y*); do
-        for boa_path in $(cd outputs/l2-ard; ls $tile_dir/*BOA.tif); do
-            export boa_path
-            export id=$(echo ${boa_path%.tif} | tr '/' '.' )
-            export size=$(ls -l outputs/l2-ard/$boa_path | cut -d ' ' -f 5)
-            export md5sum=$(md5sum outputs/l2-ard/$boa_path | cut -d ' ' -f 1)
-            export title="$(echo ${boa_path%.tif} | tr '/' ' ' | tr '_' ' ')"
-            cat /opt/apex-force-wrapper/etc/output-item-boa-asset.template | envsubst >> outputs/l2-ard/$processing_name-l2-ard.json
-        done
-        for qai_path in $(cd outputs/l2-ard; ls $tile_dir/*QAI.tif); do
-            export qai_path
-            export id=$(echo ${qai_path%.tif} | tr '/' '.' )
-            export size=$(ls -l outputs/l2-ard/$qai_path | cut -d ' ' -f 5)
-            export md5sum=$(md5sum outputs/l2-ard/$qai_path | cut -d ' ' -f 1)
-            export title="$(echo ${qai_path%.tif} | tr '/' ' ' | tr '_' ' ')"
-            cat /opt/apex-force-wrapper/etc/output-item-qai-asset.template | envsubst >> outputs/l2-ard/$processing_name-l2-ard.json
-        done
-        for ovv_path in $(cd outputs/l2-ard; ls $tile_dir/*OVV.jpg); do
-            export ovv_path
-            export id=$(echo ${ovv_path%.tif} | tr '/' '.' )
-            export size=$(ls -l outputs/l2-ard/$ovv_path | cut -d ' ' -f 5)
-            export md5sum=$(md5sum outputs/l2-ard/$ovv_path | cut -d ' ' -f 1)
-            export title="$(echo ${ovv_path%.jpg} | tr '/' ' ' | tr '_' ' ')"
-            cat /opt/apex-force-wrapper/etc/output-item-ovv-asset.template | envsubst >> outputs/l2-ard/$processing_name-l2-ard.json
-        done
-    done
-    export id=datacube-definition.prj
-    export continent_prj_path
-    export title="$continent_dir projection"
-    cat /opt/apex-force-wrapper/etc/output-item-continent.template | envsubst >> outputs/l2-ard/$processing_name-l2-ard.json
-done
-cat /opt/apex-force-wrapper/etc/output-item-footer.template | envsubst >> outputs/l2-ard/$processing_name-l2-ard.json
+uv() {
+  /opt/uv/uv "$@"
+}
+gen-stac() {
+  uv run --project /opt/force-python-tools --no-sync gen-stac "$@"
+}
+
+# TODO instead of hardcoded europe, generate stac for each continent
+mv outputs/l2-ard/CITEME* outputs/l2-ard/europe/
 
-cat /opt/apex-force-wrapper/etc/output-catalogue.template | envsubst > outputs/l2-ard/catalogue.json
+gen-stac outputs/l2-ard/europe --output-path outputs/l2-ard/europe --item-id "$processing_name-level2" --type level2

bin/force-tsa-wrapper.sh

@@ -102,6 +102,8 @@ while [ "$1" != "" ]; do
     fi
 done
 
+find "$input_data_dir"
+
 if [ "$date_range" = "" ]; then
     echo "missing date_range"
     exit 4
@@ -197,7 +199,7 @@ gen-stac() {
   uv run --project /opt/force-python-tools --no-sync gen-stac "$@"
 }
 
-gen-stac "$output_dir" --output-path "$stac_output_dir" --item-id "$processing_name-tsa"
+gen-stac "$output_dir" --output-path "$stac_output_dir" --item-id "$processing_name-tsa" --type tsa
 
 # trace
 find $output_dir -ls

cwl/docker-requirement.yaml

@@ -1 +1 @@
-quay.io/bcdev/force-eoap:0.5.2
+quay.io/bcdev/force-eoap:0.5.6

cwl/force-l2.cwl

@@ -194,7 +194,7 @@ inputs:
     type: int?
     inputBinding:
       prefix: --nproc
-    default: 4
+    default: 2
 
 outputs:
   ## Alternative with a flat file list, breaks structure

cwl/force-tsa.cwl

@@ -6,8 +6,10 @@ requirements:
     networkAccess: true # TODO is this needed, if we can download independently?
   ResourceRequirement:
     # TODO set reasonable limits
-    ramMin: 16384
-    ramMax: 16384
+    #ramMin: 24576
+    #ramMax: 24576
+    ramMin: 22000
+    ramMax: 22000
     #ramMin: 7000
     #ramMax: 7000
     coresMin: 1

examples/run-level2-and-tsa/force-level2.cwl

@@ -553,7 +553,7 @@
             "class": "CommandLineTool",
             "requirements": [
                 {
-                    "dockerPull": "quay.io/bcdev/force-eoap:0.5.2-dev1",
+                    "dockerPull": "quay.io/bcdev/force-eoap:0.5.6",
                     "class": "DockerRequirement"
                 },
                 {
@@ -791,7 +791,7 @@
                     "inputBinding": {
                         "prefix": "--nproc"
                     },
-                    "default": 4,
+                    "default": 2,
                     "id": "#force-l2.cwl/nproc"
                 },
                 {
@@ -987,7 +987,7 @@
             "class": "CommandLineTool",
             "requirements": [
                 {
-                    "dockerPull": "quay.io/bcdev/force-eoap:0.5.2-dev1",
+                    "dockerPull": "quay.io/bcdev/force-eoap:0.5.6",
                     "class": "DockerRequirement"
                 },
                 {

examples/run-level2-and-tsa/force-tsa.cwl

@@ -984,16 +984,16 @@
             "class": "CommandLineTool",
             "requirements": [
                 {
-                    "dockerPull": "quay.io/bcdev/force-eoap:0.5.2-dev1",
+                    "dockerPull": "quay.io/bcdev/force-eoap:0.5.6",
                     "class": "DockerRequirement"
                 },
                 {
                     "networkAccess": true,
                     "class": "NetworkAccess"
                 },
                 {
-                    "ramMin": 16384,
-                    "ramMax": 16384,
+                    "ramMin": 22000,
+                    "ramMax": 22000,
                     "coresMin": 1,
                     "coresMax": 4,
                     "class": "ResourceRequirement"
@@ -1699,7 +1699,7 @@
             "class": "CommandLineTool",
             "requirements": [
                 {
-                    "dockerPull": "quay.io/bcdev/force-eoap:0.5.2-dev1",
+                    "dockerPull": "quay.io/bcdev/force-eoap:0.5.6",
                     "class": "DockerRequirement"
                 },
                 {

python/.python-version

@@ -0,0 +1 @@
+3.13

python/pyproject.toml

@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "force-python-utils"
-version = "0.5.2"
+version = "0.5.6"
 requires-python = ">= 3.10"
 dependencies = [
-    "click>=8.3.2",
+    "click>=8.3.3",
     "geopandas>=1.1.3",
     "py-multihash>=3.0.0",
     "pystac[validation]>=1.14.3",
@@ -21,12 +21,13 @@ download-stac = "stac_staging.cli:download_from_stac"
 force-aoi-converter = "force_utils.force_aoi_converter:main"
 datacube-bounding-box = "force_utils.datacube_definition:compute_datacube_bounding_box"
 gen-stac = "force_utils.cli:gen_stac"
+gen-stac-old = "force_utils.cli:gen_stac_old"
 
 [tool.setuptools.packages.find]
 where = ["src"]
 include = ["stac_staging", "force_utils"]
 
 [dependency-groups]
 dev = [
-    "pytest>=9.0.2",
+    "pytest>=9.0.3",
 ]

python/src/force_utils/__init__.py

@@ -0,0 +1,6 @@
+from importlib.metadata import version, PackageNotFoundError
+
+try:
+    __version__ = version("force-python-utils")
+except PackageNotFoundError as e:
+    pass

python/src/force_utils/cli.py

@@ -1,26 +1,85 @@
-from pathlib import Path
 import logging
+from pathlib import Path
 
 import click
 import pystac
-
+from force_utils.contributor import (
+    FileExtensionMetadataStacContributor,
+    ProjExtensionMetadataStacContributor,
+    RasterExtensionMetadataStacContributor,
+    Level2StacContributor,
+    TsaStacContributor,
+)
 from force_utils.datacube_definition import ForceDataCubeDefinition
+from force_utils.datacube_store import ForceDatacubeStore
+from force_utils.force_stac import ForceStacBuilder
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+
 @click.command()
 @click.argument("datacube-root", type=click.Path(exists=True))
 @click.option("--output-path", "-o", type=click.Path(exists=True), default=Path.cwd())
 @click.option("--item-id", "-i", type=str, default="")
-def gen_stac(datacube_root, output_path, item_id):
+def gen_stac_old(datacube_root, output_path, item_id):
     output_path = Path(output_path)
     logger.info(f"Generating STAC catalog and item for {datacube_root}")
     data_cube_def = ForceDataCubeDefinition(datacube_root)
     catalog = data_cube_def.generate_stac(item_id=item_id)
     catalog.normalize_hrefs(str(output_path))
     logger.info(f"Saving STAC catalog to {output_path.resolve()}")
-    catalog.save(dest_href=str(output_path), catalog_type=pystac.CatalogType.SELF_CONTAINED)
+    catalog.save(
+        dest_href=str(output_path), catalog_type=pystac.CatalogType.SELF_CONTAINED
+    )
+
+
+@click.command()
+@click.argument("datacube-root", type=click.Path(exists=True))
+@click.option("--output-path", "-o", type=click.Path(exists=True), default=Path.cwd())
+@click.option("--item-id", "-i", type=str, default="")
+@click.option("--type", "-t", type=str, required=True)
+@click.option(
+    "--parameter-path", "-p", type=click.Path(exists=True), default=None, multiple=True
+)
+@click.option("--validate", is_flag=True)
+def gen_stac(datacube_root, output_path, item_id, type, parameter_path, validate):
+    output_path = Path(output_path)
+    logger.info(f"Generating STAC catalog and item for {datacube_root}")
+
+    citeme_path_candidates = list(Path(datacube_root).glob(r"CITEME*.txt"))
+    if len(citeme_path_candidates) != 1:
+        raise ValueError(
+            f"No CITEME files found in {list(Path(datacube_root).iterdir())}"
+        )
+
+    citeme_path = next(iter(citeme_path_candidates))
+    datacube_store = ForceDatacubeStore(
+        datacube_root, parameter_files=parameter_path, citeme_path=citeme_path
+    )
+
+    type_contributor = dict(
+        level2=Level2StacContributor,
+        tsa=TsaStacContributor,
+    ).get(type)
+    stac_builder = ForceStacBuilder(
+        datacube_store,
+        contributors=[
+            FileExtensionMetadataStacContributor(),
+            ProjExtensionMetadataStacContributor(),
+            RasterExtensionMetadataStacContributor(),
+            type_contributor(),
+        ],
+    )
+    catalog = stac_builder.generate_stac(item_id=item_id)
+    catalog.normalize_hrefs(str(output_path))
+    if validate:
+        catalog.validate_all()
+    logger.info(f"Saving STAC catalog to {output_path.resolve()}")
+    catalog.save(
+        dest_href=str(output_path), catalog_type=pystac.CatalogType.SELF_CONTAINED
+    )
+
 
 @click.command()
 @click.argument("data_cube_root", type=click.Path(exists=True))