spec updates, updates to carry locaiton thru to end and addition of p…

pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml

@@ -44,7 +44,8 @@ transform:
       DirOut=/pfs/out \
       DirErr=/pfs/out/errored_datums \
       FileSchmStats=$FILE_SCHEMA_STATS \
-      FileSchmQm=$FILE_SCHEMA_QM 
+      FileSchmQm=$FILE_SCHEMA_QM \
+      "DirSubCopy=location"
 
     echo "=== flow.concH2oSalinity.grp.split complete ==="
     EOF

pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml

@@ -41,7 +41,7 @@ transform:
       DirErr=/pfs/out/errored_datums \
       "DirData=stats|quality_metrics" \
       PathPubWb=$PUB_WORKBOOKS \
-      "DirSubCopy=science_review_flags|group"
+      "DirSubCopy=science_review_flags|group|location"
     #
     #
     # Export Level 1 data to bucket
@@ -121,10 +121,12 @@ transform:
     # folders (group, science_review_flags) sit directly under the group at index 7.
     GROUP_METADATA_INDEX: "7"
     GROUP_METADATA_NAMES: group,science_review_flags
-    # Data types (stats, quality_metrics) remain nested under the CFGLOC level,
-    # one deeper than the group metadata, at index 8.
+    # Data types (stats, quality_metrics, location) remain nested under the CFGLOC
+    # level, one deeper than the group metadata, at index 8. Location must be listed
+    # here — otherwise level1_consolidate drops the CFGLOC/location/*.json files that
+    # pub_files' CFGLOC discovery loop relies on.
     DATA_TYPE_INDEX: "8"
-    DATA_TYPE_NAMES: stats,quality_metrics
+    DATA_TYPE_NAMES: stats,quality_metrics,location
 
 # Environment variables for pub_workbook_loader
     OUT_PATH_WORKBOOK: /tmp/pub_workbooks

pipe/concH2oSoilSalinity/concH2oSoilSalinity_position_history_loader.yaml

@@ -0,0 +1,45 @@
+---
+# Production concH2oSoilSalinity position-history loader.
+# Queries PDR directly for the full CFGLOC × asset × calibration × geolocation
+# history for every enviroscan probe, applies CVALD1-derived depth adjustment
+# per VER (501-508), and emits one JSON per CFGLOC. Time-invariant output —
+# a Feb 2015 CSV, a June 2023 CSV, and a today CSV all contain the same rows.
+#
+# Loader is unfiltered by site (PDR query is cheap enough at ~200 CFGLOCs).
+# pub_format_and_package crosses this repo with monthly pub_group datums so
+# every monthly sensor_positions.csv gets the complete CFGLOC history tree.
+pipeline:
+  name: concH2oSoilSalinity_position_history_loader
+transform:
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-conc-h2o-salinity-position-history-loader:sha-fb1172f
+  cmd:
+  - /bin/bash
+  stdin:
+  - '#!/bin/bash'
+  - python3 -m concH2oSoilSalinity_position_history_loader.concH2oSoilSalinity_position_history_loader_main
+  env:
+    OUT_PATH: /pfs/out
+    ERR_PATH: /pfs/out/errored_datums
+    LOG_LEVEL: INFO
+  secrets:
+  - name: pdr-secret
+    mount_path: /var/db_secret
+input:
+  pfs:
+    repo: enviroscan_ingest_days
+    glob: /tick/
+    branch: master
+    empty_files: true
+autoscaling: true
+resource_requests:
+  memory: 300M
+  cpu: 0.2
+resource_limits:
+  memory: 800M
+  cpu: 1
+sidecar_resource_requests:
+  memory: 300M
+  cpu: 0.3
+scheduling_spec:
+  node_selector:
+    cloud.google.com/compute-class: pach-pipeline-class

pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_egress_and_publish.yaml

@@ -158,14 +158,14 @@ input:
         empty_files: true
 autoscaling: true
 resource_requests:
-  memory: 400M
-  cpu: .5
+  memory: 1G
+  cpu: 1.7
 resource_limits:
-  memory: 800M
-  cpu: 1.3
+  memory: 2G
+  cpu: 2.5
 sidecar_resource_requests:
   memory: 3G
-  cpu: 1
+  cpu: 2.5
 datum_set_spec:
   number: 1
 scheduling_spec:

pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_format_and_package.yaml

@@ -2,7 +2,7 @@
 pipeline:
   name: concH2oSoilSalinity_pub_format_and_package
 transform:
-  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-pub-grp-pack:v4.2.1
+  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-pub-grp-pack:sha-f5b2bdc
   cmd:
   - sh
   - "-c"
@@ -116,20 +116,29 @@ transform:
       mount_path: /var/github_secret
 
 input:
-  pfs:
-    name: GROUPED_PATH
-    repo: concH2oSoilSalinity_pub_group
-    # Glob must be product-monthly or product-site-monthly. Product-site-month datums reduce unneccesary republication.
-    # path structure is e.g. DP1.00098.001/2023/04/CPER/04 (product/year/month/site/day)
-    glob: /*/*/*/*
+  cross:
+  - pfs:
+      name: GROUPED_PATH
+      repo: concH2oSoilSalinity_pub_group
+      # Glob must be product-monthly or product-site-monthly. Product-site-month datums reduce unnecessary republication.
+      # path structure is e.g. DP1.00098.001/2023/04/CPER/04 (product/year/month/site/day)
+      glob: /*/*/*/*
+  - pfs:
+      name: POSITION_HISTORY_PATH
+      repo: concH2oSoilSalinity_position_history_loader
+      # Full repo as one blob per commit. Crossed with monthly datums so every
+      # month gets the complete CFGLOC position history. pub_files reads
+      # enviroscan/<CFGLOC>/position_history/<CFGLOC>_history.json per CFGLOC.
+      glob: /
+      empty_files: false
 parallelism_spec:
   constant: 10
 autoscaling: true
 resource_requests:
-  memory: 400M
+  memory: 800M
   cpu: 1
 resource_limits:
-  memory: 800M
+  memory: 1G
   cpu: 1.2
 sidecar_resource_requests:
   memory: 2G

pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml

@@ -33,6 +33,7 @@ transform:
       DirErr=/pfs/out/errored_datums \
       "WndwAgr=005|030" \
       "WghtAlphBeta=1|2" \
+      "DirSubCopy=location" \
       FileSchmQm=$FILE_SCHEMA_QM \
       Thsh=0.6 \
       "GrpQfAlph1=VSWCsoilSpecificDepth01:VSWCsoilSpecificDepth01NullQF|VSWCsoilSpecificDepth01GapQF|VSWCsoilSpecificDepth01RangeQF|VSWCsoilSpecificDepth01StepQF|VSWCsoilSpecificDepth01PersistenceQF|VSWCsoilSpecificDepth01SpikeQF|VSWCDepth01SuspectCalQF|tempTestDepth01QF" \
@@ -146,6 +147,12 @@ transform:
             glob_pattern: /pfs/QAQC_PLAUSIBILITY_PATH/*/*/*/*/*/*/flags/**
             # Join on named location (already joined below by day)
             join_indices: [6]
+        - path:
+            name: QAQC_PLAUSIBILITY_PATH
+            # Location JSONs — carried through via DirSubCopy so pub_files
+            # can discover CFGLOCs downstream in the pub_group output.
+            glob_pattern: /pfs/QAQC_PLAUSIBILITY_PATH/*/*/*/*/*/*/location/**
+            join_indices: [6]
         - path:
             name: REGULARIZED_FLAGS_PATH
             # Filter for flags directory

pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml

@@ -24,6 +24,7 @@ transform:
       DirErr=/pfs/out/errored_datums \
       FileSchmStat=$FILE_SCHEMA_STATS \
       "WndwAgr=005|030" \
+      "DirSubCopy=location" \
       "TermStat1=VSWCsoilSpecificDepth01:mean|minimum|maximum|variance|numPts|stdEr|expUncert(wrap.ucrt.dp01.cal.mult.envsc)" \
       "TermStat2=VSWCsoilSpecificDepth02:mean|minimum|maximum|variance|numPts|stdEr|expUncert(wrap.ucrt.dp01.cal.mult.envsc)" \
       "TermStat3=VSWCsoilSpecificDepth03:mean|minimum|maximum|variance|numPts|stdEr|expUncert(wrap.ucrt.dp01.cal.mult.envsc)" \
@@ -63,6 +64,12 @@ transform:
             glob_pattern: /pfs/QAQC_PATH/*/*/*/*/*/*/data/**
             # Join on named location (already joined below by day)
             join_indices: [6]
+        - path:
+            name: QAQC_PATH
+            # Location JSONs — carried through via DirSubCopy so pub_files
+            # can discover CFGLOCs downstream in the pub_group output.
+            glob_pattern: /pfs/QAQC_PATH/*/*/*/*/*/*/location/**
+            join_indices: [6]
         - path:
             name: UNCERTAINTY_PATH
             # Filter for data directory

pipe/concH2oSoilSalinity/pipe_list_concH2oSoilSalinity.txt

@@ -12,6 +12,7 @@ concH2oSoilSalinity_stats_group_and_compute.yaml
 concH2oSoilSalinity_qm_join_stats.yaml
 concH2oSoilSalinity_group_split.yaml
 concH2oSoilSalinity_level1_group_consolidate_srf.yaml
+concH2oSoilSalinity_position_history_loader.yaml
 concH2oSoilSalinity_cron_monthly_and_pub_control.yaml
 concH2oSoilSalinity_pub_group.yaml
 concH2oSoilSalinity_pub_format_and_package.yaml