From f634677bfcd05ac927ae83757ff6c00d0612d564 Mon Sep 17 00:00:00 2001 From: Teresa Burlingame Date: Wed, 8 Jul 2026 10:28:33 -0700 Subject: [PATCH 1/2] spec updates, updates to carry locaiton thru to end and addition of position loader. --- .../concH2oSoilSalinity_group_split.yaml | 3 +- ...Salinity_level1_group_consolidate_srf.yaml | 10 +++-- ...oSoilSalinity_position_history_loader.yaml | 45 +++++++++++++++++++ ...2oSoilSalinity_pub_egress_and_publish.yaml | 10 ++--- ...2oSoilSalinity_pub_format_and_package.yaml | 27 +++++++---- .../concH2oSoilSalinity_qm_join_stats.yaml | 7 +++ ...oSoilSalinity_stats_group_and_compute.yaml | 7 +++ .../pipe_list_concH2oSoilSalinity.txt | 1 + 8 files changed, 91 insertions(+), 19 deletions(-) create mode 100644 pipe/concH2oSoilSalinity/concH2oSoilSalinity_position_history_loader.yaml diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml index 4f3d41f075..ba4d149d2c 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml @@ -44,7 +44,8 @@ transform: DirOut=/pfs/out \ DirErr=/pfs/out/errored_datums \ FileSchmStats=$FILE_SCHEMA_STATS \ - FileSchmQm=$FILE_SCHEMA_QM + FileSchmQm=$FILE_SCHEMA_QM \ + "DirSubCopy=location" echo "=== flow.concH2oSalinity.grp.split complete ===" EOF diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml index 09d6e500b5..0e83e25df1 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml @@ -41,7 +41,7 @@ transform: DirErr=/pfs/out/errored_datums \ "DirData=stats|quality_metrics" \ PathPubWb=$PUB_WORKBOOKS \ - "DirSubCopy=science_review_flags|group" + "DirSubCopy=science_review_flags|group|location" # # # Export Level 1 data to bucket @@ -121,10 +121,12 @@ transform: # folders (group, science_review_flags) sit directly under the group at index 7. GROUP_METADATA_INDEX: "7" GROUP_METADATA_NAMES: group,science_review_flags - # Data types (stats, quality_metrics) remain nested under the CFGLOC level, - # one deeper than the group metadata, at index 8. + # Data types (stats, quality_metrics, location) remain nested under the CFGLOC + # level, one deeper than the group metadata, at index 8. Location must be listed + # here — otherwise level1_consolidate drops the CFGLOC/location/*.json files that + # pub_files' CFGLOC discovery loop relies on. DATA_TYPE_INDEX: "8" - DATA_TYPE_NAMES: stats,quality_metrics + DATA_TYPE_NAMES: stats,quality_metrics,location # Environment variables for pub_workbook_loader OUT_PATH_WORKBOOK: /tmp/pub_workbooks diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_position_history_loader.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_position_history_loader.yaml new file mode 100644 index 0000000000..da348aa409 --- /dev/null +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_position_history_loader.yaml @@ -0,0 +1,45 @@ +--- +# Production concH2oSoilSalinity position-history loader. +# Queries PDR directly for the full CFGLOC × asset × calibration × geolocation +# history for every enviroscan probe, applies CVALD1-derived depth adjustment +# per VER (501-508), and emits one JSON per CFGLOC. Time-invariant output — +# a Feb 2015 CSV, a June 2023 CSV, and a today CSV all contain the same rows. +# +# Loader is unfiltered by site (PDR query is cheap enough at ~200 CFGLOCs). +# pub_format_and_package crosses this repo with monthly pub_group datums so +# every monthly sensor_positions.csv gets the complete CFGLOC history tree. +pipeline: + name: concH2oSoilSalinity_position_history_loader +transform: + image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-conc-h2o-salinity-position-history-loader:sha-fb1172f + cmd: + - /bin/bash + stdin: + - '#!/bin/bash' + - python3 -m concH2oSoilSalinity_position_history_loader.concH2oSoilSalinity_position_history_loader_main + env: + OUT_PATH: /pfs/out + ERR_PATH: /pfs/out/errored_datums + LOG_LEVEL: INFO + secrets: + - name: pdr-secret + mount_path: /var/db_secret +input: + pfs: + repo: enviroscan_ingest_days + glob: /tick/ + branch: master + empty_files: true +autoscaling: true +resource_requests: + memory: 300M + cpu: 0.2 +resource_limits: + memory: 800M + cpu: 1 +sidecar_resource_requests: + memory: 300M + cpu: 0.3 +scheduling_spec: + node_selector: + cloud.google.com/compute-class: pach-pipeline-class diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_egress_and_publish.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_egress_and_publish.yaml index 846e21b448..3db142117b 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_egress_and_publish.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_egress_and_publish.yaml @@ -158,14 +158,14 @@ input: empty_files: true autoscaling: true resource_requests: - memory: 400M - cpu: .5 + memory: 1G + cpu: 1.7 resource_limits: - memory: 800M - cpu: 1.3 + memory: 2G + cpu: 2.5 sidecar_resource_requests: memory: 3G - cpu: 1 + cpu: 2.5 datum_set_spec: number: 1 scheduling_spec: diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_format_and_package.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_format_and_package.yaml index eae2ac533a..df9b743dad 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_format_and_package.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_format_and_package.yaml @@ -2,7 +2,7 @@ pipeline: name: concH2oSoilSalinity_pub_format_and_package transform: - image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-pub-grp-pack:v4.2.1 + image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-pub-grp-pack:sha-f5b2bdc cmd: - sh - "-c" @@ -116,20 +116,29 @@ transform: mount_path: /var/github_secret input: - pfs: - name: GROUPED_PATH - repo: concH2oSoilSalinity_pub_group - # Glob must be product-monthly or product-site-monthly. Product-site-month datums reduce unneccesary republication. - # path structure is e.g. DP1.00098.001/2023/04/CPER/04 (product/year/month/site/day) - glob: /*/*/*/* + cross: + - pfs: + name: GROUPED_PATH + repo: concH2oSoilSalinity_pub_group + # Glob must be product-monthly or product-site-monthly. Product-site-month datums reduce unneccesary republication. + # path structure is e.g. DP1.00098.001/2023/04/CPER/04 (product/year/month/site/day) + glob: /*/*/*/* + - pfs: + name: POSITION_HISTORY_PATH + repo: concH2oSoilSalinity_position_history_loader + # Full repo as one blob per commit. Crossed with monthly datums so every + # month gets the complete CFGLOC position history. pub_files reads + # enviroscan//position_history/_history.json per CFGLOC. + glob: / + empty_files: false parallelism_spec: constant: 10 autoscaling: true resource_requests: - memory: 400M + memory: 800M cpu: 1 resource_limits: - memory: 800M + memory: 1G cpu: 1.2 sidecar_resource_requests: memory: 2G diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml index 8c8870c6d3..533003fdd2 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml @@ -33,6 +33,7 @@ transform: DirErr=/pfs/out/errored_datums \ "WndwAgr=005|030" \ "WghtAlphBeta=1|2" \ + "DirSubCopy=location" \ FileSchmQm=$FILE_SCHEMA_QM \ Thsh=0.6 \ "GrpQfAlph1=VSWCsoilSpecificDepth01:VSWCsoilSpecificDepth01NullQF|VSWCsoilSpecificDepth01GapQF|VSWCsoilSpecificDepth01RangeQF|VSWCsoilSpecificDepth01StepQF|VSWCsoilSpecificDepth01PersistenceQF|VSWCsoilSpecificDepth01SpikeQF|VSWCDepth01SuspectCalQF|tempTestDepth01QF" \ @@ -146,6 +147,12 @@ transform: glob_pattern: /pfs/QAQC_PLAUSIBILITY_PATH/*/*/*/*/*/*/flags/** # Join on named location (already joined below by day) join_indices: [6] + - path: + name: QAQC_PLAUSIBILITY_PATH + # Location JSONs — carried through via DirSubCopy so pub_files + # can discover CFGLOCs downstream in the pub_group output. + glob_pattern: /pfs/QAQC_PLAUSIBILITY_PATH/*/*/*/*/*/*/location/** + join_indices: [6] - path: name: REGULARIZED_FLAGS_PATH # Filter for flags directory diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml index 75e7e0d96f..4e939c5382 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml @@ -24,6 +24,7 @@ transform: DirErr=/pfs/out/errored_datums \ FileSchmStat=$FILE_SCHEMA_STATS \ "WndwAgr=005|030" \ + "DirSubCopy=location" \ "TermStat1=VSWCsoilSpecificDepth01:mean|minimum|maximum|variance|numPts|stdEr|expUncert(wrap.ucrt.dp01.cal.mult.envsc)" \ "TermStat2=VSWCsoilSpecificDepth02:mean|minimum|maximum|variance|numPts|stdEr|expUncert(wrap.ucrt.dp01.cal.mult.envsc)" \ "TermStat3=VSWCsoilSpecificDepth03:mean|minimum|maximum|variance|numPts|stdEr|expUncert(wrap.ucrt.dp01.cal.mult.envsc)" \ @@ -63,6 +64,12 @@ transform: glob_pattern: /pfs/QAQC_PATH/*/*/*/*/*/*/data/** # Join on named location (already joined below by day) join_indices: [6] + - path: + name: QAQC_PATH + # Location JSONs — carried through via DirSubCopy so pub_files + # can discover CFGLOCs downstream in the pub_group output. + glob_pattern: /pfs/QAQC_PATH/*/*/*/*/*/*/location/** + join_indices: [6] - path: name: UNCERTAINTY_PATH # Filter for data directory diff --git a/pipe/concH2oSoilSalinity/pipe_list_concH2oSoilSalinity.txt b/pipe/concH2oSoilSalinity/pipe_list_concH2oSoilSalinity.txt index 0463219440..e2fcd6334f 100644 --- a/pipe/concH2oSoilSalinity/pipe_list_concH2oSoilSalinity.txt +++ b/pipe/concH2oSoilSalinity/pipe_list_concH2oSoilSalinity.txt @@ -12,6 +12,7 @@ concH2oSoilSalinity_stats_group_and_compute.yaml concH2oSoilSalinity_qm_join_stats.yaml concH2oSoilSalinity_group_split.yaml concH2oSoilSalinity_level1_group_consolidate_srf.yaml +concH2oSoilSalinity_position_history_loader.yaml concH2oSoilSalinity_cron_monthly_and_pub_control.yaml concH2oSoilSalinity_pub_group.yaml concH2oSoilSalinity_pub_format_and_package.yaml From 38d4eadaab306f0260b0efdb6025bbfe7ef4fe2a Mon Sep 17 00:00:00 2001 From: Teresa Burlingame <55860349+burlingamet@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:28:50 -0700 Subject: [PATCH 2/2] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../concH2oSoilSalinity_pub_format_and_package.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_format_and_package.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_format_and_package.yaml index df9b743dad..c4e53bfc9b 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_format_and_package.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_format_and_package.yaml @@ -120,7 +120,7 @@ input: - pfs: name: GROUPED_PATH repo: concH2oSoilSalinity_pub_group - # Glob must be product-monthly or product-site-monthly. Product-site-month datums reduce unneccesary republication. + # Glob must be product-monthly or product-site-monthly. Product-site-month datums reduce unnecessary republication. # path structure is e.g. DP1.00098.001/2023/04/CPER/04 (product/year/month/site/day) glob: /*/*/*/* - pfs: