diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml index 1430e22bc..1a083bd0a 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml @@ -44,7 +44,8 @@ transform: DirOut=/pfs/out \ DirErr=/pfs/out/errored_datums \ FileSchmStats=$FILE_SCHEMA_STATS \ - FileSchmQm=$FILE_SCHEMA_QM + FileSchmQm=$FILE_SCHEMA_QM \ + "DirSubCopy=location" echo "=== flow.concH2oSalinity.grp.split complete ===" EOF diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml index 5869c979c..5f8e4a3bc 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml @@ -41,7 +41,7 @@ transform: DirErr=/pfs/out/errored_datums \ "DirData=stats|quality_metrics" \ PathPubWb=$PUB_WORKBOOKS \ - "DirSubCopy=science_review_flags|group" + "DirSubCopy=science_review_flags|group|location" # # # Export Level 1 data to bucket @@ -121,10 +121,12 @@ transform: # folders (group, science_review_flags) sit directly under the group at index 7. GROUP_METADATA_INDEX: "7" GROUP_METADATA_NAMES: group,science_review_flags - # Data types (stats, quality_metrics) remain nested under the CFGLOC level, - # one deeper than the group metadata, at index 8. + # Data types (stats, quality_metrics, location) remain nested under the CFGLOC + # level, one deeper than the group metadata, at index 8. Location must be listed + # here — otherwise level1_consolidate drops the CFGLOC/location/*.json files that + # pub_files' CFGLOC discovery loop relies on. DATA_TYPE_INDEX: "8" - DATA_TYPE_NAMES: stats,quality_metrics + DATA_TYPE_NAMES: stats,quality_metrics,location # Environment variables for pub_workbook_loader OUT_PATH_WORKBOOK: /tmp/pub_workbooks diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_position_history_loader.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_position_history_loader.yaml new file mode 100644 index 000000000..da348aa40 --- /dev/null +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_position_history_loader.yaml @@ -0,0 +1,45 @@ +--- +# Production concH2oSoilSalinity position-history loader. +# Queries PDR directly for the full CFGLOC × asset × calibration × geolocation +# history for every enviroscan probe, applies CVALD1-derived depth adjustment +# per VER (501-508), and emits one JSON per CFGLOC. Time-invariant output — +# a Feb 2015 CSV, a June 2023 CSV, and a today CSV all contain the same rows. +# +# Loader is unfiltered by site (PDR query is cheap enough at ~200 CFGLOCs). +# pub_format_and_package crosses this repo with monthly pub_group datums so +# every monthly sensor_positions.csv gets the complete CFGLOC history tree. +pipeline: + name: concH2oSoilSalinity_position_history_loader +transform: + image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-conc-h2o-salinity-position-history-loader:sha-fb1172f + cmd: + - /bin/bash + stdin: + - '#!/bin/bash' + - python3 -m concH2oSoilSalinity_position_history_loader.concH2oSoilSalinity_position_history_loader_main + env: + OUT_PATH: /pfs/out + ERR_PATH: /pfs/out/errored_datums + LOG_LEVEL: INFO + secrets: + - name: pdr-secret + mount_path: /var/db_secret +input: + pfs: + repo: enviroscan_ingest_days + glob: /tick/ + branch: master + empty_files: true +autoscaling: true +resource_requests: + memory: 300M + cpu: 0.2 +resource_limits: + memory: 800M + cpu: 1 +sidecar_resource_requests: + memory: 300M + cpu: 0.3 +scheduling_spec: + node_selector: + cloud.google.com/compute-class: pach-pipeline-class diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_egress_and_publish.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_egress_and_publish.yaml index 846e21b44..3db142117 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_egress_and_publish.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_egress_and_publish.yaml @@ -158,14 +158,14 @@ input: empty_files: true autoscaling: true resource_requests: - memory: 400M - cpu: .5 + memory: 1G + cpu: 1.7 resource_limits: - memory: 800M - cpu: 1.3 + memory: 2G + cpu: 2.5 sidecar_resource_requests: memory: 3G - cpu: 1 + cpu: 2.5 datum_set_spec: number: 1 scheduling_spec: diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_format_and_package.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_format_and_package.yaml index eae2ac533..c4e53bfc9 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_format_and_package.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_pub_format_and_package.yaml @@ -2,7 +2,7 @@ pipeline: name: concH2oSoilSalinity_pub_format_and_package transform: - image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-pub-grp-pack:v4.2.1 + image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-pub-grp-pack:sha-f5b2bdc cmd: - sh - "-c" @@ -116,20 +116,29 @@ transform: mount_path: /var/github_secret input: - pfs: - name: GROUPED_PATH - repo: concH2oSoilSalinity_pub_group - # Glob must be product-monthly or product-site-monthly. Product-site-month datums reduce unneccesary republication. - # path structure is e.g. DP1.00098.001/2023/04/CPER/04 (product/year/month/site/day) - glob: /*/*/*/* + cross: + - pfs: + name: GROUPED_PATH + repo: concH2oSoilSalinity_pub_group + # Glob must be product-monthly or product-site-monthly. Product-site-month datums reduce unnecessary republication. + # path structure is e.g. DP1.00098.001/2023/04/CPER/04 (product/year/month/site/day) + glob: /*/*/*/* + - pfs: + name: POSITION_HISTORY_PATH + repo: concH2oSoilSalinity_position_history_loader + # Full repo as one blob per commit. Crossed with monthly datums so every + # month gets the complete CFGLOC position history. pub_files reads + # enviroscan//position_history/_history.json per CFGLOC. + glob: / + empty_files: false parallelism_spec: constant: 10 autoscaling: true resource_requests: - memory: 400M + memory: 800M cpu: 1 resource_limits: - memory: 800M + memory: 1G cpu: 1.2 sidecar_resource_requests: memory: 2G diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml index 02c42587a..8f98d86f0 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml @@ -33,6 +33,7 @@ transform: DirErr=/pfs/out/errored_datums \ "WndwAgr=005|030" \ "WghtAlphBeta=1|2" \ + "DirSubCopy=location" \ FileSchmQm=$FILE_SCHEMA_QM \ Thsh=0.6 \ "GrpQfAlph1=VSWCsoilSpecificDepth01:VSWCsoilSpecificDepth01NullQF|VSWCsoilSpecificDepth01GapQF|VSWCsoilSpecificDepth01RangeQF|VSWCsoilSpecificDepth01StepQF|VSWCsoilSpecificDepth01PersistenceQF|VSWCsoilSpecificDepth01SpikeQF|VSWCDepth01SuspectCalQF|tempTestDepth01QF" \ @@ -146,6 +147,12 @@ transform: glob_pattern: /pfs/QAQC_PLAUSIBILITY_PATH/*/*/*/*/*/*/flags/** # Join on named location (already joined below by day) join_indices: [6] + - path: + name: QAQC_PLAUSIBILITY_PATH + # Location JSONs — carried through via DirSubCopy so pub_files + # can discover CFGLOCs downstream in the pub_group output. + glob_pattern: /pfs/QAQC_PLAUSIBILITY_PATH/*/*/*/*/*/*/location/** + join_indices: [6] - path: name: REGULARIZED_FLAGS_PATH # Filter for flags directory diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml index afdc8c80f..6887e0467 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml @@ -24,6 +24,7 @@ transform: DirErr=/pfs/out/errored_datums \ FileSchmStat=$FILE_SCHEMA_STATS \ "WndwAgr=005|030" \ + "DirSubCopy=location" \ "TermStat1=VSWCsoilSpecificDepth01:mean|minimum|maximum|variance|numPts|stdEr|expUncert(wrap.ucrt.dp01.cal.mult.envsc)" \ "TermStat2=VSWCsoilSpecificDepth02:mean|minimum|maximum|variance|numPts|stdEr|expUncert(wrap.ucrt.dp01.cal.mult.envsc)" \ "TermStat3=VSWCsoilSpecificDepth03:mean|minimum|maximum|variance|numPts|stdEr|expUncert(wrap.ucrt.dp01.cal.mult.envsc)" \ @@ -63,6 +64,12 @@ transform: glob_pattern: /pfs/QAQC_PATH/*/*/*/*/*/*/data/** # Join on named location (already joined below by day) join_indices: [6] + - path: + name: QAQC_PATH + # Location JSONs — carried through via DirSubCopy so pub_files + # can discover CFGLOCs downstream in the pub_group output. + glob_pattern: /pfs/QAQC_PATH/*/*/*/*/*/*/location/** + join_indices: [6] - path: name: UNCERTAINTY_PATH # Filter for data directory diff --git a/pipe/concH2oSoilSalinity/pipe_list_concH2oSoilSalinity.txt b/pipe/concH2oSoilSalinity/pipe_list_concH2oSoilSalinity.txt index 046321944..e2fcd6334 100644 --- a/pipe/concH2oSoilSalinity/pipe_list_concH2oSoilSalinity.txt +++ b/pipe/concH2oSoilSalinity/pipe_list_concH2oSoilSalinity.txt @@ -12,6 +12,7 @@ concH2oSoilSalinity_stats_group_and_compute.yaml concH2oSoilSalinity_qm_join_stats.yaml concH2oSoilSalinity_group_split.yaml concH2oSoilSalinity_level1_group_consolidate_srf.yaml +concH2oSoilSalinity_position_history_loader.yaml concH2oSoilSalinity_cron_monthly_and_pub_control.yaml concH2oSoilSalinity_pub_group.yaml concH2oSoilSalinity_pub_format_and_package.yaml