1922 add metplus pointstat capability for met office

.pre-commit-config.yaml

@@ -8,6 +8,7 @@ repos:
     rev: 63c8f8312b7559622c0d82815639671ae42132ac  # frozen: v2.4.1
     hooks:
       - id: codespell
+        exclude: src/CSET/cset_workflow/app/metplus_point_stat/file/metoffice/PointStat_METplus_UKV.conf
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: 0839f92796ae388643a08a21640a029b322be5c2  # frozen: v0.15.2

src/CSET/cset_workflow/app/metplus_point_stat/file/metoffice/PointStat_METplus_UKV.conf

@@ -0,0 +1,151 @@
+[dir]
+INPUT_BASE_FORECAST = {ENV[METPLUS_FCST_DIR]}
+INPUT_BASE_OBSERVATION = {ENV[METPLUS_NCOBS_DIR]}
+OUTPUT_BASE = {ENV[CYLC_TASK_WORK_DIR]}
+MET_INSTALL_DIR = {ENV[MET_INSTALL_DIR]}
+TIMESTRING = {ENV[TASK_START_TIME]}
+FORECAST_LENGTH = {ENV[FORECAST_LENGTH]}
+
+[config]
+
+# List of applications to run - only PointStat for this case
+PROCESS_LIST = PointStat
+
+# time looping - options are INIT, VALID, RETRO, and REALTIME
+# If set to INIT or RETRO:
+#   INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set
+# If set to VALID or REALTIME:
+#   VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set
+LOOP_BY = VALID
+
+# Format of VALID_BEG and VALID_END using % items
+# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc.
+# see www.strftime.org for more information
+# %Y%m%d%H expands to YYYYMMDDHH
+VALID_TIME_FMT = %Y%m%dT%H
+
+# Start time for METplus run - must match VALID_TIME_FMT
+VALID_BEG = {TIMESTRING}
+
+# End time for METplus run - must match VALID_TIME_FMT
+VALID_END = {TIMESTRING}
+
+# Increment between METplus runs (in seconds if no units are specified)
+#  Must be >= 60 seconds
+VALID_INCREMENT = 60M
+
+# List of forecast leads to process for each run time (init or valid)
+# In hours if units are not specified
+# If unset, defaults to 0 (don't loop through forecast leads)
+LEAD_SEQ = {FCAST_LEADS}
+
+# Order of loops to process data - Options are times, processes
+# Not relevant if only one item is in the PROCESS_LIST
+# times = run all wrappers in the PROCESS_LIST for a single run time, then
+#   increment the run time and run all wrappers again until all times have
+#   been evaluated.
+# processes = run the first wrapper in the PROCESS_LIST for all times
+#   specified, then repeat for the next item in the PROCESS_LIST until all
+#   wrappers have been run
+LOOP_ORDER = processes
+
+# Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only
+LOG_POINT_STAT_VERBOSITY = 3
+
+# Time relative to each input file's valid time (in seconds if no units are specified) for data within the file to be
+# considered valid. Values are set in the 'obs_window' dictionary in the PointStat config file
+OBS_POINT_STAT_WINDOW_BEGIN = -1800
+OBS_POINT_STAT_WINDOW_END = 1800
+
+# Optional list of offsets to look for point observation data
+POINT_STAT_OFFSETS = 0
+
+# Model/fcst and obs name, e.g. GFS, GDAS, etc.
+MODEL = METO_UKV
+OBTYPE = METO
+
+# Regrid to specified grid.  Indicate NONE if no regridding, or the grid id
+# (e.g. G212)
+POINT_STAT_REGRID_TO_GRID = NONE
+
+POINT_STAT_OUTPUT_PREFIX = {MODEL}_G000
+
+# sets the -obs_valid_beg command line argument (optional)
+# not used for this example
+#POINT_STAT_OBS_VALID_BEG = {valid?fmt=%Y%m%d_%H}
+
+# sets the -obs_valid_end command line argument (optional)
+# not used for this example
+#POINT_STAT_OBS_VALID_END = {valid?fmt=%Y%m%d_%H}
+
+# Verification Masking regions
+# Indicate which grid and polygon masking region, if applicable
+POINT_STAT_GRID =
+
+# Message types, if all message types are to be returned, leave this empty,
+# otherwise indicate the message types of interest.
+POINT_STAT_MESSAGE_TYPE = SURFACE
+POINT_STAT_MESSAGE_TYPE_GROUP_MAP = { key = "SURFACE"; val = "11600,10101,10102,10201,10202,10204,10210,10310"; }, { key = "10101"; val = "10101"; }, { key = "10102"; val = "10102"; }, { key = "10201"; val = "10201"; }, { key = "10202"; val = "10202"; }, { key = "10204"; val = "10204"; }, { key = "10210"; val = "10210"; }, { key = "10310"; val = "10310"; }, { key = "SREW"; val = "10500"; }, { key = "UPPERAIR"; val = "50101,50102,50201,50202"; }, { key = "50101"; val = "50101"; }, { key = "50102"; val = "50102"; }, { key = "50201"; val = "50201"; }, { key = "50202"; val = "50202"; }, { key = "AIRC_AM"; val = "30100"; }, { key = "AIRC_AI"; val = "30200"; }, { key = "AIRC_ALL"; val = "30100,30200"; }, { key = "SATW"; val = "22501,22505,22503,23531,23502,23053,23501,23532"; }
+
+# Variables and levels as specified in the field dictionary of the MET
+# point_stat configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS,
+# (optional) FCST_VARn_OPTION
+
+# set to True to run PointStat once for each name/level combination
+# set to False to run PointStat once per run time including all fields
+POINT_STAT_ONCE_PER_FIELD = False
+
+
+# End of [config] section and start of [dir] section
+[dir]
+FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE_FORECAST}
+OBS_POINT_STAT_INPUT_DIR = {INPUT_BASE_OBSERVATION}
+
+# directory containing climatology mean input to PointStat
+# Not used in this example
+POINT_STAT_CLIMO_MEAN_INPUT_DIR =
+
+# directory containing climatology mean input to PointStat
+# Not used in this example
+POINT_STAT_CLIMO_STDEV_INPUT_DIR =
+
+
+POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE}
+
+
+# Select the type of output file(s) you wish to create
+#POINT_STAT_OUTPUT_FLAG_FHO =
+#POINT_STAT_OUTPUT_FLAG_CTC =
+#POINT_STAT_OUTPUT_FLAG_CTS =
+#POINT_STAT_OUTPUT_FLAG_MCTC =
+#POINT_STAT_OUTPUT_FLAG_MCTS =
+POINT_STAT_OUTPUT_FLAG_CNT = BOTH
+POINT_STAT_OUTPUT_FLAG_SL1L2 = BOTH
+#POINT_STAT_OUTPUT_FLAG_SAL1L2 =
+POINT_STAT_OUTPUT_FLAG_VL1L2 = NONE
+#POINT_STAT_OUTPUT_FLAG_VAL1L2 =
+#POINT_STAT_OUTPUT_FLAG_VCNT =
+#POINT_STAT_OUTPUT_FLAG_PCT =
+#POINT_STAT_OUTPUT_FLAG_PSTD =
+#POINT_STAT_OUTPUT_FLAG_PJC =
+#POINT_STAT_OUTPUT_FLAG_PRC =
+#POINT_STAT_OUTPUT_FLAG_ECNT =
+#POINT_STAT_OUTPUT_FLAG_RPS =
+#POINT_STAT_OUTPUT_FLAG_ECLV =
+POINT_STAT_OUTPUT_FLAG_MPR = NONE
+
+# End of [dir] section and start of [filename_templates] section
+[filename_templates]
+
+# Template to look for forecast input to PointStat relative to FCST_POINT_STAT_INPUT_DIR
+FCST_POINT_STAT_INPUT_TEMPLATE = Fcast_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}_{lead?fmt=%3H}_ukv.nc
+
+# Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_MEAN_INPUT_DIR
+# Not used in this example
+POINT_STAT_CLIMO_MEAN_INPUT_TEMPLATE =
+
+# Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_STDEV_INPUT_DIR
+# Not used in this example
+POINT_STAT_CLIMO_STDEV_INPUT_TEMPLATE =
+
+POINT_STAT_OUTPUT_TEMPLATE = pt_ukv

src/CSET/cset_workflow/app/metplus_point_stat/file/metoffice/PointStat_UKV_Areas0.conf

@@ -0,0 +1,103 @@
+[config]
+
+# Selection of qc flag entries to include in the output.
+# Leave blank to include everything
+# Need to directly match what is stored in obs file (include leading zeros if present)
+POINT_STAT_OBS_QUALITY =
+#POINT_STAT_OBS_QUALITY_INC = 08388928
+POINT_STAT_OBS_QUALITY_INC =
+POINT_STAT_OBS_QUALITY_EXC = -99
+
+# fields to compare
+# Note: If FCST_VAR<n>_* is set, then a corresponding OBS_VAR<n>_* variable must be set
+# To use one variables for both forecast and observation data, set BOTH_VAR<n>_* instead
+
+file_type=NETCDF_NCCF
+FCST_VAR1_NAME = air_pressure_at_sea_level
+FCST_VAR1_LEVELS = "(*,*)"
+FCST_VAR1_THRESH = [<=98000.0, <=98500.0, <=99000.0, <=99500.0, <=100000.0, <=100500.0, <=101000.0, <=101500.0, <=102000.0]
+FCST_VAR1_OPTIONS = set_attr_name="PMSL"; set_attr_level="Z0"
+OBS_VAR1_NAME = PRMSL
+OBS_VAR1_OPTIONS = set_attr_name="PMSL"; set_attr_level="Z0"
+OBS_VAR1_THRESH = [<=98000.0, <=98500.0, <=99000.0, <=99500.0, <=100000.0, <=100500.0, <=101000.0, <=101500.0, <=102000.0]
+OBS_VAR1_LEVELS = Z0
+FCST_VAR2_NAME = air_temperature
+FCST_VAR2_LEVELS = "(*,*)"
+FCST_VAR2_THRESH = [<=273.15, <=278.15, <=283.15, <=288.15, <=293.15]
+FCST_VAR2_OPTIONS = set_attr_name="T2M"; set_attr_level="Z0"
+OBS_VAR2_NAME = TMP
+OBS_VAR2_OPTIONS = set_attr_name="T2M"
+OBS_VAR2_THRESH = [<=273.15, <=278.15, <=283.15, <=288.15, <=293.15]
+OBS_VAR2_LEVELS = Z0
+FCST_VAR3_NAME = visibility_in_air
+FCST_VAR3_LEVELS = "(*,*)"
+FCST_VAR3_THRESH = [<=200, <=1000, <=4000, <=5000]
+FCST_VAR3_OPTIONS = set_attr_level="Z0";
+OBS_VAR3_NAME = VIS
+OBS_VAR3_OPTIONS = set_attr_level="Z0";
+OBS_VAR3_THRESH = [<=200, <=1000, <=4000, <=5000]
+OBS_VAR3_LEVELS = Z0
+FCST_VAR4_NAME = visibility_in_air
+FCST_VAR4_LEVELS = "(*,*)"
+FCST_VAR4_THRESH = [<=2.3, <=3, <=3.6, <=3.7]
+FCST_VAR4_OPTIONS = set_attr_name="vis_log10"; set_attr_level="Z0"; set_attr_units="m(log10)"; convert(a) = log10(a);
+OBS_VAR4_NAME = VIS
+OBS_VAR4_OPTIONS = set_attr_name="vis_log10"; set_attr_level="Z0"; set_attr_units="m(log10)"; convert(a) = log10(a);
+OBS_VAR4_THRESH = [<=2.3, <=3, <=3.6, <=3.7]
+OBS_VAR4_LEVELS = Z0
+FCST_VAR5_NAME = m01s09i233
+FCST_VAR5_LEVELS = "(*,*)"
+FCST_VAR5_THRESH = [<=0.3125, <=0.5625, <=0.8125]
+FCST_VAR5_OPTIONS = set_attr_level="Z0"; set_attr_name="TCC_9233"
+OBS_VAR5_NAME = TCDC
+OBS_VAR5_OPTIONS = set_attr_level="Z0"
+OBS_VAR5_THRESH = [<=0.3125, <=0.5625, <=0.8125]
+OBS_VAR5_LEVELS = Z0
+FCST_VAR6_NAME = relative_humidity
+FCST_VAR6_LEVELS = "(*,*)"
+FCST_VAR6_OPTIONS = set_attr_level="Z0"; set_attr_name="RH";
+OBS_VAR6_NAME = RH
+OBS_VAR6_OPTIONS = set_attr_level="Z0"
+OBS_VAR6_LEVELS = Z0
+
+# List of full path to poly masking files.  NOTE: Only short lists of poly
+# files work (those that fit on one line), a long list will result in an
+# environment variable that is too long, resulting in an error.  For long
+# lists of poly masking files (i.e. all the mask files in the NCEP_mask
+# directory), define these in the MET point_stat configuration file.
+POINT_STAT_MASK_POLY={MET_AREA_DIR}/poly_umukv.nc
+POINT_STAT_MASK_SID={MET_STN_DIR}/2011.stns,
+                    {MET_STN_DIR}/2014.stns,
+                    {MET_STN_DIR}/2015.stns,
+                    {MET_STN_DIR}/2020.stns,
+                    {MET_STN_DIR}/2024.stns,
+                    {MET_STN_DIR}/2103.stns
+POINT_STAT_MASK_LLPNT=
+
+# Select the type of output file(s) you wish to create
+POINT_STAT_OUTPUT_FLAG_FHO = BOTH
+POINT_STAT_OUTPUT_FLAG_CTC = BOTH
+POINT_STAT_OUTPUT_FLAG_CTS = BOTH
+POINT_STAT_OUTPUT_FLAG_MCTC = BOTH
+#POINT_STAT_OUTPUT_FLAG_MCTS =
+POINT_STAT_OUTPUT_FLAG_CNT = BOTH
+POINT_STAT_OUTPUT_FLAG_SL1L2 = BOTH
+#POINT_STAT_OUTPUT_FLAG_SAL1L2 =
+POINT_STAT_OUTPUT_FLAG_VL1L2 = NONE
+#POINT_STAT_OUTPUT_FLAG_VAL1L2 =
+POINT_STAT_OUTPUT_FLAG_VCNT = NONE
+#POINT_STAT_OUTPUT_FLAG_PCT =
+#POINT_STAT_OUTPUT_FLAG_PSTD =
+#POINT_STAT_OUTPUT_FLAG_PJC =
+#POINT_STAT_OUTPUT_FLAG_PRC =
+#POINT_STAT_OUTPUT_FLAG_ECNT =
+#POINT_STAT_OUTPUT_FLAG_RPS =
+#POINT_STAT_OUTPUT_FLAG_ECLV =
+POINT_STAT_OUTPUT_FLAG_MPR = NONE
+
+# End of [dir] section and start of [filename_templates] section
+[filename_templates]
+
+# Template to look for observation input to PointStat relative to OBS_POINT_STAT_INPUT_DIR
+#OBS_POINT_STAT_INPUT_TEMPLATE = PYTHON_NUMPY = {ODBSCRIPT_DIR}/PointStat_ReadODB.py {INPUT_BASE_OBSERVATION}/{valid?fmt=%Y%m%d}T{valid?fmt=%H%M}Z_ukv_surface_odb2 {INPUT_BASE_OBSERVATION}/{MODEL}_{valid?fmt=%Y%m%d}T{valid?fmt=%H}00Z_surf_{lead?fmt=%3H}_ukv.csv True
+OBS_POINT_STAT_INPUT_TEMPLATE = Surface_{valid?fmt=%Y%m%d}T{valid?fmt=%H}00Z.nc

src/CSET/cset_workflow/app/metplus_point_stat/file/metoffice/user_system_local.conf

@@ -0,0 +1,23 @@
+[dir]
+# any {ENV[]} variable is defined within [env] section of the rose-app.conf
+
+#INPUT_BASE_FORECAST = {ENV[PNT_FCST]}
+#INPUT_BASE_OBSERVATION = {ENV[PNT_OBVS]}
+#
+#OUTPUT_BASE={ENV[MET_OUTPUT]}
+OUTPUT_BASE_RES={ENV[MET_OUTPUT_RES]}
+
+#MET_INSTALL_DIR={ENV[VERSION_MET]}
+
+MET_AREA_DIR={ENV[METPLUS_AREA_DIR]}
+MET_STN_DIR={ENV[METPLUS_STN_DIR]}
+#ODBSCRIPT_DIR={ENV[READ_ODB]}
+
+# TIMESTRING is linked to whattime in rose-app.conf, linked to the suite.rc so that the time formats in the suite.rc and the METplus' PointStat.conf match up.
+TIMESTRING={ENV[TASK_START_TIME]}
+
+FCAST_LEADS={ENV[FORECAST_LENGTH]}
+
+## Additional config files required to run Pointstat
+#ADD_CONFIG={ENV[ADD_CONFIG_PS]}/{ENV[MET_config_PS]}
+#ADD_CONFIG={ENV[CYLC_SUITE_DEF_PATH]}/app/metplus_point_stat/file/metoffice/PointStat_MET_UKV.conf

src/CSET/cset_workflow/app/metplus_point_stat/opt/rose-app-metoffice.conf

@@ -0,0 +1,10 @@
+[command]
+default=run_metplus.py metoffice/PointStat_METplus_UKV.conf  metoffice/PointStat_UKV_Areas0.conf metoffice/user_system_local.conf
+
+[env]
+# METplus_config_PS = Pointstat_METplus_${CYLC_TASK_PARAM_model}.conf
+CONDA_VENV_LOCATION = ${CONDA_METPLUS_VENV_LOCATION}
+METPLUS_FCST_DIR = ${METPLUS_FCST_DIR}
+METPLUS_NCOBS_DIR = ${CYLC_WORKFLOW_SHARE_DIR}/nc_obs
+START_TIME=${TASK_START_TIME}
+FORECAST_LENGTH = ${FORECAST_LENGTH}

src/CSET/cset_workflow/includes/metplus_common.cylc

@@ -4,6 +4,8 @@
         METPLUS_ANA_DIR = {{METPLUS_ANA_DIR}}
         METPLUS_FCST_DIR = {{METPLUS_FCST_DIR}}
         METPLUS_OBS_DIR = {{METPLUS_OBS_DIR}}
+        METPLUS_AREA_DIR = {{METPLUS_AREA_DIR}}
+        METPLUS_STN_DIR = {{METPLUS_STN_DIR}}
         ROSE_APP_OPT_CONF_KEYS = {{METPLUS_OPT_CONFIG_KEYS}}
         TASK_START_TIME = $(cylc cyclepoint --template CCYYMMDDThh)
         FORECAST_LENGTH = $(isodatetime --as-total H -- {{ANALYSIS_LENGTH}})