diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a71df5527..6de36bbee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,6 +8,7 @@ repos: rev: 63c8f8312b7559622c0d82815639671ae42132ac # frozen: v2.4.1 hooks: - id: codespell + exclude: src/CSET/cset_workflow/app/metplus_point_stat/file/metoffice/PointStat_METplus_UKV.conf - repo: https://github.com/astral-sh/ruff-pre-commit rev: 0839f92796ae388643a08a21640a029b322be5c2 # frozen: v0.15.2 diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/file/metoffice/PointStat_METplus_UKV.conf b/src/CSET/cset_workflow/app/metplus_point_stat/file/metoffice/PointStat_METplus_UKV.conf new file mode 100755 index 000000000..50955fcc7 --- /dev/null +++ b/src/CSET/cset_workflow/app/metplus_point_stat/file/metoffice/PointStat_METplus_UKV.conf @@ -0,0 +1,151 @@ +[dir] +INPUT_BASE_FORECAST = {ENV[METPLUS_FCST_DIR]} +INPUT_BASE_OBSERVATION = {ENV[METPLUS_NCOBS_DIR]} +OUTPUT_BASE = {ENV[CYLC_TASK_WORK_DIR]} +MET_INSTALL_DIR = {ENV[MET_INSTALL_DIR]} +TIMESTRING = {ENV[TASK_START_TIME]} +FORECAST_LENGTH = {ENV[FORECAST_LENGTH]} + +[config] + +# List of applications to run - only PointStat for this case +PROCESS_LIST = PointStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = VALID + +# Format of VALID_BEG and VALID_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +VALID_TIME_FMT = %Y%m%dT%H + +# Start time for METplus run - must match VALID_TIME_FMT +VALID_BEG = {TIMESTRING} + +# End time for METplus run - must match VALID_TIME_FMT +VALID_END = {TIMESTRING} + +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +VALID_INCREMENT = 60M + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {FCAST_LEADS} + +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = processes + +# Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only +LOG_POINT_STAT_VERBOSITY = 3 + +# Time relative to each input file's valid time (in seconds if no units are specified) for data within the file to be +# considered valid. Values are set in the 'obs_window' dictionary in the PointStat config file +OBS_POINT_STAT_WINDOW_BEGIN = -1800 +OBS_POINT_STAT_WINDOW_END = 1800 + +# Optional list of offsets to look for point observation data +POINT_STAT_OFFSETS = 0 + +# Model/fcst and obs name, e.g. GFS, GDAS, etc. +MODEL = METO_UKV +OBTYPE = METO + +# Regrid to specified grid. Indicate NONE if no regridding, or the grid id +# (e.g. G212) +POINT_STAT_REGRID_TO_GRID = NONE + +POINT_STAT_OUTPUT_PREFIX = {MODEL}_G000 + +# sets the -obs_valid_beg command line argument (optional) +# not used for this example +#POINT_STAT_OBS_VALID_BEG = {valid?fmt=%Y%m%d_%H} + +# sets the -obs_valid_end command line argument (optional) +# not used for this example +#POINT_STAT_OBS_VALID_END = {valid?fmt=%Y%m%d_%H} + +# Verification Masking regions +# Indicate which grid and polygon masking region, if applicable +POINT_STAT_GRID = + +# Message types, if all message types are to be returned, leave this empty, +# otherwise indicate the message types of interest. +POINT_STAT_MESSAGE_TYPE = SURFACE +POINT_STAT_MESSAGE_TYPE_GROUP_MAP = { key = "SURFACE"; val = "11600,10101,10102,10201,10202,10204,10210,10310"; }, { key = "10101"; val = "10101"; }, { key = "10102"; val = "10102"; }, { key = "10201"; val = "10201"; }, { key = "10202"; val = "10202"; }, { key = "10204"; val = "10204"; }, { key = "10210"; val = "10210"; }, { key = "10310"; val = "10310"; }, { key = "SREW"; val = "10500"; }, { key = "UPPERAIR"; val = "50101,50102,50201,50202"; }, { key = "50101"; val = "50101"; }, { key = "50102"; val = "50102"; }, { key = "50201"; val = "50201"; }, { key = "50202"; val = "50202"; }, { key = "AIRC_AM"; val = "30100"; }, { key = "AIRC_AI"; val = "30200"; }, { key = "AIRC_ALL"; val = "30100,30200"; }, { key = "SATW"; val = "22501,22505,22503,23531,23502,23053,23501,23532"; } + +# Variables and levels as specified in the field dictionary of the MET +# point_stat configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION + +# set to True to run PointStat once for each name/level combination +# set to False to run PointStat once per run time including all fields +POINT_STAT_ONCE_PER_FIELD = False + + +# End of [config] section and start of [dir] section +[dir] +FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE_FORECAST} +OBS_POINT_STAT_INPUT_DIR = {INPUT_BASE_OBSERVATION} + +# directory containing climatology mean input to PointStat +# Not used in this example +POINT_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to PointStat +# Not used in this example +POINT_STAT_CLIMO_STDEV_INPUT_DIR = + + +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} + + +# Select the type of output file(s) you wish to create +#POINT_STAT_OUTPUT_FLAG_FHO = +#POINT_STAT_OUTPUT_FLAG_CTC = +#POINT_STAT_OUTPUT_FLAG_CTS = +#POINT_STAT_OUTPUT_FLAG_MCTC = +#POINT_STAT_OUTPUT_FLAG_MCTS = +POINT_STAT_OUTPUT_FLAG_CNT = BOTH +POINT_STAT_OUTPUT_FLAG_SL1L2 = BOTH +#POINT_STAT_OUTPUT_FLAG_SAL1L2 = +POINT_STAT_OUTPUT_FLAG_VL1L2 = NONE +#POINT_STAT_OUTPUT_FLAG_VAL1L2 = +#POINT_STAT_OUTPUT_FLAG_VCNT = +#POINT_STAT_OUTPUT_FLAG_PCT = +#POINT_STAT_OUTPUT_FLAG_PSTD = +#POINT_STAT_OUTPUT_FLAG_PJC = +#POINT_STAT_OUTPUT_FLAG_PRC = +#POINT_STAT_OUTPUT_FLAG_ECNT = +#POINT_STAT_OUTPUT_FLAG_RPS = +#POINT_STAT_OUTPUT_FLAG_ECLV = +POINT_STAT_OUTPUT_FLAG_MPR = NONE + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to PointStat relative to FCST_POINT_STAT_INPUT_DIR +FCST_POINT_STAT_INPUT_TEMPLATE = Fcast_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}_{lead?fmt=%3H}_ukv.nc + +# Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +POINT_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +POINT_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +POINT_STAT_OUTPUT_TEMPLATE = pt_ukv diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/file/metoffice/PointStat_UKV_Areas0.conf b/src/CSET/cset_workflow/app/metplus_point_stat/file/metoffice/PointStat_UKV_Areas0.conf new file mode 100644 index 000000000..8af525dc6 --- /dev/null +++ b/src/CSET/cset_workflow/app/metplus_point_stat/file/metoffice/PointStat_UKV_Areas0.conf @@ -0,0 +1,103 @@ +[config] + +# Selection of qc flag entries to include in the output. +# Leave blank to include everything +# Need to directly match what is stored in obs file (include leading zeros if present) +POINT_STAT_OBS_QUALITY = +#POINT_STAT_OBS_QUALITY_INC = 08388928 +POINT_STAT_OBS_QUALITY_INC = +POINT_STAT_OBS_QUALITY_EXC = -99 + +# fields to compare +# Note: If FCST_VAR_* is set, then a corresponding OBS_VAR_* variable must be set +# To use one variables for both forecast and observation data, set BOTH_VAR_* instead + +file_type=NETCDF_NCCF +FCST_VAR1_NAME = air_pressure_at_sea_level +FCST_VAR1_LEVELS = "(*,*)" +FCST_VAR1_THRESH = [<=98000.0, <=98500.0, <=99000.0, <=99500.0, <=100000.0, <=100500.0, <=101000.0, <=101500.0, <=102000.0] +FCST_VAR1_OPTIONS = set_attr_name="PMSL"; set_attr_level="Z0" +OBS_VAR1_NAME = PRMSL +OBS_VAR1_OPTIONS = set_attr_name="PMSL"; set_attr_level="Z0" +OBS_VAR1_THRESH = [<=98000.0, <=98500.0, <=99000.0, <=99500.0, <=100000.0, <=100500.0, <=101000.0, <=101500.0, <=102000.0] +OBS_VAR1_LEVELS = Z0 +FCST_VAR2_NAME = air_temperature +FCST_VAR2_LEVELS = "(*,*)" +FCST_VAR2_THRESH = [<=273.15, <=278.15, <=283.15, <=288.15, <=293.15] +FCST_VAR2_OPTIONS = set_attr_name="T2M"; set_attr_level="Z0" +OBS_VAR2_NAME = TMP +OBS_VAR2_OPTIONS = set_attr_name="T2M" +OBS_VAR2_THRESH = [<=273.15, <=278.15, <=283.15, <=288.15, <=293.15] +OBS_VAR2_LEVELS = Z0 +FCST_VAR3_NAME = visibility_in_air +FCST_VAR3_LEVELS = "(*,*)" +FCST_VAR3_THRESH = [<=200, <=1000, <=4000, <=5000] +FCST_VAR3_OPTIONS = set_attr_level="Z0"; +OBS_VAR3_NAME = VIS +OBS_VAR3_OPTIONS = set_attr_level="Z0"; +OBS_VAR3_THRESH = [<=200, <=1000, <=4000, <=5000] +OBS_VAR3_LEVELS = Z0 +FCST_VAR4_NAME = visibility_in_air +FCST_VAR4_LEVELS = "(*,*)" +FCST_VAR4_THRESH = [<=2.3, <=3, <=3.6, <=3.7] +FCST_VAR4_OPTIONS = set_attr_name="vis_log10"; set_attr_level="Z0"; set_attr_units="m(log10)"; convert(a) = log10(a); +OBS_VAR4_NAME = VIS +OBS_VAR4_OPTIONS = set_attr_name="vis_log10"; set_attr_level="Z0"; set_attr_units="m(log10)"; convert(a) = log10(a); +OBS_VAR4_THRESH = [<=2.3, <=3, <=3.6, <=3.7] +OBS_VAR4_LEVELS = Z0 +FCST_VAR5_NAME = m01s09i233 +FCST_VAR5_LEVELS = "(*,*)" +FCST_VAR5_THRESH = [<=0.3125, <=0.5625, <=0.8125] +FCST_VAR5_OPTIONS = set_attr_level="Z0"; set_attr_name="TCC_9233" +OBS_VAR5_NAME = TCDC +OBS_VAR5_OPTIONS = set_attr_level="Z0" +OBS_VAR5_THRESH = [<=0.3125, <=0.5625, <=0.8125] +OBS_VAR5_LEVELS = Z0 +FCST_VAR6_NAME = relative_humidity +FCST_VAR6_LEVELS = "(*,*)" +FCST_VAR6_OPTIONS = set_attr_level="Z0"; set_attr_name="RH"; +OBS_VAR6_NAME = RH +OBS_VAR6_OPTIONS = set_attr_level="Z0" +OBS_VAR6_LEVELS = Z0 + +# List of full path to poly masking files. NOTE: Only short lists of poly +# files work (those that fit on one line), a long list will result in an +# environment variable that is too long, resulting in an error. For long +# lists of poly masking files (i.e. all the mask files in the NCEP_mask +# directory), define these in the MET point_stat configuration file. +POINT_STAT_MASK_POLY={MET_AREA_DIR}/poly_umukv.nc +POINT_STAT_MASK_SID={MET_STN_DIR}/2011.stns, + {MET_STN_DIR}/2014.stns, + {MET_STN_DIR}/2015.stns, + {MET_STN_DIR}/2020.stns, + {MET_STN_DIR}/2024.stns, + {MET_STN_DIR}/2103.stns +POINT_STAT_MASK_LLPNT= + +# Select the type of output file(s) you wish to create +POINT_STAT_OUTPUT_FLAG_FHO = BOTH +POINT_STAT_OUTPUT_FLAG_CTC = BOTH +POINT_STAT_OUTPUT_FLAG_CTS = BOTH +POINT_STAT_OUTPUT_FLAG_MCTC = BOTH +#POINT_STAT_OUTPUT_FLAG_MCTS = +POINT_STAT_OUTPUT_FLAG_CNT = BOTH +POINT_STAT_OUTPUT_FLAG_SL1L2 = BOTH +#POINT_STAT_OUTPUT_FLAG_SAL1L2 = +POINT_STAT_OUTPUT_FLAG_VL1L2 = NONE +#POINT_STAT_OUTPUT_FLAG_VAL1L2 = +POINT_STAT_OUTPUT_FLAG_VCNT = NONE +#POINT_STAT_OUTPUT_FLAG_PCT = +#POINT_STAT_OUTPUT_FLAG_PSTD = +#POINT_STAT_OUTPUT_FLAG_PJC = +#POINT_STAT_OUTPUT_FLAG_PRC = +#POINT_STAT_OUTPUT_FLAG_ECNT = +#POINT_STAT_OUTPUT_FLAG_RPS = +#POINT_STAT_OUTPUT_FLAG_ECLV = +POINT_STAT_OUTPUT_FLAG_MPR = NONE + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for observation input to PointStat relative to OBS_POINT_STAT_INPUT_DIR +#OBS_POINT_STAT_INPUT_TEMPLATE = PYTHON_NUMPY = {ODBSCRIPT_DIR}/PointStat_ReadODB.py {INPUT_BASE_OBSERVATION}/{valid?fmt=%Y%m%d}T{valid?fmt=%H%M}Z_ukv_surface_odb2 {INPUT_BASE_OBSERVATION}/{MODEL}_{valid?fmt=%Y%m%d}T{valid?fmt=%H}00Z_surf_{lead?fmt=%3H}_ukv.csv True +OBS_POINT_STAT_INPUT_TEMPLATE = Surface_{valid?fmt=%Y%m%d}T{valid?fmt=%H}00Z.nc diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/file/metoffice/user_system_local.conf b/src/CSET/cset_workflow/app/metplus_point_stat/file/metoffice/user_system_local.conf new file mode 100755 index 000000000..ca84301fd --- /dev/null +++ b/src/CSET/cset_workflow/app/metplus_point_stat/file/metoffice/user_system_local.conf @@ -0,0 +1,23 @@ +[dir] +# any {ENV[]} variable is defined within [env] section of the rose-app.conf + +#INPUT_BASE_FORECAST = {ENV[PNT_FCST]} +#INPUT_BASE_OBSERVATION = {ENV[PNT_OBVS]} +# +#OUTPUT_BASE={ENV[MET_OUTPUT]} +OUTPUT_BASE_RES={ENV[MET_OUTPUT_RES]} + +#MET_INSTALL_DIR={ENV[VERSION_MET]} + +MET_AREA_DIR={ENV[METPLUS_AREA_DIR]} +MET_STN_DIR={ENV[METPLUS_STN_DIR]} +#ODBSCRIPT_DIR={ENV[READ_ODB]} + +# TIMESTRING is linked to whattime in rose-app.conf, linked to the suite.rc so that the time formats in the suite.rc and the METplus' PointStat.conf match up. +TIMESTRING={ENV[TASK_START_TIME]} + +FCAST_LEADS={ENV[FORECAST_LENGTH]} + +## Additional config files required to run Pointstat +#ADD_CONFIG={ENV[ADD_CONFIG_PS]}/{ENV[MET_config_PS]} +#ADD_CONFIG={ENV[CYLC_SUITE_DEF_PATH]}/app/metplus_point_stat/file/metoffice/PointStat_MET_UKV.conf diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/opt/rose-app-metoffice.conf b/src/CSET/cset_workflow/app/metplus_point_stat/opt/rose-app-metoffice.conf new file mode 100644 index 000000000..b698c46fd --- /dev/null +++ b/src/CSET/cset_workflow/app/metplus_point_stat/opt/rose-app-metoffice.conf @@ -0,0 +1,10 @@ +[command] +default=run_metplus.py metoffice/PointStat_METplus_UKV.conf metoffice/PointStat_UKV_Areas0.conf metoffice/user_system_local.conf + +[env] +# METplus_config_PS = Pointstat_METplus_${CYLC_TASK_PARAM_model}.conf +CONDA_VENV_LOCATION = ${CONDA_METPLUS_VENV_LOCATION} +METPLUS_FCST_DIR = ${METPLUS_FCST_DIR} +METPLUS_NCOBS_DIR = ${CYLC_WORKFLOW_SHARE_DIR}/nc_obs +START_TIME=${TASK_START_TIME} +FORECAST_LENGTH = ${FORECAST_LENGTH} diff --git a/src/CSET/cset_workflow/includes/metplus_common.cylc b/src/CSET/cset_workflow/includes/metplus_common.cylc index 93beedf8d..8f6f13037 100644 --- a/src/CSET/cset_workflow/includes/metplus_common.cylc +++ b/src/CSET/cset_workflow/includes/metplus_common.cylc @@ -4,6 +4,8 @@ METPLUS_ANA_DIR = {{METPLUS_ANA_DIR}} METPLUS_FCST_DIR = {{METPLUS_FCST_DIR}} METPLUS_OBS_DIR = {{METPLUS_OBS_DIR}} + METPLUS_AREA_DIR = {{METPLUS_AREA_DIR}} + METPLUS_STN_DIR = {{METPLUS_STN_DIR}} ROSE_APP_OPT_CONF_KEYS = {{METPLUS_OPT_CONFIG_KEYS}} TASK_START_TIME = $(cylc cyclepoint --template CCYYMMDDThh) FORECAST_LENGTH = $(isodatetime --as-total H -- {{ANALYSIS_LENGTH}})