diff --git a/cases/icon-test-euler/config.yaml b/cases/icon-test-euler/config.yaml index 28fd1269..f02fb174 100644 --- a/cases/icon-test-euler/config.yaml +++ b/cases/icon-test-euler/config.yaml @@ -1,55 +1,51 @@ -# Configuration file for the 'icon-async-test' case with ICON +# Global ICON simulation initialized from ERA5 (IC only, no LBC). -workflow: icon +workflow: icon-global-era5 constraint: EPYC_7H12 run_on: cpu compute_queue: normal.4h ntasks_per_node: 12 restart_step: PT6H startdate: 2018-01-01T00:00:00Z -enddate: 2018-01-01T12:00:00Z - -eccodes_dir: ./input/eccodes_definitions -iconremap_bin: ./ext/icontools/icontools/iconremap -iconsub_bin: ./ext/icontools/icontools/iconsub -latbc_filename: ifs_201801_lbc.nc -inidata_filename: ifs_init_2018010100.nc -output_filename: NWP_LAM +enddate: 2018-01-01T12:00:00Z +era5_partab: ./cases/icon-euler-test/partab_era5_to_icon.txt +era5_ic_runjob_filename: era5_ic_runjob.cfg +inidata_prefix: "era5_init_R02B05_" +inidata_nameformat: "%Y%m%d%H" +inidata_filename_suffix: ".nc" +output_filename: icon-test-euler filename_format: _DOM_ lateral_boundary_grid_order: lateral_boundary walltime: - prepare_icon: '00:10:00' - icontools: '00:30:00' - icon: '00:30:00' + prepare_icon: "00:10:00" + era5_ic: "00:40:00" + icon: "00:30:00" -meteo: - dir: ./input/meteo - prefix: ifs_ - nameformat: '%Y%m%d%H' - suffix: .grb - inc: 3 +era5: + dir: ./input/era5 + ml_filename: "era5_ml_{ymd}.grib" + sfc_filename: "era5_surf_{ymd}.grib" + global_nudging: False -icontools_runjobs: - - icontools_remap_ic_runjob.cfg - - icontools_remap_00_lbc_runjob.cfg - - icontools_remap_lbc_rest_runjob.cfg +meteo: + dir: ./input/meteo + prefix: ifs_ + nameformat: "%Y%m%d%H" + suffix: .grb + inc: 3 input_files: - radiation_grid_filename: ./input/icon/grid/VERIFY_DOM_DOM01.parent.nc - dynamics_grid_filename: ./input/icon/grid/VERIFY_DOM_DOM01.nc - map_file_latbc: ./input/icon/grid/map_file.latbc - lateral_boundary_grid: ./input/icon/grid/lateral_boundary.grid.nc - extpar_filename: ./input/icon/grid/external_parameter_icon_VERIFY_DOM_DOM01_tiles.nc - cldopt_filename: ./input/icon/rad/rrtm_cldopt.nc - lrtm_filename: ./input/icon/rad/rrtmg_lw.nc - map_file_ana: ./input/icon/mapping/map_file.ana + dynamics_grid_filename: ./input/icon/grid/iconR2B05-DOM01.nc + extpar_filename: ./input/icon/grid/extpar_iconR2B05-DOM01.nc + lrtm_filename: ./input/icon/rad/rrtmg_lw.nc + cldopt_filename: ./input/icon/rad/rrtm_cldopt.nc + map_file_ana: ./input/icon/mapping/map_file.ana icon: - binary_file: ./ext/icon/bin/icon - runjob_filename: icon_runjob.cfg - np_tot: 96 - np_io: 1 - np_restart: 1 - np_prefetch: 1 - + binary_file: /cluster/work/climate/lroither/icon_c2sm/icon/bin/icon + runjob_filename: icon_runjob.cfg + np_tot: 96 + np_io: 1 + np_restart: 1 + np_prefetch: 0 diff --git a/cases/icon-test-euler/era5_ic_runjob.cfg b/cases/icon-test-euler/era5_ic_runjob.cfg new file mode 100644 index 00000000..fbe2f24e --- /dev/null +++ b/cases/icon-test-euler/era5_ic_runjob.cfg @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +#SBATCH --job-name=era5_ic_{cfg.casename}_{cfg.startdate_sim_yyyymmddhh} +#SBATCH --time={cfg.walltime_era5_ic} +#SBATCH --partition={cfg.compute_queue} +#SBATCH --constraint={cfg.constraint} +#SBATCH --ntasks=1 +#SBATCH --output={cfg.logfile} +#SBATCH --open-mode=append +#SBATCH --chdir={cfg.icon_work} + +# --------------------------------------------------------------------- +# ERA5 -> ICON initial conditions (GLOBAL, IC only) +# +# This script is submitted by jobs/era5_ic.py. +# It MUST write the final IC file exactly to: +# {inidata_filename} +# +# Design: +# - Use case config for input locations + partab +# - Use staged ICON files from prepare_icon (cfg.input_files_scratch_*) +# - Keep all intermediate files inside cfg.icon_work +# --------------------------------------------------------------------- +# +set -euo pipefail +set -x +ulimit -s unlimited + +# Optional: load modules needed for cdo/nco on Euler. +# Adjust these to your environment/stack. +module load stack/2024-06 || true +module load cdo/2.2.2 nco/5.1.6 netcdf-c/4.9.2 || true + +# --------------------------------------------------------------------- +# Inputs (from cfg) +# --------------------------------------------------------------------- +ERA5_DIR="{cfg.era5_dir}" +ERA5_ML="{cfg.era5_ml_filename}" +ERA5_SFC="{cfg.era5_sfc_filename}" +# Replace {ymd} placeholder using values prepared in era5_ic.py +ERA5_ML="${{ERA5_ML/\{ymd\}/{cfg.era5_ymd}}}" +ERA5_SFC="${{ERA5_SFC/\{ymd\}/{cfg.era5_ymd}}}" +PARTAB="{cfg.era5_partab}" +GRID_TRI="{cfg.input_files_scratch_dynamics_grid_filename}" +EXTPAR="{cfg.input_files_scratch_extpar_filename}" +OUTFILE="{inidata_filename}" + +# --------------------------------------------------------------------- +# Working directory cleanup +# --------------------------------------------------------------------- +work="{cfg.icon_work}/era5_ic_work" +mkdir -p "$work" +cd "$work" + +# --------------------------------------------------------------------- +# 1) GRIB preprocessing: pick first timestep and convert to NetCDF +# --------------------------------------------------------------------- +# +# Note: your colleague used splitsel,1; keep it to ensure we only have t0. +cdo splitsel,1 "${{ERA5_DIR}}/${{ERA5_ML}}" era5_ml_ +cdo splitsel,1 "${{ERA5_DIR}}/${{ERA5_SFC}}" era5_sfc_ + +# Find the produced split files (avoid hardcoding the exact suffix) +ml_grb="$(ls -1 era5_ml_* | head -n 1)" +sfc_grb="$(ls -1 era5_sfc_* | head -n 1)" +cdo -t ecmwf -f nc copy "$ml_grb" era5_ml.nc +cdo -t ecmwf -f nc copy "$sfc_grb" era5_sfc.nc +cdo merge era5_ml.nc era5_sfc.nc era5_original.nc + +# Rename variables / metadata to ICON-consistent naming via partab +# (Your partab is the "contract" between ERA5 and ICON naming expectations.) +cdo setpartabn,"$PARTAB",convert era5_original.nc tmp.nc + +# Order variables (helps reproducibility and easier diffs) +ncks tmp.nc data_in.nc +rm -f tmp.nc era5_ml.nc era5_sfc.nc era5_original.nc "$ml_grb" "$sfc_grb" + +# --------------------------------------------------------------------- +# 2) Build the ICON triangular grid description for CDO remapping +# --------------------------------------------------------------------- +cdo -s selgrid,1 "$GRID_TRI" triangular-grid.nc + +# --------------------------------------------------------------------- +# 3) Land/sea-aware remapping for sensitive surface variables +# --------------------------------------------------------------------- +# +# Input land-sea mask from renamed ERA5 data +cdo selname,LSM data_in.nc LSM_in.nc +ncrename -h -v LSM,FR_LAND LSM_in.nc + +# Output land fraction from EXTPAR +cdo selname,FR_LAND "$EXTPAR" LSM_out_tmp.nc + +# Add time dimension and align time coordinate +ncecat -O -u time LSM_out_tmp.nc LSM_out_tmp.nc +ncks -h -A -v time LSM_in.nc LSM_out_tmp.nc + +# Build land/ocean masks (input and output) +cdo -L setctomiss,0. -ltc,0.5 LSM_in.nc oceanmask_in.nc +cdo -L setctomiss,0. -gec,0.5 LSM_in.nc landmask_in.nc +cdo -L setctomiss,0. -ltc,0.5 LSM_out_tmp.nc oceanmask_out.nc +cdo -L setctomiss,0. -gec,0.5 LSM_out_tmp.nc landmask_out.nc +cdo setrtoc2,0.5,1.0,1,0 LSM_out_tmp.nc LSM_out.nc +rm -f LSM_in.nc LSM_out_tmp.nc + +# Variables defined only on sea +ncks -h -v SST,CI data_in.nc datasea_in.nc + +# Variables defined on both but should not mix land/sea +ncks -h -v SKT,STL1,STL2,STL3,STL4,ALB_SNOW,W_SNOW,T_SNOW data_in.nc dataland_in.nc + +# Ocean part: mask -> fill -> remap -> unmask +cdo div dataland_in.nc oceanmask_in.nc tmp1_land.nc +cdo div datasea_in.nc oceanmask_in.nc tmp1_sea.nc +cdo setmisstodis tmp1_land.nc tmp2_land.nc +cdo setmisstodis tmp1_sea.nc tmp2_sea.nc +cdo remapdis,triangular-grid.nc tmp2_land.nc tmp3_land.nc +cdo remapdis,triangular-grid.nc tmp2_sea.nc tmp3_sea.nc +cdo div tmp3_land.nc oceanmask_out.nc dataland_ocean_out.nc +cdo div tmp3_sea.nc oceanmask_out.nc datasea_ocean_out.nc +rm -f tmp*_land.nc tmp*_sea.nc oceanmask_in.nc oceanmask_out.nc + +# Land part: mask -> fill -> remap -> unmask +cdo div dataland_in.nc landmask_in.nc tmp1.nc +cdo setmisstodis tmp1.nc tmp2.nc +cdo remapdis,triangular-grid.nc tmp2.nc tmp3.nc +cdo div tmp3.nc landmask_out.nc dataland_land_out.nc +rm -f tmp*.nc landmask_in.nc landmask_out.nc dataland_in.nc datasea_in.nc + +# Merge land+ocean contributions +cdo ifthenelse LSM_out.nc dataland_land_out.nc dataland_ocean_out.nc dataland_out.nc +rm -f dataland_ocean_out.nc dataland_land_out.nc + +# Remap all remaining variables together +ncks -h -x -v SKT,STL1,STL2,STL3,STL4,SMIL1,SMIL2,SMIL3,SMIL4,ALB_SNOW,W_SNOW,T_SNOW,SST,CI,LSM data_in.nc datarest_in.nc +cdo -s remapdis,triangular-grid.nc datarest_in.nc era5_final.nc +rm -f datarest_in.nc + +# Fill missing SST/CI after remap (sea-only fields) +cdo setmisstodis -selname,SST,CI datasea_ocean_out.nc datasea_ocean_out_filled.nc +rm -f datasea_ocean_out.nc + +# Merge special remapped variables + land fraction (renamed back to LSM) +ncks -h -A dataland_out.nc era5_final.nc +ncks -h -A datasea_ocean_out_filled.nc era5_final.nc +ncks -h -A -v FR_LAND LSM_out.nc era5_final.nc +ncrename -h -v FR_LAND,LSM era5_final.nc +rm -f LSM_out.nc dataland_out.nc datasea_ocean_out_filled.nc + +# --------------------------------------------------------------------- +# 4) Soil moisture index conversion (ERA5 SWVL -> ICON-style SMIL) +# --------------------------------------------------------------------- +# Extract soil moisture and soil type +ncks -h -v SMIL1,SMIL2,SMIL3,SMIL4,SLT data_in.nc swvl.nc +rm -f data_in.nc + +# IFS soil constants (documented in ERA5 data documentation) +wiltingp=(0 0.059 0.151 0.133 0.279 0.335 0.267 0.151) +fieldcap=(0 0.244 0.347 0.383 0.448 0.541 0.663 0.347) +smi_equation="" +for ilev in {1..4}; do + smi_equation="${smi_equation}SMIL${ilev}=(SMIL${ilev}-${wiltingp[1]})/(${fieldcap[1]}-${wiltingp[1]})*(SLT==1)" + for ist in {2..7}; do + smi_equation="${smi_equation}+(SMIL${ilev}-${wiltingp[$ist]})/(${fieldcap[$ist]}-${wiltingp[$ist]})*(SLT==${ist})" + done + smi_equation="${smi_equation};" +done +cdo expr,"${smi_equation}" swvl.nc smil_in.nc +rm -f swvl.nc + +# Remap SMIL to triangular grid and overwrite +cdo -s remapdis,triangular-grid.nc smil_in.nc smil_out.nc +ncks -A -v SMIL1,SMIL2,SMIL3,SMIL4 smil_out.nc era5_final.nc +rm -f smil_in.nc smil_out.nc + +# --------------------------------------------------------------------- +# 5) Create LNPS (log surface pressure) and finalize dimensions +# --------------------------------------------------------------------- +cdo expr,'LNPS=ln(PS);' era5_final.nc tmp.nc +ncks -A -v LNPS tmp.nc era5_final.nc +rm -f tmp.nc + +# ICON conventions: rename dimensions +ncrename -h -d cell,ncells era5_final.nc +ncrename -h -d nv,vertices era5_final.nc + +# Write final IC file to the exact path ICON will read later +mkdir -p "$(dirname "$OUTFILE")" +ncks era5_final.nc "$OUTFILE" + +# cleanup +rm -f era5_final.nc triangular-grid.nc +echo "Wrote ICON initial condition file: $OUTFILE" diff --git a/cases/icon-test-euler/icon_runjob.cfg b/cases/icon-test-euler/icon_runjob.cfg index a2bfbb46..d881a0cc 100755 --- a/cases/icon-test-euler/icon_runjob.cfg +++ b/cases/icon-test-euler/icon_runjob.cfg @@ -8,38 +8,31 @@ #SBATCH --chdir={cfg.icon_work} set +x -ulimit -s unlimited +ulimit -s unlimited -# OpenMP environment variables -# ---------------------------- -export OMP_NUM_THREADS=1 -export ICON_THREADS=1 +# OpenMP environment variables +export OMP_NUM_THREADS=1 +export ICON_THREADS=1 export OMP_SCHEDULE=static -export OMP_DYNAMIC="false" -export OMP_STACKSIZE=200M -# -# MPI variables -# ------------- +export OMP_DYNAMIC="false" +export OMP_STACKSIZE=200M + +# MPI settings no_of_nodes=1 -mpi_procs_pernode={cfg.icon_np_tot} -((mpi_total_procs=no_of_nodes * mpi_procs_pernode)) -# -# blocking length -# --------------- +mpi_procs_pernode={cfg.icon_np_tot} +((mpi_total_procs=no_of_nodes * mpi_procs_pernode)) + +# blocking length nproma=64 # ---------------------------------------------------------------------------- -# create ICON master namelist +# ICON master namelist # ---------------------------------------------------------------------------- - cat > icon_master.namelist << EOF -! master_nml: ---------------------------------------------------------------- &master_nml - lrestart = {cfg.lrestart} ! .TRUE.=current experiment is resumed - read_restart_namelists = .true. + lrestart = {cfg.lrestart} + read_restart_namelists = .true. / - -! master_time_control_nml: --------------------------------------------------- &master_time_control_nml calendar = 'proleptic gregorian' restartTimeIntval = '{cfg.restart_step}' @@ -47,295 +40,118 @@ cat > icon_master.namelist << EOF experimentStartDate = '{cfg.ini_datetime_string}' experimentStopDate = '{cfg.end_datetime_string}' / - -! master_model_nml: repeated for each model ---------------------------------- &master_model_nml - model_type = 1 ! identifies which component to run (atmosphere,ocean,...) - model_name = "ATMO" ! character string for naming this component. - model_namelist_filename = "NAMELIST_{cfg.casename}" ! file name containing the model namelists - model_min_rank = 1 ! start MPI rank for this model - model_max_rank = 65536 ! end MPI rank for this model - model_inc_rank = 1 ! stride of MPI ranks + model_type = 1 + model_name = "ATMO" + model_namelist_filename = "NAMELIST_{cfg.casename}" + model_min_rank = 1 + model_max_rank = 65536 + model_inc_rank = 1 / EOF - -# ---------------------------------------------------------------------- -# model namelists -# ---------------------------------------------------------------------- - +# ---------------------------------------------------------------------------- +# Model namelist +# ---------------------------------------------------------------------------- cat > NAMELIST_{cfg.casename} << EOF -! parallel_nml: MPI parallelization ------------------------------------------- ¶llel_nml - nproma = 128 ! loop chunk length - p_test_run = .FALSE. ! .TRUE. means verification run for MPI parallelization - num_io_procs = {cfg.icon_np_io} ! number of I/O processors - num_restart_procs = {cfg.icon_np_restart} ! number of restart processors - num_prefetch_proc = {cfg.icon_np_prefetch} ! number of processors for LBC prefetching - iorder_sendrecv = 3 ! sequence of MPI send/receive calls + nproma = 128 + p_test_run = .FALSE. + num_io_procs = {cfg.icon_np_io} + num_restart_procs = {cfg.icon_np_restart} + num_prefetch_proc = {cfg.icon_np_prefetch} ! must be 0 for global IC-only + iorder_sendrecv = 3 / - - -! run_nml: general switches --------------------------------------------------- &run_nml - ltestcase = .FALSE. ! real case run - num_lev = 60 ! number of full levels (atm.) for each domain - lvert_nest = .FALSE. ! no vertical nesting - dtime = 60. ! timestep in seconds - ldynamics = .TRUE. ! compute adiabatic dynamic tendencies - ltransport = .TRUE. ! compute large-scale tracer transport - ntracer = 0 ! number of advected tracers - iforcing = 3 ! forcing of dynamics and transport by parameterized processes - msg_level = 13 ! detailed report during integration - ltimer = .TRUE. ! timer for monitoring the runtime of specific routines - timers_level = 10 ! performance timer granularity - check_uuid_gracefully = .TRUE. ! give only warnings for non-matching uuids - output = "nml" ! main switch for enabling/disabling components of the model output - lart = .FALSE. ! main switch for ART - debug_check_level = 10 - restart_filename = "{cfg.icon_restart_out}/{cfg.output_filename}_.nc" - activate_sync_timers = .TRUE. + ltestcase = .FALSE. + num_lev = 60 + lvert_nest = .FALSE. + dtime = 60. + ldynamics = .TRUE. + ltransport = .TRUE. + ntracer = 0 + iforcing = 3 + msg_level = 13 + ltimer = .TRUE. + timers_level = 10 + check_uuid_gracefully = .TRUE. + output = "nml" + lart = .FALSE. + debug_check_level = 10 + restart_filename = "{cfg.icon_restart_out}/{cfg.output_filename}_.nc" + activate_sync_timers = .TRUE. / - -! diffusion_nml: horizontal (numerical) diffusion ---------------------------- -&diffusion_nml - lhdiff_vn = .TRUE. ! diffusion on the horizontal wind field - lhdiff_temp = .TRUE. ! diffusion on the temperature field - lhdiff_w = .TRUE. ! diffusion on the vertical wind field - hdiff_order = 5 ! order of nabla operator for diffusion - itype_vn_diffu = 1 ! reconstruction method used for Smagorinsky diffusion - itype_t_diffu = 2 ! discretization of temperature diffusion - hdiff_efdt_ratio = 24.0 ! ratio of e-folding time to time step - hdiff_smag_fac = 0.025 ! scaling factor for Smagorinsky diffusion -/ - -! dynamics_nml: dynamical core ----------------------------------------------- &dynamics_nml - iequations = 3 ! type of equations and prognostic variables - divavg_cntrwgt = 0.50 ! weight of central cell for divergence averaging - lcoriolis = .TRUE. ! Coriolis force + iequations = 3 + divavg_cntrwgt = 0.50 + lcoriolis = .TRUE. / - -! extpar_nml: external data -------------------------------------------------- &extpar_nml - itopo = 1 ! topography (0:analytical) - extpar_filename = '{cfg.input_files_scratch_extpar_filename}' ! filename of external parameter input file - n_iter_smooth_topo = 1,1 ! iterations of topography smoother - heightdiff_threshold = 3000. ! height difference between neighb. grid points - hgtdiff_max_smooth_topo = 750.,750. ! see Namelist doc - heightdiff_threshold = 2250.,1500. + itopo = 1 + extpar_filename = '{cfg.input_files_scratch_extpar_filename}' + n_iter_smooth_topo = 1,1 + heightdiff_threshold = 3000. + hgtdiff_max_smooth_topo = 750.,750. / - -! initicon_nml: specify read-in of initial state ------------------------------ &initicon_nml - init_mode = 2 ! 7: start from DWD fg with subsequent vertical remapping - lread_ana = .FALSE. ! no analysis data will be read - ifs2icon_filename = "{inidata_filename}" ! initial data filename - ana_varnames_map_file = "{cfg.input_files_scratch_map_file_ana}" ! dictionary mapping internal names onto GRIB2 shortNames - ltile_coldstart = .TRUE. ! coldstart for surface tiles - ltile_init = .FALSE. ! set it to .TRUE. if FG data originate from run without tiles + init_mode = 2 + lread_ana = .FALSE. + ifs2icon_filename = "{inidata_filename}" ! produced by era5_ic job + ana_varnames_map_file = "{cfg.input_files_scratch_map_file_ana}" + ltile_coldstart = .TRUE. + ltile_init = .FALSE. / - -! grid_nml: horizontal grid -------------------------------------------------- &grid_nml - dynamics_grid_filename = "{cfg.input_files_scratch_dynamics_grid_filename}" ! array of the grid filenames for the dycore - radiation_grid_filename = "{cfg.input_files_scratch_radiation_grid_filename}" ! array of the grid filenames for the radiation model - dynamics_parent_grid_id = 0 ! array of the indexes of the parent grid filenames - lredgrid_phys = .TRUE. ! .true.=radiation is calculated on a reduced grid - lfeedback = .TRUE. ! specifies if feedback to parent grid is performed - l_limited_area = .TRUE. ! .TRUE. performs limited area run - ifeedback_type = 2 ! feedback type (incremental/relaxation-based) - start_time = 0. ! Time when a nested domain starts to be active [s] + dynamics_grid_filename = "{cfg.input_files_scratch_dynamics_grid_filename}" + radiation_grid_filename = "{cfg.input_files_scratch_dynamics_grid_filename}" + dynamics_parent_grid_id = 0 + lredgrid_phys = .TRUE. + l_limited_area = .FALSE. ! IMPORTANT: global mode / - -! gridref_nml: grid refinement settings -------------------------------------- -&gridref_nml - denom_diffu_v = 150. ! denominator for lateral boundary diffusion of velocity -/ - -! interpol_nml: settings for internal interpolation methods ------------------ -&interpol_nml - nudge_zone_width = 8 ! width of lateral boundary nudging zone - support_baryctr_intp = .FALSE. ! barycentric interpolation support for output - nudge_max_coeff = 0.07 - nudge_efold_width = 2.0 -/ - -! io_nml: general switches for model I/O ------------------------------------- &io_nml - itype_pres_msl = 5 ! method for computation of mean sea level pressure - itype_rh = 1 ! method for computation of relative humidity - lmask_boundary = .TRUE. ! mask out interpolation zone in output - restart_file_type = 5 + itype_pres_msl = 5 + itype_rh = 1 + restart_file_type = 5 / - -! limarea_nml: settings for limited area mode --------------------------------- -&limarea_nml - itype_latbc = 1 ! 1: time-dependent lateral boundary conditions - dtime_latbc = 10800 ! time difference between 2 consecutive boundary data - nlev_latbc = 90 ! Number of vertical levels in boundary data - latbc_boundary_grid = "{cfg.input_files_scratch_lateral_boundary_grid}" ! Grid file defining the lateral boundary - latbc_path = "{cfg.icon_input_icbc}" ! Absolute path to boundary data - latbc_varnames_map_file = "{cfg.input_files_scratch_map_file_latbc}" - latbc_filename = "{cfg.latbc_filename}" ! boundary data input filename - init_latbc_from_fg = .FALSE. ! .TRUE.: take lbc for initial time from first guess -/ - -! lnd_nml: land scheme switches ----------------------------------------------- -&lnd_nml - ntiles = 1 ! number of tiles - nlev_snow = 3 ! number of snow layers - lmulti_snow = .FALSE. ! .TRUE. for use of multi-layer snow model - idiag_snowfrac = 20 ! type of snow-fraction diagnosis - lsnowtile = .TRUE. ! .TRUE.=consider snow-covered and snow-free separately - itype_root = 2 ! root density distribution - itype_heatcond = 3 ! type of soil heat conductivity - itype_lndtbl = 4 ! table for associating surface parameters - itype_evsl = 4 ! type of bare soil evaporation - itype_root = 2 ! root density distribution - cwimax_ml = 5.e-4 ! scaling parameter for max. interception storage - c_soil = 1.75 ! surface area density of the evaporative soil surface - c_soil_urb = 0.5 ! same for urban areas - lseaice = .TRUE. ! .TRUE. for use of sea-ice model - llake = .TRUE. ! .TRUE. for use of lake model -/ - -! nonhydrostatic_nml: nonhydrostatic model ----------------------------------- -&nonhydrostatic_nml - iadv_rhotheta = 2 ! advection method for rho and rhotheta - ivctype = 2 ! type of vertical coordinate - itime_scheme = 4 ! time integration scheme - ndyn_substeps = 5 ! number of dynamics steps per fast-physics step - exner_expol = 0.333 ! temporal extrapolation of Exner function - vwind_offctr = 0.2 ! off-centering in vertical wind solver - damp_height = 12500.0 ! height at which Rayleigh damping of vertical wind starts - rayleigh_coeff = 1.5 ! Rayleigh damping coefficient - divdamp_order = 24 ! order of divergence damping - divdamp_type = 3 ! type of divergence damping - divdamp_fac = 0.004 ! scaling factor for divergence damping - igradp_method = 3 ! discretization of horizontal pressure gradient - l_zdiffu_t = .TRUE. ! specifies computation of Smagorinsky temperature diffusion - thslp_zdiffu = 0.02 ! slope threshold (temperature diffusion) - thhgtd_zdiffu = 125.0 ! threshold of height difference (temperature diffusion) - htop_moist_proc = 22500.0 ! max. height for moist physics - hbot_qvsubstep = 22500.0 ! height above which QV is advected with substepping scheme -/ - -! nwp_phy_nml: switches for the physics schemes ------------------------------ &nwp_phy_nml - inwp_gscp = 2 ! cloud microphysics and precipitation - inwp_convection = 1 ! convection - lshallowconv_only = .FALSE. ! only shallow convection - inwp_radiation = 1 ! radiation - inwp_cldcover = 1 ! cloud cover scheme for radiation - inwp_turb = 1 ! vertical diffusion and transfer - inwp_satad = 1 ! saturation adjustment - inwp_sso = 1 ! subgrid scale orographic drag - inwp_gwd = 0 ! non-orographic gravity wave drag - inwp_surface = 1 ! surface scheme - latm_above_top = .TRUE. ! take into account atmosphere above model top for radiation computation - ldetrain_conv_prec = .TRUE. - efdt_min_raylfric = 7200. ! minimum e-folding time of Rayleigh friction - itype_z0 = 2 ! type of roughness length data - icapdcycl = 3 ! apply CAPE modification to improve diurnalcycle over tropical land - icpl_aero_conv = 1 ! coupling between autoconversion and Tegen aerosol climatology - icpl_aero_gscp = 1 ! coupling between autoconversion and Tegen aerosol climatology - lrtm_filename = '{cfg.input_files_scratch_lrtm_filename}' ! longwave absorption coefficients for RRTM_LW - cldopt_filename = '{cfg.input_files_scratch_cldopt_filename}' ! RRTM cloud optical properties - dt_rad = 720. ! time step for radiation in s - dt_conv = 120.,90.,90. ! time step for convection in s (domain specific) - dt_sso = 120.,360.,360. ! time step for SSO parameterization - dt_gwd = 360.,360.,360. ! time step for gravity wave drag parameterization -/ - -! nwp_tuning_nml: additional tuning parameters ---------------------------------- -&nwp_tuning_nml - itune_albedo = 1 ! reduced albedo (w.r.t. MODIS data) over Sahara - tune_gkwake = 1.8 - tune_gkdrag = 0.01 - tune_minsnowfrac = 0.3 + inwp_gscp = 2 + inwp_convection = 1 + inwp_radiation = 1 + inwp_cldcover = 1 + inwp_turb = 1 + inwp_satad = 1 + inwp_sso = 1 + inwp_gwd = 0 + inwp_surface = 1 + latm_above_top = .TRUE. + efdt_min_raylfric = 7200. + itype_z0 = 2 + lrtm_filename = '{cfg.input_files_scratch_lrtm_filename}' + cldopt_filename = '{cfg.input_files_scratch_cldopt_filename}' + dt_rad = 720. + dt_conv = 120. + dt_sso = 120. + dt_gwd = 360. / - -! output_nml: specifies an output stream -------------------------------------- &output_nml - filetype = 4 ! output format: 2=GRIB2, 4=NETCDFv2 - dom = 1 ! write domain 1 only - output_bounds = 0., 10000000., 3600. ! start, end, increment - steps_per_file = 1 ! number of steps per file - mode = 1 ! 1: forecast mode (relative t-axis), 2: climate mode (absolute t-axis) - include_last = .TRUE. - output_filename = '{cfg.output_filename}' - filename_format = '{cfg.icon_output}/_DOM_' ! file name base - steps_per_file_inclfirst = .FALSE. - output_grid = .TRUE. - remap = 1 ! 1: remap to lat-lon grid - !north_pole = -170.,40. ! definition of north_pole for rotated lat-lon grid - reg_lon_def = -16.0,0.13,36.0 ! - reg_lat_def = 32.0,0.12,74.0 ! - ml_varlist = 'group:PBL_VARS', - 'group:ATMO_ML_VARS', - 'group:precip_vars', - 'group:land_vars', - 'group:nh_prog_vars', - 'z_mc', 'z_ifc', + filetype = 4 + dom = 1 + output_bounds = 0., 10000000., 3600. + steps_per_file = 1 + mode = 1 + include_last = .TRUE. + output_filename = '{cfg.output_filename}' + filename_format = '{cfg.icon_output}/_DOM_' + output_grid = .TRUE. + remap = 1 + reg_lon_def = -179.5,1.0,179.5 + reg_lat_def = 89.5,-1.0,-89.5 / - -! radiation_nml: radiation scheme --------------------------------------------- -&radiation_nml - irad_o3 = 7 ! ozone climatology - irad_aero = 6 ! aerosols - albedo_type = 2 ! type of surface albedo - vmr_co2 = 390.e-06 - vmr_ch4 = 1800.e-09 - vmr_n2o = 322.0e-09 - vmr_o2 = 0.20946 - vmr_cfc11 = 240.e-12 - vmr_cfc12 = 532.e-12 -/ - -! sleve_nml: vertical level specification ------------------------------------- -&sleve_nml - min_lay_thckn = 20.0 ! layer thickness of lowermost layer - top_height = 23000.0 ! height of model top - stretch_fac = 0.65 ! stretching factor to vary distribution of model levels - decay_scale_1 = 4000.0 ! decay scale of large-scale topography component - decay_scale_2 = 2500.0 ! decay scale of small-scale topography component - decay_exp = 1.2 ! exponent of decay function - flat_height = 16000.0 ! height above which the coordinate surfaces are flat -/ - -! transport_nml: tracer transport --------------------------------------------- -&transport_nml - npassive_tracer = 0 ! number of additional passive tracers - ivadv_tracer = 3, 3, 3, 3, 3, 3 ! tracer specific method to compute vertical advection - itype_hlimit = 3, 4, 4, 4, 4, 4 ! type of limiter for horizontal transport - ihadv_tracer = 52, 2, 2, 2, 2, 22 ! tracer specific method to compute horizontal advection - llsq_svd = .TRUE. ! use SV decomposition for least squares design matrix -/ - -! turbdiff_nml: turbulent diffusion ------------------------------------------- -&turbdiff_nml - tkhmin = 0.75 ! scaling factor for minimum vertical diffusion coefficient - tkmmin = 0.75 ! scaling factor for minimum vertical diffusion coefficient - pat_len = 750.0 ! effective length scale of thermal surface patterns - c_diff = 0.2 ! length scale factor for vertical diffusion of TKE - rat_sea = 7.5 ! controls laminar resistance for sea surface - rlam_heat = 1.5 - ltkesso = .TRUE. ! consider TKE-production by sub-grid SSO wakes - frcsmot = 0.2 ! these 2 switches together apply vertical smoothing of the TKE source terms - imode_frcsmot = 2 ! in the tropics (only), which reduces the moist bias in the tropical lower troposphere - itype_sher = 3 ! type of shear forcing used in turbulence - ltkeshs = .TRUE. ! include correction term for coarse grids in hor. shear production term - a_hshr = 2.0 ! length scale factor for separated horizontal shear mode - icldm_turb = 1 ! mode of cloud water representation in turbulence - ldiff_qi = .TRUE. -/ - EOF -# ---------------------------------------------------------------------- -# run the model! -# ---------------------------------------------------------------------- +# ---------------------------------------------------------------------------- +# Run ICON +# ---------------------------------------------------------------------------- source {cfg.chain_src_dir}/ext/icon/modules.env set -x -mpirun -n $mpi_total_procs ./icon +mpirun -n $mpi_total_procs ./{cfg.icon_execname} set +x diff --git a/cases/icon-test-euler/icontools_remap_00_lbc_runjob.cfg b/cases/icon-test-euler/icontools_remap_00_lbc_runjob.cfg deleted file mode 100755 index 0708bd96..00000000 --- a/cases/icon-test-euler/icontools_remap_00_lbc_runjob.cfg +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env bash -#SBATCH --job-name=iconsub -#SBATCH --chdir={cfg.icon_work} -#SBATCH --partition={cfg.compute_queue} -#SBATCH --constraint={cfg.constraint} -#SBATCH --ntasks=1 -#SBATCH --output={logfile} -#SBATCH --open-mode=append - -ulimit -s unlimited - -set -x - -export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions - -#----------------------------------------------------------------------------- -# PART I: Create auxiliary grid file which contains only the cells of the -# boundary zone. -#----------------------------------------------------------------------------- -cat > NAMELIST_ICONSUB_{cfg.startdate_sim_yyyymmddhh} << EOF_1 -&iconsub_nml - grid_filename = '{cfg.input_files_scratch_dynamics_grid_filename}', - output_type = 4, - lwrite_grid = .TRUE., -/ -&subarea_nml - ORDER = "{cfg.lateral_boundary_grid_order}", - grf_info_file = '{cfg.input_files_scratch_dynamics_grid_filename}', - min_refin_c_ctrl = 1 - max_refin_c_ctrl = 14 -/ -EOF_1 - -srun -n 1 {cfg.iconsub_bin} \ - --nml NAMELIST_ICONSUB_{cfg.startdate_sim_yyyymmddhh} 2>&1 - -#----------------------------------------------------------------------------- -# PART II: Extract boundary data -#----------------------------------------------------------------------------- -rm -f ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}* - -set +x -cat > NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} << EOF -! -&input_field_nml ! temperature - inputname = "T" - outputname = "T" - code = 130 - intp_method = 3 -/ -&input_field_nml ! horiz. wind comp. u - inputname = "U" - outputname = "U" - intp_method = 3 -/ -&input_field_nml ! horiz. wind comp. u - inputname = "V" - outputname = "V" - intp_method = 3 -/ -&input_field_nml ! vertical velocity - inputname = "OMEGA" - outputname = "W" - code = 135 - intp_method = 3 -/ -&input_field_nml ! surface pressure - inputname = "LNSP" - outputname = "LNPS" - code = 152 - intp_method = 3 -/ -&input_field_nml ! geopotential - inputname = "Z" - outputname = "GEOSP" - code = 129 - intp_method = 3 -/ -&input_field_nml ! specific humidity - inputname = "QV" - outputname = "QV" - code = 133 - intp_method = 3 -/ -&input_field_nml ! cloud liquid water content - inputname = "CLWC" - outputname = "QC" - code = 246 - intp_method = 3 -/ -&input_field_nml ! cloud ice water content - inputname = "CIWC" - outputname = "QI" - code = 247 - intp_method = 3 -/ -&input_field_nml ! rain water content - inputname = "CRWC" - outputname = "QR" - code = 75 - intp_method = 3 -/ -&input_field_nml ! snow water content - inputname = "CSWC" - outputname = "QS" - code = 76 - intp_method = 3 -/ -EOF - -#----------------------------------------------------------------------------- -# loop over file list: - -echo "DATAFILELIST is {datafile_list}" -for datafilename in {datafile_list} ; do - datafile="${{datafilename##*/}}" # get filename without path - outdatafile=${{datafile%.*}} # get filename without suffix - cat > NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} << EOF_2C -&remap_nml - in_grid_filename = '{cfg.meteo_dir}/{cfg.meteo_prefix}{cfg.startdate_sim_yyyymmddhh}{cfg.meteo_suffix}' - in_filename = '{cfg.meteo_dir}/${{datafile}}' - in_type = 1 - out_grid_filename = '{cfg.input_files_scratch_lateral_boundary_grid}' - out_filename = '{cfg.icon_input_icbc}/${{outdatafile}}_lbc.nc' - out_type = 2 - out_filetype = 4 - l_have3dbuffer = .false. -! ncstorage_file = "ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}" -/ -EOF_2C - - srun -n 1 {cfg.iconremap_bin} -q \ - --remap_nml NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} \ - --input_field_nml NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} 2>&1 - -done - -#----------------------------------------------------------------------------- -# clean-up - -rm -f nml.log -rm -f NAMELIST_ICONSUB_{cfg.startdate_sim_yyyymmddhh} NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} - -#----------------------------------------------------------------------------- -exit -#----------------------------------------------------------------------------- - diff --git a/cases/icon-test-euler/icontools_remap_ic_runjob.cfg b/cases/icon-test-euler/icontools_remap_ic_runjob.cfg deleted file mode 100755 index f8d2275d..00000000 --- a/cases/icon-test-euler/icontools_remap_ic_runjob.cfg +++ /dev/null @@ -1,238 +0,0 @@ -#!/usr/bin/env bash -#SBATCH --job-name=iconremap_ic -#SBATCH --chdir={cfg.icon_work} -#SBATCH --partition={cfg.compute_queue} -#SBATCH --constraint={cfg.constraint} -#SBATCH --ntasks=1 -#SBATCH --output={logfile} -#SBATCH --open-mode=append - -ulimit -s unlimited - -set -x - -export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions - -#----------------------------------------------------------------------------- -# Remap inital data onto local (limited-area) grid -#----------------------------------------------------------------------------- -cat > NAMELIST_ICONREMAP_FIELDS << EOF -! -&input_field_nml ! temperature - inputname = "T" - outputname = "T" - code = 130 - intp_method = 3 -/ -&input_field_nml ! horiz. wind comp. u - inputname = "U" - outputname = "U" - intp_method = 3 -/ -&input_field_nml ! horiz. wind comp. u - inputname = "V" - outputname = "V" - intp_method = 3 -/ -&input_field_nml ! vertical velocity - inputname = "OMEGA" - outputname = "W" - code = 135 - intp_method = 3 -/ -&input_field_nml ! surface pressure - inputname = "LNSP" - outputname = "LNPS" - code = 152 - intp_method = 3 -/ -&input_field_nml ! geopotential - inputname = "Z" - outputname = "GEOSP" - code = 129 - intp_method = 3 -/ -&input_field_nml ! geopotential - inputname = "Z" - outputname = "GEOP_SFC" - code = 129 - intp_method = 3 -/ -&input_field_nml ! specific humidity - inputname = "QV" - outputname = "QV" - code = 133 - intp_method = 3 -/ -&input_field_nml ! cloud liquid water content - inputname = "CLWC" - outputname = "QC" - code = 246 - intp_method = 3 -/ -&input_field_nml ! cloud ice water content - inputname = "CIWC" - outputname = "QI" - code = 247 - intp_method = 3 -/ -&input_field_nml ! rain water content - inputname = "CRWC" - outputname = "QR" - code = 75 - intp_method = 3 -/ -&input_field_nml ! snow water content - inputname = "CSWC" - outputname = "QS" - code = 76 - intp_method = 3 -/ -&input_field_nml ! snow temperature - inputname = "TSN" - outputname = "T_SNOW" - code = 238 - intp_method = 3 -/ -&input_field_nml ! water content of snow - inputname = "SD" - outputname = "W_SNOW" - code = 141 - intp_method = 3 -/ -&input_field_nml ! density of snow - inputname = "RSN" - outputname = "RHO_SNOW" - code = 33 - intp_method = 3 -/ -&input_field_nml ! snow albedo - inputname = "ASN" - outputname = "ALB_SNOW" - code = 32 - intp_method = 3 -/ -&input_field_nml ! skin temperature - inputname = "SKT" - outputname = "SKT" - code = 235 - intp_method = 3 -/ -&input_field_nml ! sea surface temperature - inputname = "SST" - outputname = "SST" - code = 34 - intp_method = 3 -/ -&input_field_nml ! soil temperature level 1 - inputname = "STL1" - outputname = "STL1" - code = 139 - intp_method = 3 -/ -&input_field_nml ! soil temperature level 2 - inputname = "STL2" - outputname = "STL2" - code = 170 - intp_method = 3 -/ -&input_field_nml ! soil temperature level 3 - inputname = "STL3" - outputname = "STL3" - code = 183 - intp_method = 3 -/ -&input_field_nml ! soil temperature level 4 - inputname = "STL4" - outputname = "STL4" - code = 236 - intp_method = 3 -/ -&input_field_nml ! sea-ice cover - inputname = "CI" - outputname = "CI" - code = 31 - intp_method = 3 -/ -&input_field_nml ! water cont. of interception storage - inputname = "SRC" - outputname = "W_I" - code = 198 - intp_method = 3 -/ -&input_field_nml ! surface roughness - inputname = "SR" - outputname = "Z0" - code = 173 - intp_method = 3 -/ -&input_field_nml ! Land/sea mask - inputname = "LSM" - outputname = "LSM" - code = 172 - intp_method = 3 -/ -&input_field_nml ! soil moisture index layer 1 - inputname = "SWVL1" - outputname = "SMIL1" - code = 80 - intp_method = 3 -/ -&input_field_nml ! soil moisture index layer 2 - inputname = "SWVL2" - outputname = "SMIL2" - code = 81 - intp_method = 3 -/ -&input_field_nml ! soil moisture index layer 3 - inputname = "SWVL3" - outputname = "SMIL3" - code = 82 - intp_method = 3 -/ -&input_field_nml ! soil moisture index layer 4 - inputname = "SWVL4" - outputname = "SMIL4" - code = 83 - intp_method = 3 -/ -EOF - -#----------------------------------------------------------------------------- -# loop over file list: - -datafilename={cfg.meteo_dir}/{cfg.meteo_prefix}{cfg.startdate_sim_yyyymmddhh}{cfg.meteo_suffix} -datafile="${{datafilename##*/}}" # get filename without path -outdatafile=${{datafile%.*}} # get filename without suffix - -# create ICON master namelist -# ------------------------ -# For a complete list see Namelist_overview and Namelist_overview.pdf - -cat > NAMELIST_ICONREMAP << EOF -&remap_nml - in_grid_filename = '{cfg.meteo_dir}/{cfg.meteo_prefix}{cfg.startdate_sim_yyyymmddhh}{cfg.meteo_suffix}' - in_filename = '{cfg.meteo_dir}/${{datafile}}' - in_type = 1 - out_grid_filename = '{cfg.input_files_scratch_dynamics_grid_filename}' - out_filename = '{cfg.icon_input_icbc}/${{outdatafile}}.nc' - out_type = 2 - out_filetype = 4 - l_have3dbuffer = .false. -/ -EOF - -srun -n 1 {cfg.iconremap_bin} \ - -vvvvv -q --remap_nml NAMELIST_ICONREMAP \ - --input_field_nml NAMELIST_ICONREMAP_FIELDS 2>&1 - - -#----------------------------------------------------------------------------- -# clean-up - -rm -f ncstorage.tmp* -rm -f nml.log NAMELIST_SUB NAMELIST_ICONREMAP NAMELIST_ICONREMAP_FIELDS - -#----------------------------------------------------------------------------- -exit -#----------------------------------------------------------------------------- diff --git a/cases/icon-test-euler/icontools_remap_lbc_rest_runjob.cfg b/cases/icon-test-euler/icontools_remap_lbc_rest_runjob.cfg deleted file mode 100755 index 7f1d7968..00000000 --- a/cases/icon-test-euler/icontools_remap_lbc_rest_runjob.cfg +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash -#SBATCH --job-name=iconremap_lbc -#SBATCH --chdir={cfg.icon_work} -#SBATCH --partition={cfg.compute_queue} -#SBATCH --constraint={cfg.constraint} -#SBATCH --ntasks=1 -#SBATCH --output={logfile} -#SBATCH --open-mode=append - -ulimit -s unlimited - -set -x - -export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions - -#----------------------------------------------------------------------------- -# Extract boundary data -#----------------------------------------------------------------------------- - -rm -f ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}* - -cat > NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} << EOF -! -&input_field_nml ! temperature - inputname = "T" - outputname = "T" - code = 130 - intp_method = 3 -/ -&input_field_nml ! horiz. wind comp. u - inputname = "U" - outputname = "U" - intp_method = 3 -/ -&input_field_nml ! horiz. wind comp. v - inputname = "V" - outputname = "V" - intp_method = 3 -/ -&input_field_nml ! vertical velocity - inputname = "OMEGA" - outputname = "W" - code = 135 - intp_method = 3 -/ -&input_field_nml ! surface pressure - inputname = "LNSP" - outputname = "LNPS" - code = 152 - intp_method = 3 -/ -&input_field_nml ! specific humidity - inputname = "QV" - outputname = "QV" - code = 133 - intp_method = 3 -/ -&input_field_nml ! cloud liquid water content - inputname = "CLWC" - outputname = "QC" - code = 246 - intp_method = 3 -/ -&input_field_nml ! cloud ice water content - inputname = "CIWC" - outputname = "QI" - code = 247 - intp_method = 3 -/ -&input_field_nml ! rain water content - inputname = "CRWC" - outputname = "QR" - code = 75 - intp_method = 3 -/ -&input_field_nml ! snow water content - inputname = "CSWC" - outputname = "QS" - code = 76 - intp_method = 3 -/ -EOF - -#----------------------------------------------------------------------------- -# loop over file list: - -echo "DATAFILELIST is {datafile_list_rest}" -for datafilename in {datafile_list_rest} ; do - datafile="${{datafilename##*/}}" # get filename without path - outdatafile=${{datafile%.*}} # get filename without suffix - cat > NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} << EOF_2C -&remap_nml - in_grid_filename = '{cfg.meteo_dir}/{cfg.meteo_prefix}{cfg.startdate_sim_yyyymmddhh}{cfg.meteo_suffix}' - in_filename = '{cfg.meteo_dir}/${{datafile}}' - in_type = 1 - out_grid_filename = '{cfg.input_files_scratch_lateral_boundary_grid}' - out_filename = '{cfg.icon_input_icbc}/${{outdatafile}}_lbc.nc' - out_type = 2 - out_filetype = 4 - l_have3dbuffer = .false. -! ncstorage_file = "ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}" -/ -EOF_2C - - srun -n 1 {cfg.iconremap_bin} -q \ - --remap_nml NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} \ - --input_field_nml NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} 2>&1 - -done - -#----------------------------------------------------------------------------- -# clean-up - -#rm -f ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}* -rm -f nml.log -rm -f NAMELIST_ICONSUB_{cfg.startdate_sim_yyyymmddhh} NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} - -#----------------------------------------------------------------------------- -exit -#----------------------------------------------------------------------------- - diff --git a/cases/parent-global-r2b7-era5-update-euler/README.md b/cases/parent-global-r2b7-era5-update-euler/README.md new file mode 100644 index 00000000..8ae36cd5 --- /dev/null +++ b/cases/parent-global-r2b7-era5-update-euler/README.md @@ -0,0 +1,8 @@ +# Parent global R2B7 ERA5 case (scaffold) + +This case will: +- stage static ICON inputs (grid/extpar/radiation tables) +- generate ERA5-based initial conditions on R2B7 +- run a global NWP-style ICON simulation +- write hourly evaluation outputs +- write daily restarts (at least for 2013-05-05 .. 2013-05-22) diff --git a/cases/parent-global-r2b7-era5-update-euler/config.yaml b/cases/parent-global-r2b7-era5-update-euler/config.yaml new file mode 100644 index 00000000..e142ef0c --- /dev/null +++ b/cases/parent-global-r2b7-era5-update-euler/config.yaml @@ -0,0 +1,81 @@ +# Parent global run: ERA5 IC -> ICON R2B7 (global, no LBC) +workflow: icon-global-era5-parent-r2b7-nosst + +# Simulation window +startdate: 2013-05-25T00:00:00Z +enddate: 2013-05-25T06:00:00Z + +# HPC settings +constraint: EPYC_7H12 +compute_queue: normal.4h +run_on: cpu +ntasks_per_node: 128 + +# Restart cadence (ISO-8601 duration). P1D == PT24H. +restart_step: PT6H +# For the very first parent run from ERA5, lrestart must be false. +# (later chunks can set this automatically via the chain feature "restart") +lrestart: '.FALSE.' + +walltime: + prepare_icon: "00:10:00" + era5_ic: "00:40:00" + icon: "02:00:00" + +# ------------------------------------------------------------------- +# ERA5 inputs for the era5_ic job (MATCHES jobs/era5_ic.py EXPECTATIONS) +# ------------------------------------------------------------------- +era5_dir: /cluster/work/climate/lroither/icon_c2sm/icon_era5_global/era5_raw +era5_ml_filename: "era5_ml_{ymd}.grib" +era5_sfc_filename: "era5_surf_{ymd}.grib" + +# Partab: case-relative path (do NOT hardcode /cases/... here) +era5_partab: tables/partab_era5_to_icon.txt + +# Script template used by jobs/era5_ic.py (path relative to case_path) +era5_ic_runjob_filename: templates/era5_ic_runjob.cfg + +# ------------------------------------------------------------------- +# IC naming contract used by jobs/icon.py when generating inidata filename +# (era5_ic_runjob must write exactly this file) +# ------------------------------------------------------------------- +inidata_prefix: "era5_init_R2B7_" +inidata_nameformat: "%Y%m%d%H" +inidata_filename_suffix: ".nc" + +# ------------------------------------------------------------------- +# Files staged by prepare_icon into icon/input and referenced by runjobs +# Use case-relative paths so the case is self-contained. +# ------------------------------------------------------------------- +input_files: + dynamics_grid_filename: ../../input/icon/grid/zonda_output_I/I_DOM01.nc + radiation_grid_filename: ../../input/icon/grid/zonda_output_I/I_DOM01.parent.nc + extpar_filename: ../../input/icon/grid/zonda_output_I/I_DOM01_external_parameter.nc + + # radiation coeffs + lrtm_filename: /cluster/work/climate/lroither/icon_c2sm/icon/data/rrtmg_lw.nc + cldopt_filename: /cluster/work/climate/lroither/icon_c2sm/icon/data/ECHAM6_CldOptProps.nc + map_file_ana: /cluster/work/climate/lroither/icon_c2sm/icon/run/ana_varnames_map_file.txt + + +ecrad_data_path: /cluster/work/climate/lroither/icon_c2sm/icon/externals/ecrad/data +icon_data_path: /cluster/work/climate/lroither/icon_c2sm/icon/data +start_year: 2013 + +output_filename: "parent_R2B7" +parent_eval_reg_lon_def: "-179.875,0.25,179.875" +parent_eval_reg_lat_def: "89.875,-0.25,-89.875" +parent_eval_p_levels_pa: "100000,92500,85000,70000,50000,30000,20000,10000" + + +# ------------------------------------------------------------------- +# ICON execution config +# runjob_filename is case-relative (recommended). +# ------------------------------------------------------------------- +icon: + binary_file: /cluster/work/climate/lroither/icon_c2sm/icon/bin/icon + runjob_filename: templates/icon_runjob_parent_global.cfg + np_tot: 512 + np_io: 4 + np_restart: 4 + np_prefetch: 0 diff --git a/cases/parent-global-r2b7-era5-update-euler/load_links.txt b/cases/parent-global-r2b7-era5-update-euler/load_links.txt new file mode 100644 index 00000000..29db256d --- /dev/null +++ b/cases/parent-global-r2b7-era5-update-euler/load_links.txt @@ -0,0 +1,37 @@ +in /processing-chain/work/CASE/YYYYMMDD_CHUNK/icon/run do this: + +# Base dirs you showed +ECRAD=/cluster/work/climate/lroither/icon_c2sm/icon/externals/ecrad/data +ICONDATA=/cluster/work/climate/lroither/icon_c2sm/icon/data + +# Kinne (coa) +ln -sf $ECRAD/bc_aeropt_kinne_sw_b14_coa.nc bc_aeropt_kinne_sw_b14_coa.nc +ln -sf $ECRAD/bc_aeropt_kinne_lw_b16_coa.nc bc_aeropt_kinne_lw_b16_coa.nc + +# Kinne (fin) -> ICON expects the generic name without year +ln -sf $ECRAD/bc_aeropt_kinne_sw_b14_fin_2013.nc bc_aeropt_kinne_sw_b14_fin.nc + +# CMIP6 volcanic aerosols for 2013 +ln -sf $ECRAD/bc_aeropt_cmip6_volc_lw_b16_sw_b14_2013.nc bc_aeropt_cmip6_volc_lw_b16_sw_b14_2013.nc + +# MACv2 dataset +ln -sf $ICONDATA/MACv2.0-SP_v1.nc MACv2.0-SP_v1.nc + +# bc_ozone dataset +ln -sf $ECRAD/bc_ozone_ssp370_2013.nc bc_ozone_2013.nc + +then check: + +ls -l bc_aeropt_kinne_sw_b14_coa.nc \ + bc_aeropt_kinne_lw_b16_coa.nc \ + bc_aeropt_kinne_sw_b14_fin.nc \ + bc_aeropt_cmip6_volc_lw_b16_sw_b14_2013.nc \ + MACv2.0-SP_v1.nc + +# and actually test file exists behind link +for f in bc_aeropt_kinne_sw_b14_coa.nc bc_aeropt_kinne_lw_b16_coa.nc bc_aeropt_kinne_sw_b14_fin.nc \ + bc_aeropt_cmip6_volc_lw_b16_sw_b14_2013.nc MACv2.0-SP_v1.nc +do + test -r "$f" && echo "OK: $f" || echo "MISSING: $f" +done + diff --git a/cases/parent-global-r2b7-era5-update-euler/tables/mypartab b/cases/parent-global-r2b7-era5-update-euler/tables/mypartab new file mode 100644 index 00000000..8944c095 --- /dev/null +++ b/cases/parent-global-r2b7-era5-update-euler/tables/mypartab @@ -0,0 +1,117 @@ +¶meter ! temperature +name = "t" +out_name = "T" +/ +¶meter ! horiz. wind comp. u +name = "u" +out_name = "U" +/ +¶meter ! horiz. wind comp. u +name = "v" +out_name = "V" +/ +¶meter ! vertical velocity +name = "w" +out_name = "W" +/ +¶meter ! specific humidity +name = "q" +out_name = "QV" +/ +¶meter ! cloud liquid water content +name = "clwc" +out_name = "QC" +/ +¶meter ! cloud ice water content +name = "ciwc" +out_name = "QI" +/ +¶meter ! rain water content +name = "crwc" +out_name = "QR" +/ +¶meter ! snow water content +name = "cswc" +out_name = "QS" +/ +¶meter ! snow temperature +name = "TSN" +out_name = "T_SNOW" +/ +¶meter ! water content of snow +name = "SD" +out_name = "W_SNOW" +/ +¶meter ! density of snow +name = "RSN" +out_name = "RHO_SNOW" +/ +¶meter ! snow albedo +name = "ASN" +out_name = "ALB_SNOW" +/ +¶meter ! skin temperature +name = "SKT" +out_name = "SKT" +/ +¶meter ! sea surface temperature +name = "SSTK" +out_name = "SST" +/ +¶meter ! soil temperature level 1 +name = "STL1" +out_name = "STL1" +/ +¶meter ! soil temperature level 2 +name = "STL2" +out_name = "STL2" +/ +¶meter ! soil temperature level 3 +name = "STL3" +out_name = "STL3" +/ +¶meter ! soil temperature level 4 +name = "STL4" +out_name = "STL4" +/ +¶meter ! sea-ice cover +name = "CI" +out_name = "CI" +/ +¶meter ! water cont. of interception storage +name = "SRC" +out_name = "W_I" +/ +¶meter ! Land/sea mask +name = "LSM" +out_name = "LSM" +/ +¶meter ! soil moisture index layer 1 +name = "SWVL1" +out_name = "SMIL1" +/ +¶meter ! soil moisture index layer 2 +name = "SWVL2" +out_name = "SMIL2" +/ +¶meter ! soil moisture index layer 3 +name = "SWVL3" +out_name = "SMIL3" +/ +¶meter ! soil moisture index layer 4 +name = "SWVL4" +out_name = "SMIL4" +/ +¶meter ! logarithm of surface pressure +name = "LNSP" +out_name = "LNPS" +/ +¶meter ! logarithm of surface pressure +name = "SP" +out_name = "PS" +/ +¶meter +name = "Z" +out_name = "GEOSP" +/ + diff --git a/cases/parent-global-r2b7-era5-update-euler/tables/partab_era5_to_icon.txt b/cases/parent-global-r2b7-era5-update-euler/tables/partab_era5_to_icon.txt new file mode 100644 index 00000000..8944c095 --- /dev/null +++ b/cases/parent-global-r2b7-era5-update-euler/tables/partab_era5_to_icon.txt @@ -0,0 +1,117 @@ +¶meter ! temperature +name = "t" +out_name = "T" +/ +¶meter ! horiz. wind comp. u +name = "u" +out_name = "U" +/ +¶meter ! horiz. wind comp. u +name = "v" +out_name = "V" +/ +¶meter ! vertical velocity +name = "w" +out_name = "W" +/ +¶meter ! specific humidity +name = "q" +out_name = "QV" +/ +¶meter ! cloud liquid water content +name = "clwc" +out_name = "QC" +/ +¶meter ! cloud ice water content +name = "ciwc" +out_name = "QI" +/ +¶meter ! rain water content +name = "crwc" +out_name = "QR" +/ +¶meter ! snow water content +name = "cswc" +out_name = "QS" +/ +¶meter ! snow temperature +name = "TSN" +out_name = "T_SNOW" +/ +¶meter ! water content of snow +name = "SD" +out_name = "W_SNOW" +/ +¶meter ! density of snow +name = "RSN" +out_name = "RHO_SNOW" +/ +¶meter ! snow albedo +name = "ASN" +out_name = "ALB_SNOW" +/ +¶meter ! skin temperature +name = "SKT" +out_name = "SKT" +/ +¶meter ! sea surface temperature +name = "SSTK" +out_name = "SST" +/ +¶meter ! soil temperature level 1 +name = "STL1" +out_name = "STL1" +/ +¶meter ! soil temperature level 2 +name = "STL2" +out_name = "STL2" +/ +¶meter ! soil temperature level 3 +name = "STL3" +out_name = "STL3" +/ +¶meter ! soil temperature level 4 +name = "STL4" +out_name = "STL4" +/ +¶meter ! sea-ice cover +name = "CI" +out_name = "CI" +/ +¶meter ! water cont. of interception storage +name = "SRC" +out_name = "W_I" +/ +¶meter ! Land/sea mask +name = "LSM" +out_name = "LSM" +/ +¶meter ! soil moisture index layer 1 +name = "SWVL1" +out_name = "SMIL1" +/ +¶meter ! soil moisture index layer 2 +name = "SWVL2" +out_name = "SMIL2" +/ +¶meter ! soil moisture index layer 3 +name = "SWVL3" +out_name = "SMIL3" +/ +¶meter ! soil moisture index layer 4 +name = "SWVL4" +out_name = "SMIL4" +/ +¶meter ! logarithm of surface pressure +name = "LNSP" +out_name = "LNPS" +/ +¶meter ! logarithm of surface pressure +name = "SP" +out_name = "PS" +/ +¶meter +name = "Z" +out_name = "GEOSP" +/ + diff --git a/cases/parent-global-r2b7-era5-update-euler/templates/era5_ic_runjob.cfg b/cases/parent-global-r2b7-era5-update-euler/templates/era5_ic_runjob.cfg new file mode 100644 index 00000000..ca8ae5d6 --- /dev/null +++ b/cases/parent-global-r2b7-era5-update-euler/templates/era5_ic_runjob.cfg @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +#SBATCH --job-name=era5_ic_{cfg.casename}_{cfg.era5_yyyymmddhh} +#SBATCH --time={cfg.walltime_era5_ic} +#SBATCH --partition={cfg.compute_queue} +#SBATCH --constraint={cfg.constraint} +#SBATCH --ntasks=1 +#SBATCH --output={cfg.logfile} +#SBATCH --open-mode=append +#SBATCH --chdir={cfg.icon_work} + +# --------------------------------------------------------------------- +# ERA5 -> ICON initial conditions (GLOBAL, IC only) +# +# This script is submitted by jobs/era5_ic.py. +# It MUST write the final IC file exactly to: +# {inidata_filename} +# --------------------------------------------------------------------- + +set -euo pipefail +set -x +ulimit -s unlimited + + +# Load modules if available (safe on Euler; no hard failure) +module load stack/2024-06 || true +module load cdo/2.2.2 nco/5.1.6 netcdf-c/4.9.2 || true +export CDO_FILETYPE=nc2 + + +# --------------------------------------------------------------------- +# Inputs (from cfg) +# --------------------------------------------------------------------- +ERA5_DIR="{cfg.era5_dir}" +ERA5_ML="{cfg.era5_ml_file}" +ERA5_SFC="{cfg.era5_sfc_file}" + +PARTAB="{cfg.era5_partab_path}" + +GRID_TRI="{cfg.input_files_scratch_dynamics_grid_filename}" +EXTPAR="{cfg.input_files_scratch_extpar_filename}" + +OUTFILE="{inidata_filename}" + +work="{cfg.icon_work}/era5_ic_work" +mkdir -p "$work" +cd "$work" + +# --------------------------------------------------------------------- +# 1) GRIB preprocessing: pick first timestep and convert to NetCDF +# --------------------------------------------------------------------- +cdo splitsel,1 "${{ERA5_DIR}}/${{ERA5_ML}}" era5_ml_ +cdo splitsel,1 "${{ERA5_DIR}}/${{ERA5_SFC}}" era5_sfc_ + +ml_grb="$(ls -1 era5_ml_* | head -n 1)" +sfc_grb="$(ls -1 era5_sfc_* | head -n 1)" + +cdo -t ecmwf -f nc copy "$ml_grb" era5_ml.nc +cdo -t ecmwf -f nc copy "$sfc_grb" era5_sfc.nc + +cdo merge era5_ml.nc era5_sfc.nc era5_original.nc + +# Rename variables to ICON naming using partab +cdo setpartabn,"$PARTAB",convert era5_original.nc tmp.nc + +# Stable var order +ncks tmp.nc data_in.nc + +rm -f tmp.nc era5_ml.nc era5_sfc.nc era5_original.nc "$ml_grb" "$sfc_grb" + +# --------------------------------------------------------------------- +# 2) Triangular grid file for CDO remapping +# --------------------------------------------------------------------- +cdo -s selgrid,1 "$GRID_TRI" triangular-grid.nc + +# --------------------------------------------------------------------- +# 3) Land/sea-aware remapping for surface variables +# --------------------------------------------------------------------- +cdo selname,LSM data_in.nc LSM_in.nc +ncrename -h -v LSM,FR_LAND LSM_in.nc + +cdo selname,FR_LAND "$EXTPAR" LSM_out_tmp.nc +ncecat -O -u time LSM_out_tmp.nc LSM_out_tmp.nc +ncks -h -A -v time LSM_in.nc LSM_out_tmp.nc + +# Build 0/1 masks explicitly (robust across CDO/HDF5 quirks) +cdo -L expr,'FR_LAND=FR_LAND<0.5 ? 1 : 0;' LSM_in.nc oceanmask_in.nc +cdo -L expr,'FR_LAND=FR_LAND>=0.5 ? 1 : 0;' LSM_in.nc landmask_in.nc +cdo -L expr,'FR_LAND=FR_LAND<0.5 ? 1 : 0;' LSM_out_tmp.nc oceanmask_out.nc +cdo -L expr,'FR_LAND=FR_LAND>=0.5 ? 1 : 0;' LSM_out_tmp.nc landmask_out.nc +if [ ! -s oceanmask_in.nc ]; then + echo "ERROR: oceanmask_in.nc not created" + exit 1 +fi +if [ ! -s landmask_in.nc ]; then + echo "ERROR: landmask_in.nc not created" + exit 1 +fi +if [ ! -s oceanmask_out.nc ]; then + echo "ERROR: oceanmask_out.nc not created" + exit 1 +fi +if [ ! -s landmask_out.nc ]; then + echo "ERROR: landmask_out.nc not created" + exit 1 +fi + +cdo setrtoc2,0.5,1.0,1,0 LSM_out_tmp.nc LSM_out.nc +rm -f LSM_in.nc LSM_out_tmp.nc + +ncks -h -v SST,CI data_in.nc datasea_in.nc +ncks -h -v SKT,STL1,STL2,STL3,STL4,ALB_SNOW,W_SNOW,T_SNOW data_in.nc dataland_in.nc + +# ocean masked remap +cdo div dataland_in.nc oceanmask_in.nc tmp1_land.nc +cdo div datasea_in.nc oceanmask_in.nc tmp1_sea.nc +cdo setmisstodis tmp1_land.nc tmp2_land.nc +cdo setmisstodis tmp1_sea.nc tmp2_sea.nc +cdo remapdis,triangular-grid.nc tmp2_land.nc tmp3_land.nc +cdo remapdis,triangular-grid.nc tmp2_sea.nc tmp3_sea.nc +cdo div tmp3_land.nc oceanmask_out.nc dataland_ocean_out.nc +cdo div tmp3_sea.nc oceanmask_out.nc datasea_ocean_out.nc +rm -f tmp*_land.nc tmp*_sea.nc oceanmask_in.nc oceanmask_out.nc + +# land masked remap +cdo div dataland_in.nc landmask_in.nc tmp1.nc +cdo setmisstodis tmp1.nc tmp2.nc +cdo remapdis,triangular-grid.nc tmp2.nc tmp3.nc +cdo div tmp3.nc landmask_out.nc dataland_land_out.nc +rm -f tmp*.nc landmask_in.nc landmask_out.nc dataland_in.nc datasea_in.nc + +cdo ifthenelse LSM_out.nc dataland_land_out.nc dataland_ocean_out.nc dataland_out.nc +rm -f dataland_ocean_out.nc dataland_land_out.nc + +# Remap the rest +ncks -h -x -v SKT,STL1,STL2,STL3,STL4,SMIL1,SMIL2,SMIL3,SMIL4,ALB_SNOW,W_SNOW,T_SNOW,SST,CI,LSM data_in.nc datarest_in.nc +cdo -s remapdis,triangular-grid.nc datarest_in.nc era5_final.nc +rm -f datarest_in.nc + +# Fill missing SST/CI +rm -f sst_ci.nc datasea_ocean_out_filled.nc +cdo -s selname,SST,CI datasea_ocean_out.nc sst_ci.nc +cdo -s setmisstodis sst_ci.nc datasea_ocean_out_filled.nc +rm -f sst_ci.nc +test -s datasea_ocean_out_filled.nc + +rm -f datasea_ocean_out.nc + +# Merge special variables + land fraction +ncks -h -A dataland_out.nc era5_final.nc +ncks -h -A datasea_ocean_out_filled.nc era5_final.nc +ncks -h -A -v FR_LAND LSM_out.nc era5_final.nc +ncrename -h -v FR_LAND,LSM era5_final.nc +rm -f LSM_out.nc dataland_out.nc datasea_ocean_out_filled.nc + +# --------------------------------------------------------------------- +# 4) Soil moisture index conversion (SMIL + SLT required) +# --------------------------------------------------------------------- +ncks -h -v SMIL1,SMIL2,SMIL3,SMIL4,SLT data_in.nc swvl.nc +rm -f data_in.nc + +wiltingp=(0 0.059 0.151 0.133 0.279 0.335 0.267 0.151) +fieldcap=(0 0.244 0.347 0.383 0.448 0.541 0.663 0.347) + +smi_equation="" +for ilev in {{1..4}}; do + smi_equation="${{smi_equation}}SMIL${{ilev}}=(SMIL${{ilev}}-${{wiltingp[1]}})/(${{fieldcap[1]}}-${{wiltingp[1]}})*(SLT==1)" + for ist in {{2..7}}; do + smi_equation="${{smi_equation}}+(SMIL${{ilev}}-${{wiltingp[$ist]}})/(${{fieldcap[$ist]}}-${{wiltingp[$ist]}})*(SLT==${{ist}})" + done + smi_equation="${{smi_equation}};" +done + +cdo expr,"${{smi_equation}}" swvl.nc smil_in.nc + +rm -f swvl.nc + +cdo -s remapdis,triangular-grid.nc smil_in.nc smil_out.nc +ncks -A -v SMIL1,SMIL2,SMIL3,SMIL4 smil_out.nc era5_final.nc +rm -f smil_in.nc smil_out.nc + +# --------------------------------------------------------------------- +# 5) Create LNPS and finalize dimensions +# --------------------------------------------------------------------- +cdo expr,'LNPS=ln(PS);' era5_final.nc tmp.nc +ncks -A -v LNPS tmp.nc era5_final.nc +rm -f tmp.nc + +ncrename -h -d cell,ncells era5_final.nc || true +ncrename -h -d nv,vertices era5_final.nc || true + +# sanity: ensure directory exists +mkdir -p "$(dirname "$OUTFILE")" + +ncdump -k era5_final.nc + +# cheap + no-RAM: move the file into place +mv -f era5_final.nc "$OUTFILE" + +# cleanup +rm -f triangular-grid.nc + +echo "Wrote ICON initial condition file: $OUTFILE" diff --git a/cases/parent-global-r2b7-era5-update-euler/templates/icon_runjob_parent_global.cfg b/cases/parent-global-r2b7-era5-update-euler/templates/icon_runjob_parent_global.cfg new file mode 100644 index 00000000..5c152387 --- /dev/null +++ b/cases/parent-global-r2b7-era5-update-euler/templates/icon_runjob_parent_global.cfg @@ -0,0 +1,387 @@ +#!/usr/bin/env bash +#SBATCH --job-name=icon_parent_global_np{cfg.icon_np_tot} +#SBATCH --time={cfg.walltime_icon} +#SBATCH --ntasks={cfg.icon_np_tot} +#SBATCH --constraint={cfg.constraint} +#SBATCH --partition={cfg.compute_queue} +#SBATCH --output=slurm-%x-%j.out +#SBATCH --chdir={cfg.icon_work} +#SBATCH --ntasks-per-node={cfg.ntasks_per_node} +#SBATCH --hint=nomultithread +#SBATCH --cpus-per-task=1 +#SBATCH --open-mode=append +#SBATCH --mem-per-cpu=1800 + + + +set -euo pipefail +ulimit -s unlimited + +export OMP_NUM_THREADS=1 +export ICON_THREADS=1 +export OMP_SCHEDULE=static +export OMP_DYNAMIC="false" +export OMP_STACKSIZE=200M + + + +# ----------------------------------------------------------------------------- +# Read varlists (brace-free, safe with Python .format templates) +# ----------------------------------------------------------------------------- +PARENT_EVAL_SFC_VARS="$(grep -v -E '^\s*($|#)' "{cfg.case_path}/varlists/parent_eval_hourly_sfc.txt" \ + | sed -e 's/\r$//' -e "s/.*/'&'/" | paste -sd, -)" + +PARENT_EVAL_PL_VARS="$(grep -v -E '^\s*($|#)' "{cfg.case_path}/varlists/parent_eval_hourly_pl.txt" \ + | sed -e 's/\r$//' -e "s/.*/'&'/" | paste -sd, -)" + +PARENT_NATIVE_3D_VARS="$(grep -v -E '^\s*($|#)' "{cfg.case_path}/varlists/parent_native_3d_1D_ml.txt" \ + | sed -e 's/\r$//' -e "s/.*/'&'/" | paste -sd, -)" + +# ----------------------------------------------------------------------------- +# Stage required radiation/aerosol/ozone datasets (brace-free) +# ----------------------------------------------------------------------------- +ECRAD="{cfg.ecrad_data_path}" +ICONDATA="{cfg.icon_data_path}" +YEAR="{cfg.start_year}" + +test -e "$ECRAD/bc_aeropt_kinne_sw_b14_coa.nc" || exit 2 +ln -sf "$ECRAD/bc_aeropt_kinne_sw_b14_coa.nc" bc_aeropt_kinne_sw_b14_coa.nc + +test -e "$ECRAD/bc_aeropt_kinne_lw_b16_coa.nc" || exit 2 +ln -sf "$ECRAD/bc_aeropt_kinne_lw_b16_coa.nc" bc_aeropt_kinne_lw_b16_coa.nc + +test -e "$ECRAD/bc_aeropt_kinne_sw_b14_fin_${{YEAR}}.nc" || exit 2 +ln -sf "$ECRAD/bc_aeropt_kinne_sw_b14_fin_${{YEAR}}.nc" bc_aeropt_kinne_sw_b14_fin.nc + +test -e "$ECRAD/bc_aeropt_cmip6_volc_lw_b16_sw_b14_${{YEAR}}.nc" || exit 2 +ln -sf "$ECRAD/bc_aeropt_cmip6_volc_lw_b16_sw_b14_${{YEAR}}.nc" bc_aeropt_cmip6_volc_lw_b16_sw_b14_${{YEAR}}.nc + +test -e "$ICONDATA/MACv2.0-SP_v1.nc" || exit 2 +ln -sf "$ICONDATA/MACv2.0-SP_v1.nc" MACv2.0-SP_v1.nc + +test -e "$ECRAD/bc_ozone_ssp370_${{YEAR}}.nc" || exit 2 +ln -sf "$ECRAD/bc_ozone_ssp370_${{YEAR}}.nc" bc_ozone_${{YEAR}}.nc + + + +# ----------------------------------------------------------------------------- +# ICON master namelist +# ----------------------------------------------------------------------------- +cat > icon_master.namelist << EOF +&master_nml + lrestart = {cfg.lrestart} + read_restart_namelists = .true. +/ + +&master_time_control_nml + calendar = 'proleptic gregorian' + experimentStartDate = '{cfg.ini_datetime_string}' + experimentStopDate = '{cfg.end_datetime_string}' + restartTimeIntval = '{cfg.restart_step}' + checkpointTimeIntval = '{cfg.restart_step}' +/ + +&master_model_nml + model_type = 1 + model_name = "ATMO" + model_namelist_filename = "NAMELIST_{cfg.casename}" + model_min_rank = 1 + model_max_rank = 65536 + model_inc_rank = 1 +/ +EOF + +# ----------------------------------------------------------------------------- +# Model namelist (ATMO) +# ----------------------------------------------------------------------------- +cat > NAMELIST_{cfg.casename} << EOF + +¶llel_nml + nproma = 128 + ! nblocks_e = 0 + l_log_checks = .FALSE. + num_io_procs = {cfg.icon_np_io} + num_restart_procs = {cfg.icon_np_restart} + iorder_sendrecv = 3 +/ + +&grid_nml + dynamics_grid_filename = "{cfg.input_files_scratch_dynamics_grid_filename}" + radiation_grid_filename = "{cfg.input_files_scratch_radiation_grid_filename}" + dynamics_parent_grid_id = 0 + lredgrid_phys = .TRUE. + l_limited_area = .FALSE. + lfeedback = .FALSE. +/ + +&initicon_nml + init_mode = 2 + lread_ana = .FALSE. + ifs2icon_filename = "{inidata_filename}" + ana_varnames_map_file = "{cfg.input_files_scratch_map_file_ana}" + zpbl1 = 500. + zpbl2 = 1000. + ltile_init = .TRUE. + ltile_coldstart = .TRUE. +/ + +&run_nml + modelTimeStep = "PT120S" + num_lev = 120 + lvert_nest = .FALSE. + ldynamics = .TRUE. + ltransport = .TRUE. + iforcing = 3 + lart = .FALSE. + ltestcase = .FALSE. + msg_level = 10 + ltimer = .TRUE. + activate_sync_timers = .TRUE. + timers_level = 10 + output = "nml" + check_uuid_gracefully = .TRUE. + restart_filename = "{cfg.icon_restart_out}/{cfg.output_filename}_.nc" +/ + +&io_nml + itype_pres_msl = 5 + itype_rh = 1 + restart_file_type = 5 + restart_write_mode = "joint procs multifile" + lflux_avg = .TRUE. + lnetcdf_flt64_output = .FALSE. + precip_interval = "PT1H" + runoff_interval = "PT3H" + maxt_interval = "PT3H" + melt_interval = "PT3H" + lmask_boundary = .FALSE. +/ + +&extpar_nml + extpar_filename = "{cfg.input_files_scratch_extpar_filename}" + itopo = 1 + n_iter_smooth_topo = 1 + heightdiff_threshold = 3000. + hgtdiff_max_smooth_topo = 750. + itype_vegetation_cycle = 3 + itype_lwemiss = 2 +/ + +&nwp_phy_nml + inwp_gscp = 2 + inwp_convection = 1 + lshallowconv_only = .FALSE. + inwp_radiation = 4 + inwp_cldcover = 1 + inwp_turb = 1 + inwp_satad = 1 + inwp_sso = 1 + inwp_gwd = 1 + inwp_surface = 1 + icapdcycl = 3 + itype_z0 = 2 + + ! cadence consistent with modelTimeStep=PT120S + dt_conv = 120 + dt_sso = 120 + dt_gwd = 120 + dt_ccov = 120 + dt_rad = 600 + + latm_above_top = .FALSE. + efdt_min_raylfric = 7200.0 + icpl_aero_conv = 0 + icpl_aero_gscp = 0 + ldetrain_conv_prec = .FALSE. + + lrtm_filename = "{cfg.input_files_scratch_lrtm_filename}" + cldopt_filename = "{cfg.input_files_scratch_cldopt_filename}" +/ + +&radiation_nml + ecrad_isolver = 0 + irad_o3 = 5 + irad_o2 = 2 ! Tracer variable (CLM commnity) + irad_cfc11 = 2 ! Tracer variableTracer variable (co2, ch4,n20,o2,cfc11,cfc12)) + irad_cfc12 = 2 ! Tracer Variable (cfc12) + irad_aero = 18 ! was 18, change back if we get file + albedo_type = 2 + direct_albedo = 4 + albedo_whitecap = 1 + vmr_co2 = 390.e-06 + vmr_ch4 = 1800.e-09 + vmr_n2o = 322.0e-09 + vmr_o2 = 0.20946 + vmr_cfc11 = 240.e-12 + vmr_cfc12 = 532.e-12 + ecrad_data_path = "{cfg.ecrad_data_path}" +/ + +&nonhydrostatic_nml + iadv_rhotheta = 2 + ivctype = 2 + itime_scheme = 4 + exner_expol = 0.333 + vwind_offctr = 0.2 + damp_height = 30000. + rayleigh_coeff = 0.5 + divdamp_order = 24 + divdamp_type = 32 + divdamp_fac = 0.004 + divdamp_trans_start = 12500. + divdamp_trans_end = 17500. + igradp_method = 3 + l_zdiffu_t = .TRUE. + thslp_zdiffu = 0.02 + thhgtd_zdiffu = 125. + htop_moist_proc = 22500. + hbot_qvsubstep = 16000. + ndyn_substeps = 5 +/ + +&sleve_nml + min_lay_thckn = 50. + htop_thcknlimit = 15000. + top_height = 85000. + stretch_fac = 0.9 + decay_scale_1 = 4000. + decay_scale_2 = 2500. + decay_exp = 1.2 + flat_height = 25000. +/ + +&dynamics_nml + iequations = 3 + divavg_cntrwgt = 0.50 + lcoriolis = .TRUE. +/ + +&transport_nml + ihadv_tracer = 2,2,2,2,2,2 + itype_hlimit = 4,4,4,4,4,4 + ivadv_tracer = 3,3,3,3,3,3 + itype_vlimit = 1,1,1,1,1,1 + ivlimit_selective = 1,1,1,1,1,1 + llsq_svd = .TRUE. +/ + +&diffusion_nml + hdiff_order = 5 + itype_vn_diffu = 1 + itype_t_diffu = 2 + hdiff_efdt_ratio = 32.0 + hdiff_smag_fac = 0.025 + lhdiff_vn = .TRUE. + lhdiff_temp = .TRUE. +/ + +&turbdiff_nml + tkhmin = 0.6 + tkhmin_strat = 1.0 + tkmmin = 0.75 + pat_len = 750. + c_diff = 0.2 + rlam_heat = 10.0 + rat_sea = 0.8 + ltkesso = .TRUE. + frcsmot = 0.2 + imode_frcsmot = 2 + alpha1 = 0.125 + icldm_turb = 1 + itype_sher = 1 + ltkeshs = .TRUE. + a_hshr = 2.0 +/ + +&lnd_nml + sstice_mode = 1 ! update sst and ice at intervals is mode 6 + ntiles = 3 + nlev_snow = 1 + zml_soil = 0.005,0.02,0.06,0.18,0.54,1.62,4.86,14.58 + lmulti_snow = .FALSE. + itype_heatcond = 3 + idiag_snowfrac = 20 + itype_snowevap = 3 + lsnowtile = .TRUE. + lseaice = .TRUE. + llake = .TRUE. + itype_lndtbl = 4 + itype_evsl = 4 + itype_trvg = 3 + itype_root = 2 + itype_canopy = 2 + cwimax_ml = 5.e-4 + c_soil = 1.25 + c_soil_urb = 0.5 + lprog_albsi = .TRUE. +/ + +! ============================================================ +! OUTPUT STREAM 1a: hourly eval (remapped) - MODEL LEVELS +! ============================================================ +&output_nml + filetype = 5 + dom = 1 + steps_per_file = 1 + include_last = .TRUE. + + output_filename = '{cfg.output_filename}_eval_hourly_sfc' + filename_format = '{cfg.icon_output}/{cfg.output_filename}_eval_hourly_sfc_DOM_' + + output_bounds = 0., 1.0e15, 3600. + + remap = 1 + reg_lon_def = {cfg.parent_eval_reg_lon_def} + reg_lat_def = {cfg.parent_eval_reg_lat_def} + + ml_varlist = ${{PARENT_EVAL_SFC_VARS}} +/ + +! ============================================================ +! OUTPUT STREAM 1b: hourly eval (remapped) - PRESSURE LEVELS +! ============================================================ +&output_nml + filetype = 5 + dom = 1 + steps_per_file = 1 + include_last = .FALSE. ! back to true in actual runs + + output_filename = '{cfg.output_filename}_eval_hourly_pl' + filename_format = '{cfg.icon_output}/{cfg.output_filename}_eval_hourly_pl_DOM_' + + output_bounds = 0., 1.0e15, 86400. ! longer interval than run, no output + + remap = 1 + reg_lon_def = {cfg.parent_eval_reg_lon_def} + reg_lat_def = {cfg.parent_eval_reg_lat_def} + + p_levels = {cfg.parent_eval_p_levels_pa} + pl_varlist = ${{PARENT_EVAL_PL_VARS}} +/ + +! ============================================================ +! OUTPUT STREAM 2: daily native 3D (ICON grid) +! ============================================================ +&output_nml + filetype = 5 + dom = 1 + steps_per_file = 1 + include_last = .FALSE. + + output_filename = '{cfg.output_filename}_native_3d_1D' + filename_format = '{cfg.icon_output}/{cfg.output_filename}_native_3d_1D_DOM_' + + output_bounds = 0., 1.0e15, 86400. + remap = 0 + + ml_varlist = ${{PARENT_NATIVE_3D_VARS}} +/ +EOF + +# ----------------------------------------------------------------------------- +# Run ICON +# ----------------------------------------------------------------------------- +source {cfg.chain_src_dir}/ext/icon/modules.env +set -x +srun -n {cfg.icon_np_tot} ./{cfg.icon_execname} +set +x diff --git a/cases/parent-global-r2b7-era5-update-euler/varlists/parent_eval_hourly_pl.txt b/cases/parent-global-r2b7-era5-update-euler/varlists/parent_eval_hourly_pl.txt new file mode 100644 index 00000000..beda0d19 --- /dev/null +++ b/cases/parent-global-r2b7-era5-update-euler/varlists/parent_eval_hourly_pl.txt @@ -0,0 +1,6 @@ +geopot +temp +u +v +qv +w diff --git a/cases/parent-global-r2b7-era5-update-euler/varlists/parent_eval_hourly_sfc.txt b/cases/parent-global-r2b7-era5-update-euler/varlists/parent_eval_hourly_sfc.txt new file mode 100644 index 00000000..ea29305b --- /dev/null +++ b/cases/parent-global-r2b7-era5-update-euler/varlists/parent_eval_hourly_sfc.txt @@ -0,0 +1,5 @@ +tot_prec_d +pres_sfc +t_2m +u_10m +v_10m diff --git a/cases/parent-global-r2b7-era5-update-euler/varlists/parent_native_3d_1D_ml.txt b/cases/parent-global-r2b7-era5-update-euler/varlists/parent_native_3d_1D_ml.txt new file mode 100644 index 00000000..30e8cd0f --- /dev/null +++ b/cases/parent-global-r2b7-era5-update-euler/varlists/parent_native_3d_1D_ml.txt @@ -0,0 +1,12 @@ +temp +u +v +w +pres +rho +qv +qc +qi +geopot +z_mc +z_ifc diff --git a/config.py b/config.py index 8e1a445f..dbcbf652 100644 --- a/config.py +++ b/config.py @@ -627,7 +627,7 @@ def get_previous_slurm_summary(self, # Get job info for all jobs self.slurm_info = {} for job_name in self.jobs: - for job_id in self.job_ids['previous'][job_name]: + for job_id in self.job_ids['previous'].get(job_name, []): self.slurm_info[job_name] = [] self.slurm_info[job_name].append( self.get_job_info(job_id, slurm_keys=info_keys, @@ -665,7 +665,7 @@ def print_previous_slurm_summary(self): f.write(table_header) f.write('\n') for job_name in self.jobs: - for info in self.slurm_info[job_name]: + for info in self.slurm_info.get(job_name, []): f.write(line_format.format(**info)) f.write('\n') f.write('\n') diff --git a/env/environment.yml b/env/environment.yml index 98b90e44..0070da5b 100644 --- a/env/environment.yml +++ b/env/environment.yml @@ -1,7 +1,6 @@ name: proc-chain channels: - conda-forge - - defaults dependencies: - python=3.11 - cdo diff --git a/jenkins/Jenkinsfile b/jenkins/Jenkinsfile index 19875218..b34b0b64 100644 --- a/jenkins/Jenkinsfile +++ b/jenkins/Jenkinsfile @@ -8,17 +8,6 @@ pipeline { } } stages { - stage('Setup spack') { - steps { - sh './jenkins/scripts/setup-spack.sh' - } - post { - failure { - echo 'Cleaning up workspace' - deleteDir() - } - } - } stage('Preparation') { parallel { stage('Setup miniconda') { @@ -29,8 +18,10 @@ pipeline { sh 'wget -O ${WORKSPACE}/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh' sh 'rm -fr ${WORKSPACE}/miniconda' sh 'bash miniconda.sh -b -p $WORKSPACE/miniconda' + sh 'conda config --remove channels defaults' sh 'conda config --set always_yes yes --set changeps1 no' sh 'conda config --add channels conda-forge' + sh 'conda config --set channel_priority strict' sh 'conda update -n base -c defaults conda' sh 'conda env create -f env/environment.yml' sh '''source ${WORKSPACE}/miniconda/etc/profile.d/conda.sh diff --git a/jenkins/scripts/build_icon-art.sh b/jenkins/scripts/build_icon-art.sh deleted file mode 100755 index 68d09fd2..00000000 --- a/jenkins/scripts/build_icon-art.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -set -e -x - -# Check if script is called correctly -[[ $(git rev-parse --show-toplevel 2>/dev/null) = $(pwd) ]] || error "$0 not launched from toplevel of repository" - -source jenkins/scripts/common.sh - -BRANCH=art -GIT_REMOTE=git@github.com:C2SM/icon.git -MODEL=icon-art - -pushd ext - -# Clone the repo if not already existing -if [[ ! -d "${MODEL}" ]]; then - git clone --depth 1 --recurse-submodules -b ${BRANCH} ${GIT_REMOTE} ${MODEL} -fi - -pushd ${MODEL} - -if [[ $(hostname) == eu-* ]]; then - ./jenkins/scripts/jenkins_euler.sh -b -fc gcc --configure euler.cpu.gcc.O2 -elif [[ $(hostname) == daint* ]]; then - SPACK_TAG=`cat config/cscs/SPACK_TAG_DAINT` - . ../spack-c2sm/setup-env.sh - spack env activate -d config/cscs/spack/${SPACK_TAG}/daint_cpu_nvhpc - spack install -u build -elif [[ $(hostname) == balfrin* ]]; then - SPACK_TAG=`cat config/cscs/SPACK_TAG_BALFRIN` - . ../spack-c2sm/setup-env.sh - spack env activate -d config/cscs/spack/${SPACK_TAG}/daint_cpu_nvhpc - spack install -u build -else - error "Unknown hostname: $(hostname)" -fi - -popd - -popd diff --git a/jenkins/scripts/build_icon.sh b/jenkins/scripts/build_icon.sh index ea1ab959..f4c3fb67 100755 --- a/jenkins/scripts/build_icon.sh +++ b/jenkins/scripts/build_icon.sh @@ -7,8 +7,8 @@ set -e -x source jenkins/scripts/common.sh -BRANCH=main -GIT_REMOTE=git@github.com:C2SM/icon.git +BRANCH=release-2025.10-public +GIT_REMOTE=https://gitlab.dkrz.de/icon/icon-model.git MODEL=icon pushd ext @@ -21,17 +21,15 @@ fi pushd ${MODEL} if [[ $(hostname) == eu-* ]]; then - ./jenkins/scripts/jenkins_euler.sh -b -fc gcc --configure euler.cpu.gcc.O2 -elif [[ $(hostname) == daint* ]]; then - SPACK_TAG=`cat config/cscs/SPACK_TAG_DAINT` - . ../spack-c2sm/setup-env.sh - spack env activate -d config/cscs/spack/${SPACK_TAG}/daint_cpu_nvhpc - spack install -u build -elif [[ $(hostname) == balfrin* ]]; then - SPACK_TAG=`cat config/cscs/SPACK_TAG_BALFRIN` - . ../spack-c2sm/setup-env.sh - spack env activate -d config/cscs/spack/${SPACK_TAG}/daint_cpu_nvhpc - spack install -u build + # Load necessary modules + module load stack/2025-06 git eth_proxy + # Setup spack + SPACK_TAG=$(cat "config/ethz/SPACK_TAG_EULER") + git clone --depth 1 --recurse-submodules --shallow-submodules -b ${SPACK_TAG} https://github.com/C2SM/spack-c2sm.git + . spack-c2sm/setup-env.sh + # Build ICON + spack env activate -d config/ethz/spack/${SPACK_TAG}/euler_cpu_gcc + srun -N 1 -n 12 --mem-per-cpu=1G spack install -j 12 else error "Unknown hostname: $(hostname)" fi diff --git a/jenkins/scripts/jenkins.sh b/jenkins/scripts/jenkins.sh index 8fbe49ff..aeff8650 100755 --- a/jenkins/scripts/jenkins.sh +++ b/jenkins/scripts/jenkins.sh @@ -21,25 +21,16 @@ set -e -x # Check if we are on Euler if [[ $(hostname) == eu-* ]]; then host=euler -elif [[ $(hostname) == daint* ]]; then - host=daint +elif [[ $(hostname) == santis* ]]; then + host=santis +else + echo "Unknown hostname: $(hostname)" fi - # Activate conda environment eval "$(conda shell.bash hook)" conda activate proc-chain -# Setup spack -if [[ -d ext/spack-c2sm ]]; then - echo spack folder already exists - skipping build... -else - echo building spack... - ./jenkins/scripts/setup-spack.sh -fi -echo activating spack... -. ext/spack-c2sm/setup-env.sh - # Preparation size=$(du -sb input | awk '{print $1}') if [[ $size -gt 12000000000 ]]; then @@ -49,34 +40,6 @@ else ./jenkins/scripts/get_data.sh fi -# Build icontools -if [[ -f ext/icontools/icontools/iconremap ]]; then - echo icontools already installed - skipping build... -else - echo building icontools... - ./jenkins/scripts/build_icontools.sh -fi - -# Build int2lm -if [[ "$host" == euler ]]; then - echo skipping int2lm build on Euler... -elif [[ -f ext/int2lm/TESTSUITE/int2lm ]]; then - echo int2lm executable already exists - skipping build... -else - echo building int2lm... - ./jenkins/scripts/build_int2lm.sh -fi - -# Build COSMO-GHG -if [[ "$host" == euler ]]; then - echo skipping cosmo-ghg build on Euler... -elif [[ -f ext/cosmo-ghg/cosmo/ACC/cosmo_gpu ]]; then - echo cosmo executable already exists - skipping build. -else - echo building cosmo... - ./jenkins/scripts/build_cosmo-ghg.sh -fi - # Build ICON if [[ -f ext/icon/bin/icon ]]; then echo icon executable already exists - skipping build. @@ -85,65 +48,7 @@ else ./jenkins/scripts/build_icon.sh fi -# Build ICON-ART -if [[ -f ext/icon-art/bin/icon ]]; then - echo icon-art executable already exists - skipping build. -else - echo building icon-art... - ./jenkins/scripts/build_icon-art.sh -fi - -# Test COSMO-GHG -if [[ "$host" == euler ]]; then - echo skipping cosmo-ghg test on Euler... -elif [[ -f work/cosmo-ghg-test/2015010106_2015010112/checkpoints/finished/post_cosmo && "$force_execution" == false ]]; then - echo cosmo-ghg test case already finished - skipping test. -else - echo running cosmo-ghg test case... - ./jenkins/scripts/test_cosmo-ghg.sh -fi - -# Test COSMO-GHG (spinup) -if [[ "$host" == euler ]]; then - echo skipping cosmo-ghg-spinup test on Euler... -elif [[ -f work/cosmo-ghg-spinup-test/2015010109_2015010118/checkpoints/finished/post_cosmo && "$force_execution" == false ]]; then - echo cosmo-ghg-spinup test case already finished - skipping test. -else - echo running cosmo-ghg-spinup test case... - ./jenkins/scripts/test_cosmo-ghg-spinup.sh -fi - # Test ICON -if [[ "$host" == euler ]]; then - echo skipping icon test on Euler... -elif [[ -f work/icon-test/2018010106_2018010112/checkpoints/finished/icon && "$force_execution" == false ]]; then - echo icon test case already finished - skipping test. -else - echo running icon test case... - ./jenkins/scripts/test_icon.sh -fi - -# Test ICON-ART -if [[ "$host" == euler ]]; then - echo skipping icon-art-oem test on Euler... -elif [[ -f work/icon-art-oem-test/2018010106_2018010112/checkpoints/finished/icon && "$force_execution" == false ]]; then - echo icon-art test case already finished - skipping test. -else - echo running icon-art-oem test case... - ./jenkins/scripts/test_icon-art-oem.sh -fi - -# Test ICON-ART-GLOBAL -if [[ "$host" == euler ]]; then - echo skipping icon-art-global test on Euler... -elif [[ -f work/icon-art-global-test/2018010106_2018010112/checkpoints/finished/icon && "$force_execution" == false ]]; then - echo icon-art-global test case already finished - skipping test. -else - echo running icon-art-global test case... - ./jenkins/scripts/test_icon-art-global.sh -fi - -# Test ICON (Euler) if [[ "$host" == euler ]]; then if [[ -f work/icon-test-euler/2018010106_2018010112/checkpoints/finished/icon && "$force_execution" == false ]]; then echo icon test case already finished - skipping test. diff --git a/jenkins/scripts/setup-spack.sh b/jenkins/scripts/setup-spack.sh deleted file mode 100755 index 76ed1a11..00000000 --- a/jenkins/scripts/setup-spack.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -e -x - -source jenkins/scripts/common.sh - -# Check if script is called correctly -[[ $(git rev-parse --show-toplevel 2>/dev/null) = $(pwd) ]] || error "$0 not launched from toplevel of repository" - -SPACK_TAG_COSMO=v0.18.1.12 - -if [[ $(hostname) == eu-* ]]; then - source /cluster/apps/local/env2lmod.sh - module load git/2.31.1 - SPACK_TAG=main -elif [[ $(hostname) == daint* ]]; then - git clone --depth 1 git@github.com:C2SM/icon.git icon-tag - SPACK_TAG=`cat icon-tag/config/cscs/SPACK_TAG_DAINT` - rm -fr icon-tag -elif [[ $(hostname) == balfrin* ]]; then - git clone --depth 1 git@github.com:C2SM/icon.git icon-tag - SPACK_TAG=`cat icon-tag/config/cscs/SPACK_TAG_BALFRIN` - rm -fr icon-tag -else - error "Unknown hostname: $(hostname)" -fi - -GIT_REMOTE=https://github.com/C2SM/spack-c2sm.git - -rm -fr ext/spack-c2sm - -pushd ext - -# Clone Spack for ICON -git clone --depth 1 --recurse-submodules --shallow-submodules -b ${SPACK_TAG} ${GIT_REMOTE} spack-c2sm - -# Clone Spack for COSMO-GHG -git clone --depth 1 --recurse-submodules --shallow-submodules -b ${SPACK_TAG_COSMO} ${GIT_REMOTE} spack-c2sm-cosmo - -popd diff --git a/jobs/__init__.py b/jobs/__init__.py index a3d75541..393e0ffb 100644 --- a/jobs/__init__.py +++ b/jobs/__init__.py @@ -5,6 +5,7 @@ from . import check_output from . import cosmo from . import emissions +from . import era5_ic from . import icon from . import icontools from . import int2lm diff --git a/jobs/era5_ic.py b/jobs/era5_ic.py new file mode 100644 index 00000000..d3acc641 --- /dev/null +++ b/jobs/era5_ic.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +era5_ic.py +Processing-chain job that generates ICON initial conditions (IC) from ERA5 input. + +Design goals +------------ +- Keep the job modular: + * all paths and filenames come from cfg / case templates + * the actual transformation is executed in a Slurm job script produced + from a case-provided template (similar to icontools runjobs) +- Support global runs: + * generate a single IC file on the ICON grid + * do NOT generate LBC files +""" + +import logging +from pathlib import Path +from . import tools, prepare_icon + +BASIC_PYTHON_JOB = True + + +def _compute_inidata_filename(cfg) -> Path: + """ + Replicate the same IC filename logic used by jobs/icon.py, + so that era5_ic produces *exactly* the file ICON will read. + """ + if hasattr(cfg, 'inicond_filename'): + return cfg.icon_input_icbc / cfg.inicond_filename + if (hasattr(cfg, 'inidata_prefix') and hasattr(cfg, 'inidata_nameformat') + and hasattr(cfg, 'inidata_filename_suffix')): + return cfg.icon_input_icbc / str( + cfg.startdate.strftime(cfg.inidata_prefix + + cfg.inidata_nameformat + + cfg.inidata_filename_suffix)) + # fallback: match icon.py default behavior + return cfg.icon_input_icbc / str( + cfg.startdate_sim.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '.nc') + + +def main(cfg): + """ + 1) Prepare standard ICON paths (same helper as other jobs) + 2) Create a Slurm script from the case template cfg.era5_ic_runjob_filename + 3) Submit it + The Slurm script is responsible for: + - converting ERA5 GRIB -> NetCDF + - renaming variables to ICON-like naming (via a partab) + - remapping to the ICON triangular grid + - writing the final IC file to cfg.icon_input/icbc + """ + + prepare_icon.set_cfg_variables(cfg) + tools.change_logfile(cfg.logfile) + logging.info( + "Generate global ICON initial conditions from ERA5 (IC only).") + + # Ensure run + icbc directories exist (prepare_icon usually created them, + # but being explicit makes the job robust if invoked in isolation). + tools.create_dir(cfg.icon_work, "icon_work") + tools.create_dir(cfg.icon_input_icbc, "icon_input_icbc") + + # Useful formatted dates for the template (avoid bash date gymnastics) + cfg.era5_ymd = cfg.startdate_sim.strftime('%Y-%m-%d') # e.g. 2021-01-01 + cfg.era5_yyyymmddhh = cfg.startdate_sim.strftime('%Y%m%d%H') # 2021010100 + + # ------------------------------------------------------------------ + # Expand ERA5 input filename patterns from config.yaml + # Supports placeholders like {ymd} and {yyyymmddhh}. + # This is critical because bash will NOT expand "{ymd}". + # ------------------------------------------------------------------ + if hasattr(cfg, "era5_ml_filename"): + cfg.era5_ml_file = cfg.era5_ml_filename.format( + ymd=cfg.era5_ymd, + yyyymmddhh=cfg.era5_yyyymmddhh, + ) + if hasattr(cfg, "era5_sfc_filename"): + cfg.era5_sfc_file = cfg.era5_sfc_filename.format( + ymd=cfg.era5_ymd, + yyyymmddhh=cfg.era5_yyyymmddhh, + ) + + # Make the partab path absolute (case-relative -> absolute) + if hasattr(cfg, "era5_partab"): + p = Path(str(cfg.era5_partab)) + cfg.era5_partab_path = p if p.is_absolute() else (cfg.case_path / p) + + # Compute the *exact* file that ICON will later read + inidata_filename = _compute_inidata_filename(cfg) + + # Case template name (kept configurable) + # Put in config.yaml: era5_ic_runjob_filename: era5_ic_runjob.cfg + template_name = getattr(cfg, 'era5_ic_runjob_filename', + 'era5_ic_runjob.cfg') + template = (cfg.case_path / template_name).read_text() + script_str = template.format( + cfg=cfg, + inidata_filename=inidata_filename, + ) + script = (cfg.icon_work / 'run_era5_ic.job') + script.write_text(script_str) + logging.info(f"Submitting ERA5 IC generation job: {script}") + cfg.submit('era5_ic', script) + logging.info("OK") diff --git a/jobs/prepare_icon.py b/jobs/prepare_icon.py index 41fc4d25..ca041b72 100644 --- a/jobs/prepare_icon.py +++ b/jobs/prepare_icon.py @@ -22,9 +22,18 @@ def set_cfg_variables(cfg): cfg.input_files_scratch = {} for dsc, file in cfg.input_files.items(): - cfg.input_files[dsc] = (p := Path(file)) + p = Path(file) + if not p.is_absolute(): + # resolve relative input_files paths relative to the case directory + p = (cfg.case_path / p).resolve() + cfg.input_files[dsc] = p cfg.input_files_scratch[dsc] = cfg.icon_input / p.name + +# for dsc, file in cfg.input_files.items(): +# cfg.input_files[dsc] = (p := Path(file)) +# cfg.input_files_scratch[dsc] = cfg.icon_input / p.name + cfg.create_vars_from_dicts() cfg.ini_datetime_string = cfg.startdate.strftime('%Y-%m-%dT%H:00:00Z') @@ -51,6 +60,8 @@ def set_cfg_variables(cfg): cfg, 'species_inicond') and cfg.species_inicond else 0 cfg.startdate_sim_yyyymmdd_hh = cfg.startdate_sim.strftime('%Y%m%d_%H') + cfg.startdate_sim_yyyymmddhh = cfg.startdate_sim.strftime('%Y%m%d%H') + cfg.enddate_sim_yyyymmddhh = cfg.enddate_sim.strftime('%Y%m%d%H') def main(cfg): diff --git a/proc-chain-environment.yml b/proc-chain-environment.yml new file mode 100644 index 00000000..9282a3be --- /dev/null +++ b/proc-chain-environment.yml @@ -0,0 +1,237 @@ +name: proc-chain +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1 + - _openmp_mutex=4.5 + - alabaster=1.0.0 + - alsa-lib=1.2.14 + - attr=2.5.2 + - attrs=25.4.0 + - babel=2.17.0 + - blosc=1.21.6 + - brotli=1.2.0 + - brotli-bin=1.2.0 + - brotli-python=1.2.0 + - bzip2=1.0.8 + - c-ares=1.34.5 + - ca-certificates=2026.1.4 + - cairo=1.18.4 + - cartopy=0.25.0 + - cdo=2.2.0 + - cdsapi=0.7.7 + - certifi=2026.1.4 + - cffi=2.0.0 + - cftime=1.6.4 + - charset-normalizer=3.4.4 + - colorama=0.4.6 + - contourpy=1.3.3 + - cycler=0.12.1 + - cyrus-sasl=2.1.28 + - dbus=1.16.2 + - docutils=0.22.3 + - double-conversion=3.3.1 + - eccodes=2.44.0 + - ecmwf-datastores-client=0.4.1 + - esmf=8.9.0 + - expat=2.7.3 + - f90nml=1.5 + - fftw=3.3.10 + - findlibs=0.1.2 + - font-ttf-dejavu-sans-mono=2.37 + - font-ttf-inconsolata=3.000 + - font-ttf-source-code-pro=2.038 + - font-ttf-ubuntu=0.83 + - fontconfig=2.15.0 + - fonts-conda-ecosystem=1 + - fonts-conda-forge=1 + - fonttools=4.61.0 + - freeglut=3.2.2 + - freetype=2.14.1 + - fribidi=1.0.16 + - geos=3.14.1 + - graphite2=1.3.14 + - gsl=2.7 + - h2=4.3.0 + - harfbuzz=12.2.0 + - hdf4=4.2.15 + - hdf5=1.14.6 + - hpack=4.1.0 + - hyperframe=6.1.0 + - icu=75.1 + - idna=3.11 + - imagesize=1.4.1 + - jasper=4.2.8 + - jinja2=3.1.6 + - joblib=1.5.2 + - keyutils=1.6.3 + - kiwisolver=1.4.9 + - krb5=1.21.3 + - lcms2=2.17 + - ld_impl_linux-64=2.45 + - lerc=4.0.0 + - libaec=1.1.4 + - libblas=3.11.0 + - libbrotlicommon=1.2.0 + - libbrotlidec=1.2.0 + - libbrotlienc=1.2.0 + - libcblas=3.11.0 + - libclang-cpp21.1=21.1.7 + - libclang13=21.1.7 + - libcups=2.3.3 + - libcurl=8.17.0 + - libdeflate=1.25 + - libdrm=2.4.125 + - libedit=3.1.20250104 + - libegl=1.7.0 + - libev=4.33 + - libexpat=2.7.3 + - libffi=3.5.2 + - libfreetype=2.14.1 + - libfreetype6=2.14.1 + - libgcc=15.2.0 + - libgcc-ng=15.2.0 + - libgfortran=15.2.0 + - libgfortran5=15.2.0 + - libgl=1.7.0 + - libglib=2.86.2 + - libglu=9.0.3 + - libglvnd=1.7.0 + - libglx=1.7.0 + - libgomp=15.2.0 + - libiconv=1.18 + - libjpeg-turbo=3.1.2 + - liblapack=3.11.0 + - libllvm21=21.1.7 + - liblzma=5.8.1 + - libnetcdf=4.9.3 + - libnghttp2=1.67.0 + - libnsl=2.0.1 + - libntlm=1.8 + - libopenblas=0.3.30 + - libopengl=1.7.0 + - libpciaccess=0.18 + - libpng=1.6.51 + - libpq=18.1 + - libsqlite=3.51.1 + - libssh2=1.11.1 + - libstdcxx=15.2.0 + - libstdcxx-ng=15.2.0 + - libtiff=4.7.1 + - libudunits2=2.2.28 + - libuuid=2.41.2 + - libvulkan-loader=1.4.328.1 + - libwebp-base=1.6.0 + - libxcb=1.17.0 + - libxcrypt=4.4.36 + - libxkbcommon=1.13.1 + - libxml2=2.15.1 + - libxml2-16=2.15.1 + - libxml2-devel=2.15.1 + - libxslt=1.1.43 + - libzip=1.11.2 + - libzlib=1.3.1 + - lz4-c=1.10.0 + - magics=4.16.0 + - magics-python=1.5.8 + - markupsafe=3.0.3 + - matplotlib=3.10.8 + - matplotlib-base=3.10.8 + - multiurl=0.3.7 + - munkres=1.1.4 + - nco=5.3.6 + - ncurses=6.5 + - netcdf-fortran=4.6.2 + - netcdf4=1.7.4 + - numpy=2.4.1 + - openjpeg=2.5.4 + - openldap=2.6.10 + - openssl=3.6.0 + - packaging=25.0 + - pandas=2.3.3 + - pango=1.56.4 + - pcre2=10.46 + - pillow=12.0.0 + - pip=25.3 + - pixman=0.46.4 + - proj=9.7.1 + - pthread-stubs=0.4 + - pycparser=2.22 + - pygments=2.19.2 + - pyparsing=3.2.5 + - pyproj=3.7.2 + - pyshp=3.0.3 + - pyside6=6.9.3 + - pysocks=1.7.1 + - python=3.11.14 + - python-dateutil=2.9.0.post0 + - python-tzdata=2025.2 + - python_abi=3.11 + - pytz=2025.2 + - pyyaml=6.0.3 + - qhull=2020.2 + - qt6-main=6.9.3 + - readline=8.2 + - requests=2.32.5 + - roman-numerals=3.1.0 + - scikit-learn=1.8.0 + - scipy=1.17.0 + - setuptools=80.9.0 + - shapely=2.1.2 + - simplejson=3.20.2 + - six=1.17.0 + - snappy=1.2.2 + - snowballstemmer=3.0.1 + - sphinx=9.0.4 + - sphinx-copybutton=0.5.2 + - sphinx_rtd_theme=0.4.3 + - sphinxcontrib-applehelp=2.0.0 + - sphinxcontrib-devhelp=2.0.0 + - sphinxcontrib-htmlhelp=2.1.0 + - sphinxcontrib-jsmath=1.0.1 + - sphinxcontrib-qthelp=2.0.0 + - sphinxcontrib-serializinghtml=1.1.10 + - sqlite=3.51.1 + - tempest-remap=2.2.0 + - threadpoolctl=3.6.0 + - tk=8.6.13 + - tornado=6.5.2 + - tqdm=4.67.1 + - typing_extensions=4.15.0 + - tzdata=2025b + - udunits2=2.2.28 + - unicodedata2=17.0.0 + - urllib3=2.5.0 + - wayland=1.24.0 + - wheel=0.45.1 + - xarray=2025.12.0 + - xcb-util=0.4.1 + - xcb-util-cursor=0.1.6 + - xcb-util-image=0.4.0 + - xcb-util-keysyms=0.4.1 + - xcb-util-renderutil=0.3.10 + - xcb-util-wm=0.4.2 + - xkeyboard-config=2.46 + - xorg-libice=1.1.2 + - xorg-libsm=1.2.6 + - xorg-libx11=1.8.12 + - xorg-libxau=1.0.12 + - xorg-libxcomposite=0.4.6 + - xorg-libxcursor=1.2.3 + - xorg-libxdamage=1.1.6 + - xorg-libxdmcp=1.1.5 + - xorg-libxext=1.3.6 + - xorg-libxfixes=6.0.2 + - xorg-libxi=1.8.2 + - xorg-libxrandr=1.5.4 + - xorg-libxrender=0.9.12 + - xorg-libxtst=1.2.5 + - xorg-libxxf86vm=1.1.6 + - xorg-xorgproto=2024.1 + - yaml=0.2.5 + - zlib=1.3.1 + - zlib-ng=2.3.2 + - zstandard=0.25.0 + - zstd=1.5.7 +prefix: /cluster/project/climate/lroither/envs/proc-chain diff --git a/workflows.yaml b/workflows.yaml index abe778d7..ccefb2ec 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -1,3 +1,46 @@ + +icon-global-era5-parent-r2b7: + features: + - restart + jobs: + - prepare_icon + - era5_ic + - era5_sstice + - icon + dependencies: + era5_ic: + current: + - prepare_icon + era5_sstice: + current: + - prepare_icon + icon: + current: + - prepare_icon + - era5_ic + - era5_sstice + previous: + - icon + +icon-global-era5-parent-r2b7-nosst: + features: + - restart + jobs: + - prepare_icon + - era5_ic + - icon + dependencies: + era5_ic: + current: + - prepare_icon + icon: + current: + - prepare_icon + - era5_ic + previous: + - icon + + cosmo: features: - restart @@ -165,6 +208,24 @@ icon: previous: - icon +icon-global-era5: + features: + - restart + jobs: + - prepare_icon + - era5_ic + - icon + dependencies: + era5_ic: + current: + - prepare_icon + icon: + current: + - prepare_icon + - era5_ic + previous: + - icon + icon-art: features: - restart