diff --git a/.github/workflows/build_mpas_intel.yml b/.github/workflows/build_mpas_intel.yml index f7e1a46c9d..80a45e8ea9 100644 --- a/.github/workflows/build_mpas_intel.yml +++ b/.github/workflows/build_mpas_intel.yml @@ -42,6 +42,29 @@ jobs: install-oneapi: false mpi-wrapper-setup: classic + - name: Install MKL + run: | + wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ + | gpg --dearmor \ + | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null + echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \ + | sudo tee /etc/apt/sources.list.d/oneAPI.list + sudo apt-get update -q + sudo apt-get install -y intel-oneapi-mkl-devel + + - name: Find MKL library location + run: | + echo "=== /opt/intel/oneapi contents ===" + ls /opt/intel/oneapi/ 2>/dev/null || echo "No /opt/intel/oneapi dir" + echo "=== broad libmkl search ===" + find /opt/intel /usr -name "libmkl_core*" 2>/dev/null | head -20 || echo "libmkl_core not found" + echo "=== installed intel-oneapi packages ===" + dpkg -l | grep intel-oneapi || echo "No intel-oneapi packages found" + echo "=== MKLROOT / MKL_ROOT env ===" + echo "MKLROOT=${MKLROOT}" + echo "MKL_ROOT=${MKL_ROOT}" + echo "Bye" + - name: Cache Intel id: cache-INTEL uses: actions/cache@v4 @@ -129,6 +152,10 @@ jobs: echo "LD_LIBRARY_PATH=${AUTOTOOLS}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV echo "PATH=${PNETCDF}/bin:$PATH" >> $GITHUB_ENV echo "LD_LIBRARY_PATH=${PNETCDF}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "MKLROOT=/opt/intel/oneapi/mkl/2025.3" >> $GITHUB_ENV + echo "MKL_LIBDIR=/opt/intel/oneapi/mkl/2025.3/lib" >> $GITHUB_ENV + echo "LIBRARY_PATH=/opt/intel/oneapi/mkl/2025.3/lib:$LIBRARY_PATH" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/2025.3/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - name: Build MPAS Standalone (make) if: matrix.build-system == 'make' @@ -144,4 +171,4 @@ jobs: mkdir build cd build cmake -DMPAS_DOUBLE_PRECISION=OFF -DMPAS_CORES="init_atmosphere;atmosphere" .. - make -j8 \ No newline at end of file + make -j8 diff --git a/.gitmodules b/.gitmodules index bdf041fe6b..f19f95960e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,3 +22,6 @@ [submodule "src/core_atmosphere/physics/physics_mmm"] path = src/core_atmosphere/physics/physics_mmm url = https://github.com/NCAR/MMM-physics.git +[submodule "src/core_atmosphere/stochastic_physics"] + path = src/core_atmosphere/stochastic_physics + url = https://github.com/dtcenter/stochastic_physics.git diff --git a/src/Makefile b/src/Makefile index b9c037c8cc..305047d837 100644 --- a/src/Makefile +++ b/src/Makefile @@ -9,7 +9,7 @@ else all: mpas mpas: $(AUTOCLEAN_DEPS) externals frame ops dycore drver - $(LINKER) $(LDFLAGS) -o $(EXE_NAME) driver/*.o -L. -ldycore -lops -lframework $(LIBS) -I./external/esmf_time_f90 -L./external/esmf_time_f90 -lesmf_time + $(LINKER) $(LDFLAGS) -o $(EXE_NAME) driver/*.o -L. -ldycore -lops -lframework $(LIBS) -I./external/esmf_time_f91 -L./external/esmf_time_f90 -lesmf_time -lmkl_intel_lp64 -lmkl_core -lmkl_sequential externals: $(AUTOCLEAN_DEPS) ( cd external; $(MAKE) FC="$(FC)" SFC="$(SFC)" CC="$(CC)" SCC="$(SCC)" FFLAGS="$(FFLAGS)" CFLAGS="$(CFLAGS)" CPP="$(CPP)" NETCDF="$(NETCDF)" CORE="$(CORE)" all ) diff --git a/src/core_atmosphere/Makefile b/src/core_atmosphere/Makefile index 966027bc77..92bb34cfa3 100644 --- a/src/core_atmosphere/Makefile +++ b/src/core_atmosphere/Makefile @@ -28,6 +28,7 @@ core_reg: core_input_gen: if [ ! -e default_inputs ]; then mkdir default_inputs; fi + rm -f default_inputs/streams.atmosphere default_inputs/stream_list.atmosphere.* ( cd default_inputs; $(NL_GEN) ../Registry_processed.xml namelist.atmosphere in_defaults=true ) ( cd default_inputs; $(ST_GEN) ../Registry_processed.xml streams.atmosphere stream_list.atmosphere. listed in_defaults=true) @@ -37,12 +38,21 @@ gen_includes: core_reg post_build: if [ ! -e $(ROOT_DIR)/default_inputs ]; then mkdir $(ROOT_DIR)/default_inputs; fi + rm -f $(ROOT_DIR)/default_inputs/streams.$(CORE) $(ROOT_DIR)/default_inputs/stream_list.$(CORE).* cp default_inputs/* $(ROOT_DIR)/default_inputs/. - ( cd $(ROOT_DIR)/default_inputs; for FILE in `ls -1`; do if [ ! -e ../$$FILE ]; then cp $$FILE ../.; fi; done ) + ( cd $(ROOT_DIR)/default_inputs; for FILE in `ls -1`; do \ + if [ -e ../$$FILE ]; then \ + N=1; \ + while [ -e "../$$FILE.old$$(printf '%03d' $$N)" ]; do N=$$((N+1)); done; \ + mv "../$$FILE" "../$$FILE.old$$(printf '%03d' $$N)"; \ + fi; \ + cp $$FILE ../.; \ + done ) physcore: mpas_atm_dimensions.o ( cd physics; $(MAKE) all ) - ( mkdir libphys; cd libphys; ar -x ../physics/libphys.a ) + ( cd stochastic_physics; $(MAKE) -f Makefile all ) + ( mkdir libphys; cd libphys; ar -x ../physics/libphys.a; ar -x ../stochastic_physics/libstochphys.a ) ( cd ../..; ln -sf ./src/core_atmosphere/physics/physics_wrf/files/*TBL .) ( cd ../..; ln -sf ./src/core_atmosphere/physics/physics_wrf/files/*DATA* .) ( cd ../..; ln -sf ./src/core_atmosphere/physics/physics_noahmp/parameters/*TBL .) @@ -68,6 +78,12 @@ mpas_atm_dimensions.o: clean: ( cd physics; $(MAKE) clean ) ( cd dynamics; $(MAKE) clean ) + @# The stochastic_physics directory contains both a makefile (lower case + @# "m") and a Makefile (upper case "M"). The former is used for FV3 and + @# the latter for MPAS. The default for "make" is to use "makefile", but + @# here, we want to use the one for MPAS (with an upper case "M"). Indicate + @# that using the -f flag. + ( cd stochastic_physics; $(MAKE) -f Makefile clean ) ( cd diagnostics; $(MAKE) clean ) ( cd utils; $(MAKE) clean ) ( cd ../..; rm -f *TBL ) @@ -84,7 +100,7 @@ clean: $(RM) $@ $*.mod ifeq "$(GEN_F90)" "true" $(CPP) $(CPPFLAGS) $(PHYSICS) $(CPPINCLUDES) -I./inc $< > $*.f90 - $(FC) $(FFLAGS) -c $*.f90 $(FCINCLUDES) -I../framework -I../operators -I./physics -I./dynamics -I./diagnostics -I./physics/physics_wrf -I./physics/physics_mmm -I./physics/physics_noaa/UGWP -I../external/esmf_time_f90 + $(FC) $(FFLAGS) -c $*.f90 $(FCINCLUDES) -I../framework -I../operators -I./physics -I./dynamics -I./diagnostics -I./physics/physics_wrf -I./physics/physics_mmm -I./physics/physics_noaa/UGWP -I./stochastic_physics -I../external/esmf_time_f90 else - $(FC) $(CPPFLAGS) $(PHYSICS) $(FFLAGS) -c $*.F $(CPPINCLUDES) $(FCINCLUDES) -I./inc -I../framework -I../operators -I./physics -I./dynamics -I./diagnostics -I./physics/physics_wrf -I./physics/physics_mmm -I./physics/physics_noaa/UGWP -I../external/esmf_time_f90 + $(FC) $(CPPFLAGS) $(PHYSICS) $(FFLAGS) -c $*.F $(CPPINCLUDES) $(FCINCLUDES) -I./inc -I../framework -I../operators -I./physics -I./dynamics -I./diagnostics -I./physics/physics_wrf -I./physics/physics_mmm -I./physics/physics_noaa/UGWP -I./stochastic_physics -I../external/esmf_time_f90 endif diff --git a/src/core_atmosphere/Registry.xml b/src/core_atmosphere/Registry.xml index 4851e1cae7..985cabd5dc 100644 --- a/src/core_atmosphere/Registry.xml +++ b/src/core_atmosphere/Registry.xml @@ -32,6 +32,10 @@ description="The number of atmospheric layers"/> + + #ifdef DO_PHYSICS + + @@ -4162,6 +4168,11 @@ description="Total cell-centered meridional wind tendency from physics" persistence="scratch" /> + + + #ifdef DO_PHYSICS @@ -4672,5 +4683,6 @@ #include "physics/Registry_noahmp.xml" #include "physics/Registry_tempo.xml" #include "physics/registry.chemistry.xml" +#include "stochastic_physics/Registry_stoch_physics.xml" #endif diff --git a/src/core_atmosphere/dynamics/Makefile b/src/core_atmosphere/dynamics/Makefile index 6892633c68..62d595c90a 100644 --- a/src/core_atmosphere/dynamics/Makefile +++ b/src/core_atmosphere/dynamics/Makefile @@ -22,5 +22,5 @@ ifeq "$(GEN_F90)" "true" $(CPP) $(CPPFLAGS) $(PHYSICS) $(CPPINCLUDES) $< > $*.f90 $(FC) $(FFLAGS) -c $*.f90 $(FCINCLUDES) -I.. -I../../framework -I../../operators -I../physics -I../physics/physics_wrf -I../physics/physics_mmm -I../../external/esmf_time_f90 else - $(FC) $(CPPFLAGS) $(PHYSICS) $(FFLAGS) -c $*.F $(CPPINCLUDES) $(FCINCLUDES) -I.. -I../../framework -I../../operators -I../physics -I../physics/physics_wrf -I../physics/physics_mmm -I../../external/esmf_time_f90 + $(FC) $(CPPFLAGS) $(PHYSICS) $(FFLAGS) -c $*.F $(CPPINCLUDES) $(FCINCLUDES) -I.. -I../../framework -I../../operators -I../physics -I../physics/physics_wrf -I../physics/physics_mmm -I../stochastic_physics -I../../external/esmf_time_f90 endif diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 0601020cde..d49d58d1ea 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -31,6 +31,7 @@ module atm_time_integration use mpas_atmphys_driver_microphysics use mpas_atmphys_todynamics use mpas_atmphys_utilities + use mpas_stochastic_physics, only : stochastic_physics_pattern_apply, dosppt #endif use mpas_atm_boundaries, only : nSpecZone, nRelaxZone, nBdyZone, mpas_atm_get_bdy_state, mpas_atm_get_bdy_tend ! regional_MPAS addition @@ -880,6 +881,8 @@ subroutine atm_srk3(domain, dt, itimestep, exchange_halo_group) #endif real (kind=RKIND) :: time_dyn_step + character(len=32) :: tend_names(4) + integer :: ierr logical, parameter :: debug = .false. @@ -1077,11 +1080,28 @@ subroutine atm_srk3(domain, dt, itimestep, exchange_halo_group) call mpas_timer_start('physics_get_tend') rk_step = 1 dynamics_substep = 1 + +! apply random perturbation pattern to the tendency + tend_names(1) = "rucuten" + tend_names(2) = "rvcuten" + tend_names(3) = "rublten" + tend_names(4) = "rvblten" + if (dosppt(domain)) then + call stochastic_physics_pattern_apply(domain, 4, tend_names, ierr) + endif + call physics_get_tend( block, mesh, state, diag, tend, tend_physics, & block % configs, rk_step, dynamics_substep, & tend_ru_physics, tend_rtheta_physics, tend_rho_physics, & exchange_halo_group ) call mpas_timer_stop('physics_get_tend') + +! apply random perturbation pattern to the tendency + tend_names(1) = "tend_rtheta_physics" +! tend_names(2) = "tend_rho_physics" + if (dosppt(domain)) then + call stochastic_physics_pattern_apply(domain, 1, tend_names, ierr) + endif #else #ifndef MPAS_CAM_DYCORE ! diff --git a/src/core_atmosphere/mpas_atm_core.F b/src/core_atmosphere/mpas_atm_core.F index e10171e8c8..c0ac5c6cc8 100644 --- a/src/core_atmosphere/mpas_atm_core.F +++ b/src/core_atmosphere/mpas_atm_core.F @@ -48,6 +48,7 @@ function atm_core_init(domain, startTimeStamp) result(ierr) use mpas_attlist, only : mpas_modify_att use mpas_string_utils, only : mpas_string_replace use mpas_atm_halos, only: atm_build_halo_groups, exchange_halo_group + use mpas_stochastic_physics, only : stochastic_physics_pattern_init, dosppt implicit none @@ -301,6 +302,11 @@ function atm_core_init(domain, startTimeStamp) result(ierr) ! Prepare the dynamics for integration ! call mpas_atm_dynamics_init(domain) + ! + ! init stochastic pattern generation + if (dosppt(domain)) then + call stochastic_physics_pattern_init (domain, ierr) + endif end function atm_core_init @@ -1014,6 +1020,7 @@ subroutine atm_do_timestep(domain, dt, itimestep) use mpas_atmphys_update #endif use mpas_atm_halos, only: exchange_halo_group + use mpas_stochastic_physics, only : stochastic_physics_pattern_adv, dosppt implicit none @@ -1046,6 +1053,9 @@ subroutine atm_do_timestep(domain, dt, itimestep) endif #endif + if (dosppt(domain)) then + call stochastic_physics_pattern_adv(domain, itimestep, ierr) + endif call atm_timestep(domain, dt, currTime, itimestep, exchange_halo_group) end subroutine atm_do_timestep diff --git a/src/core_atmosphere/physics/Makefile b/src/core_atmosphere/physics/Makefile index dfe7ea1e8c..6b81f38f01 100644 --- a/src/core_atmosphere/physics/Makefile +++ b/src/core_atmosphere/physics/Makefile @@ -18,6 +18,26 @@ OBJS_init = \ mpas_atmphys_functions.o \ mpas_atmphys_utilities.o +# Set TEMPO_TBLS_MAKEFILE_SUFFIX needed to build TEMPO's executable for +# generating microphysics tables. Note that when the "clean" target is +# invoked by upstream Makefies, FC_SERIAL and FC_PARALLEL will not be set, +# in which case TEMPO_TBLS_MAKEFILE_SUFFIX will be set to its default +# value of "none". +TEMPO_TBLS_MAKEFILE_SUFFIX = none + +ifeq "$(FC_SERIAL)" "gfortran" +ifeq "$(FC_PARALLEL)" "mpif90" +TEMPO_TBLS_MAKEFILE_SUFFIX = gfortran +endif +endif + +ifeq "$(FC_SERIAL)" "ifort" +ifeq "$(FC_PARALLEL)" "mpiifort" +TEMPO_TBLS_MAKEFILE_SUFFIX = intel +endif +endif +$(info TEMPO_TBLS_MAKEFILE_SUFFIX = $(TEMPO_TBLS_MAKEFILE_SUFFIX)) + OBJS = \ mpas_atmphys_camrad_init.o \ mpas_atmphys_control.o \ @@ -65,7 +85,8 @@ core_physics_mmm: core_physics_init (cd physics_mmm; $(MAKE) -f Makefile.mpas all) core_microphysics: core_physics_init core_physics_mmm - (cd physics_noaa/TEMPO; $(MAKE) -f Makefile.mpas) + (cd physics_noaa/TEMPO; $(MAKE) -f Makefile.mpas; cp tables/ccn_activate.bin ../../../../../) + (cd physics_noaa/TEMPO; $(MAKE) -f Makefile.$(TEMPO_TBLS_MAKEFILE_SUFFIX) run_build_tables ; cp run_build_tables ../../../../../) core_SMOKE: core_physics_init (cd physics_noaa/SMOKE; cp ./MPAS/Makefile .; cp ./MPAS/mpas_smoke_wrapper.F90 .; $(MAKE) all) @@ -290,6 +311,13 @@ clean: ( cd physics_noahmp/src; $(MAKE) clean ) ( cd physics_noahmp/utility; $(MAKE) clean ) ( if [ -d physics_noaa/TEMPO ]; then cd physics_noaa/TEMPO; $(MAKE) -f Makefile.mpas clean; fi ) + @# For cleaning the TEMPO executable for generating microphysics tables, call both + @# the intel and the gfortran Makefiles. This is because when the "clean" target + @# is invoked in upstream Makefiles, a compiler is not specified, so it is not + @# clear which of the two Makefiles to use here. By calling both, we ensure that + @# the directory is properly cleaned regardless of which compiler was used to build + @# the executable. + ( if [ -d physics_noaa/TEMPO ]; then cd physics_noaa/TEMPO; $(MAKE) -f Makefile.gfortran clean; $(MAKE) -f Makefile.intel clean; fi ) ( if [ -d physics_noaa/UGWP ]; then cd physics_noaa/UGWP; $(MAKE) clean; fi ) ( if [ -d physics_noaa/RUCLSM ]; then cd physics_noaa/RUCLSM; $(MAKE) -f MPAS/Makefile clean; fi ) ( if [ -d physics_noaa/MYNN-EDMF ]; then cd physics_noaa/MYNN-EDMF; cp ./MPAS/Makefile .; $(MAKE) clean; fi ) diff --git a/src/core_atmosphere/physics/registry.chemistry.xml b/src/core_atmosphere/physics/registry.chemistry.xml index 6b5141613f..760577afaf 100644 --- a/src/core_atmosphere/physics/registry.chemistry.xml +++ b/src/core_atmosphere/physics/registry.chemistry.xml @@ -199,9 +199,9 @@ - diff --git a/src/core_atmosphere/stochastic_physics b/src/core_atmosphere/stochastic_physics new file mode 160000 index 0000000000..16d9eca781 --- /dev/null +++ b/src/core_atmosphere/stochastic_physics @@ -0,0 +1 @@ +Subproject commit 16d9eca781528eda6d4636436256a59ef6fa0e27