diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 815369e0a..09d293e20 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -112,7 +112,7 @@ jobs: # Concretize and check for duplicates spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME} - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env -i fms # Add and update source cache spack mirror add local-source file://${SOURCE_CACHE_PATH}/ @@ -190,7 +190,7 @@ jobs: # Concretize and check for duplicates spack concretize --force 2>&1 | tee log.concretize.${ENVNAME} - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env -i fms # Add binary cache back in spack mirror add local-binary file://${BUILD_CACHE_PATH}/ diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml index a1ebeab0b..55da517e2 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml @@ -128,7 +128,7 @@ jobs: # Concretize and check for duplicates spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME} - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env -i fms # Add and update source cache spack mirror add local-source file://${SOURCE_CACHE_PATH}/ @@ -209,7 +209,7 @@ jobs: # Concretize and check for duplicates spack concretize --force 2>&1 | tee log.concretize.${ENVNAME} - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env -i fms # Add binary cache back in spack mirror add local-binary file://${BUILD_CACHE_PATH}/ diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index c4cbd7094..7777ddece 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -128,7 +128,7 @@ jobs: # Concretize and check for duplicates spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME} - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env -i fms # Add and update source cache spack mirror add local-source file://${SOURCE_CACHE_PATH}/ @@ -209,7 +209,7 @@ jobs: # Concretize and check for duplicates spack concretize --force 2>&1 | tee log.concretize.${ENVNAME} - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i neptune-env -i fms # Add binary cache back in spack mirror add local-binary file://${BUILD_CACHE_PATH}/ diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 8bdad785c..d5a605872 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -142,6 +142,10 @@ modules: ^esmf@8.8.0+debug snapshot=none: 'esmf-8.8.0-debug' ^esmf@8.9.0~debug snapshot=none: 'esmf-8.9.0' ^esmf@8.9.0+debug snapshot=none: 'esmf-8.9.0-debug' + fms: + suffixes: + constants=GFS: 'gfs-constants' + constants=GEOS: 'geos-constants' openmpi: environment: set: diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index cab7b3190..1cf5cd4d5 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -161,6 +161,10 @@ modules: ^esmf@8.8.0+debug snapshot=none: 'esmf-8.8.0-debug' ^esmf@8.9.0~debug snapshot=none: 'esmf-8.9.0' ^esmf@8.9.0+debug snapshot=none: 'esmf-8.9.0-debug' + fms: + suffixes: + constants=GFS: 'gfs-constants' + constants=GEOS: 'geos-constants' openmpi: environment: set: diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 33916b603..c184c8eda 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -107,7 +107,10 @@ packages: fms: require: - '@2024.02' - - precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +deprecated_io + - precision=32,64 +quad_precision +openmp +pic build_type=Release +deprecated_io + - any_of: + - +gfs_phys constants=GFS + - ~gfs_phys constants=GEOS fontconfig: require: - +pic @@ -206,8 +209,6 @@ packages: require: - '@2.53' - ~shared ~f2py +pflogger - # Note: Please check the sites/tier1/nas/packages_gcc.yaml if - # this is changed, as it has a custom variant combination met: require: - '@12.0.1' diff --git a/configs/sites/tier1/nas/README.md b/configs/sites/tier1/nas/README.md index b3f570619..70aa6e8f8 100644 --- a/configs/sites/tier1/nas/README.md +++ b/configs/sites/tier1/nas/README.md @@ -3,10 +3,15 @@ In the commands below some will be run on login nodes (with internet access) and some on compute nodes as, at NAS, you aren't allowed more than 2 processes on a login node. +## Machines + +For the below you will need to login to both an `afe01` node for one step. You'll +also want to get a Rome compute node for the rest of the steps. + ## Clone spack-stack ``` -git clone --recursive https://github.com/JCSDA/spack-stack.git -b release/1.9.0 spack-stack-1.9.3 +git clone --recurse-submodules https://github.com/mathomp4/spack-stack.git -b feature/nas_install_spack_v1 spack-stack-2.0.0-test ``` ## Grab interactive node @@ -23,7 +28,7 @@ We will start on a login node with internet access. This is mainly needed for th `spack mirror create` command which downloads all the source code for the packages. ``` -cd spack-stack-1.9.3 +cd spack-stack-2.0.0-test . setup.sh ``` @@ -37,7 +42,7 @@ are used to create the environments. You only need to do this once. To create the oneAPI environment, do: ``` -spack stack create env --name ue-oneapi-2024.2.0 --template unified-dev --site nas --compiler oneapi +spack stack create env --name ue-oneapi-2024.2.0 --template unified-dev --site nas --compiler=oneapi-2024.2.0 cd envs/ue-oneapi-2024.2.0 ``` @@ -46,8 +51,8 @@ cd envs/ue-oneapi-2024.2.0 To create the GCC environment, do: ``` -spack stack create env --name ue-gcc-12.3.0 --template unified-dev --site nas --compiler gcc -cd envs/ue-gcc-12.3.0 +spack stack create env --name ue-gcc-13.2.0 --template unified-dev --site nas --compiler gcc-13.2.0 +cd envs/ue-gcc-13.2.0 ``` ## Activate environment @@ -67,41 +72,75 @@ below with this environment. spack concretize 2>&1 | tee log.concretize ``` +NOTE: The first time you do this on a new build, you should do it on a *LOGIN* node. This is because +it might need to bootstrap things and so it will reach out to the internet. + ## Create source cache (LOGIN NODE ONLY) Because this step downloads all the source code for all packages and all versions, it should be done on a login node with internet access. ``` -spack mirror create -a -d /nobackup/gmao_SIteam/spack-stack/source-cache +spack mirror create -a -d /swbuild/gmao_SIteam/spack-stack/source-cache ``` NOTE: Make sure you are in an environment when you run that `spack mirror create` command. Otherwise, you will download *EVERY* package and *EVERY* version in spack! +## Pre-fetch cargo packages (LOGIN NODE ONLY) + +Some packages use Rust/Cargo for dependencies. These need internet access to build. So we pre-fetch them here. + +We need to set `CARGO_HOME` to a location where the Cargo deps have been downloaded + +``` +export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache +../../util/fetch_cargo_deps.py +``` + +NOTE: `CARGO_HOME` should be set as well on the COMPUTE node! + ## Install packages Our install process will actually have (at least) three steps. This is because of the `crtm` package which requires internet access at build time. -### Install crtm dependencies (COMPUTE NODE) +### Install Step 1: Dependencies of Rust codes and ecflow (COMPUTE NODE) + +We currently have some codes that use rust/cargo for dependencies. And, for some reason, +even doing the "cargo dependencies" as above, they still need internet +access to build/install. + +As for ecflow, we built QT on a login node (as it was the only complete node), so we +then have to build ecflow on a login node as well. + +So we first install all the dependencies of then codes. ``` -spack install -j 10 --verbose --fail-fast --show-log-on-error --no-check-signature --only dependencies crtm 2>&1 | tee log.install.crtm_dependencies +export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache +spack install -j 16 --verbose --fail-fast --show-log-on-error --no-check-signature --only dependencies py-cryptography py-maturin py-rpds-py ecflow 2>&1 | tee log.install.deps-for-rust-and-ecflow ``` -### Install crtm (LOGIN NODE) +### Install Step 2: Rust Codes and ecflow (AFE LOGIN NODE) + +NOTE: You *MUST* run this on an afe login node. The reason is the pfe login nodes are Sandy +Bridge but we are building Spack with `x86_64_v3` and these are too old (`_v2`). So +you will get an illegal instruction error when the install below calls python3. + +So go back to an afe login node and run: ``` -spack install -j 2 --verbose --fail-fast --show-log-on-error --no-check-signature crtm 2>&1 | tee log.install.crtm +export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache +spack install -j 2 -p 1 --verbose --fail-fast --show-log-on-error --no-check-signature py-cryptography py-maturin py-rpds-py ecflow 2>&1 | tee log.install.rust-and-ecflow ``` Note we are only using 2 processes here because NAS limits you to 2 processes on a login node. -### Install rest of packages (COMPUTE NODE) +### Install Step 3: The rest (COMPUTE NODE) ``` -spack install -j 10 --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install.after_crtm +export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache +spack install -j 16 --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install.after-cargo ``` NOTE: You might need to run the `spack install` command multiple times because sometimes @@ -120,7 +159,7 @@ Then, once that package is built, you can go back to the compute node and run th ## Update module files and setup meta-modules ``` -spack module tcl refresh -y +spack module tcl refresh -y --delete-tree spack stack setup-meta-modules ``` @@ -129,3 +168,13 @@ spack stack setup-meta-modules ``` spack env deactivate ``` + +# Debugging a package + +When things go wrong, a good way to debug a failure is: + +``` +spack clean +spack stage +spack build-env -- bash --norc --noprofile +``` diff --git a/configs/sites/tier1/nas/compilers.yaml b/configs/sites/tier1/nas/compilers.yaml deleted file mode 100644 index 6842f9719..000000000 --- a/configs/sites/tier1/nas/compilers.yaml +++ /dev/null @@ -1,57 +0,0 @@ -compilers: -- compiler: - spec: oneapi@=2024.2.0 - paths: - cc: /nobackup/gmao_SIteam/intel/oneapi/compiler/2024.2/bin/icx - cxx: /nobackup/gmao_SIteam/intel/oneapi/compiler/2024.2/bin/icpx - f77: /nobackup/gmao_SIteam/intel/oneapi/compiler/2024.2/bin/ifort - fc: /nobackup/gmao_SIteam/intel/oneapi/compiler/2024.2/bin/ifort - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - comp-intel/2024.2.0-ifort - environment: - prepend_path: - PATH: '/nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/bin' - CPATH: '/nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/include' - LD_LIBRARY_PATH: '/nobackup/gmao_SIteam/intel/oneapi/compiler/2024.2/lib/:/nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/lib64' - set: - MPI_COLL_REPRODUCIBLE: 'yes' - MPI_MEMMAP_OFF: 'yes' - MPI_XPMEM_ENABLED: 'yes' - MPI_LAUNCH_TIMEOUT: '40' - MPI_COMM_MAX: '1024' - MPI_GROUP_MAX: '1024' - MPI_BUFS_PER_PROC: '256' - unset: - - MPI_NUM_MEMORY_REGIONS - - SUPPRESS_XPMEM_TRIM_THRESH - - PMI_RANK - extra_rpaths: [] -- compiler: - spec: gcc@=12.3.0 - paths: - cc: /nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/bin/gcc - cxx: /nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/bin/g++ - f77: /nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/bin/gfortran - fc: /nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/bin/gfortran - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - comp-gcc/12.3.0-TOSS4 - environment: - set: - OMPI_MCA_mpi_preconnect_all: 1 - OMPI_MCA_coll_tuned_bcast_algorithm: 7 - OMPI_MCA_coll_tuned_scatter_algorithm: 2 - OMPI_MCA_coll_tuned_reduce_scatter_algorithm: 3 - OMPI_MCA_coll_tuned_allreduce_algorithm: 3 - OMPI_MCA_coll_tuned_allgather_algorithm: 4 - OMPI_MCA_coll_tuned_allgatherv_algorithm: 3 - OMPI_MCA_coll_tuned_gather_algorithm: 1 - OMPI_MCA_coll_tuned_barrier_algorithm: 0 - OMPI_MCA_coll_tuned_use_dynamic_rules: 1 - OMPI_MCA_sharedfp: '^lockedfile,individual' - extra_rpaths: [] diff --git a/configs/sites/tier1/nas/config.yaml b/configs/sites/tier1/nas/config.yaml index 7e559f33f..714b3c10e 100644 --- a/configs/sites/tier1/nas/config.yaml +++ b/configs/sites/tier1/nas/config.yaml @@ -1,2 +1,9 @@ config: build_jobs: 6 + + # Overrides for spack build and staging areas to speed up builds + # and avoid errors with Lustre file locking and xattr issues + build_stage: /swbuild/gmao_SIteam/spack-stack/cache/build_stage + test_stage: /swbuild/gmao_SIteam/spack-stack/cache/test_stage + source_cache: /swbuild/gmao_SIteam/spack-stack/cache/source_cache + misc_cache: /swbuild/gmao_SIteam/spack-stack/cache/misc_cache diff --git a/configs/sites/tier1/nas/mirrors.yaml b/configs/sites/tier1/nas/mirrors.yaml index 2be6fcc16..d9ccc4914 100644 --- a/configs/sites/tier1/nas/mirrors.yaml +++ b/configs/sites/tier1/nas/mirrors.yaml @@ -1,18 +1,3 @@ mirrors: - local-source: - fetch: - url: file:///nobackup/gmao_SIteam/spack-stack/source-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null - push: - url: file:///nobackup/gmao_SIteam/spack-stack/source-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null + local-source: file:///swbuild/gmao_SIteam/spack-stack/source-cache + local-binary: file:///swbuild/gmao_SIteam/spack-stack/binary-cache-spack-v1 diff --git a/configs/sites/tier1/nas/packages.yaml b/configs/sites/tier1/nas/packages.yaml index 7aed2e62e..854d4a164 100644 --- a/configs/sites/tier1/nas/packages.yaml +++ b/configs/sites/tier1/nas/packages.yaml @@ -7,6 +7,8 @@ packages: variants: ~shared met: variants: +python +grib2 +graphics +lidar2nc +modis + zlib-api: + buildable: False ### All other external packages listed alphabetically automake: externals: @@ -20,6 +22,10 @@ packages: externals: - spec: bison@3.0.4 prefix: /usr + cmake: + externals: + - spec: cmake@4.0.2 + prefix: /nobackup/gmao_SIteam/cmake/cmake-4.0.2 coreutils: externals: - spec: coreutils@8.30 @@ -45,10 +51,12 @@ packages: - spec: gh@2.78.0 prefix: /nobackup/gmao_SIteam/gh/2.78.0 git: + buildable: False externals: - spec: git@2.43.5~tcltk prefix: /usr git-lfs: + buildable: False externals: - spec: git-lfs@3.7.0 prefix: /nobackup/gmao_SIteam/git-lfs/3.7.0 @@ -80,6 +88,11 @@ packages: externals: - spec: pkgconf@1.4.2 prefix: /usr + qt: + buildable: False + externals: + - spec: qt@5.15.18 + prefix: /nobackup/gmao_SIteam/qt5/5.15.18 sed: externals: - spec: sed@4.5 @@ -97,6 +110,7 @@ packages: - spec: wget@1.19.5 prefix: /usr zlib: + buildable: False externals: - spec: zlib@1.2.11 prefix: /usr diff --git a/configs/sites/tier1/nas/packages_gcc-13.2.0.yaml b/configs/sites/tier1/nas/packages_gcc-13.2.0.yaml new file mode 100644 index 000000000..58a198650 --- /dev/null +++ b/configs/sites/tier1/nas/packages_gcc-13.2.0.yaml @@ -0,0 +1,49 @@ +packages: + mpi: + buildable: false + require: + - "openmpi@4.1.8" + gcc-runtime: + buildable: false + externals: + - spec: gcc-runtime@13.2.0%gcc@13.2.0 + prefix: /nobackup/gmao_SIteam/gcc/gcc-13.2.0 + gcc: + buildable: false + externals: + - spec: gcc@13.2.0 languages:='c,c++,fortran' + prefix: /nobackup/gmao_SIteam/gcc/gcc-13.2.0 + modules: + - comp-gcc/13.2.0 + extra_attributes: + compilers: + c: /nobackup/gmao_SIteam/gcc/gcc-13.2.0/bin/gcc + cxx: /nobackup/gmao_SIteam/gcc/gcc-13.2.0/bin/g++ + fortran: /nobackup/gmao_SIteam/gcc/gcc-13.2.0/bin/gfortran + openmpi: + externals: + - spec: openmpi@4.1.8 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath+lustre fabrics=ucx schedulers=tm %gcc@13.2.0 + prefix: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0 + modules: + - mpi-openmpi/4.1.8/gcc/13.2.0 + environment: + set: + OPAL_PREFIX: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0 + UCX_MAX_EAGER_LANES: 1 + UCX_MAX_RNDV_LANES: 1 + OMPI_MCA_pml: ucx + OMPI_MCA_btl: ^openib + OMPI_MCA_oob_tcp_listen_mode: listen_thread + OMPI_MCA_oob_tcp_if_include: ib0,ib1 + OMPI_MCA_btl_tcp_if_include: ib0,ib1 + OMPI_MCA_opal_warn_on_missing_libcuda: 0 + UCX_IB_SUBNET_PREFIX: fec0:0000:0000:0008 + OMPI_MCA_shmem_mmap_enable_nfs_warning: 0 + OPENMPI: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0 + prepend_path: + PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0/bin + LIBRARY_PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0/lib + LD_LIBRARY_PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0/lib + CPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0/include + FPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0/include + MANPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0/share/man diff --git a/configs/sites/tier1/nas/packages_gcc.yaml b/configs/sites/tier1/nas/packages_gcc.yaml deleted file mode 100644 index a84dd36bc..000000000 --- a/configs/sites/tier1/nas/packages_gcc.yaml +++ /dev/null @@ -1,39 +0,0 @@ -packages: - all: - compiler:: [gcc@12.3.0] - providers: - mpi:: [openmpi@4.1.8] - mpi: - buildable: False - openmpi: - externals: - - spec: openmpi@4.1.8%gcc@=12.3.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath+lustre - fabrics=ucx schedulers=tm - prefix: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-12.3.0 - environment: - set: - OPAL_PREFIX: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-12.3.0 - UCX_MAX_EAGER_LANES: 1 - UCX_MAX_RNDV_LANES: 1 - OMPI_MCA_pml: ucx - OMPI_MCA_btl: ^openib - OMPI_MCA_oob_tcp_listen_mode: listen_thread - OMPI_MCA_oob_tcp_if_include: ib0,ib1 - OMPI_MCA_btl_tcp_if_include: ib0,ib1 - OMPI_MCA_opal_warn_on_missing_libcuda: 0 - UCX_IB_SUBNET_PREFIX: fec0:0000:0000:0008 - OMPI_MCA_shmem_mmap_enable_nfs_warning: 0 - OPENMPI: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-12.3.0 - prepend_path: - PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-12.3.0/bin - LIBRARY_PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-12.3.0/lib - LD_LIBRARY_PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-12.3.0/lib - CPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-12.3.0/include - FPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-12.3.0/include - MANPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-12.3.0/share/man - # MAT: I cannot figure out how to get hdf-eos2 to build with GCC on NAS - # When I try, it dies with an error trying to *patch* the code! - # This turns off the variants of met that need it - met: - variants:: - - '+python +grib2 +graphics' diff --git a/configs/sites/tier1/nas/packages_oneapi-2024.2.0.yaml b/configs/sites/tier1/nas/packages_oneapi-2024.2.0.yaml new file mode 100644 index 000000000..a37f2788e --- /dev/null +++ b/configs/sites/tier1/nas/packages_oneapi-2024.2.0.yaml @@ -0,0 +1,59 @@ +packages: + mpi: + buildable: false + require: + - "mpt@2.30" + intel-oneapi-runtime: + buildable: false + externals: + - spec: intel-oneapi-runtime@2024.2.1 + prefix: /nobackup/gmao_SIteam/intel/oneapi/compiler/2024.2/bin/ifort + modules: + - comp-intel/2024.2.0-ifort + intel-oneapi-compilers: + externals: + - spec: intel-oneapi-compilers@2024.2.0 + prefix: /nobackup/gmao_SIteam/intel/oneapi + modules: + - comp-intel/2024.2.0-ifort + extra_attributes: + compilers: + c: /nobackup/gmao_SIteam/intel/oneapi/compiler/2024.2/bin/icx + cxx: /nobackup/gmao_SIteam/intel/oneapi/compiler/2024.2/bin/icpx + fortran: /nobackup/gmao_SIteam/intel/oneapi/compiler/2024.2/bin/ifort + environment: + prepend_path: + PATH: /nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/bin + CPATH: /nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/include + LD_LIBRARY_PATH: '/nobackup/gmao_SIteam/intel/oneapi/compiler/2024.2/lib/:/nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/lib64' + extra_rpaths: + - /nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/lib64 + - /nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/lib + gcc-runtime: + buildable: false + externals: + - spec: gcc-runtime@12.3.0%gcc@12.3.0 + prefix: /nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4 + gcc: + externals: + - spec: gcc@12.3.0 languages:='c,c++' + prefix: /nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4 + modules: + - comp-gcc/12.3.0-TOSS4 + extra_attributes: + compilers: + c: /nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/bin/gcc + cxx: /nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/bin/g++ + fortran: /nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/bin/gfortran + mpt: + buildable: false + externals: + - spec: mpt@2.30%oneapi@=2024.2.0 + prefix: /nasa/hpe/mpt/2.30_rhel810 + modules: + - mpi-hpe/mpt.2.30 + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2024.2.0%oneapi@2024.2.0 + prefix: /nobackup/gmao_SIteam/intel/oneapi diff --git a/configs/sites/tier1/nas/packages_oneapi.yaml b/configs/sites/tier1/nas/packages_oneapi.yaml deleted file mode 100644 index 5ec68c87f..000000000 --- a/configs/sites/tier1/nas/packages_oneapi.yaml +++ /dev/null @@ -1,15 +0,0 @@ -packages: - all: - compiler:: [oneapi@2024.2.0,gcc@12.3.0] - providers: - mpi:: [mpt@2.30] - mpi: - buildable: False - mpt: - externals: - - spec: mpt@2.30%oneapi@=2024.2.0 - prefix: /nasa/hpe/mpt/2.30_rhel810 - intel-oneapi-mkl: - externals: - - spec: intel-oneapi-mkl@2024.2.0%oneapi@2024.2.0 - prefix: /nobackup/gmao_SIteam/intel/oneapi diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 64597db3f..e51d9a6ca 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -10,12 +10,12 @@ spack: specs: - ewok-env #- ai-env - - geos-gcm-env ^esmf@=8.6.1 + - geos-gcm-env ^esmf@=8.9.0 - global-workflow-env ^esmf@=8.6.1 ^crtm@=3.1.2 - gmao-swell-env ^crtm@=v2.4.1-jedi.2 - gsi-env ^crtm@=3.1.2 - jedi-fv3-env - - jedi-geos-env ^esmf@=8.6.1 + - jedi-geos-env ^esmf@=8.9.0 - jedi-mpas-env - jedi-neptune-env ^esmf@=8.9.0 #- jedi-tools-env @@ -40,6 +40,10 @@ spack: - esmf@=8.8.0 - esmf@=8.9.0 + # Various fms builds + - fms +gfs_phys constants=GFS + - fms ~gfs_phys constants=GEOS + # MADIS for WCOSS2 decoders. - madis@4.5 diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/gmao_swell_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/gmao_swell_env/package.py index 29cd73402..bd5292977 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/gmao_swell_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/gmao_swell_env/package.py @@ -25,7 +25,7 @@ class GmaoSwellEnv(BundlePackage): depends_on("crtm", type="run") # Additional dependencies for JEDI used by swell - depends_on("fms", type="run") + depends_on("fms ~gfs_phys constants=GEOS", type="run") depends_on("nco", type="run") # GEOS diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/jedi_fv3_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_fv3_env/package.py index c02be8f42..27ffc1477 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/jedi_fv3_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_fv3_env/package.py @@ -18,6 +18,6 @@ class JediFv3Env(BundlePackage): version("1.0.0") depends_on("jedi-base-env", type="run") - depends_on("fms", type="run") + depends_on("fms +gfs_phys constants=GFS", type="run") # There is no need for install() since there is no code. diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/jedi_ufs_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_ufs_env/package.py index 783f196a6..bf788588f 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/jedi_ufs_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/jedi_ufs_env/package.py @@ -18,7 +18,7 @@ class JediUfsEnv(BundlePackage): version("1.0.0") depends_on("jedi-base-env", type="run") - depends_on("fms", type="run") + depends_on("fms +gfs_phys constants=GFS", type="run") depends_on("bacio", type="run") depends_on("g2", type="run") diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/ufs_srw_app_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/ufs_srw_app_env/package.py index 4afc2b015..7b93000f0 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/ufs_srw_app_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/ufs_srw_app_env/package.py @@ -26,7 +26,7 @@ class UfsSrwAppEnv(BundlePackage): depends_on("netcdf-fortran") depends_on("parallelio") depends_on("esmf") - depends_on("fms") + depends_on("fms +gfs_phys constants=GFS") depends_on("bacio") depends_on("crtm") depends_on("g2") diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/ufs_weather_model_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/ufs_weather_model_env/package.py index 07c8c36a6..f00d05cf1 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/ufs_weather_model_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/ufs_weather_model_env/package.py @@ -27,7 +27,7 @@ class UfsWeatherModelEnv(BundlePackage): depends_on("base-env", type="run") depends_on("ufs-pyenv", type="run", when="+python") - depends_on("fms", type="run") + depends_on("fms +gfs_phys constants=GFS", type="run") depends_on("bacio", type="run") depends_on("crtm", type="run") depends_on("g2", type="run")