diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index f6438ca..31c6dfc 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -11,21 +11,21 @@ jobs: lint: runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v3.5.2 - with: - submodules: 'recursive' - - name: Step Python - uses: actions/setup-python@v4.6.0 - with: - python-version: '3.11.7' - - name: Install OpenMPI for gt4py - run: | - sudo apt-get install libopenmpi-dev netcdf-bin libnetcdf-dev libnetcdff-dev nco libyaml-dev diffutils - - name: Install Python packages - run: | - python -m pip install --upgrade pip setuptools wheel - pip install .[develop] - - name: Run lint via pre-commit - run: | - pre-commit run --all-files + - name: Checkout repository + uses: actions/checkout@v3.5.2 + with: + submodules: 'recursive' + - name: Step Python + uses: actions/setup-python@v4.6.0 + with: + python-version: '3.12.13' + - name: Install OpenMPI + run: | + sudo apt-get update && sudo apt-get install libopenmpi-dev netcdf-bin libnetcdf-dev libnetcdff-dev nco libyaml-dev diffutils + - name: Install Python packages + run: | + python -m pip install --upgrade pip setuptools wheel + pip install .[dev] + - name: Run lint via pre-commit + run: | + pre-commit run --all-files diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index aa69eb2..5bcbfb6 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -12,21 +12,21 @@ jobs: env: LD_LIBRARY_PATH: steps: - - name: Checkout out hash that triggered CI - uses: actions/checkout@v4 - with: - submodules: 'recursive' - - name: Step Python - uses: actions/setup-python@v4.6.0 - with: - python-version: '3.11.7' - - name: Install OpenMPI for gt4py - run: | - sudo apt-get install libopenmpi-dev netcdf-bin libnetcdf-dev libnetcdff-dev nco libyaml-dev diffutils - - name: install packages - run: | - pip install .[test] - - name: run tests - run: | - cd tests - ./run_tests.sh -o + - name: Checkout out hash that triggered CI + uses: actions/checkout@v4 + with: + submodules: 'recursive' + - name: Step Python + uses: actions/setup-python@v4.6.0 + with: + python-version: '3.12.13' + - name: Install Dependencies for Build + run: | + sudo apt-get update && sudo apt-get install libopenmpi-dev netcdf-bin libnetcdf-dev libnetcdff-dev nco libyaml-dev diffutils + - name: install packages + run: | + pip install -v .[test] + - name: run tests + run: | + cd tests + ./run_tests.sh -o diff --git a/.gitmodules b/.gitmodules index 31f95b1..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,5 +0,0 @@ - -[submodule "cFMS"] - path = cFMS - url = https://github.com/NOAA-GFDL/cFMS.git - branch = main diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6953431..60db865 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,38 +2,30 @@ default_language_version: python: python3 repos: -- repo: https://github.com/psf/black + - repo: https://github.com/psf/black rev: 20.8b1 hooks: - - id: black - additional_dependencies: ["click==8.0.4"] + - id: black + additional_dependencies: ["click==8.0.4"] -- repo: https://github.com/pre-commit/mirrors-isort - rev: v5.4.2 + - repo: https://github.com/pre-commit/mirrors-isort + rev: v5.10.1 hooks: - - id: isort - args: ["--profile", "black"] + - id: isort + args: ["--profile", "black"] -- repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.3.0 hooks: - - id: check-toml - - id: check-yaml - - id: end-of-file-fixer - - id: trailing-whitespace -- repo: https://github.com/pycqa/flake8 - rev: 3.9.2 + - id: check-toml + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + + - repo: https://github.com/pycqa/flake8 + rev: 7.3.0 hooks: - - id: flake8 - name: flake8 - language_version: python3 - args: [--config, setup.cfg] - exclude: | - (?x)^( - .*/__init__.py | - )$ - - id: flake8 - name: flake8 __init__.py files - files: "__init__.py" - # ignore unused import error in __init__.py files - args: ["--ignore=F401,E203", --config, setup.cfg] + - id: flake8 + name: flake8 + language_version: python3 + additional_dependencies: [Flake8-pyproject, flake8-bugbear] diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..38ec891 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.18) + +project(pyfms LANGUAGES C Fortran) + +include(FetchContent) + +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +FetchContent_Declare( + cFMS + GIT_REPOSITORY https://github.com/NOAA-GFDL/cFMS.git + GIT_TAG main +) +FetchContent_MakeAvailable(cFMS) + +install(TARGETS cFMS DESTINATION ${SKBUILD_PLATLIB_DIR}) diff --git a/README.md b/README.md index fdcb2a6..efa5641 100644 --- a/README.md +++ b/README.md @@ -1 +1,61 @@ -# fms2py +# pyFMS + +The `pyFMS` package is a Python-C-Fortran interface (through the use of the `ctypes` package), for access to select methods from the Flexible Modeling System (`FMS`) developed by NOAA/GFDL (https://github.com/NOAA-GFDL/FMS.git). + +## Quickstart - bare metal + +### Build + +The build backend of `pyFMS` is `scikit-build-core`, which allows for the compilation of extension modules for Python packages through CMake. + +`pyFMS` requires: + +- GCC > 9.2 +- MPI +- Python 3.12 + +To clone `pyFMS` from its GitHub repository: + +```shell +git clone https://github.com/NOAA-GFDL/pyFMS.git +``` + +We recommend creating a Python `venv` or `conda` environment for your installation. + +```shell +python -m venv +source ./bin/activate +``` + +Inside of your Python environment, to install `pyFMS` and its dependencies: + +```shell +pip install .[install_options] +``` + +The available `install_options` are: +- `test` +- `dev` (which installs the `test` dependecies as well) +- `extras` (which installs the `dev` dependencies as well) + +For developers we suggest installing with the editable flag `-e` and the verbose flag `-v`: + +```shell +pip install -e -v .[dev] +``` + +use of the editable installation method will allow updates to the Python source code to be reflected in the installation without re-installing. Subsequent installs of `pyFMS` will recompile all extension modules, due to the methods of compilation used by `scikit-build-core`. + +The `CMakeLists.txt` file will be accessed by the build backend during the installation to compile the necessary shared object file from the C-Fortran interface, `cFMS`, to be loaded by the `ctypes` package for use by `pyFMS`. + +### Editing the extension modules of pyFMS + +If you require a different version of `cFMS` and/or its dependency `FMS`, depending on the use case, two methods of use are available: + +1. Edit the `CMakeLists.txt` file to point to the repository containing your custom `cFMS` and/or `FMS` to install it alongside `pyFMS` through the methods outlined above +2. Perform a separate compilation of `cFMS` and/or `FMS` outside of the provided build tools and point to the desired shared object file when running `pyfms.cfms.init(libpath=)` + +## pyFMS as a dependency + +This package can be treated like any Python package when used needed as a dependency; ensure it is defined as a dependency for your project, and import as usual. +Subsequent installs of the dependent package will not trigger a recompilation of the extension modules of pyFMS, unless directly called to due so by a reinstall of pyFMS, as the dependency manager of the build backend for the dependent project will see the requirements of pyFMS are satisfied. If an editable version of pyFMS is needed for the dependent package, please use the method of editable install detailed above. diff --git a/cFMS b/cFMS deleted file mode 160000 index ea76360..0000000 --- a/cFMS +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ea76360bd1356168994f5a2bd1c182a9b8eab90a diff --git a/cmake/FindNetCDF.cmake b/cmake/FindNetCDF.cmake new file mode 100644 index 0000000..1439ae8 --- /dev/null +++ b/cmake/FindNetCDF.cmake @@ -0,0 +1,337 @@ +# (C) Copyright 2011- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +# Try to find NetCDF includes and library. +# Supports static and shared libaries and allows each component to be found in sepearte prefixes. +# +# This module defines +# +# - NetCDF_FOUND - System has NetCDF +# - NetCDF_INCLUDE_DIRS - the NetCDF include directories +# - NetCDF_VERSION - the version of NetCDF +# - NetCDF_CONFIG_EXECUTABLE - the netcdf-config executable if found +# - NetCDF_PARALLEL - Boolean True if NetCDF4 has parallel IO support via hdf5 and/or pnetcdf +# - NetCDF_HAS_PNETCDF - Boolean True if NetCDF4 has pnetcdf support +# +# Deprecated Defines +# - NetCDF_LIBRARIES - [Deprecated] Use NetCDF::NetCDF_ targets instead. +# +# +# Following components are available: +# +# - C - C interface to NetCDF (netcdf) +# - CXX - CXX4 interface to NetCDF (netcdf_c++4) +# - Fortran - Fortran interface to NetCDF (netcdff) +# +# For each component the following are defined: +# +# - NetCDF__FOUND - whether the component is found +# - NetCDF__LIBRARIES - the libraries for the component +# - NetCDF__LIBRARY_SHARED - Boolean is true if libraries for component are shared +# - NetCDF__INCLUDE_DIRS - the include directories for specified component +# - NetCDF::NetCDF_ - target of component to be used with target_link_libraries() +# +# The following paths will be searched in order if set in CMake (first priority) or environment (second priority) +# +# - NetCDF_ROOT - root of NetCDF installation +# - NetCDF_PATH - root of NetCDF installation +# +# The search process begins with locating NetCDF Include headers. If these are in a non-standard location, +# set one of the following CMake or environment variables to point to the location: +# +# - NetCDF_INCLUDE_DIR or NetCDF_${comp}_INCLUDE_DIR +# - NetCDF_INCLUDE_DIRS or NetCDF_${comp}_INCLUDE_DIR +# +# Notes: +# +# - Use "NetCDF::NetCDF_" targets only. NetCDF_LIBRARIES exists for backwards compatibility and should not be used. +# - These targets have all the knowledge of include directories and library search directories, and a single +# call to target_link_libraries will provide all these transitive properties to your target. Normally all that is +# needed to build and link against NetCDF is, e.g.: +# target_link_libraries(my_c_tgt PUBLIC NetCDF::NetCDF_C) +# - "NetCDF" is always the preferred naming for this package, its targets, variables, and environment variables +# - For compatibility, some variables are also set/checked using alternate names NetCDF4, NETCDF, or NETCDF4 +# - Environments relying on these older environment variable names should move to using a "NetCDF_ROOT" environment variable +# - Preferred component capitalization follows the CMake LANGUAGES variables: i.e., C, Fortran, CXX +# - For compatibility, alternate capitalizations are supported but should not be used. +# - If no components are defined, all components will be searched +# + +list( APPEND _possible_components C CXX Fortran ) + +## Include names for each component +set( NetCDF_C_INCLUDE_NAME netcdf.h ) +set( NetCDF_CXX_INCLUDE_NAME netcdf ) +set( NetCDF_Fortran_INCLUDE_NAME netcdf.mod ) + +## Library names for each component +set( NetCDF_C_LIBRARY_NAME netcdf ) +set( NetCDF_CXX_LIBRARY_NAME netcdf_c++4 ) +set( NetCDF_Fortran_LIBRARY_NAME netcdff ) + +## Enumerate search components +foreach( _comp ${_possible_components} ) + string( TOUPPER "${_comp}" _COMP ) + set( _arg_${_COMP} ${_comp} ) + set( _name_${_COMP} ${_comp} ) +endforeach() + +set( _search_components C) +foreach( _comp ${${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS} ) + string( TOUPPER "${_comp}" _COMP ) + set( _arg_${_COMP} ${_comp} ) + list( APPEND _search_components ${_name_${_COMP}} ) + if( NOT _name_${_COMP} ) + message(SEND_ERROR "Find${CMAKE_FIND_PACKAGE_NAME}: COMPONENT ${_comp} is not a valid component. Valid components: ${_possible_components}" ) + endif() +endforeach() +list( REMOVE_DUPLICATES _search_components ) + +## Search hints for finding include directories and libraries +foreach( _comp IN ITEMS "_" "_C_" "_Fortran_" "_CXX_" ) + foreach( _name IN ITEMS NetCDF4 NetCDF NETCDF4 NETCDF ) + foreach( _var IN ITEMS ROOT PATH ) + list(APPEND _search_hints ${${_name}${_comp}${_var}} $ENV{${_name}${_comp}${_var}} ) + list(APPEND _include_search_hints + ${${_name}${_comp}INCLUDE_DIR} $ENV{${_name}${_comp}INCLUDE_DIR} + ${${_name}${_comp}INCLUDE_DIRS} $ENV{${_name}${_comp}INCLUDE_DIRS} ) + endforeach() + endforeach() +endforeach() +#Old-school HPC module env variable names +foreach( _name IN ITEMS NetCDF4 NetCDF NETCDF4 NETCDF ) + foreach( _comp IN ITEMS "_C" "_Fortran" "_CXX" ) + list(APPEND _search_hints ${${_name}} $ENV{${_name}}) + list(APPEND _search_hints ${${_name}${_comp}} $ENV{${_name}${_comp}}) + endforeach() +endforeach() + +## Find headers for each component +set(NetCDF_INCLUDE_DIRS) +set(_new_search_components) +foreach( _comp IN LISTS _search_components ) + if(NOT ${PROJECT_NAME}_NetCDF_${_comp}_FOUND) + list(APPEND _new_search_components ${_comp}) + endif() + find_file(NetCDF_${_comp}_INCLUDE_FILE + NAMES ${NetCDF_${_comp}_INCLUDE_NAME} + DOC "NetCDF ${_comp} include directory" + HINTS ${_include_search_hints} ${_search_hints} + PATH_SUFFIXES include include/netcdf + ) + mark_as_advanced(NetCDF_${_comp}_INCLUDE_FILE) + message(DEBUG "NetCDF_${_comp}_INCLUDE_FILE: ${NetCDF_${_comp}_INCLUDE_FILE}") + if( NetCDF_${_comp}_INCLUDE_FILE ) + get_filename_component(NetCDF_${_comp}_INCLUDE_FILE ${NetCDF_${_comp}_INCLUDE_FILE} ABSOLUTE) + get_filename_component(NetCDF_${_comp}_INCLUDE_DIR ${NetCDF_${_comp}_INCLUDE_FILE} DIRECTORY) + list(APPEND NetCDF_INCLUDE_DIRS ${NetCDF_${_comp}_INCLUDE_DIR}) + endif() +endforeach() +if(NetCDF_INCLUDE_DIRS) + list(REMOVE_DUPLICATES NetCDF_INCLUDE_DIRS) +endif() +set(NetCDF_INCLUDE_DIRS "${NetCDF_INCLUDE_DIRS}" CACHE STRING "NetCDF Include directory paths" FORCE) + +## Find n*-config executables for search components +foreach( _comp IN LISTS _search_components ) + if( _comp MATCHES "^(C)$" ) + set(_conf "c") + elseif( _comp MATCHES "^(Fortran)$" ) + set(_conf "f") + elseif( _comp MATCHES "^(CXX)$" ) + set(_conf "cxx4") + endif() + find_program( NetCDF_${_comp}_CONFIG_EXECUTABLE + NAMES n${_conf}-config + HINTS ${NetCDF_INCLUDE_DIRS} ${_include_search_hints} ${_search_hints} + PATH_SUFFIXES bin Bin ../bin ../../bin + DOC "NetCDF n${_conf}-config helper" ) + message(DEBUG "NetCDF_${_comp}_CONFIG_EXECUTABLE: ${NetCDF_${_comp}_CONFIG_EXECUTABLE}") +endforeach() + +set(_C_libs_flag --libs) +set(_Fortran_libs_flag --flibs) +set(_CXX_libs_flag --libs) +set(_C_includes_flag --includedir) +set(_Fortran_includes_flag --includedir) +set(_CXX_includes_flag --includedir) +function(netcdf_config exec flag output_var) + set(${output_var} False PARENT_SCOPE) + if( exec ) + execute_process( COMMAND ${exec} ${flag} RESULT_VARIABLE _ret OUTPUT_VARIABLE _val) + if( _ret EQUAL 0 ) + string( STRIP ${_val} _val ) + set( ${output_var} ${_val} PARENT_SCOPE ) + endif() + endif() +endfunction() + +## Find libraries for each component +set( NetCDF_LIBRARIES ) +foreach( _comp IN LISTS _search_components ) + string( TOUPPER "${_comp}" _COMP ) + + find_library( NetCDF_${_comp}_LIBRARY + NAMES ${NetCDF_${_comp}_LIBRARY_NAME} + DOC "NetCDF ${_comp} library" + HINTS ${NetCDF_${_comp}_INCLUDE_DIRS} ${_search_hints} + PATH_SUFFIXES lib64 lib ../lib64 ../lib ../../lib64 ../../lib ) + mark_as_advanced( NetCDF_${_comp}_LIBRARY ) + get_filename_component(NetCDF_${_comp}_LIBRARY ${NetCDF_${_comp}_LIBRARY} ABSOLUTE) + set(NetCDF_${_comp}_LIBRARY ${NetCDF_${_comp}_LIBRARY} CACHE STRING "NetCDF ${_comp} library" FORCE) + message(DEBUG "NetCDF_${_comp}_LIBRARY: ${NetCDF_${_comp}_LIBRARY}") + + if( NetCDF_${_comp}_LIBRARY ) + if( NetCDF_${_comp}_LIBRARY MATCHES ".a$" ) + set( NetCDF_${_comp}_LIBRARY_SHARED FALSE ) + set( _library_type STATIC) + else() + list( APPEND NetCDF_LIBRARIES ${NetCDF_${_comp}_LIBRARY} ) + set( NetCDF_${_comp}_LIBRARY_SHARED TRUE ) + set( _library_type SHARED) + endif() + endif() + + #Use nc-config to set per-component LIBRARIES variable if possible + netcdf_config( ${NetCDF_${_comp}_CONFIG_EXECUTABLE} ${_${_comp}_libs_flag} _val ) + if( _val ) + set( NetCDF_${_comp}_LIBRARIES ${_val} ) + if(NOT NetCDF_${_comp}_LIBRARY_SHARED AND NOT NetCDF_${_comp}_FOUND) #Static targets should use nc_config to get a proper link line with all necessary static targets. + list( APPEND NetCDF_LIBRARIES ${NetCDF_${_comp}_LIBRARIES} ) + endif() + else() + set( NetCDF_${_comp}_LIBRARIES ${NetCDF_${_comp}_LIBRARY} ) + if(NOT NetCDF_${_comp}_LIBRARY_SHARED) + message(SEND_ERROR "Unable to properly find NetCDF. Found static libraries at: ${NetCDF_${_comp}_LIBRARY} but could not run nc-config: ${NetCDF_CONFIG_EXECUTABLE}") + endif() + endif() + + #Use nc-config to set per-component INCLUDE_DIRS variable if possible + netcdf_config( ${NetCDF_${_comp}_CONFIG_EXECUTABLE} ${_${_comp}_includes_flag} _val ) + if( _val ) + string( REPLACE " " ";" _val ${_val} ) + set( NetCDF_${_comp}_INCLUDE_DIRS ${_val} ) + else() + set( NetCDF_${_comp}_INCLUDE_DIRS ${NetCDF_${_comp}_INCLUDE_DIR} ) + endif() + + if( NetCDF_${_comp}_LIBRARIES AND NetCDF_${_comp}_INCLUDE_DIRS ) + set( ${CMAKE_FIND_PACKAGE_NAME}_${_arg_${_COMP}}_FOUND TRUE ) + if (NOT TARGET NetCDF::NetCDF_${_comp}) + add_library(NetCDF::NetCDF_${_comp} ${_library_type} IMPORTED) + set_target_properties(NetCDF::NetCDF_${_comp} PROPERTIES + IMPORTED_LOCATION ${NetCDF_${_comp}_LIBRARY} + INTERFACE_INCLUDE_DIRECTORIES "${NetCDF_${_comp}_INCLUDE_DIRS}" + INTERFACE_LINK_LIBRARIES ${NetCDF_${_comp}_LIBRARIES} ) + endif() + endif() +endforeach() +if(NetCDF_LIBRARIES AND NetCDF_${_comp}_LIBRARY_SHARED) + list(REMOVE_DUPLICATES NetCDF_LIBRARIES) +endif() +set(NetCDF_LIBRARIES "${NetCDF_LIBRARIES}" CACHE STRING "NetCDF library targets" FORCE) + +## Find version via netcdf-config if possible +if (NetCDF_INCLUDE_DIRS) + if( NetCDF_C_CONFIG_EXECUTABLE ) + netcdf_config( ${NetCDF_C_CONFIG_EXECUTABLE} --version _vers ) + if( _vers ) + string(REGEX REPLACE ".* ((([0-9]+)\\.)+([0-9]+)).*" "\\1" NetCDF_VERSION "${_vers}" ) + endif() + else() + foreach( _dir IN LISTS NetCDF_INCLUDE_DIRS) + if( EXISTS "${_dir}/netcdf_meta.h" ) + file(STRINGS "${_dir}/netcdf_meta.h" _netcdf_version_lines + REGEX "#define[ \t]+NC_VERSION_(MAJOR|MINOR|PATCH|NOTE)") + string(REGEX REPLACE ".*NC_VERSION_MAJOR *\([0-9]*\).*" "\\1" _netcdf_version_major "${_netcdf_version_lines}") + string(REGEX REPLACE ".*NC_VERSION_MINOR *\([0-9]*\).*" "\\1" _netcdf_version_minor "${_netcdf_version_lines}") + string(REGEX REPLACE ".*NC_VERSION_PATCH *\([0-9]*\).*" "\\1" _netcdf_version_patch "${_netcdf_version_lines}") + string(REGEX REPLACE ".*NC_VERSION_NOTE *\"\([^\"]*\)\".*" "\\1" _netcdf_version_note "${_netcdf_version_lines}") + set(NetCDF_VERSION "${_netcdf_version_major}.${_netcdf_version_minor}.${_netcdf_version_patch}${_netcdf_version_note}") + unset(_netcdf_version_major) + unset(_netcdf_version_minor) + unset(_netcdf_version_patch) + unset(_netcdf_version_note) + unset(_netcdf_version_lines) + endif() + endforeach() + endif() +endif () + +## Detect additional package properties +netcdf_config(${NetCDF_C_CONFIG_EXECUTABLE} --has-parallel4 _val) +if( NOT _val MATCHES "^(yes|no)$" ) + netcdf_config(${NetCDF_C_CONFIG_EXECUTABLE} --has-parallel _val) +endif() +if( _val MATCHES "^(yes)$" ) + set(NetCDF_PARALLEL TRUE CACHE STRING "NetCDF has parallel IO capability via pnetcdf or hdf5." FORCE) +else() + set(NetCDF_PARALLEL FALSE CACHE STRING "NetCDF has no parallel IO capability." FORCE) +endif() + +## Finalize find_package +include(FindPackageHandleStandardArgs) + +if(NOT NetCDF_FOUND OR _new_search_components) + find_package_handle_standard_args( ${CMAKE_FIND_PACKAGE_NAME} + REQUIRED_VARS NetCDF_INCLUDE_DIRS NetCDF_LIBRARIES + VERSION_VAR NetCDF_VERSION + HANDLE_COMPONENTS ) +endif() + +foreach( _comp IN LISTS _search_components ) + if( NetCDF_${_comp}_FOUND ) + #Record found components to avoid duplication in NetCDF_LIBRARIES for static libraries + set(NetCDF_${_comp}_FOUND ${NetCDF_${_comp}_FOUND} CACHE BOOL "NetCDF ${_comp} Found" FORCE) + #Set a per-package, per-component found variable to communicate between multiple calls to find_package() + set(${PROJECT_NAME}_NetCDF_${_comp}_FOUND True) + endif() +endforeach() + +if( ${CMAKE_FIND_PACKAGE_NAME}_FOUND AND NOT ${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY AND _new_search_components) + message( STATUS "Find${CMAKE_FIND_PACKAGE_NAME} defines targets:" ) + message( STATUS " - NetCDF_VERSION [${NetCDF_VERSION}]") + message( STATUS " - NetCDF_PARALLEL [${NetCDF_PARALLEL}]") + foreach( _comp IN LISTS _new_search_components ) + string( TOUPPER "${_comp}" _COMP ) + message( STATUS " - NetCDF_${_comp}_CONFIG_EXECUTABLE [${NetCDF_${_comp}_CONFIG_EXECUTABLE}]") + if( ${CMAKE_FIND_PACKAGE_NAME}_${_arg_${_COMP}}_FOUND ) + get_filename_component(_root ${NetCDF_${_comp}_INCLUDE_DIR}/.. ABSOLUTE) + if( NetCDF_${_comp}_LIBRARY_SHARED ) + message( STATUS " - NetCDF::NetCDF_${_comp} [SHARED] [Root: ${_root}] Lib: ${NetCDF_${_comp}_LIBRARY} ") + else() + message( STATUS " - NetCDF::NetCDF_${_comp} [STATIC] [Root: ${_root}] Lib: ${NetCDF_${_comp}_LIBRARY} ") + endif() + endif() + endforeach() +endif() + +foreach( _prefix NetCDF NetCDF4 NETCDF NETCDF4 ${CMAKE_FIND_PACKAGE_NAME} ) + set( ${_prefix}_INCLUDE_DIRS ${NetCDF_INCLUDE_DIRS} ) + set( ${_prefix}_LIBRARIES ${NetCDF_LIBRARIES}) + set( ${_prefix}_VERSION ${NetCDF_VERSION} ) + set( ${_prefix}_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_FOUND} ) + set( ${_prefix}_CONFIG_EXECUTABLE ${NetCDF_CONFIG_EXECUTABLE} ) + set( ${_prefix}_PARALLEL ${NetCDF_PARALLEL} ) + + foreach( _comp ${_search_components} ) + string( TOUPPER "${_comp}" _COMP ) + set( _arg_comp ${_arg_${_COMP}} ) + set( ${_prefix}_${_comp}_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_${_arg_comp}_FOUND} ) + set( ${_prefix}_${_COMP}_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_${_arg_comp}_FOUND} ) + set( ${_prefix}_${_arg_comp}_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_${_arg_comp}_FOUND} ) + + set( ${_prefix}_${_comp}_LIBRARIES ${NetCDF_${_comp}_LIBRARIES} ) + set( ${_prefix}_${_COMP}_LIBRARIES ${NetCDF_${_comp}_LIBRARIES} ) + set( ${_prefix}_${_arg_comp}_LIBRARIES ${NetCDF_${_comp}_LIBRARIES} ) + + set( ${_prefix}_${_comp}_INCLUDE_DIRS ${NetCDF_${_comp}_INCLUDE_DIRS} ) + set( ${_prefix}_${_COMP}_INCLUDE_DIRS ${NetCDF_${_comp}_INCLUDE_DIRS} ) + set( ${_prefix}_${_arg_comp}_INCLUDE_DIRS ${NetCDF_${_comp}_INCLUDE_DIRS} ) + endforeach() +endforeach() diff --git a/compile_c_libs.sh b/compile_c_libs.sh deleted file mode 100755 index 409c4ad..0000000 --- a/compile_c_libs.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -set -ex -#set -o posix - -#yaml includes -YAML_FLAGS+="" - -#yaml libraries -YAML_LDFLAGS+="" - -#fortran netcdf includes -NF_FLAGS+=$(nf-config --fflags) - -#c netcdf includes -NC_FLAGS+=$(nc-config --cflags) - -#netcdf libraries -NC_LDFLAGS=$(nc-config --libs | nf-config --flibs) - -#fortran and c compiler -FC=mpif90 -CC=mpicc - -#fms fortran, c, library compiler flags -FMS_FCFLAGS+="$NF_FLAGS -fPIC" -FMS_CFLAGS+="$NC_FLAGS $YAML_FLAGS -fPIC" -FMS_LDFLAGS+="$NC_LDFLAGS $YAML_LDFLAGS" - -#cfms fortran, c, library compiler flags -#cfms does not need the netcdf flags. -#these will be removed once cfms configure.ac is updated -cFMS_FCFLAGS+="-fPIC $NF_FLAGS" -cFMS_CFLAGS+="-fPIC $NC_FLAGS" -cFMS_LDFLAGS+="" - -curr_dir=$PWD - -#fms installation path -fms_install=$curr_dir/pyfms/lib/FMS - -#cfms installation path -cfms_install=$curr_dir/pyfms/lib/cFMS - -#cfms to compile -cfms_dir=$curr_dir/cFMS - -#fms to compile -fms_dir=$cfms_dir/FMS - -#compile fms with autotools -cd $fms_dir -autoreconf -iv -./configure --enable-portable-kinds \ - --with-yaml \ - --prefix=$fms_install \ - FC=$FC \ - CC=$CC \ - FCFLAGS="$FMS_FCFLAGS" \ - CFLAGS="$FMS_CFLAGS" \ - LDFLAGS="$FMS_LDFLAGS" -make install - -cd $currdir - -#compile cFMS with autotools -cd $cfms_dir -autoreconf -iv -./configure --with-fms=$fms_install \ - --prefix=$cfms_install \ - FC=$FC \ - CC=$CC \ - FCFLAGS="$cFMS_FCFLAGS" \ - CFLAGS="$cFMS_CFLAGS" \ - LDFLAGS="$cFMS_LDFLAGS" -make install - -cd $currdir diff --git a/pyfms/.pre-commit-config.yaml b/pyfms/.pre-commit-config.yaml deleted file mode 100644 index 6f269ff..0000000 --- a/pyfms/.pre-commit-config.yaml +++ /dev/null @@ -1,48 +0,0 @@ -default_language_version: - python: python3 - -repos: -- repo: https://github.com/psf/black - rev: 20.8b1 - hooks: - - id: black - additional_dependencies: ["click==8.0.4"] - -- repo: https://github.com/pre-commit/mirrors-isort - rev: v5.4.2 - hooks: - - id: isort - args: ["--profile", "black"] - -- repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.4.1 - hooks: - - id: mypy - additional_dependencies: [types-pyyaml] - name: mypy-pace - files: pace - args: [--config-file, setup.cfg, --disallow-untyped-calls] - -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 - hooks: - - id: check-toml - - id: check-yaml - - id: end-of-file-fixer - - id: trailing-whitespace -- repo: https://github.com/pycqa/flake8 - rev: 3.9.2 - hooks: - - id: flake8 - name: flake8 - language_version: python3 - args: [--config, setup.cfg] - exclude: | - (?x)^( - .*/__init__.py | - )$ - - id: flake8 - name: flake8 __init__.py files - files: "__init__.py" - # ignore unused import error in __init__.py files - args: ["--ignore=F401,E203", --config, setup.cfg] diff --git a/pyfms/cfms.py b/pyfms/cfms.py index 72a570e..30dfec1 100644 --- a/pyfms/cfms.py +++ b/pyfms/cfms.py @@ -1,5 +1,5 @@ import ctypes -import os +import sysconfig import pyfms @@ -26,14 +26,11 @@ def init(libpath: str = None): # todo reset all _function parameters to None if libpath is None: - _libpath = os.path.dirname(__file__) + "/lib/cFMS/lib/libcFMS.so" + _libpath = sysconfig.get_paths()["purelib"] + "/libcFMS.so" try: _lib = ctypes.cdll.LoadLibrary(_libpath) - except OSError: - print( - f"{_libpath} does not exist. Please provide a path to cFMS with\ - pyfms.cfms.init(libpath=path_to_cfms)" - ) + except Exception as e: + print(f"{type(e).__name__}: {e}") return else: _libpath = libpath diff --git a/pyfms/py_field_manager/py_field_manager.py b/pyfms/py_field_manager/py_field_manager.py index 9f1ce94..27a5e70 100644 --- a/pyfms/py_field_manager/py_field_manager.py +++ b/pyfms/py_field_manager/py_field_manager.py @@ -139,7 +139,7 @@ def get_subparam_list(self, module: str, varname: str) -> List[Dict]: var = self.get_var(module=module, varname=varname) for key in var: - if type(var[key]) == list: + if isinstance(var[key], list): subparamlist.append(key) return subparamlist @@ -155,7 +155,7 @@ def get_num_subparam(self, module: str, varname: str) -> int: var = self.get_var(module=module, varname=varname) for key in var: - if type(var[key]) == list: + if isinstance(var[key], list): subparamlist.append(key) return len(subparamlist) diff --git a/pyfms/py_mpp/_mpp_functions.py b/pyfms/py_mpp/_mpp_functions.py index 6bf6f19..ef3fbb5 100644 --- a/pyfms/py_mpp/_mpp_functions.py +++ b/pyfms/py_mpp/_mpp_functions.py @@ -24,6 +24,7 @@ def define(lib): POINTER(c_int), # npes ndpointer(dtype=np.int32, ndim=(1), flags=C), # pelist c_char_p, # name + POINTER(c_int), # commID ] # cFMS_error diff --git a/pyfms/py_mpp/mpp.py b/pyfms/py_mpp/mpp.py index e7a9e10..ee8384d 100644 --- a/pyfms/py_mpp/mpp.py +++ b/pyfms/py_mpp/mpp.py @@ -90,7 +90,9 @@ def gather( else: rbuf_shape, rbuf = None, None - pelist = get_current_pelist(npes()) if pelist is None else pelist + pelist = ( + get_current_pelist(npes(), get_commID=True) if pelist is None else pelist + ) set_c_int(domain.isc, arglist) set_c_int(domain.iec, arglist) diff --git a/pyproject.toml b/pyproject.toml index ad0679d..637e8a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,85 @@ [build-system] -requires = ["setuptools >= 64"] -build-backend = "setuptools.build_meta" +requires = ["scikit-build-core >= 0.3.3"] +build-backend = "scikit_build_core.build" + +[project] +name = "pyfms" +version = "2026.03.00" +description = "A Python interface to the FMS Fortran software package." +readme = "README.md" +requires-python = ">=3.12,<3.13" +# when updating the tested Python versions, please also update README.md, +dependencies = [ + "dacite", + "h5netcdf", + "numpy", + "pyyaml", + "mpi4py", + "xarray", + "netcdf4", +] +classifiers = [ + "Development Status :: 1 - Pre-Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Programming Language :: Python :: 3", +] + +[project.optional-dependencies] +test = [ + "pytest", + "pytest-subtests", + "coverage", +] +dev = [ + "pyfms[test]", + "pre-commit", + "flake8-pyproject", +] +extras = [ + "pyfms[test]", + "pyfms[dev]", +] + +[tool.aliases] + +[tool.scikit-build] +wheel.expand-macos-universal-tags = true + +[tool.scikit-build.cmake.define] +CMAKE_GENERATOR = "Unix Makefiles" + +[tool.isort] +line_length = "88" +force_grid_wrap = "0" +include_trailing_comma = "true" +multi_line_output = "3" +use_parentheses = "true" +lines_after_imports = "2" +default_section = "THIRDPARTY" +sections = "FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER" +known_first_party = "pyfms" +known_third_party = "pytest,dacite,xarray,numpy" + +[tool.mypy] +ignore_missing_imports = true +follow_imports = "normal" +namespace_packages = true +strict_optional = false +warn_unreachable = true +explicit_package_bases = true + +[tool.coverage.run] +parallel = true +branch = true +omit = [ + "tests/*", + "__init__.py", +] +source_pkgs = ["pyfms"] + +[tool.flake8] +exclude = ["docs"] +extend-ignore = ["E203","E501","W293","W503","E302","E203","F841", "F401", "F824", "B007"] +max-line-length = "88" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index aefb6d4..0000000 --- a/setup.cfg +++ /dev/null @@ -1,34 +0,0 @@ -[flake8] -exclude = docs -ignore = E203,E501,W293,W503,E302,E203,F841 -max-line-length = 88 - -[aliases] - -[tool:isort] -line_length = 88 -force_grid_wrap = 0 -include_trailing_comma = true -multi_line_output = 3 -use_parentheses = true -lines_after_imports = 2 -default_section = THIRDPARTY -sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER -known_first_party = pyfms -known_third_party = pytest,dacite,xarray,numpy - -[mypy] -ignore_missing_imports = True -follow_imports = normal -namespace_packages = True -strict_optional = False -warn_unreachable = True -explicit_package_bases = True - -[coverage:run] -parallel = true -branch = true -omit = - tests/* - __init__.py -source_pkgs = pyfms diff --git a/setup.py b/setup.py deleted file mode 100644 index 749db05..0000000 --- a/setup.py +++ /dev/null @@ -1,52 +0,0 @@ -import subprocess - -from setuptools import find_namespace_packages, setup -from setuptools.command.build import build - - -class CustomBuild(build): - def run(self): - with open("install.log", "w") as f: - subprocess.run(["./compile_c_libs.sh"], stdout=f, check=True) - build.run(self) - - -test_requirements = ["pytest", "pytest-subtests", "coverage", "dask"] -develop_requirements = test_requirements + ["pre-commit"] - -extras_requires = { - "test": test_requirements, - "develop": develop_requirements, -} - -requirements = [ - "dacite", - "h5netcdf", - "numpy", - "pyyaml", - "mpi4py", - "xarray", - "netcdf4", -] - -setup( - author="NOAA/GFDL", - python_requires=">=3.11", - classifiers=[ - "Development Status :: 1 - Pre-Alpha", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Natural Language :: English", - "Programming Language :: Python :: 3", - ], - install_requires=requirements, - extras_require=extras_requires, - name="pyfms", - license="", - packages=find_namespace_packages(include=["pyfms", "pyfms.*"]), - cmdclass={"build": CustomBuild}, - include_package_data=True, - url="https://github.com/fmalatino/pyFMS.git", - version="2024.02.0", - zip_safe=False, -) diff --git a/tests/py_diag_manager/test_diag_manager.py b/tests/py_diag_manager/test_diag_manager.py index 082d652..ffaa05c 100644 --- a/tests/py_diag_manager/test_diag_manager.py +++ b/tests/py_diag_manager/test_diag_manager.py @@ -177,7 +177,7 @@ def test_send_data(): """ if pyfms.mpp.pe() == pyfms.mpp.root_pe(): assert os.path.isfile("test_send_data.nc") - ds = xr.open_mfdataset("test_send_data.nc", decode_times=True) + ds = xr.open_dataset("test_send_data.nc", decode_times=True) assert "var2_avg" in ds assert "var3_avg" in ds assert ds["var2_avg"].dims == ("time", "y", "x") diff --git a/tests/py_horiz_interp/test_horiz_interp.py b/tests/py_horiz_interp/test_horiz_interp.py index 56f2f06..7143249 100755 --- a/tests/py_horiz_interp/test_horiz_interp.py +++ b/tests/py_horiz_interp/test_horiz_interp.py @@ -81,7 +81,7 @@ def test_horiz_interp_conservative(): domain = pyfms.mpp_domains.define_domains( global_indices=global_indices, layout=layout, - pelist=pyfms.mpp.get_current_pelist(npes=pes), + pelist=pyfms.mpp.get_current_pelist(npes=pes, get_commID=True)[0], name="horiz_interp_conservative_test", xflags=pyfms.mpp_domains.CYCLIC_GLOBAL_DOMAIN, yflags=pyfms.mpp_domains.CYCLIC_GLOBAL_DOMAIN, @@ -190,7 +190,7 @@ def test_horiz_interp_bilinear(): global_indices=[0, ni_src - 1, 0, nj_src - 1], ndivs=pes, ), - pelist=pyfms.mpp.get_current_pelist(npes=pes), + pelist=pyfms.mpp.get_current_pelist(npes=pes, get_commID=True)[0], name="horiz_interp_bilinear_test", whalo=halo, ehalo=halo, diff --git a/tests/py_mpp/test_define_domains.py b/tests/py_mpp/test_define_domains.py index b3fc995..41a793d 100644 --- a/tests/py_mpp/test_define_domains.py +++ b/tests/py_mpp/test_define_domains.py @@ -33,7 +33,9 @@ def test_define_domains(): """get global pelist""" - global_pelist = pyfms.mpp.get_current_pelist(npes=pyfms.mpp.npes()) + global_pelist = pyfms.mpp.get_current_pelist( + npes=pyfms.mpp.npes(), get_commID=True + )[0] """set coarse domain as tile=0""" diff --git a/tests/py_mpp/test_gather.py b/tests/py_mpp/test_gather.py index b3ff8cb..37c1ebc 100644 --- a/tests/py_mpp/test_gather.py +++ b/tests/py_mpp/test_gather.py @@ -35,7 +35,7 @@ def test_gather_2d(): else: rbuf_shape = [ny, nx] - pelist = pyfms.mpp.get_current_pelist(pyfms.mpp.npes()) + pelist = pyfms.mpp.get_current_pelist(npes=pyfms.mpp.npes(), get_commID=True)[0] gathered = pyfms.mpp.gather( send, rbuf_shape=rbuf_shape,