From 9cdeffcbe24f67a12644c48c2d9e8555d79fd908 Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 5 May 2025 12:07:41 -0400 Subject: [PATCH 01/29] Pre-lim install changes --- compile_c_libs.sh | 9 +++++---- pyfms/cfms.py | 3 ++- setup.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/compile_c_libs.sh b/compile_c_libs.sh index 15eda09..999d294 100755 --- a/compile_c_libs.sh +++ b/compile_c_libs.sh @@ -1,12 +1,13 @@ #!/bin/bash -curr_dir=$PWD/cFMS -install_fms=$curr_dir/FMS/LIBFMS +top_dir=$PWD +cfms_dir=$PWD/cFMS +install_fms=$cfms_dir/FMS/LIBFMS export FC=mpif90 export CC=mpicc -cd $curr_dir/FMS +cd $cfms_dir/FMS autoreconf -iv export FCFLAGS="$FCFLAGS -fPIC" export CFLAGS="$CFLAGS -fPIC" @@ -21,5 +22,5 @@ export CFLAGS="$CFLAGS -I$install_fms/include" export LDFLAGS="$LDFLAGS -lFMS -L$install_fms/lib" autoreconf -iv -./configure --prefix=$curr_dir/cLIBFMS +./configure --prefix=$top_dir/pyfms/cLIBFMS make install diff --git a/pyfms/cfms.py b/pyfms/cfms.py index cfc07d9..ffd6570 100644 --- a/pyfms/cfms.py +++ b/pyfms/cfms.py @@ -6,7 +6,8 @@ class cfms: - __libpath: str = os.path.dirname(__file__) + "/../cFMS/cLIBFMS/lib/libcFMS.so" + # __libpath: str = os.path.dirname(__file__) + "/../cFMS/cLIBFMS/lib/libcFMS.so" + __libpath: str = os.path.dirname(__file__) + "/cLIBFMS/lib/libcFMS.so" __lib: type[ctypes.CDLL] = ctypes.cdll.LoadLibrary(__libpath) lib = None diff --git a/setup.py b/setup.py index 40cedb8..2ee1ffe 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ import subprocess from typing import List -from setuptools import find_namespace_packages, setup +from setuptools import find_namespace_packages, setup, find_packages from setuptools.command.install import install From a40dc3026afe71dc8a13cf736be70d1e1b80dace Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Tue, 30 Sep 2025 12:04:32 -0400 Subject: [PATCH 02/29] Testing conda install of cFMS locally --- .gitmodules | 4 ---- cFMS | 1 - pyfms/cfms.py | 22 +++++++++++++--------- setup.py | 11 +++++------ 4 files changed, 18 insertions(+), 20 deletions(-) delete mode 160000 cFMS diff --git a/.gitmodules b/.gitmodules index 31f95b1..8b13789 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,5 +1 @@ -[submodule "cFMS"] - path = cFMS - url = https://github.com/NOAA-GFDL/cFMS.git - branch = main diff --git a/cFMS b/cFMS deleted file mode 160000 index b8d268f..0000000 --- a/cFMS +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b8d268fb5223d043cea6ebd9dee7e6e14466f4e7 diff --git a/pyfms/cfms.py b/pyfms/cfms.py index ebef266..a626fd8 100644 --- a/pyfms/cfms.py +++ b/pyfms/cfms.py @@ -22,15 +22,19 @@ def init(libpath: str = None): global _libpath, _lib if libpath is None: - _libpath = os.path.dirname(__file__) + "/lib/cFMS/lib/libcFMS.so" - try: - _lib = ctypes.cdll.LoadLibrary(_libpath) - except OSError: - print( - f"{_libpath} does not exist. Please provide a path to cFMS with\ - pyfms.cfms.init(libpath=path_to_cfms)" - ) - return + # _libpath = os.path.dirname(__file__) + "/lib/cFMS/lib/libcFMS.so" + # _libpath = "/home/Frank.Malatino/pyfms_fork/pyfms/lib/cFMS/lib/libcFMS.so" + # _libpath = os.getenv('CONDA_PREFIX') + "/lib/libcFMS.so" + _libpath = "libcFMS.so" + _lib = ctypes.cdll.LoadLibrary(_libpath) + # try: + # _lib = ctypes.cdll.LoadLibrary(_libpath) + # except OSError: + # print( + # f"{_libpath} does not exist. Please provide a path to cFMS with\ + # pyfms.cfms.init(libpath=path_to_cfms)" + # ) + # return else: _libpath = libpath _lib = ctypes.cdll.LoadLibrary(_libpath) diff --git a/setup.py b/setup.py index e8d16e2..9c3d0a1 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,13 @@ import subprocess from setuptools import find_namespace_packages, setup -from setuptools.command.build import build +from setuptools.command.install import install -class CustomBuild(build): +class CustomInstall(install): def run(self): - with open("install.log", "w") as f: - subprocess.run(["./compile_c_libs.sh"], stdout=f, check=True) - build.run(self) + subprocess.run(["conda", "install", "-c", "/home/Frank.Malatino/.conda/envs/throw4/conda-bld"]) + install.run(self) test_requirements = ["pytest", "pytest-subtests", "coverage"] @@ -44,7 +43,7 @@ def run(self): name="pyfms", license="", packages=find_namespace_packages(include=["pyfms", "pyfms.*"]), - cmdclass={"build": CustomBuild}, + cmdclass={"install": CustomInstall}, include_package_data=True, url="https://github.com/fmalatino/pyFMS.git", version="2024.02.0", From c28e60689d832d58677b394538a3eeefb398444b Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Wed, 15 Oct 2025 14:48:45 -0400 Subject: [PATCH 03/29] Switching to main --- pyfms/cfms.py | 18 +++++------------- setup.py | 8 +++++++- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/pyfms/cfms.py b/pyfms/cfms.py index a626fd8..d4cb0ed 100644 --- a/pyfms/cfms.py +++ b/pyfms/cfms.py @@ -1,5 +1,4 @@ import ctypes -import os import pyfms @@ -22,19 +21,12 @@ def init(libpath: str = None): global _libpath, _lib if libpath is None: - # _libpath = os.path.dirname(__file__) + "/lib/cFMS/lib/libcFMS.so" - # _libpath = "/home/Frank.Malatino/pyfms_fork/pyfms/lib/cFMS/lib/libcFMS.so" - # _libpath = os.getenv('CONDA_PREFIX') + "/lib/libcFMS.so" _libpath = "libcFMS.so" - _lib = ctypes.cdll.LoadLibrary(_libpath) - # try: - # _lib = ctypes.cdll.LoadLibrary(_libpath) - # except OSError: - # print( - # f"{_libpath} does not exist. Please provide a path to cFMS with\ - # pyfms.cfms.init(libpath=path_to_cfms)" - # ) - # return + try: + _lib = ctypes.cdll.LoadLibrary(_libpath) + except Exception as e: + print(f"{type(e).__name__}: {e}") + return else: _libpath = libpath _lib = ctypes.cdll.LoadLibrary(_libpath) diff --git a/setup.py b/setup.py index 9c3d0a1..5857961 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,13 @@ class CustomInstall(install): def run(self): - subprocess.run(["conda", "install", "-c", "/home/Frank.Malatino/.conda/envs/throw4/conda-bld"]) + print("Installing cFMS") + try: + subprocess.run(["conda", "install", "-c", "/home/Frank.Malatino/.conda/envs/throw4/conda-bld"], check=True, capture_output=True, text=True) + except subprocess.CalledProcessError as e: + print(f"Error during conda install of cFMS: {e}") + print("STDOUT:", e.stdout) + print("STDERR:", e.stderr) install.run(self) From fdd3d252b7e7acab5fae32464498ad3c8a7f45ef Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 27 Oct 2025 16:15:12 -0400 Subject: [PATCH 04/29] Branch for installing cFMS from conda package --- compile_c_libs.sh | 78 ----------------------------------------------- setup.py | 2 +- 2 files changed, 1 insertion(+), 79 deletions(-) delete mode 100755 compile_c_libs.sh diff --git a/compile_c_libs.sh b/compile_c_libs.sh deleted file mode 100755 index 409c4ad..0000000 --- a/compile_c_libs.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -set -ex -#set -o posix - -#yaml includes -YAML_FLAGS+="" - -#yaml libraries -YAML_LDFLAGS+="" - -#fortran netcdf includes -NF_FLAGS+=$(nf-config --fflags) - -#c netcdf includes -NC_FLAGS+=$(nc-config --cflags) - -#netcdf libraries -NC_LDFLAGS=$(nc-config --libs | nf-config --flibs) - -#fortran and c compiler -FC=mpif90 -CC=mpicc - -#fms fortran, c, library compiler flags -FMS_FCFLAGS+="$NF_FLAGS -fPIC" -FMS_CFLAGS+="$NC_FLAGS $YAML_FLAGS -fPIC" -FMS_LDFLAGS+="$NC_LDFLAGS $YAML_LDFLAGS" - -#cfms fortran, c, library compiler flags -#cfms does not need the netcdf flags. -#these will be removed once cfms configure.ac is updated -cFMS_FCFLAGS+="-fPIC $NF_FLAGS" -cFMS_CFLAGS+="-fPIC $NC_FLAGS" -cFMS_LDFLAGS+="" - -curr_dir=$PWD - -#fms installation path -fms_install=$curr_dir/pyfms/lib/FMS - -#cfms installation path -cfms_install=$curr_dir/pyfms/lib/cFMS - -#cfms to compile -cfms_dir=$curr_dir/cFMS - -#fms to compile -fms_dir=$cfms_dir/FMS - -#compile fms with autotools -cd $fms_dir -autoreconf -iv -./configure --enable-portable-kinds \ - --with-yaml \ - --prefix=$fms_install \ - FC=$FC \ - CC=$CC \ - FCFLAGS="$FMS_FCFLAGS" \ - CFLAGS="$FMS_CFLAGS" \ - LDFLAGS="$FMS_LDFLAGS" -make install - -cd $currdir - -#compile cFMS with autotools -cd $cfms_dir -autoreconf -iv -./configure --with-fms=$fms_install \ - --prefix=$cfms_install \ - FC=$FC \ - CC=$CC \ - FCFLAGS="$cFMS_FCFLAGS" \ - CFLAGS="$cFMS_CFLAGS" \ - LDFLAGS="$cFMS_LDFLAGS" -make install - -cd $currdir diff --git a/setup.py b/setup.py index 5857961..bdf37a9 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ class CustomInstall(install): def run(self): print("Installing cFMS") try: - subprocess.run(["conda", "install", "-c", "/home/Frank.Malatino/.conda/envs/throw4/conda-bld"], check=True, capture_output=True, text=True) + subprocess.run(["conda", "install", "-y", "-c", "file:///home/Frank.Malatino/cfms_conda_channel", "cfms"], check=True, capture_output=True, text=True) except subprocess.CalledProcessError as e: print(f"Error during conda install of cFMS: {e}") print("STDOUT:", e.stdout) From 4da5be8e6602cd68cca49bd77d238080dc40913f Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Tue, 28 Oct 2025 15:50:31 -0400 Subject: [PATCH 05/29] Adding override for editable_wheel for setuptools for editable installs --- setup.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index bdf37a9..74b5f25 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,11 @@ import subprocess from setuptools import find_namespace_packages, setup -from setuptools.command.install import install +from setuptools.command.editable_wheel import editable_wheel as _editable_wheel +from setuptools.command.install import install as _install -class CustomInstall(install): +class editable_wheel(_editable_wheel): def run(self): print("Installing cFMS") try: @@ -13,7 +14,18 @@ def run(self): print(f"Error during conda install of cFMS: {e}") print("STDOUT:", e.stdout) print("STDERR:", e.stderr) - install.run(self) + _editable_wheel.run(self) + +class install(_install): + def run(self): + print("Installing cFMS") + try: + subprocess.run(["conda", "install", "-y", "-c", "file:///home/Frank.Malatino/cfms_conda_channel", "cfms"], check=True, capture_output=True, text=True) + except subprocess.CalledProcessError as e: + print(f"Error during conda install of cFMS: {e}") + print("STDOUT:", e.stdout) + print("STDERR:", e.stderr) + _install.run(self) test_requirements = ["pytest", "pytest-subtests", "coverage"] @@ -49,7 +61,7 @@ def run(self): name="pyfms", license="", packages=find_namespace_packages(include=["pyfms", "pyfms.*"]), - cmdclass={"install": CustomInstall}, + cmdclass={"editable_wheel": editable_wheel, "install": install}, include_package_data=True, url="https://github.com/fmalatino/pyFMS.git", version="2024.02.0", From 691d3be971e87b20031e01eee28d4b2a2e311f59 Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Fri, 13 Mar 2026 15:02:56 -0400 Subject: [PATCH 06/29] Pre-lim work to use cmake install of cfms --- CMakeLists.txt | 26 ++++ cmake/FindNetCDF.cmake | 337 +++++++++++++++++++++++++++++++++++++++++ pyfms/cfms.py | 4 +- pyproject.toml | 90 ++++++++++- setup.cfg | 34 ----- setup.py | 69 --------- 6 files changed, 453 insertions(+), 107 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 cmake/FindNetCDF.cmake delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..47b93f4 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 3.18) + +project(pyfms LANGUAGES C Fortran) + +include(FetchContent) + +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +find_library(cFMS_LIBRARY NAMES "libcFMS.so" PATHS ${SKBUILD_PLATLIB_DIR}) + +if(NOT cFMS_LIBRARY) + message(STATUS "Did not find cFMS, using FetchContent to build") + FetchContent_Declare( + cFMS_LIBRARY + GIT_REPOSITORY https://github.com/fmalatino/cFMS.git + GIT_TAG cmake + ) + FetchContent_MakeAvailable(cFMS_LIBRARY) +else() + message(STATUS "Found cFMS library, including it in install") + add_library(cFMS_LIBRARY SHARED IMPORTED) + message(STATUS "Added cFMS_LIBRARY as a target") + set_property(TARGET cFMS_LIBRARY PROPERTY IMPORTED_LOCATION ${SKBUILD_PLATLIB_DIR}) +endif() + +install(TARGETS cFMS_LIBRARY LIBRARY DESTINATION ${SKBUILD_PLATLIB_DIR}) \ No newline at end of file diff --git a/cmake/FindNetCDF.cmake b/cmake/FindNetCDF.cmake new file mode 100644 index 0000000..82fe366 --- /dev/null +++ b/cmake/FindNetCDF.cmake @@ -0,0 +1,337 @@ +# (C) Copyright 2011- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +# Try to find NetCDF includes and library. +# Supports static and shared libaries and allows each component to be found in sepearte prefixes. +# +# This module defines +# +# - NetCDF_FOUND - System has NetCDF +# - NetCDF_INCLUDE_DIRS - the NetCDF include directories +# - NetCDF_VERSION - the version of NetCDF +# - NetCDF_CONFIG_EXECUTABLE - the netcdf-config executable if found +# - NetCDF_PARALLEL - Boolean True if NetCDF4 has parallel IO support via hdf5 and/or pnetcdf +# - NetCDF_HAS_PNETCDF - Boolean True if NetCDF4 has pnetcdf support +# +# Deprecated Defines +# - NetCDF_LIBRARIES - [Deprecated] Use NetCDF::NetCDF_ targets instead. +# +# +# Following components are available: +# +# - C - C interface to NetCDF (netcdf) +# - CXX - CXX4 interface to NetCDF (netcdf_c++4) +# - Fortran - Fortran interface to NetCDF (netcdff) +# +# For each component the following are defined: +# +# - NetCDF__FOUND - whether the component is found +# - NetCDF__LIBRARIES - the libraries for the component +# - NetCDF__LIBRARY_SHARED - Boolean is true if libraries for component are shared +# - NetCDF__INCLUDE_DIRS - the include directories for specified component +# - NetCDF::NetCDF_ - target of component to be used with target_link_libraries() +# +# The following paths will be searched in order if set in CMake (first priority) or environment (second priority) +# +# - NetCDF_ROOT - root of NetCDF installation +# - NetCDF_PATH - root of NetCDF installation +# +# The search process begins with locating NetCDF Include headers. If these are in a non-standard location, +# set one of the following CMake or environment variables to point to the location: +# +# - NetCDF_INCLUDE_DIR or NetCDF_${comp}_INCLUDE_DIR +# - NetCDF_INCLUDE_DIRS or NetCDF_${comp}_INCLUDE_DIR +# +# Notes: +# +# - Use "NetCDF::NetCDF_" targets only. NetCDF_LIBRARIES exists for backwards compatibility and should not be used. +# - These targets have all the knowledge of include directories and library search directories, and a single +# call to target_link_libraries will provide all these transitive properties to your target. Normally all that is +# needed to build and link against NetCDF is, e.g.: +# target_link_libraries(my_c_tgt PUBLIC NetCDF::NetCDF_C) +# - "NetCDF" is always the preferred naming for this package, its targets, variables, and environment variables +# - For compatibility, some variables are also set/checked using alternate names NetCDF4, NETCDF, or NETCDF4 +# - Environments relying on these older environment variable names should move to using a "NetCDF_ROOT" environment variable +# - Preferred component capitalization follows the CMake LANGUAGES variables: i.e., C, Fortran, CXX +# - For compatibility, alternate capitalizations are supported but should not be used. +# - If no components are defined, all components will be searched +# + +list( APPEND _possible_components C CXX Fortran ) + +## Include names for each component +set( NetCDF_C_INCLUDE_NAME netcdf.h ) +set( NetCDF_CXX_INCLUDE_NAME netcdf ) +set( NetCDF_Fortran_INCLUDE_NAME netcdf.mod ) + +## Library names for each component +set( NetCDF_C_LIBRARY_NAME netcdf ) +set( NetCDF_CXX_LIBRARY_NAME netcdf_c++4 ) +set( NetCDF_Fortran_LIBRARY_NAME netcdff ) + +## Enumerate search components +foreach( _comp ${_possible_components} ) + string( TOUPPER "${_comp}" _COMP ) + set( _arg_${_COMP} ${_comp} ) + set( _name_${_COMP} ${_comp} ) +endforeach() + +set( _search_components C) +foreach( _comp ${${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS} ) + string( TOUPPER "${_comp}" _COMP ) + set( _arg_${_COMP} ${_comp} ) + list( APPEND _search_components ${_name_${_COMP}} ) + if( NOT _name_${_COMP} ) + message(SEND_ERROR "Find${CMAKE_FIND_PACKAGE_NAME}: COMPONENT ${_comp} is not a valid component. Valid components: ${_possible_components}" ) + endif() +endforeach() +list( REMOVE_DUPLICATES _search_components ) + +## Search hints for finding include directories and libraries +foreach( _comp IN ITEMS "_" "_C_" "_Fortran_" "_CXX_" ) + foreach( _name IN ITEMS NetCDF4 NetCDF NETCDF4 NETCDF ) + foreach( _var IN ITEMS ROOT PATH ) + list(APPEND _search_hints ${${_name}${_comp}${_var}} $ENV{${_name}${_comp}${_var}} ) + list(APPEND _include_search_hints + ${${_name}${_comp}INCLUDE_DIR} $ENV{${_name}${_comp}INCLUDE_DIR} + ${${_name}${_comp}INCLUDE_DIRS} $ENV{${_name}${_comp}INCLUDE_DIRS} ) + endforeach() + endforeach() +endforeach() +#Old-school HPC module env variable names +foreach( _name IN ITEMS NetCDF4 NetCDF NETCDF4 NETCDF ) + foreach( _comp IN ITEMS "_C" "_Fortran" "_CXX" ) + list(APPEND _search_hints ${${_name}} $ENV{${_name}}) + list(APPEND _search_hints ${${_name}${_comp}} $ENV{${_name}${_comp}}) + endforeach() +endforeach() + +## Find headers for each component +set(NetCDF_INCLUDE_DIRS) +set(_new_search_components) +foreach( _comp IN LISTS _search_components ) + if(NOT ${PROJECT_NAME}_NetCDF_${_comp}_FOUND) + list(APPEND _new_search_components ${_comp}) + endif() + find_file(NetCDF_${_comp}_INCLUDE_FILE + NAMES ${NetCDF_${_comp}_INCLUDE_NAME} + DOC "NetCDF ${_comp} include directory" + HINTS ${_include_search_hints} ${_search_hints} + PATH_SUFFIXES include include/netcdf + ) + mark_as_advanced(NetCDF_${_comp}_INCLUDE_FILE) + message(DEBUG "NetCDF_${_comp}_INCLUDE_FILE: ${NetCDF_${_comp}_INCLUDE_FILE}") + if( NetCDF_${_comp}_INCLUDE_FILE ) + get_filename_component(NetCDF_${_comp}_INCLUDE_FILE ${NetCDF_${_comp}_INCLUDE_FILE} ABSOLUTE) + get_filename_component(NetCDF_${_comp}_INCLUDE_DIR ${NetCDF_${_comp}_INCLUDE_FILE} DIRECTORY) + list(APPEND NetCDF_INCLUDE_DIRS ${NetCDF_${_comp}_INCLUDE_DIR}) + endif() +endforeach() +if(NetCDF_INCLUDE_DIRS) + list(REMOVE_DUPLICATES NetCDF_INCLUDE_DIRS) +endif() +set(NetCDF_INCLUDE_DIRS "${NetCDF_INCLUDE_DIRS}" CACHE STRING "NetCDF Include directory paths" FORCE) + +## Find n*-config executables for search components +foreach( _comp IN LISTS _search_components ) + if( _comp MATCHES "^(C)$" ) + set(_conf "c") + elseif( _comp MATCHES "^(Fortran)$" ) + set(_conf "f") + elseif( _comp MATCHES "^(CXX)$" ) + set(_conf "cxx4") + endif() + find_program( NetCDF_${_comp}_CONFIG_EXECUTABLE + NAMES n${_conf}-config + HINTS ${NetCDF_INCLUDE_DIRS} ${_include_search_hints} ${_search_hints} + PATH_SUFFIXES bin Bin ../bin ../../bin + DOC "NetCDF n${_conf}-config helper" ) + message(DEBUG "NetCDF_${_comp}_CONFIG_EXECUTABLE: ${NetCDF_${_comp}_CONFIG_EXECUTABLE}") +endforeach() + +set(_C_libs_flag --libs) +set(_Fortran_libs_flag --flibs) +set(_CXX_libs_flag --libs) +set(_C_includes_flag --includedir) +set(_Fortran_includes_flag --includedir) +set(_CXX_includes_flag --includedir) +function(netcdf_config exec flag output_var) + set(${output_var} False PARENT_SCOPE) + if( exec ) + execute_process( COMMAND ${exec} ${flag} RESULT_VARIABLE _ret OUTPUT_VARIABLE _val) + if( _ret EQUAL 0 ) + string( STRIP ${_val} _val ) + set( ${output_var} ${_val} PARENT_SCOPE ) + endif() + endif() +endfunction() + +## Find libraries for each component +set( NetCDF_LIBRARIES ) +foreach( _comp IN LISTS _search_components ) + string( TOUPPER "${_comp}" _COMP ) + + find_library( NetCDF_${_comp}_LIBRARY + NAMES ${NetCDF_${_comp}_LIBRARY_NAME} + DOC "NetCDF ${_comp} library" + HINTS ${NetCDF_${_comp}_INCLUDE_DIRS} ${_search_hints} + PATH_SUFFIXES lib64 lib ../lib64 ../lib ../../lib64 ../../lib ) + mark_as_advanced( NetCDF_${_comp}_LIBRARY ) + get_filename_component(NetCDF_${_comp}_LIBRARY ${NetCDF_${_comp}_LIBRARY} ABSOLUTE) + set(NetCDF_${_comp}_LIBRARY ${NetCDF_${_comp}_LIBRARY} CACHE STRING "NetCDF ${_comp} library" FORCE) + message(DEBUG "NetCDF_${_comp}_LIBRARY: ${NetCDF_${_comp}_LIBRARY}") + + if( NetCDF_${_comp}_LIBRARY ) + if( NetCDF_${_comp}_LIBRARY MATCHES ".a$" ) + set( NetCDF_${_comp}_LIBRARY_SHARED FALSE ) + set( _library_type STATIC) + else() + list( APPEND NetCDF_LIBRARIES ${NetCDF_${_comp}_LIBRARY} ) + set( NetCDF_${_comp}_LIBRARY_SHARED TRUE ) + set( _library_type SHARED) + endif() + endif() + + #Use nc-config to set per-component LIBRARIES variable if possible + netcdf_config( ${NetCDF_${_comp}_CONFIG_EXECUTABLE} ${_${_comp}_libs_flag} _val ) + if( _val ) + set( NetCDF_${_comp}_LIBRARIES ${_val} ) + if(NOT NetCDF_${_comp}_LIBRARY_SHARED AND NOT NetCDF_${_comp}_FOUND) #Static targets should use nc_config to get a proper link line with all necessary static targets. + list( APPEND NetCDF_LIBRARIES ${NetCDF_${_comp}_LIBRARIES} ) + endif() + else() + set( NetCDF_${_comp}_LIBRARIES ${NetCDF_${_comp}_LIBRARY} ) + if(NOT NetCDF_${_comp}_LIBRARY_SHARED) + message(SEND_ERROR "Unable to properly find NetCDF. Found static libraries at: ${NetCDF_${_comp}_LIBRARY} but could not run nc-config: ${NetCDF_CONFIG_EXECUTABLE}") + endif() + endif() + + #Use nc-config to set per-component INCLUDE_DIRS variable if possible + netcdf_config( ${NetCDF_${_comp}_CONFIG_EXECUTABLE} ${_${_comp}_includes_flag} _val ) + if( _val ) + string( REPLACE " " ";" _val ${_val} ) + set( NetCDF_${_comp}_INCLUDE_DIRS ${_val} ) + else() + set( NetCDF_${_comp}_INCLUDE_DIRS ${NetCDF_${_comp}_INCLUDE_DIR} ) + endif() + + if( NetCDF_${_comp}_LIBRARIES AND NetCDF_${_comp}_INCLUDE_DIRS ) + set( ${CMAKE_FIND_PACKAGE_NAME}_${_arg_${_COMP}}_FOUND TRUE ) + if (NOT TARGET NetCDF::NetCDF_${_comp}) + add_library(NetCDF::NetCDF_${_comp} ${_library_type} IMPORTED) + set_target_properties(NetCDF::NetCDF_${_comp} PROPERTIES + IMPORTED_LOCATION ${NetCDF_${_comp}_LIBRARY} + INTERFACE_INCLUDE_DIRECTORIES "${NetCDF_${_comp}_INCLUDE_DIRS}" + INTERFACE_LINK_LIBRARIES ${NetCDF_${_comp}_LIBRARIES} ) + endif() + endif() +endforeach() +if(NetCDF_LIBRARIES AND NetCDF_${_comp}_LIBRARY_SHARED) + list(REMOVE_DUPLICATES NetCDF_LIBRARIES) +endif() +set(NetCDF_LIBRARIES "${NetCDF_LIBRARIES}" CACHE STRING "NetCDF library targets" FORCE) + +## Find version via netcdf-config if possible +if (NetCDF_INCLUDE_DIRS) + if( NetCDF_C_CONFIG_EXECUTABLE ) + netcdf_config( ${NetCDF_C_CONFIG_EXECUTABLE} --version _vers ) + if( _vers ) + string(REGEX REPLACE ".* ((([0-9]+)\\.)+([0-9]+)).*" "\\1" NetCDF_VERSION "${_vers}" ) + endif() + else() + foreach( _dir IN LISTS NetCDF_INCLUDE_DIRS) + if( EXISTS "${_dir}/netcdf_meta.h" ) + file(STRINGS "${_dir}/netcdf_meta.h" _netcdf_version_lines + REGEX "#define[ \t]+NC_VERSION_(MAJOR|MINOR|PATCH|NOTE)") + string(REGEX REPLACE ".*NC_VERSION_MAJOR *\([0-9]*\).*" "\\1" _netcdf_version_major "${_netcdf_version_lines}") + string(REGEX REPLACE ".*NC_VERSION_MINOR *\([0-9]*\).*" "\\1" _netcdf_version_minor "${_netcdf_version_lines}") + string(REGEX REPLACE ".*NC_VERSION_PATCH *\([0-9]*\).*" "\\1" _netcdf_version_patch "${_netcdf_version_lines}") + string(REGEX REPLACE ".*NC_VERSION_NOTE *\"\([^\"]*\)\".*" "\\1" _netcdf_version_note "${_netcdf_version_lines}") + set(NetCDF_VERSION "${_netcdf_version_major}.${_netcdf_version_minor}.${_netcdf_version_patch}${_netcdf_version_note}") + unset(_netcdf_version_major) + unset(_netcdf_version_minor) + unset(_netcdf_version_patch) + unset(_netcdf_version_note) + unset(_netcdf_version_lines) + endif() + endforeach() + endif() +endif () + +## Detect additional package properties +netcdf_config(${NetCDF_C_CONFIG_EXECUTABLE} --has-parallel4 _val) +if( NOT _val MATCHES "^(yes|no)$" ) + netcdf_config(${NetCDF_C_CONFIG_EXECUTABLE} --has-parallel _val) +endif() +if( _val MATCHES "^(yes)$" ) + set(NetCDF_PARALLEL TRUE CACHE STRING "NetCDF has parallel IO capability via pnetcdf or hdf5." FORCE) +else() + set(NetCDF_PARALLEL FALSE CACHE STRING "NetCDF has no parallel IO capability." FORCE) +endif() + +## Finalize find_package +include(FindPackageHandleStandardArgs) + +if(NOT NetCDF_FOUND OR _new_search_components) + find_package_handle_standard_args( ${CMAKE_FIND_PACKAGE_NAME} + REQUIRED_VARS NetCDF_INCLUDE_DIRS NetCDF_LIBRARIES + VERSION_VAR NetCDF_VERSION + HANDLE_COMPONENTS ) +endif() + +foreach( _comp IN LISTS _search_components ) + if( NetCDF_${_comp}_FOUND ) + #Record found components to avoid duplication in NetCDF_LIBRARIES for static libraries + set(NetCDF_${_comp}_FOUND ${NetCDF_${_comp}_FOUND} CACHE BOOL "NetCDF ${_comp} Found" FORCE) + #Set a per-package, per-component found variable to communicate between multiple calls to find_package() + set(${PROJECT_NAME}_NetCDF_${_comp}_FOUND True) + endif() +endforeach() + +if( ${CMAKE_FIND_PACKAGE_NAME}_FOUND AND NOT ${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY AND _new_search_components) + message( STATUS "Find${CMAKE_FIND_PACKAGE_NAME} defines targets:" ) + message( STATUS " - NetCDF_VERSION [${NetCDF_VERSION}]") + message( STATUS " - NetCDF_PARALLEL [${NetCDF_PARALLEL}]") + foreach( _comp IN LISTS _new_search_components ) + string( TOUPPER "${_comp}" _COMP ) + message( STATUS " - NetCDF_${_comp}_CONFIG_EXECUTABLE [${NetCDF_${_comp}_CONFIG_EXECUTABLE}]") + if( ${CMAKE_FIND_PACKAGE_NAME}_${_arg_${_COMP}}_FOUND ) + get_filename_component(_root ${NetCDF_${_comp}_INCLUDE_DIR}/.. ABSOLUTE) + if( NetCDF_${_comp}_LIBRARY_SHARED ) + message( STATUS " - NetCDF::NetCDF_${_comp} [SHARED] [Root: ${_root}] Lib: ${NetCDF_${_comp}_LIBRARY} ") + else() + message( STATUS " - NetCDF::NetCDF_${_comp} [STATIC] [Root: ${_root}] Lib: ${NetCDF_${_comp}_LIBRARY} ") + endif() + endif() + endforeach() +endif() + +foreach( _prefix NetCDF NetCDF4 NETCDF NETCDF4 ${CMAKE_FIND_PACKAGE_NAME} ) + set( ${_prefix}_INCLUDE_DIRS ${NetCDF_INCLUDE_DIRS} ) + set( ${_prefix}_LIBRARIES ${NetCDF_LIBRARIES}) + set( ${_prefix}_VERSION ${NetCDF_VERSION} ) + set( ${_prefix}_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_FOUND} ) + set( ${_prefix}_CONFIG_EXECUTABLE ${NetCDF_CONFIG_EXECUTABLE} ) + set( ${_prefix}_PARALLEL ${NetCDF_PARALLEL} ) + + foreach( _comp ${_search_components} ) + string( TOUPPER "${_comp}" _COMP ) + set( _arg_comp ${_arg_${_COMP}} ) + set( ${_prefix}_${_comp}_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_${_arg_comp}_FOUND} ) + set( ${_prefix}_${_COMP}_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_${_arg_comp}_FOUND} ) + set( ${_prefix}_${_arg_comp}_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_${_arg_comp}_FOUND} ) + + set( ${_prefix}_${_comp}_LIBRARIES ${NetCDF_${_comp}_LIBRARIES} ) + set( ${_prefix}_${_COMP}_LIBRARIES ${NetCDF_${_comp}_LIBRARIES} ) + set( ${_prefix}_${_arg_comp}_LIBRARIES ${NetCDF_${_comp}_LIBRARIES} ) + + set( ${_prefix}_${_comp}_INCLUDE_DIRS ${NetCDF_${_comp}_INCLUDE_DIRS} ) + set( ${_prefix}_${_COMP}_INCLUDE_DIRS ${NetCDF_${_comp}_INCLUDE_DIRS} ) + set( ${_prefix}_${_arg_comp}_INCLUDE_DIRS ${NetCDF_${_comp}_INCLUDE_DIRS} ) + endforeach() +endforeach() \ No newline at end of file diff --git a/pyfms/cfms.py b/pyfms/cfms.py index fe962d2..f5858f3 100644 --- a/pyfms/cfms.py +++ b/pyfms/cfms.py @@ -1,5 +1,5 @@ import ctypes - +from pathlib import Path import pyfms @@ -25,7 +25,7 @@ def init(libpath: str = None): # todo reset all _function parameters to None if libpath is None: - _libpath = "libcFMS.so" + _libpath = "/home/Frank.Malatino/.conda/envs/throw3/lib/python3.11/site-packages/pyfms/lib/libcFMS.so" try: _lib = ctypes.cdll.LoadLibrary(_libpath) except Exception as e: diff --git a/pyproject.toml b/pyproject.toml index ad0679d..86e03db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,89 @@ [build-system] -requires = ["setuptools >= 64"] -build-backend = "setuptools.build_meta" +requires = ["scikit-build-core >= 0.3.3"] +build-backend = "scikit_build_core.build" + +[project] +name = "pyfms" +version = "2026.03.00" +description = "A Python interface to the FMS Fortran software package." +readme = "README.md" +requires-python = ">=3.11,<3.13" +# when updating the tested Python versions, please also update README.md, +dependencies = [ + "dacite", + "h5netcdf", + "numpy", + "pyyaml", + "mpi4py", + "xarray", + "netcdf4", + "pyyaml", +] +classifiers = [ + "Development Status :: 1 - Pre-Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Programming Language :: Python :: 3", +] + +[project.optional-dependencies] +test = [ + "pytest", + "pytest-subtests", + "coverage", + "dask", +] +develop = [ + "pyfms[test]", + "pre-commit" +] +extras = [ + "pyfms[test]", + "pyfms[develop]", +] + +[tool.aliases] + +[tool.scikit-build] +wheel.expand-macos-universal-tags = true + +[tool.scikit-build.cmake.define] +CMAKE_GENERATOR = "Unix Makefiles" + +[tool.isort] +line_length = "88" +force_grid_wrap = "0" +include_trailing_comma = "true" +multi_line_output = "3" +use_parentheses = "true" +lines_after_imports = "2" +default_section = "THIRDPARTY" +sections = "FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER" +known_first_party = "pyfms" +known_third_party = "pytest,dacite,xarray,numpy" + +[tool.mypy] +ignore_missing_imports = true +follow_imports = "normal" +namespace_packages = true +strict_optional = false +warn_unreachable = true +explicit_package_bases = true + +[tool.coverage.run] +parallel = true +branch = true +omit = [ + "tests/*", + "__init__.py", +] +source_pkgs = ["pyfms"] + +[tool.flake8] +exclude = ["docs"] +extend-ignore = ["E203","E501","W293","W503","E302","E203","F841"] +max-line-length = "88" + +[tool.scikit-build-core] +cmake.generator = "Unix Makefiles" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index aefb6d4..0000000 --- a/setup.cfg +++ /dev/null @@ -1,34 +0,0 @@ -[flake8] -exclude = docs -ignore = E203,E501,W293,W503,E302,E203,F841 -max-line-length = 88 - -[aliases] - -[tool:isort] -line_length = 88 -force_grid_wrap = 0 -include_trailing_comma = true -multi_line_output = 3 -use_parentheses = true -lines_after_imports = 2 -default_section = THIRDPARTY -sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER -known_first_party = pyfms -known_third_party = pytest,dacite,xarray,numpy - -[mypy] -ignore_missing_imports = True -follow_imports = normal -namespace_packages = True -strict_optional = False -warn_unreachable = True -explicit_package_bases = True - -[coverage:run] -parallel = true -branch = true -omit = - tests/* - __init__.py -source_pkgs = pyfms diff --git a/setup.py b/setup.py deleted file mode 100644 index 74b5f25..0000000 --- a/setup.py +++ /dev/null @@ -1,69 +0,0 @@ -import subprocess - -from setuptools import find_namespace_packages, setup -from setuptools.command.editable_wheel import editable_wheel as _editable_wheel -from setuptools.command.install import install as _install - - -class editable_wheel(_editable_wheel): - def run(self): - print("Installing cFMS") - try: - subprocess.run(["conda", "install", "-y", "-c", "file:///home/Frank.Malatino/cfms_conda_channel", "cfms"], check=True, capture_output=True, text=True) - except subprocess.CalledProcessError as e: - print(f"Error during conda install of cFMS: {e}") - print("STDOUT:", e.stdout) - print("STDERR:", e.stderr) - _editable_wheel.run(self) - -class install(_install): - def run(self): - print("Installing cFMS") - try: - subprocess.run(["conda", "install", "-y", "-c", "file:///home/Frank.Malatino/cfms_conda_channel", "cfms"], check=True, capture_output=True, text=True) - except subprocess.CalledProcessError as e: - print(f"Error during conda install of cFMS: {e}") - print("STDOUT:", e.stdout) - print("STDERR:", e.stderr) - _install.run(self) - - -test_requirements = ["pytest", "pytest-subtests", "coverage"] -develop_requirements = test_requirements + ["pre-commit"] - -extras_requires = { - "test": test_requirements, - "develop": develop_requirements, -} - -requirements = [ - "dacite", - "h5netcdf", - "numpy", - "pyyaml", - "mpi4py", - "xarray", - "netcdf4", -] - -setup( - author="NOAA/GFDL", - python_requires=">=3.11", - classifiers=[ - "Development Status :: 1 - Pre-Alpha", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Natural Language :: English", - "Programming Language :: Python :: 3", - ], - install_requires=requirements, - extras_require=extras_requires, - name="pyfms", - license="", - packages=find_namespace_packages(include=["pyfms", "pyfms.*"]), - cmdclass={"editable_wheel": editable_wheel, "install": install}, - include_package_data=True, - url="https://github.com/fmalatino/pyFMS.git", - version="2024.02.0", - zip_safe=False, -) From 711f80d79229c9001982105df5c4dedcf686b143 Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 16 Mar 2026 10:18:42 -0400 Subject: [PATCH 07/29] Checking use as a dependency --- CMakeLists.txt | 24 +++++++----------------- pyproject.toml | 4 ---- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 47b93f4..2074560 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,21 +6,11 @@ include(FetchContent) set(CMAKE_POSITION_INDEPENDENT_CODE ON) -find_library(cFMS_LIBRARY NAMES "libcFMS.so" PATHS ${SKBUILD_PLATLIB_DIR}) +FetchContent_Declare( + cFMS + GIT_REPOSITORY https://github.com/fmalatino/cFMS.git + GIT_TAG cmake +) +FetchContent_MakeAvailable(cFMS) -if(NOT cFMS_LIBRARY) - message(STATUS "Did not find cFMS, using FetchContent to build") - FetchContent_Declare( - cFMS_LIBRARY - GIT_REPOSITORY https://github.com/fmalatino/cFMS.git - GIT_TAG cmake - ) - FetchContent_MakeAvailable(cFMS_LIBRARY) -else() - message(STATUS "Found cFMS library, including it in install") - add_library(cFMS_LIBRARY SHARED IMPORTED) - message(STATUS "Added cFMS_LIBRARY as a target") - set_property(TARGET cFMS_LIBRARY PROPERTY IMPORTED_LOCATION ${SKBUILD_PLATLIB_DIR}) -endif() - -install(TARGETS cFMS_LIBRARY LIBRARY DESTINATION ${SKBUILD_PLATLIB_DIR}) \ No newline at end of file +install(TARGETS cFMS DESTINATION ${SKBUILD_PLATLIB_DIR}) diff --git a/pyproject.toml b/pyproject.toml index 86e03db..de45f02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ dependencies = [ "mpi4py", "xarray", "netcdf4", - "pyyaml", ] classifiers = [ "Development Status :: 1 - Pre-Alpha", @@ -84,6 +83,3 @@ source_pkgs = ["pyfms"] exclude = ["docs"] extend-ignore = ["E203","E501","W293","W503","E302","E203","F841"] max-line-length = "88" - -[tool.scikit-build-core] -cmake.generator = "Unix Makefiles" From 23f7628844538b510cd0cd4ee411e3f996be9025 Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 16 Mar 2026 10:57:51 -0400 Subject: [PATCH 08/29] Pre-lim workflow file for new install --- .github/workflows/cmake_install.yaml | 24 ++++++++++++++++++++++++ pyfms/cfms.py | 4 ++-- 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/cmake_install.yaml diff --git a/.github/workflows/cmake_install.yaml b/.github/workflows/cmake_install.yaml new file mode 100644 index 0000000..5034a04 --- /dev/null +++ b/.github/workflows/cmake_install.yaml @@ -0,0 +1,24 @@ +name: Install libcFMS with scikit-build-core + +on: [push, pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + tests: + runs-on: ubuntu-latest + container: + image: ghcr.io/noaa-gfdl/fms/fms-ci-rocky-gnu:13.2.0 + env: + PKG_CONFIG_PATH: "/opt/views/view/lib64/pkgconfig:/opt/views/view/lib/pkgconfig:/opt/views/view/share/pkgconfig" + steps: + - name: checkout code + uses: actions/checkout@v4 + - name: Run pip install + run: | + export LIBYAML_ROOT=/opt/view + pip install .[develop] + - name: Run tests + run: cd tests && ./run_tests.sh -o \ No newline at end of file diff --git a/pyfms/cfms.py b/pyfms/cfms.py index f5858f3..f3079f7 100644 --- a/pyfms/cfms.py +++ b/pyfms/cfms.py @@ -1,5 +1,5 @@ import ctypes -from pathlib import Path +import sysconfig import pyfms @@ -25,7 +25,7 @@ def init(libpath: str = None): # todo reset all _function parameters to None if libpath is None: - _libpath = "/home/Frank.Malatino/.conda/envs/throw3/lib/python3.11/site-packages/pyfms/lib/libcFMS.so" + _libpath = sysconfig.get_paths()["purelib"] + "/libcFMS.so" try: _lib = ctypes.cdll.LoadLibrary(_libpath) except Exception as e: From f2d3aab8772e9c610a8aa1b774d428b5ec125b5f Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 16 Mar 2026 11:03:21 -0400 Subject: [PATCH 09/29] Installing Python in cmake workflow --- .github/workflows/cmake_install.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/cmake_install.yaml b/.github/workflows/cmake_install.yaml index 5034a04..9782ab6 100644 --- a/.github/workflows/cmake_install.yaml +++ b/.github/workflows/cmake_install.yaml @@ -16,8 +16,14 @@ jobs: steps: - name: checkout code uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' - name: Run pip install run: | + python -m pip install --upgrade pip export LIBYAML_ROOT=/opt/view pip install .[develop] - name: Run tests From d393bb8771109f7769c724b3ddf8f75541c1e84e Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 16 Mar 2026 11:08:39 -0400 Subject: [PATCH 10/29] Adding installation of lsb-release to cmake workflow --- .github/workflows/cmake_install.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/cmake_install.yaml b/.github/workflows/cmake_install.yaml index 9782ab6..6270073 100644 --- a/.github/workflows/cmake_install.yaml +++ b/.github/workflows/cmake_install.yaml @@ -16,6 +16,9 @@ jobs: steps: - name: checkout code uses: actions/checkout@v4 + - name: Install lsb-release + run: | + apt-get update && apt-get install -y lsb-release && apt-get clean all - name: Set up Python 3.11 uses: actions/setup-python@v5 with: From ff9df298b259d2374ca547b0f7cf276c5c7fa2fa Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 16 Mar 2026 11:14:03 -0400 Subject: [PATCH 11/29] Trying sudo in cmake workflow for install of lsb-release --- .github/workflows/cmake_install.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake_install.yaml b/.github/workflows/cmake_install.yaml index 6270073..8dc410b 100644 --- a/.github/workflows/cmake_install.yaml +++ b/.github/workflows/cmake_install.yaml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v4 - name: Install lsb-release run: | - apt-get update && apt-get install -y lsb-release && apt-get clean all + sudo apt-get update && sudo apt-get install -y lsb-release && sudo apt-get clean all - name: Set up Python 3.11 uses: actions/setup-python@v5 with: From d5180e0fcaae936102e38c42418842ca3318843b Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Wed, 18 Mar 2026 14:21:05 -0400 Subject: [PATCH 12/29] Bumping Python required version to >=3.12 --- pyproject.toml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index de45f02..9adc1ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "pyfms" version = "2026.03.00" description = "A Python interface to the FMS Fortran software package." readme = "README.md" -requires-python = ">=3.11,<3.13" +requires-python = ">=3.12,<3.13" # when updating the tested Python versions, please also update README.md, dependencies = [ "dacite", @@ -30,16 +30,15 @@ classifiers = [ test = [ "pytest", "pytest-subtests", - "coverage", - "dask", + "coverage", ] -develop = [ +dev = [ "pyfms[test]", "pre-commit" ] extras = [ "pyfms[test]", - "pyfms[develop]", + "pyfms[dev]", ] [tool.aliases] From 47be94aa2782fd6d81d10370210ee0d195cab62d Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 23 Mar 2026 14:15:16 -0400 Subject: [PATCH 13/29] Amending workflows for new Python --- .github/workflows/cmake_install.yaml | 33 ---------------------------- .github/workflows/unit_tests.yaml | 2 +- 2 files changed, 1 insertion(+), 34 deletions(-) delete mode 100644 .github/workflows/cmake_install.yaml diff --git a/.github/workflows/cmake_install.yaml b/.github/workflows/cmake_install.yaml deleted file mode 100644 index 8dc410b..0000000 --- a/.github/workflows/cmake_install.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: Install libcFMS with scikit-build-core - -on: [push, pull_request] - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - tests: - runs-on: ubuntu-latest - container: - image: ghcr.io/noaa-gfdl/fms/fms-ci-rocky-gnu:13.2.0 - env: - PKG_CONFIG_PATH: "/opt/views/view/lib64/pkgconfig:/opt/views/view/lib/pkgconfig:/opt/views/view/share/pkgconfig" - steps: - - name: checkout code - uses: actions/checkout@v4 - - name: Install lsb-release - run: | - sudo apt-get update && sudo apt-get install -y lsb-release && sudo apt-get clean all - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: '3.11' - cache: 'pip' - - name: Run pip install - run: | - python -m pip install --upgrade pip - export LIBYAML_ROOT=/opt/view - pip install .[develop] - - name: Run tests - run: cd tests && ./run_tests.sh -o \ No newline at end of file diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index aa69eb2..c117ba9 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -19,7 +19,7 @@ jobs: - name: Step Python uses: actions/setup-python@v4.6.0 with: - python-version: '3.11.7' + python-version: '3.12.13' - name: Install OpenMPI for gt4py run: | sudo apt-get install libopenmpi-dev netcdf-bin libnetcdf-dev libnetcdff-dev nco libyaml-dev diffutils From b9d6845f378987632234d7261709af97f93bbf15 Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 23 Mar 2026 14:21:04 -0400 Subject: [PATCH 14/29] Updating before dependency install in unit test workflow --- .github/workflows/unit_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index c117ba9..8ee8aa6 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -20,9 +20,9 @@ jobs: uses: actions/setup-python@v4.6.0 with: python-version: '3.12.13' - - name: Install OpenMPI for gt4py + - name: Install Dependencies for Build run: | - sudo apt-get install libopenmpi-dev netcdf-bin libnetcdf-dev libnetcdff-dev nco libyaml-dev diffutils + sudo apt-get update && sudo apt-get install libopenmpi-dev netcdf-bin libnetcdf-dev libnetcdff-dev nco libyaml-dev diffutils - name: install packages run: | pip install .[test] From 2dd802319020b47497242d92a1e14769c570d581 Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 23 Mar 2026 14:23:29 -0400 Subject: [PATCH 15/29] Making install verbose in workflow --- .github/workflows/unit_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 8ee8aa6..5d157e6 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -25,7 +25,7 @@ jobs: sudo apt-get update && sudo apt-get install libopenmpi-dev netcdf-bin libnetcdf-dev libnetcdff-dev nco libyaml-dev diffutils - name: install packages run: | - pip install .[test] + pip install -v .[test] - name: run tests run: | cd tests From 7e0d56ae6b369143ad4ba2d577418281f8c57920 Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 23 Mar 2026 14:53:35 -0400 Subject: [PATCH 16/29] Linting and fixing of test_define_domains --- .github/workflows/lint.yaml | 2 +- .gitmodules | 1 - .pre-commit-config.yaml | 4 ++-- cmake/FindNetCDF.cmake | 2 +- pyfms/cfms.py | 1 + pyproject.toml | 7 ++++--- tests/py_mpp/test_define_domains.py | 2 +- 7 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index f6438ca..be3f226 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -21,7 +21,7 @@ jobs: python-version: '3.11.7' - name: Install OpenMPI for gt4py run: | - sudo apt-get install libopenmpi-dev netcdf-bin libnetcdf-dev libnetcdff-dev nco libyaml-dev diffutils + sudo apt-get update && sudo apt-get install libopenmpi-dev netcdf-bin libnetcdf-dev libnetcdff-dev nco libyaml-dev diffutils - name: Install Python packages run: | python -m pip install --upgrade pip setuptools wheel diff --git a/.gitmodules b/.gitmodules index 8b13789..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1 +0,0 @@ - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6953431..c975714 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: - id: flake8 name: flake8 language_version: python3 - args: [--config, setup.cfg] + args: ["--ignore=F401,E203,E501,F841,E302",--config, pyproject.toml] exclude: | (?x)^( .*/__init__.py | @@ -36,4 +36,4 @@ repos: name: flake8 __init__.py files files: "__init__.py" # ignore unused import error in __init__.py files - args: ["--ignore=F401,E203", --config, setup.cfg] + args: ["--ignore=F401,E203", --config, pyproject.toml] diff --git a/cmake/FindNetCDF.cmake b/cmake/FindNetCDF.cmake index 82fe366..1439ae8 100644 --- a/cmake/FindNetCDF.cmake +++ b/cmake/FindNetCDF.cmake @@ -334,4 +334,4 @@ foreach( _prefix NetCDF NetCDF4 NETCDF NETCDF4 ${CMAKE_FIND_PACKAGE_NAME} ) set( ${_prefix}_${_COMP}_INCLUDE_DIRS ${NetCDF_${_comp}_INCLUDE_DIRS} ) set( ${_prefix}_${_arg_comp}_INCLUDE_DIRS ${NetCDF_${_comp}_INCLUDE_DIRS} ) endforeach() -endforeach() \ No newline at end of file +endforeach() diff --git a/pyfms/cfms.py b/pyfms/cfms.py index f3079f7..30dfec1 100644 --- a/pyfms/cfms.py +++ b/pyfms/cfms.py @@ -1,5 +1,6 @@ import ctypes import sysconfig + import pyfms diff --git a/pyproject.toml b/pyproject.toml index 9adc1ae..4368b3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,13 +28,14 @@ classifiers = [ [project.optional-dependencies] test = [ - "pytest", - "pytest-subtests", + "pytest", + "pytest-subtests", "coverage", ] dev = [ "pyfms[test]", - "pre-commit" + "pre-commit", + "flake8-pyproject", ] extras = [ "pyfms[test]", diff --git a/tests/py_mpp/test_define_domains.py b/tests/py_mpp/test_define_domains.py index b3fc995..76b5e27 100644 --- a/tests/py_mpp/test_define_domains.py +++ b/tests/py_mpp/test_define_domains.py @@ -33,7 +33,7 @@ def test_define_domains(): """get global pelist""" - global_pelist = pyfms.mpp.get_current_pelist(npes=pyfms.mpp.npes()) + global_pelist = pyfms.mpp.get_current_pelist(npes=pyfms.mpp.npes(), get_commID=True) """set coarse domain as tile=0""" From a51efb686022a06903df66db777546bf60bf8537 Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 23 Mar 2026 15:29:45 -0400 Subject: [PATCH 17/29] Amending interface fro declare_pelist --- pyfms/py_mpp/_mpp_functions.py | 1 + tests/py_mpp/test_define_domains.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyfms/py_mpp/_mpp_functions.py b/pyfms/py_mpp/_mpp_functions.py index 6bf6f19..ef3fbb5 100644 --- a/pyfms/py_mpp/_mpp_functions.py +++ b/pyfms/py_mpp/_mpp_functions.py @@ -24,6 +24,7 @@ def define(lib): POINTER(c_int), # npes ndpointer(dtype=np.int32, ndim=(1), flags=C), # pelist c_char_p, # name + POINTER(c_int), # commID ] # cFMS_error diff --git a/tests/py_mpp/test_define_domains.py b/tests/py_mpp/test_define_domains.py index 76b5e27..848863d 100644 --- a/tests/py_mpp/test_define_domains.py +++ b/tests/py_mpp/test_define_domains.py @@ -37,7 +37,7 @@ def test_define_domains(): """set coarse domain as tile=0""" - coarse_pelist = global_pelist[:coarse_npes] + coarse_pelist = global_pelist[0][:coarse_npes] pyfms.mpp.declare_pelist(pelist=coarse_pelist, name="test coarse pelist") if pyfms.mpp.pe() in coarse_pelist: @@ -71,7 +71,7 @@ def test_define_domains(): """set fine domain as tile=1""" - fine_pelist = global_pelist[coarse_npes : coarse_npes + fine_npes] + fine_pelist = global_pelist[0][coarse_npes : coarse_npes + fine_npes] pyfms.mpp.declare_pelist(pelist=fine_pelist, name="test fine pelist") if pyfms.mpp.pe() in fine_pelist: From 92ebaeb1a2db73671075da9c4a57930ad8bf6afc Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 23 Mar 2026 15:30:38 -0400 Subject: [PATCH 18/29] Amending lint workflow --- .github/workflows/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index be3f226..2a9d12e 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -18,7 +18,7 @@ jobs: - name: Step Python uses: actions/setup-python@v4.6.0 with: - python-version: '3.11.7' + python-version: '3.12.13' - name: Install OpenMPI for gt4py run: | sudo apt-get update && sudo apt-get install libopenmpi-dev netcdf-bin libnetcdf-dev libnetcdff-dev nco libyaml-dev diffutils From 2e35b98700bba1d4470178fa5e5e7b623a763284 Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 23 Mar 2026 15:34:43 -0400 Subject: [PATCH 19/29] Amending test_gather use of get_current_pelist --- tests/py_mpp/test_gather.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/py_mpp/test_gather.py b/tests/py_mpp/test_gather.py index b3ff8cb..e196d4d 100644 --- a/tests/py_mpp/test_gather.py +++ b/tests/py_mpp/test_gather.py @@ -35,7 +35,7 @@ def test_gather_2d(): else: rbuf_shape = [ny, nx] - pelist = pyfms.mpp.get_current_pelist(pyfms.mpp.npes()) + pelist = pyfms.mpp.get_current_pelist(npes=pyfms.mpp.npes(), get_commID=True) gathered = pyfms.mpp.gather( send, rbuf_shape=rbuf_shape, From b8c57b81594b2a92632353058c9c9529c1d2e4a6 Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 23 Mar 2026 15:36:31 -0400 Subject: [PATCH 20/29] Using dev build option when linting --- .github/workflows/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 2a9d12e..34ac033 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -25,7 +25,7 @@ jobs: - name: Install Python packages run: | python -m pip install --upgrade pip setuptools wheel - pip install .[develop] + pip install .[dev] - name: Run lint via pre-commit run: | pre-commit run --all-files From 0d861ab56bb973e4092d7bb262f012333eca7a2d Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 23 Mar 2026 15:47:09 -0400 Subject: [PATCH 21/29] Amending horiz_interp test use of get_current_pelist --- pyfms/py_mpp/mpp.py | 4 +++- tests/py_horiz_interp/test_horiz_interp.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pyfms/py_mpp/mpp.py b/pyfms/py_mpp/mpp.py index e7a9e10..ee8384d 100644 --- a/pyfms/py_mpp/mpp.py +++ b/pyfms/py_mpp/mpp.py @@ -90,7 +90,9 @@ def gather( else: rbuf_shape, rbuf = None, None - pelist = get_current_pelist(npes()) if pelist is None else pelist + pelist = ( + get_current_pelist(npes(), get_commID=True) if pelist is None else pelist + ) set_c_int(domain.isc, arglist) set_c_int(domain.iec, arglist) diff --git a/tests/py_horiz_interp/test_horiz_interp.py b/tests/py_horiz_interp/test_horiz_interp.py index 56f2f06..768498a 100755 --- a/tests/py_horiz_interp/test_horiz_interp.py +++ b/tests/py_horiz_interp/test_horiz_interp.py @@ -81,7 +81,7 @@ def test_horiz_interp_conservative(): domain = pyfms.mpp_domains.define_domains( global_indices=global_indices, layout=layout, - pelist=pyfms.mpp.get_current_pelist(npes=pes), + pelist=pyfms.mpp.get_current_pelist(npes=pes, get_commID=True), name="horiz_interp_conservative_test", xflags=pyfms.mpp_domains.CYCLIC_GLOBAL_DOMAIN, yflags=pyfms.mpp_domains.CYCLIC_GLOBAL_DOMAIN, @@ -190,7 +190,7 @@ def test_horiz_interp_bilinear(): global_indices=[0, ni_src - 1, 0, nj_src - 1], ndivs=pes, ), - pelist=pyfms.mpp.get_current_pelist(npes=pes), + pelist=pyfms.mpp.get_current_pelist(npes=pes, get_commID=True), name="horiz_interp_bilinear_test", whalo=halo, ehalo=halo, From d0b3096d81f0b01f635499fc89139174c8f7b472 Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 23 Mar 2026 16:02:59 -0400 Subject: [PATCH 22/29] Amending all tests for use of get_current_pelist --- tests/py_horiz_interp/test_horiz_interp.py | 4 ++-- tests/py_mpp/test_define_domains.py | 8 +++++--- tests/py_mpp/test_gather.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/py_horiz_interp/test_horiz_interp.py b/tests/py_horiz_interp/test_horiz_interp.py index 768498a..7143249 100755 --- a/tests/py_horiz_interp/test_horiz_interp.py +++ b/tests/py_horiz_interp/test_horiz_interp.py @@ -81,7 +81,7 @@ def test_horiz_interp_conservative(): domain = pyfms.mpp_domains.define_domains( global_indices=global_indices, layout=layout, - pelist=pyfms.mpp.get_current_pelist(npes=pes, get_commID=True), + pelist=pyfms.mpp.get_current_pelist(npes=pes, get_commID=True)[0], name="horiz_interp_conservative_test", xflags=pyfms.mpp_domains.CYCLIC_GLOBAL_DOMAIN, yflags=pyfms.mpp_domains.CYCLIC_GLOBAL_DOMAIN, @@ -190,7 +190,7 @@ def test_horiz_interp_bilinear(): global_indices=[0, ni_src - 1, 0, nj_src - 1], ndivs=pes, ), - pelist=pyfms.mpp.get_current_pelist(npes=pes, get_commID=True), + pelist=pyfms.mpp.get_current_pelist(npes=pes, get_commID=True)[0], name="horiz_interp_bilinear_test", whalo=halo, ehalo=halo, diff --git a/tests/py_mpp/test_define_domains.py b/tests/py_mpp/test_define_domains.py index 848863d..41a793d 100644 --- a/tests/py_mpp/test_define_domains.py +++ b/tests/py_mpp/test_define_domains.py @@ -33,11 +33,13 @@ def test_define_domains(): """get global pelist""" - global_pelist = pyfms.mpp.get_current_pelist(npes=pyfms.mpp.npes(), get_commID=True) + global_pelist = pyfms.mpp.get_current_pelist( + npes=pyfms.mpp.npes(), get_commID=True + )[0] """set coarse domain as tile=0""" - coarse_pelist = global_pelist[0][:coarse_npes] + coarse_pelist = global_pelist[:coarse_npes] pyfms.mpp.declare_pelist(pelist=coarse_pelist, name="test coarse pelist") if pyfms.mpp.pe() in coarse_pelist: @@ -71,7 +73,7 @@ def test_define_domains(): """set fine domain as tile=1""" - fine_pelist = global_pelist[0][coarse_npes : coarse_npes + fine_npes] + fine_pelist = global_pelist[coarse_npes : coarse_npes + fine_npes] pyfms.mpp.declare_pelist(pelist=fine_pelist, name="test fine pelist") if pyfms.mpp.pe() in fine_pelist: diff --git a/tests/py_mpp/test_gather.py b/tests/py_mpp/test_gather.py index e196d4d..37c1ebc 100644 --- a/tests/py_mpp/test_gather.py +++ b/tests/py_mpp/test_gather.py @@ -35,7 +35,7 @@ def test_gather_2d(): else: rbuf_shape = [ny, nx] - pelist = pyfms.mpp.get_current_pelist(npes=pyfms.mpp.npes(), get_commID=True) + pelist = pyfms.mpp.get_current_pelist(npes=pyfms.mpp.npes(), get_commID=True)[0] gathered = pyfms.mpp.gather( send, rbuf_shape=rbuf_shape, From 4ce868ad08de0aad6fa281c01a9dc2dfe8562c50 Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Mon, 23 Mar 2026 16:19:28 -0400 Subject: [PATCH 23/29] Using xarray open_dataset in test_diag_manager --- tests/py_diag_manager/test_diag_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/py_diag_manager/test_diag_manager.py b/tests/py_diag_manager/test_diag_manager.py index 082d652..ffaa05c 100644 --- a/tests/py_diag_manager/test_diag_manager.py +++ b/tests/py_diag_manager/test_diag_manager.py @@ -177,7 +177,7 @@ def test_send_data(): """ if pyfms.mpp.pe() == pyfms.mpp.root_pe(): assert os.path.isfile("test_send_data.nc") - ds = xr.open_mfdataset("test_send_data.nc", decode_times=True) + ds = xr.open_dataset("test_send_data.nc", decode_times=True) assert "var2_avg" in ds assert "var3_avg" in ds assert ds["var2_avg"].dims == ("time", "y", "x") From 05086e03504bf0cabd770320e3e944dcfa55d48c Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Tue, 24 Mar 2026 10:23:39 -0400 Subject: [PATCH 24/29] Fixing linting issues --- .github/workflows/lint.yaml | 36 ++++++++--------- .github/workflows/unit_tests.yaml | 36 ++++++++--------- .pre-commit-config.yaml | 46 +++++++++------------- pyfms/py_field_manager/py_field_manager.py | 4 +- pyproject.toml | 2 +- 5 files changed, 58 insertions(+), 66 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 34ac033..151e688 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -11,21 +11,21 @@ jobs: lint: runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v3.5.2 - with: - submodules: 'recursive' - - name: Step Python - uses: actions/setup-python@v4.6.0 - with: - python-version: '3.12.13' - - name: Install OpenMPI for gt4py - run: | - sudo apt-get update && sudo apt-get install libopenmpi-dev netcdf-bin libnetcdf-dev libnetcdff-dev nco libyaml-dev diffutils - - name: Install Python packages - run: | - python -m pip install --upgrade pip setuptools wheel - pip install .[dev] - - name: Run lint via pre-commit - run: | - pre-commit run --all-files + - name: Checkout repository + uses: actions/checkout@v3.5.2 + with: + submodules: 'recursive' + - name: Step Python + uses: actions/setup-python@v4.6.0 + with: + python-version: '3.12.13' + - name: Install OpenMPI for gt4py + run: | + sudo apt-get update && sudo apt-get install libopenmpi-dev netcdf-bin libnetcdf-dev libnetcdff-dev nco libyaml-dev diffutils + - name: Install Python packages + run: | + python -m pip install --upgrade pip setuptools wheel + pip install .[dev] + - name: Run lint via pre-commit + run: | + pre-commit run --all-files diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 5d157e6..5bcbfb6 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -12,21 +12,21 @@ jobs: env: LD_LIBRARY_PATH: steps: - - name: Checkout out hash that triggered CI - uses: actions/checkout@v4 - with: - submodules: 'recursive' - - name: Step Python - uses: actions/setup-python@v4.6.0 - with: - python-version: '3.12.13' - - name: Install Dependencies for Build - run: | - sudo apt-get update && sudo apt-get install libopenmpi-dev netcdf-bin libnetcdf-dev libnetcdff-dev nco libyaml-dev diffutils - - name: install packages - run: | - pip install -v .[test] - - name: run tests - run: | - cd tests - ./run_tests.sh -o + - name: Checkout out hash that triggered CI + uses: actions/checkout@v4 + with: + submodules: 'recursive' + - name: Step Python + uses: actions/setup-python@v4.6.0 + with: + python-version: '3.12.13' + - name: Install Dependencies for Build + run: | + sudo apt-get update && sudo apt-get install libopenmpi-dev netcdf-bin libnetcdf-dev libnetcdff-dev nco libyaml-dev diffutils + - name: install packages + run: | + pip install -v .[test] + - name: run tests + run: | + cd tests + ./run_tests.sh -o diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c975714..60db865 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,38 +2,30 @@ default_language_version: python: python3 repos: -- repo: https://github.com/psf/black + - repo: https://github.com/psf/black rev: 20.8b1 hooks: - - id: black - additional_dependencies: ["click==8.0.4"] + - id: black + additional_dependencies: ["click==8.0.4"] -- repo: https://github.com/pre-commit/mirrors-isort - rev: v5.4.2 + - repo: https://github.com/pre-commit/mirrors-isort + rev: v5.10.1 hooks: - - id: isort - args: ["--profile", "black"] + - id: isort + args: ["--profile", "black"] -- repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.3.0 hooks: - - id: check-toml - - id: check-yaml - - id: end-of-file-fixer - - id: trailing-whitespace -- repo: https://github.com/pycqa/flake8 - rev: 3.9.2 + - id: check-toml + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + + - repo: https://github.com/pycqa/flake8 + rev: 7.3.0 hooks: - - id: flake8 - name: flake8 - language_version: python3 - args: ["--ignore=F401,E203,E501,F841,E302",--config, pyproject.toml] - exclude: | - (?x)^( - .*/__init__.py | - )$ - - id: flake8 - name: flake8 __init__.py files - files: "__init__.py" - # ignore unused import error in __init__.py files - args: ["--ignore=F401,E203", --config, pyproject.toml] + - id: flake8 + name: flake8 + language_version: python3 + additional_dependencies: [Flake8-pyproject, flake8-bugbear] diff --git a/pyfms/py_field_manager/py_field_manager.py b/pyfms/py_field_manager/py_field_manager.py index 9f1ce94..27a5e70 100644 --- a/pyfms/py_field_manager/py_field_manager.py +++ b/pyfms/py_field_manager/py_field_manager.py @@ -139,7 +139,7 @@ def get_subparam_list(self, module: str, varname: str) -> List[Dict]: var = self.get_var(module=module, varname=varname) for key in var: - if type(var[key]) == list: + if isinstance(var[key], list): subparamlist.append(key) return subparamlist @@ -155,7 +155,7 @@ def get_num_subparam(self, module: str, varname: str) -> int: var = self.get_var(module=module, varname=varname) for key in var: - if type(var[key]) == list: + if isinstance(var[key], list): subparamlist.append(key) return len(subparamlist) diff --git a/pyproject.toml b/pyproject.toml index 4368b3d..637e8a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,5 +81,5 @@ source_pkgs = ["pyfms"] [tool.flake8] exclude = ["docs"] -extend-ignore = ["E203","E501","W293","W503","E302","E203","F841"] +extend-ignore = ["E203","E501","W293","W503","E302","E203","F841", "F401", "F824", "B007"] max-line-length = "88" From 82c947e98af5e1ce5a48625fd01dc2369f15500a Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Thu, 23 Apr 2026 14:19:38 -0400 Subject: [PATCH 25/29] Pointing to branch of cFMS for CMake build --- CMakeLists.txt | 2 +- pyfms/.pre-commit-config.yaml | 48 ----------------------------------- 2 files changed, 1 insertion(+), 49 deletions(-) delete mode 100644 pyfms/.pre-commit-config.yaml diff --git a/CMakeLists.txt b/CMakeLists.txt index 2074560..dd27bcf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) FetchContent_Declare( cFMS GIT_REPOSITORY https://github.com/fmalatino/cFMS.git - GIT_TAG cmake + GIT_TAG main ) FetchContent_MakeAvailable(cFMS) diff --git a/pyfms/.pre-commit-config.yaml b/pyfms/.pre-commit-config.yaml deleted file mode 100644 index 6f269ff..0000000 --- a/pyfms/.pre-commit-config.yaml +++ /dev/null @@ -1,48 +0,0 @@ -default_language_version: - python: python3 - -repos: -- repo: https://github.com/psf/black - rev: 20.8b1 - hooks: - - id: black - additional_dependencies: ["click==8.0.4"] - -- repo: https://github.com/pre-commit/mirrors-isort - rev: v5.4.2 - hooks: - - id: isort - args: ["--profile", "black"] - -- repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.4.1 - hooks: - - id: mypy - additional_dependencies: [types-pyyaml] - name: mypy-pace - files: pace - args: [--config-file, setup.cfg, --disallow-untyped-calls] - -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 - hooks: - - id: check-toml - - id: check-yaml - - id: end-of-file-fixer - - id: trailing-whitespace -- repo: https://github.com/pycqa/flake8 - rev: 3.9.2 - hooks: - - id: flake8 - name: flake8 - language_version: python3 - args: [--config, setup.cfg] - exclude: | - (?x)^( - .*/__init__.py | - )$ - - id: flake8 - name: flake8 __init__.py files - files: "__init__.py" - # ignore unused import error in __init__.py files - args: ["--ignore=F401,E203", --config, setup.cfg] From 02085a310a4d5b4ab8a39cf61b367ff807284c51 Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Thu, 23 Apr 2026 14:21:09 -0400 Subject: [PATCH 26/29] Using correct URL for cFMS --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dd27bcf..38ec891 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) FetchContent_Declare( cFMS - GIT_REPOSITORY https://github.com/fmalatino/cFMS.git + GIT_REPOSITORY https://github.com/NOAA-GFDL/cFMS.git GIT_TAG main ) FetchContent_MakeAvailable(cFMS) From 578ddf911599e48563f9992a4d40716a5a58dc03 Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Tue, 28 Apr 2026 11:33:13 -0400 Subject: [PATCH 27/29] Added installation instructions to README --- README.md | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fdcb2a6..694b0d9 100644 --- a/README.md +++ b/README.md @@ -1 +1,61 @@ -# fms2py +# pyFMS + +The `pyFMS` package is a Python-C-Fortran interface (through the use of the `ctypes` package), for access to select methods from the Flexible Modeling System (`FMS`) developed by NOAA/GFDL (https://github.com/NOAA-GFDL/FMS.git). + +## Quickstart - bare metal + +### Build + +The build backend of `pyFMS` is `scikit-build-core`, which allows for the compilation of extension modules for Python packages through CMake. + +`pyFMS` requires: + +- GCC > 9.2 +- MPI +- Python 3.12 + +To clone `pyFMS` from its GitHub repository: + +```shell +git clone https://github.com/NOAA-GFDL/pyFMS.git +``` + +We recommend creating a Python `venv` or `conda` environment for your installation. + +```shell +python -m venv +source ./bin/activate +``` + +Inside of your Python environment, to install `pyFMS` and its dependencies: + +```shell +pip install .[install_options] +``` + +The available `install_options` are: +- `test` +- `dev` (which installs the `test` dependecies as well) +- `extras` (which installs the `dev` dependencies as well) + +For developers we suggest installing with the editable flag `-e` and the verbose flag `-v`: + +```shell +pip install -e -v .[dev] +``` + +use of the editable installation method will allow updates to the Python source code to be reflected in the installation without re-installing. Subsequent installs of `pyFMS` will recompile all extension modules, due to the methods of compilation used by `scikit-build-core`. + +The `CMakeLists.txt` file will be accessed by the build backend during the installation to compile the necessary shared object file from the C-Fortran interface, `cFMS`, to be loaded by the `ctypes` package for use by `pyFMS`. + +### Editing the extension modules of pyFMS + +If you require a different version of `cFMS` and/or its dependency `FMS`, depending on the use case, two methods of use are available: + +1. Edit the `CMakeLists.txt` file to point to the repository containing your custom `cFMS` and/or `FMS` to install it alongside `pyFMS` through the methods outlined above +2. Perform a separate compilation of `cFMS` and/or `FMS` outside of the provided build tools and point to the desired shared object file when running `pyfms.cfms.init(libpath=)` + +## pyFMS as a dependency + +This package can be treated like any Python package when used needed as a dependency; ensure it is defined as a dependency for your project, and import as usual. +Subsequent installs of the dependent package will not trigger a recompilation of the extension modules of pyFMS, unless directly called to due so by a reinstall of pyFMS, as the dependency manager of the build backend for the dependent project will see the requirements of pyFMS are satisfied. If an editable version of pyFMS is needed for the dependent package, please use the method of editable install detailed above. From 5a279b55d3e5150f768c9b046afbd86b1ff4ef31 Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Tue, 28 Apr 2026 11:45:05 -0400 Subject: [PATCH 28/29] Linting --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 694b0d9..efa5641 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The `pyFMS` package is a Python-C-Fortran interface (through the use of the `cty ### Build -The build backend of `pyFMS` is `scikit-build-core`, which allows for the compilation of extension modules for Python packages through CMake. +The build backend of `pyFMS` is `scikit-build-core`, which allows for the compilation of extension modules for Python packages through CMake. `pyFMS` requires: @@ -57,5 +57,5 @@ If you require a different version of `cFMS` and/or its dependency `FMS`, depend ## pyFMS as a dependency -This package can be treated like any Python package when used needed as a dependency; ensure it is defined as a dependency for your project, and import as usual. -Subsequent installs of the dependent package will not trigger a recompilation of the extension modules of pyFMS, unless directly called to due so by a reinstall of pyFMS, as the dependency manager of the build backend for the dependent project will see the requirements of pyFMS are satisfied. If an editable version of pyFMS is needed for the dependent package, please use the method of editable install detailed above. +This package can be treated like any Python package when used needed as a dependency; ensure it is defined as a dependency for your project, and import as usual. +Subsequent installs of the dependent package will not trigger a recompilation of the extension modules of pyFMS, unless directly called to due so by a reinstall of pyFMS, as the dependency manager of the build backend for the dependent project will see the requirements of pyFMS are satisfied. If an editable version of pyFMS is needed for the dependent package, please use the method of editable install detailed above. From 08a116419b58746d2ec817df537237625974ddb6 Mon Sep 17 00:00:00 2001 From: Frank Malatino Date: Tue, 28 Apr 2026 11:48:43 -0400 Subject: [PATCH 29/29] Updating action steps names in Lint workflow --- .github/workflows/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 151e688..31c6dfc 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -19,7 +19,7 @@ jobs: uses: actions/setup-python@v4.6.0 with: python-version: '3.12.13' - - name: Install OpenMPI for gt4py + - name: Install OpenMPI run: | sudo apt-get update && sudo apt-get install libopenmpi-dev netcdf-bin libnetcdf-dev libnetcdff-dev nco libyaml-dev diffutils - name: Install Python packages