Update: GPU knn BVH search for irregular grid

.github/workflows/nvc-build.yml

@@ -2,40 +2,146 @@ name: nvc build
 
 on:
   push:
-    branches: [ "master", "*gpu*" ]
+    branches:
+      - "**gpu**"
+      - "**GPU**"
+      - "**cuda**"
+      - "**CUDA**"
     paths-ignore:
       - '**/README.md'
+      - '*.md'
+      - '*.py'
+
   pull_request:
-    branches: [ "master" ]
+    branches:
+      - "master"
     paths-ignore:
       - '**/README.md'
+      - '*.md'
+      - '*.py'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.ref_name }}
+  cancel-in-progress: true
 
 jobs:
-  build:
+  check-duplicate:
+    runs-on: ubuntu-latest
+    outputs:
+      should_skip: ${{ steps.check.outputs.skip }}
+    steps:
+      - id: check
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if [[ "${{ github.event_name }}" == "push" ]]; then
+            # Check if there is an open PR associated with the current branch
+            PR_COUNT=$(gh pr list --repo ${{ github.repository }} --head ${{ github.ref_name }} --state open --json number | jq length)
+
+            if [[ "$PR_COUNT" -gt 0 ]]; then
+              echo "Open PR found. Skipping this push CI."
+              echo "skip=true" >> $GITHUB_OUTPUT
+            else
+              echo "skip=false" >> $GITHUB_OUTPUT
+            fi
+          else
+            echo "skip=false" >> $GITHUB_OUTPUT
+          fi
+
+  detect-changes:
+    needs: check-duplicate
+    # Only run if the interceptor did not flag to skip
+    if: ${{ needs.check-duplicate.outputs.should_skip != 'true' }}
+    runs-on: ubuntu-latest
+    outputs:
+      docker_changed: ${{ steps.filter.outputs.docker_changed }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            docker_changed:
+              - 'docker/Dockerfile.cuda'
+
+  build-and-push-image:
+    needs: detect-changes
+    if: ${{ needs.detect-changes.outputs.docker_changed == 'true' }}
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Lowercase repository name
+        id: repo_name
+        run: |
+          echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./docker/Dockerfile.cuda
+          push: true
+          tags: ghcr.io/${{ env.REPO_LC }}/nvhpc-build-env:latest
+
+          cache-from: type=registry,ref=ghcr.io/${{ env.REPO_LC }}/nvhpc-build-env:buildcache
+          cache-to: type=registry,ref=ghcr.io/${{ env.REPO_LC }}/nvhpc-build-env:buildcache,mode=max
+
+  nvc-build:
+    needs: [check-duplicate, detect-changes, build-and-push-image]
+    # Skip if the interceptor flagged it (prevents downstream execution)
+    if: ${{ needs.check-duplicate.outputs.should_skip != 'true' }}
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - target_name: "2D_All_Targets"
+            ndims: 2
+          - target_name: "3D_All_Targets"
+            ndims: 3
+
+    name: Build ${{ matrix.target_name }}
+
     container:
-      image: nvcr.io/nvidia/nvhpc:25.7-devel-cuda_multi-ubuntu22.04
+      # need lowercase for ghcr.io
+      image: ghcr.io/geoflac/dynearthsol/nvhpc-build-env:latest
 
     steps:
     - uses: actions/checkout@v4
 
-    - name: Install dependencies
-      run: |
-        apt-get update
-        apt-get install -y libboost-program-options-dev libhdf5-serial-dev make pkg-config
+    - name: Fix git safe directory
+      run: git config --global --add safe.directory '*'
 
-    - name: Build all targets (CPU & GPU)
+    - name: set path for nvc and hdf5 (${{ matrix.ndims }}D)
       run: |
-        NVARCH=$(uname -s)_$(uname -m)
-        export NVHPC_DIR=/opt/nvidia/hpc_sdk/${NVARCH}/25.7
-        export HDF5_INCLUDE_DIR=$(pkg-config --cflags-only-I hdf5-serial | sed 's/-I//g' | tr -d ' ')
-        export HDF5_LIB_DIR=$(pkg-config --libs-only-L hdf5-serial | sed 's/-L//g' | tr -d ' ')
-
-        BASE_MAKE="make hdf5=1 NVHPC_DIR=$NVHPC_DIR HDF5_LIB_DIR=$HDF5_LIB_DIR HDF5_INCLUDE_DIR=$HDF5_INCLUDE_DIR"
-
-        # make cleanall && $BASE_MAKE ndims=2 nprof=1 opt=0
-        # make cleanall && $BASE_MAKE ndims=3 nprof=1 opt=0
-        make cleanall && $BASE_MAKE ndims=2 openacc=1 -j
-        make cleanall && $BASE_MAKE ndims=3 openacc=1 -j
-        make cleanall && $BASE_MAKE ndims=2 openacc=1 nprof=1 -j
-        make cleanall && $BASE_MAKE ndims=3 openacc=1 nprof=1 -j
+        HDF5_INC=$(pkg-config --cflags-only-I hdf5-serial | sed 's/-I//g' | tr -d ' ')
+        HDF5_LIB=$(pkg-config --libs-only-L hdf5-serial | sed 's/-L//g' | tr -d ' ')
+
+        echo "HDF5_INCLUDE_DIR=$HDF5_INC" >> $GITHUB_ENV
+        echo "HDF5_LIB_DIR=$HDF5_LIB" >> $GITHUB_ENV
+        echo "BASE_MAKE=make hdf5=1 NVHPC_DIR=$NVHPC_DIR HDF5_LIB_DIR=$HDF5_LIB HDF5_INCLUDE_DIR=$HDF5_INC" >> $GITHUB_ENV
+
+    - name: make ndims=${{ matrix.ndims }} openacc=1 SM=80 hdf5=1
+      run: make cleanall && $BASE_MAKE ndims=${{ matrix.ndims }} openacc=1 SM=80
+
+    - name: make ndims=${{ matrix.ndims }} openacc=1 nprof=1 SM=80 hdf5=1
+      run: make cleanall && $BASE_MAKE ndims=${{ matrix.ndims }} openacc=1 nprof=1 SM=80
+
+    - name: make ndims=${{ matrix.ndims }} nprof=1 opt=0 hdf5=1
+      run: make cleanall && $BASE_MAKE ndims=${{ matrix.ndims }} nprof=1 opt=0

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "knn-bvh"]
+	path = knn-bvh
+	url = https://github.com/GeoFLAC/knn-bvh.git
+	ignore = dirty

DEVELOPING.md

@@ -0,0 +1,55 @@
+# Development Notes
+
+The main repository is on:
+  https://github.com/GeoFLAC/DynEarthSol
+
+To get the most timely progress, using Github:
+  git clone https://github.com/GeoFLAC/DynEarthSol.git
+
+## To-do list:
+
+High priority:
+* Simple benchmarks for rheology
+* Local remeshing
+* Different init_marker_spacing in regions
+
+Low priority:
+* Save output as vtk format directly
+* Stress BC, esp no-stress sidewall
+* Heatflux BC
+* Frictional heating
+* Adiabatic cooling (adiabatic temperature profile)
+* Internal heating
+* Volume changed induced stress
+
+
+## Design notes:
+
+* Avoid C++ stream for bulk output, as stream is slower than C-style IO.
+* Avoid creating/destroying objects in inner for-loops.
+* Avoid static variables and global variables.
+* Meaning of error codes:
+   1: User input error
+   2: IO error
+  10: Triangulation/tetrahedralization error
+  11: Runtime error
+  12: Assertion error (due to programming)
+
+## Git Submodule Workflow
+
+Since this repository relies on Git submodules (e.g., `knn-bvh`), your development workflow needs to account for them. Submodules are essentially pointers to specific commits in other repositories.
+
+**1. Pulling the latest changes:**
+When you pull updates from the main repository, Git does not automatically update the submodule contents by default. To fetch and update everything in one command, use:
+```bash
+git pull --recurse-submodules
+```
+*(If you simply ran `git pull` and notice your submodules are out of sync, you can fix it by running `git submodule update`.)*
+
+**2. Modifying a submodule:**
+By default, submodules are in a "detached HEAD" state. If you need to modify the code inside a submodule:
+1. `cd` into the submodule directory.
+2. Checkout the appropriate branch (e.g., `git checkout main`).
+3. Make your changes, `git add`, `git commit`, and `git push` from **within** the submodule directory.
+4. `cd` back to the main repository root. The main repo will now see that the submodule pointer has changed. 
+5. `git add` the submodule directory and commit this pointer update to the main repository.

Makefile

@@ -402,16 +402,42 @@ C3X3_LIBNAME = 3x3$(suffix)
 ANN_DIR = nanoflann
 CXXFLAGS += -I$(ANN_DIR)/include
 
+KNN_BVH_DIR = knn-bvh
+ifeq ($(openacc), 1)
+	CXXFLAGS += -I$(KNN_BVH_DIR)/include
+	KNN_BVH_LIB = $(KNN_BVH_DIR)/lib/libknn_bvh.$(ndims)d.a
+	LDFLAGS += $(KNN_BVH_LIB)
+endif
+
 # Enable Array2D structure of Array
 CXXFLAGS += -DSOA
 
 ## Action
 
-.PHONY: all clean take-snapshot
+.PHONY: all clean take-snapshot prepare build
+
+all: prepare
+	$(MAKE) build
+
+prepare:
+ifeq ($(openacc), 1)
+	@if command -v git >/dev/null 2>&1 && git rev-parse --is-inside-work-tree >/dev/null 2>&1; then \
+		if git submodule status $(KNN_BVH_DIR) | grep -q '^[-+]'; then \
+			echo "   Status mismatch. Updating submodule $(KNN_BVH_DIR)..."; \
+			git submodule update --init --recursive $(KNN_BVH_DIR); \
+		fi; \
+	elif [ -f "$(KNN_BVH_DIR)/Makefile" ]; then \
+		:; \
+	else \
+		echo "Error: OpenACC build requires $(KNN_BVH_DIR), but git is unavailable or this source tree is not a git checkout."; \
+		echo "       Please initialize/provide $(KNN_BVH_DIR) before building with openacc=1."; \
+		exit 1; \
+	fi
+endif
 
-all: $(EXE) tetgen/tetgen triangle/triangle take-snapshot
+build: $(EXE) tetgen/tetgen triangle/triangle take-snapshot
 
-$(EXE): $(M_OBJS) $(OBJS) $(C3X3_DIR)/lib$(C3X3_LIBNAME).a
+$(EXE): $(M_OBJS) $(OBJS) $(C3X3_DIR)/lib$(C3X3_LIBNAME).a $(KNN_BVH_LIB)
 		$(CXX) $(M_OBJS) $(OBJS) $(LDFLAGS) $(BOOST_LDFLAGS) \
 			-L$(C3X3_DIR) -l$(C3X3_LIBNAME) \
 			-o $@
@@ -471,6 +497,9 @@ endif
 $(OBJS): %.$(ndims)d$(suffix).o : %.cxx $(INCS)
 	$(CXX) $(CXXFLAGS) $(BOOST_CXXFLAGS) -c $< -o $@
 
+$(KNN_BVH_LIB):
+	$(MAKE) -C $(KNN_BVH_DIR) NDIM=$(ndims)
+
 $(TRI_OBJS): %$(suffix).o : %.c $(TRI_INCS)
 	@# Triangle cannot be compiled with -O2
 	$(CXX) $(CXXFLAGS) -O1 -DTRILIBRARY -DREDUCED -DANSI_DECLARATORS -c $< -o $@
@@ -500,6 +529,9 @@ deepclean:
 cleanall: clean
 	@rm -f $(TET_OBJS) $(TRI_OBJS) $(OBJS) $(EXE)
 	@+$(MAKE) -C $(C3X3_DIR) clean openacc=$(openacc)
+ifeq ($(openacc), 1)
+	@+$(MAKE) -C $(KNN_BVH_DIR) clean NDIM=$(ndims)
+endif
 
 clean:
 	@rm -f $(OBJS) $(EXE)

README.md

@@ -2,6 +2,7 @@
 [![Exodus build](https://github.com/GeoFLAC/DynEarthSol/actions/workflows/exodus-build.yml/badge.svg)](https://github.com/GeoFLAC/DynEarthSol/actions/workflows/exodus-build.yml)
 [![MMG build](https://github.com/GeoFLAC/DynEarthSol/actions/workflows/mmg-build.yml/badge.svg)](https://github.com/GeoFLAC/DynEarthSol/actions/workflows/mmg-build.yml)
 [![macOS build](https://github.com/GeoFLAC/DynEarthSol/actions/workflows/macos-build.yml/badge.svg)](https://github.com/GeoFLAC/DynEarthSol/actions/workflows/macos-build.yml)
+[![nvc build](https://github.com/GeoFLAC/DynEarthSol/actions/workflows/nvc-build.yml/badge.svg)](https://github.com/GeoFLAC/DynEarthSol/actions/workflows/nvc-build.yml)
 
 # Overview
 
@@ -11,6 +12,22 @@ used to study the long-term deformation of Earth's lithosphere and problems
 alike.
 
 # Building DES3D
+
+## Getting the Source Code
+This repository uses Git submodules (e.g., `knn-bvh` for GPU version) to manage certain internal libraries. By default, the Makefile automatically prepares these submodules, fetching them via the internet before the build process begins. 
+
+For environments without internet access (such as certain HPC compute nodes), pre-downloading all dependencies is highly recommended. To ensure all necessary source files are downloaded upfront, clone the repository with its submodules using the `--recurse-submodules` flag:
+
+```bash
+git clone --recurse-submodules https://github.com/GeoFLAC/DynEarthSol.git
+```
+
+If you have already cloned the repository without the submodules, you can initialize and update them by running the following command inside the DynEarthSol directory:
+
+```bash
+git submodule update --init --recursive
+```
+
 ## Requirements
 * You will need a recent C++ compiler that supports C++11 standard. (GNU g++
   4.4 or newer version will suffice.)
@@ -22,6 +39,10 @@ alike.
      the library.
 * You will need Python 2.6+ or 3.2+ and the Numpy package.
 * **macOS users**: For OpenMP support on macOS, see the [LLVM OpenMP library build instructions](#llvm) below.
+### Submodules
+* [knn-bvh](https://github.com/GeoFLAC/knn-bvh): A library for K-Nearest Neighbors (KNN) searches utilizing Bounding Volume Hierarchies (BVH). This submodule provides GPU acceleration for intensive spatial queries (e.g., particle locating, mesh interpolation, or contact detection)
+  * Required when compiling with `openacc=1` (GPU acceleration enabled).
+
 ### Optional packages
 * [Exodus](https://github.com/gsjaardema/seacas/) for importing a mesh in the ExodusII format
   * Suggested building procedure