diff --git a/single-node-refactor/CMakeLists.txt b/single-node-refactor/CMakeLists.txt index 98a745760..fae8c911d 100755 --- a/single-node-refactor/CMakeLists.txt +++ b/single-node-refactor/CMakeLists.txt @@ -29,8 +29,8 @@ endif() if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE RelWithDebInfo) - # set(CMAKE_BUILD_TYPE Release) + # set(CMAKE_BUILD_TYPE RelWithDebInfo) + set(CMAKE_BUILD_TYPE Release) endif(NOT CMAKE_BUILD_TYPE) include_directories(../lib/Elements/matar) @@ -40,20 +40,22 @@ add_subdirectory(../lib/Elements/matar cbin) # include_directories(Mesh-Builder) # add_subdirectory(Mesh-Builder) +find_package(MPI REQUIRED) +add_definitions(-DHAVE_MPI=1) +set(LINKING_LIBRARIES MPI::MPI_CXX) +if (GPU_AWARE_MPI) + add_definitions(-DHAVE_GPU_AWARE_MPI) +# elseif (GPU_SHARED_MEM) +# add_definitions(-DHAVE_GPU_SHARED_MEM) +endif() if (FIERRO_ENABLE_TRILINOS) + find_package(Trilinos REQUIRED) #new # Assume if the CXX compiler exists, the rest do too. - if (EXISTS ${Trilinos_CXX_COMPILER}) set(CMAKE_CXX_COMPILER ${Trilinos_CXX_COMPILER}) set(CMAKE_C_COMPILER ${Trilinos_C_COMPILER}) set(CMAKE_Fortran_COMPILER ${Trilinos_Fortran_COMPILER}) - endif() - if(NOT DISTRIBUTION) - # Make sure to use same compilers and flags as Trilinos - set(CMAKE_CXX_FLAGS "${Trilinos_CXX_COMPILER_FLAGS} ${CMAKE_CXX_FLAGS}") - set(CMAKE_C_FLAGS "${Trilinos_C_COMPILER_FLAGS} ${CMAKE_C_FLAGS}") - set(CMAKE_Fortran_FLAGS "${Trilinos_Fortran_COMPILER_FLAGS} ${CMAKE_Fortran_FLAGS}") - endif() + message("\nFound Trilinos! Here are the details: ") message(" Trilinos_DIR = ${Trilinos_DIR}") @@ -71,12 +73,14 @@ if (FIERRO_ENABLE_TRILINOS) include_directories(${Trilinos_INCLUDE_DIRS} ${Trilinos_TPL_INCLUDE_DIRS}) list(APPEND LINKING_LIBRARIES Trilinos::all_selected_libs) - add_definitions(-DTRILINOS_INTERFACE=1 -DHAVE_MPI=1) + add_definitions(-DTRILINOS_INTERFACE=1 -DHAVE_KOKKOS=1) else() find_package(Kokkos REQUIRED) list(APPEND LINKING_LIBRARIES Kokkos::kokkos) + add_definitions(-DHAVE_KOKKOS=1) endif() -find_package(Matar REQUIRED) +#find_package(Matar REQUIRED) +include_directories(${MATAR_SOURCE_DIR}/src/include) include_directories(src/material_models/artificial_viscosity) include_directories(src/material_models/eos) @@ -96,13 +100,8 @@ include_directories(src/common) include_directories(src/input) add_subdirectory(src/input) - - add_subdirectory(src) - - - # Add uninstall target if(NOT TARGET uninstall) configure_file( diff --git a/single-node-refactor/regression_tests/standard_inputs/confined_preload.yaml b/single-node-refactor/regression_tests/standard_inputs/confined_preload.yaml index 57c43f9cd..52dca23fa 100755 --- a/single-node-refactor/regression_tests/standard_inputs/confined_preload.yaml +++ b/single-node-refactor/regression_tests/standard_inputs/confined_preload.yaml @@ -10,7 +10,7 @@ dynamic_options: mesh_options: source: file - file_path: ./standard_inputs/meshes/preload.geo + file_path: ./standard_inputs/meshes/preload.vtk num_dims: 3 #mesh_options: diff --git a/single-node-refactor/regression_tests/standard_inputs/edge_flat_contact.yaml b/single-node-refactor/regression_tests/standard_inputs/edge_flat_contact.yaml index 8a3b931eb..06309a5c8 100644 --- a/single-node-refactor/regression_tests/standard_inputs/edge_flat_contact.yaml +++ b/single-node-refactor/regression_tests/standard_inputs/edge_flat_contact.yaml @@ -10,7 +10,7 @@ dynamic_options: mesh_options: source: file - file_path: ./standard_inputs/meshes/edge_flat_contact.geo + file_path: ./standard_inputs/meshes/edge_flat_contact.vtk num_dims: 3 #mesh_options: diff --git a/single-node-refactor/regression_tests/standard_inputs/meshes/edge_flat_contact.vtk b/single-node-refactor/regression_tests/standard_inputs/meshes/edge_flat_contact.vtk new file mode 100644 index 000000000..8e748c358 --- /dev/null +++ b/single-node-refactor/regression_tests/standard_inputs/meshes/edge_flat_contact.vtk @@ -0,0 +1,42 @@ +# vtk DataFile Version 3.0 +Hexahedral element example +ASCII +DATASET UNSTRUCTURED_GRID + +POINTS 24 float +-0.500000 -2.000000 1.000000 +-0.500000 0.000000 1.000000 +-0.500000 2.000000 1.000000 +-0.500000 -2.000000 0.000000 +-0.500000 0.000000 0.000000 +-0.500000 2.000000 0.000000 +0.500000 -2.000000 1.000000 +0.500000 0.000000 1.000000 +0.500000 2.000000 1.000000 +0.500000 -2.000000 0.000000 +0.500000 0.000000 0.000000 +0.500000 2.000000 0.000000 +-0.500000 -1.000000 2.500000 +-0.500000 0.000000 2.500000 +-0.500000 1.000000 2.500000 +-0.500000 1.000000 3.500000 +-0.500000 0.000000 3.500000 +-0.500000 -1.000000 3.500000 +0.500000 -1.000000 2.500000 +0.500000 0.000000 2.500000 +0.500000 1.000000 2.500000 +0.500000 1.000000 3.500000 +0.500000 0.000000 3.500000 +0.500000 -1.000000 3.500000 + +CELLS 4 32 +8 6 7 10 9 0 1 4 3 +8 7 8 11 10 1 2 5 4 +8 12 13 16 17 18 19 22 23 +8 13 14 15 16 19 20 21 22 + +CELL_TYPES 4 +12 +12 +12 +12 \ No newline at end of file diff --git a/single-node-refactor/regression_tests/standard_inputs/meshes/preload.vtk b/single-node-refactor/regression_tests/standard_inputs/meshes/preload.vtk new file mode 100644 index 000000000..73d7e0a6a --- /dev/null +++ b/single-node-refactor/regression_tests/standard_inputs/meshes/preload.vtk @@ -0,0 +1,46 @@ +# vtk DataFile Version 3.0 +Hexahedral element example +ASCII +DATASET UNSTRUCTURED_GRID + +POINTS 26 float +1.000000 -1.000000 0.000000 +1.000000 0.000000 0.000000 +1.000000 1.000000 0.000000 +1.000000 1.000000 1.000000 +1.000000 0.000000 1.000000 +1.000000 -1.000000 1.000000 +0.000000 -1.000000 0.000000 +0.000000 0.000000 0.000000 +0.000000 1.000000 0.000000 +0.000000 1.000000 1.000000 +0.000000 0.000000 1.000000 +0.000000 -1.000000 1.000000 +-1.000000 -1.000000 0.000000 +-1.000000 0.000000 0.000000 +-1.000000 1.000000 0.000000 +-1.000000 1.000000 1.000000 +-1.000000 0.000000 1.000000 +-1.000000 -1.000000 1.000000 +-0.250000 -0.250000 0.500000 +-0.250000 0.250000 0.500000 +-0.250000 0.250000 1.500000 +-0.250000 -0.250000 1.500000 +0.250000 -0.250000 0.500000 +0.250000 0.250000 0.500000 +0.250000 0.250000 1.500000 +0.250000 -0.250000 1.500000 + +CELLS 5 40 +8 6 7 10 11 0 1 4 5 +8 7 8 9 10 1 2 3 4 +8 12 13 16 17 6 7 10 11 +8 13 14 15 16 7 8 9 10 +8 18 19 20 21 22 23 24 25 + +CELL_TYPES 5 +12 +12 +12 +12 +12 \ No newline at end of file diff --git a/single-node-refactor/regression_tests/standard_inputs/meshes/sie_expansion_contact.vtk b/single-node-refactor/regression_tests/standard_inputs/meshes/sie_expansion_contact.vtk new file mode 100644 index 000000000..6e552fffe --- /dev/null +++ b/single-node-refactor/regression_tests/standard_inputs/meshes/sie_expansion_contact.vtk @@ -0,0 +1,46 @@ +# vtk DataFile Version 3.0 +Hexahedral element example +ASCII +DATASET UNSTRUCTURED_GRID + +POINTS 26 float +1.000000 -1.000000 0.000000 +1.000000 0.000000 0.000000 +1.000000 1.000000 0.000000 +1.000000 1.000000 1.000000 +1.000000 0.000000 1.000000 +1.000000 -1.000000 1.000000 +0.000000 -1.000000 0.000000 +0.000000 0.000000 0.000000 +0.000000 1.000000 0.000000 +0.000000 1.000000 1.000000 +0.000000 0.000000 1.000000 +0.000000 -1.000000 1.000000 +-1.000000 -1.000000 0.000000 +-1.000000 0.000000 0.000000 +-1.000000 1.000000 0.000000 +-1.000000 1.000000 1.000000 +-1.000000 0.000000 1.000000 +-1.000000 -1.000000 1.000000 +-0.250000 -0.250000 1.000010 +-0.250000 0.250000 1.000010 +-0.250000 0.250000 1.500000 +-0.250000 -0.250000 1.500000 +0.250000 -0.250000 1.000010 +0.250000 0.250000 1.000010 +0.250000 0.250000 1.500000 +0.250000 -0.250000 1.500000 + +CELLS 5 40 +8 6 7 10 11 0 1 4 5 +8 7 8 9 10 1 2 3 4 +8 12 13 16 17 6 7 10 11 +8 13 14 15 16 7 8 9 10 +8 18 19 20 21 22 23 24 25 + +CELL_TYPES 5 +12 +12 +12 +12 +12 \ No newline at end of file diff --git a/single-node-refactor/regression_tests/standard_inputs/meshes/slanted_bounce_contact.vtk b/single-node-refactor/regression_tests/standard_inputs/meshes/slanted_bounce_contact.vtk new file mode 100644 index 000000000..7669220f3 --- /dev/null +++ b/single-node-refactor/regression_tests/standard_inputs/meshes/slanted_bounce_contact.vtk @@ -0,0 +1,36 @@ +# vtk DataFile Version 3.0 +Hexahedral element example +ASCII +DATASET UNSTRUCTURED_GRID + +POINTS 20 float +-0.500000 -2.000000 1.000000 +-0.500000 0.000000 2.000000 +-0.500000 2.000000 1.000000 +-0.500000 -2.000000 0.000000 +-0.500000 0.000000 1.000000 +-0.500000 2.000000 0.000000 +0.500000 -2.000000 1.000000 +0.500000 0.000000 2.000000 +0.500000 2.000000 1.000000 +0.500000 -2.000000 0.000000 +0.500000 0.000000 1.000000 +0.500000 2.000000 0.000000 +0.000000 1.000000 1.600000 +0.000000 2.000000 1.600000 +0.000000 2.000000 2.850000 +0.000000 1.000000 2.850000 +0.450000 1.000000 1.600000 +0.450000 2.000000 1.600000 +0.450000 2.000000 2.850000 +0.450000 1.000000 2.850000 + +CELLS 3 24 +8 6 7 10 9 0 1 4 3 +8 7 8 11 10 1 2 5 4 +8 12 13 14 15 16 17 18 19 + +CELL_TYPES 3 +12 +12 +12 \ No newline at end of file diff --git a/single-node-refactor/regression_tests/standard_inputs/meshes/slanted_impact_contact.vtk b/single-node-refactor/regression_tests/standard_inputs/meshes/slanted_impact_contact.vtk new file mode 100644 index 000000000..cbdc76def --- /dev/null +++ b/single-node-refactor/regression_tests/standard_inputs/meshes/slanted_impact_contact.vtk @@ -0,0 +1,36 @@ +# vtk DataFile Version 3.0 +Hexahedral element example +ASCII +DATASET UNSTRUCTURED_GRID + +POINTS 20 float +-0.500000 -2.000000 1.000000 +-0.500000 0.000000 2.000000 +-0.500000 2.000000 1.000000 +-0.500000 -2.000000 0.000000 +-0.500000 0.000000 0.000000 +-0.500000 2.000000 0.000000 +0.500000 -2.000000 1.000000 +0.500000 0.000000 2.000000 +0.500000 2.000000 1.000000 +0.500000 -2.000000 0.000000 +0.500000 0.000000 0.000000 +0.500000 2.000000 0.000000 +-0.250000 0.000000 2.500000 +-0.250000 1.000000 2.500000 +-0.250000 1.000000 3.500000 +-0.250000 0.000000 3.500000 +0.250000 0.000000 2.500000 +0.250000 1.000000 2.500000 +0.250000 1.000000 3.500000 +0.250000 0.000000 3.500000 + +CELLS 3 24 +8 6 7 10 9 0 1 4 3 +8 7 8 11 10 1 2 5 4 +8 12 13 14 15 16 17 18 19 + +CELL_TYPES 3 +12 +12 +12 \ No newline at end of file diff --git a/single-node-refactor/regression_tests/standard_inputs/sie_expansion_contact.yaml b/single-node-refactor/regression_tests/standard_inputs/sie_expansion_contact.yaml index c488a26c2..b027afe6d 100755 --- a/single-node-refactor/regression_tests/standard_inputs/sie_expansion_contact.yaml +++ b/single-node-refactor/regression_tests/standard_inputs/sie_expansion_contact.yaml @@ -10,7 +10,7 @@ dynamic_options: mesh_options: source: file - file_path: ./standard_inputs/meshes/sie_expansion_contact.geo + file_path: ./standard_inputs/meshes/sie_expansion_contact.vtk num_dims: 3 #mesh_options: diff --git a/single-node-refactor/regression_tests/standard_inputs/slanted_bounce_contact.yaml b/single-node-refactor/regression_tests/standard_inputs/slanted_bounce_contact.yaml index 757047b84..756371543 100644 --- a/single-node-refactor/regression_tests/standard_inputs/slanted_bounce_contact.yaml +++ b/single-node-refactor/regression_tests/standard_inputs/slanted_bounce_contact.yaml @@ -10,7 +10,7 @@ dynamic_options: mesh_options: source: file - file_path: ./standard_inputs/meshes/slanted_bounce_contact.geo + file_path: ./standard_inputs/meshes/slanted_bounce_contact.vtk num_dims: 3 #mesh_options: diff --git a/single-node-refactor/regression_tests/standard_inputs/slanted_impact_contact.yaml b/single-node-refactor/regression_tests/standard_inputs/slanted_impact_contact.yaml index a28b388ac..df1d18652 100644 --- a/single-node-refactor/regression_tests/standard_inputs/slanted_impact_contact.yaml +++ b/single-node-refactor/regression_tests/standard_inputs/slanted_impact_contact.yaml @@ -10,7 +10,7 @@ dynamic_options: mesh_options: source: file - file_path: ./standard_inputs/meshes/slanted_impact_contact.geo + file_path: ./standard_inputs/meshes/slanted_impact_contact.vtk num_dims: 3 #mesh_options: diff --git a/single-node-refactor/regression_tests/standard_inputs/unconfined_preload.yaml b/single-node-refactor/regression_tests/standard_inputs/unconfined_preload.yaml index 24f7b4faf..8ded950df 100755 --- a/single-node-refactor/regression_tests/standard_inputs/unconfined_preload.yaml +++ b/single-node-refactor/regression_tests/standard_inputs/unconfined_preload.yaml @@ -10,7 +10,7 @@ dynamic_options: mesh_options: source: file - file_path: ./standard_inputs/meshes/preload.geo + file_path: ./standard_inputs/meshes/preload.vtk num_dims: 3 #mesh_options: diff --git a/single-node-refactor/regression_tests/test_refactor.py b/single-node-refactor/regression_tests/test_refactor.py index e304c893d..4dec6a3ce 100644 --- a/single-node-refactor/regression_tests/test_refactor.py +++ b/single-node-refactor/regression_tests/test_refactor.py @@ -13,9 +13,17 @@ solvers = ["Fierro"] # Add names of each test +# tests = ["TaylorAnvil", "TaylorAnvil_rz", "Compaction", "Compaction_rz", \ +# "Sedov", "Sod_X", "Sod_Y", "Sod_Z", "Sedov_Erosion", \ +# "Sedov_Read_Ensight", "Sedov_rz_polar", "Abaqus_read", \ +# "Pressure_bc_box","vtu_read","SGTM_cooling_cube", \ +# "lin_vol_frac_two_mat", "Bending-3D-plate", "Vel_bc_box", \ +# "slanted_bounce_contact", "slanted_impact_contact", \ +# "sie_expansion_contact", "confined_preload", "unconfined_preload", \ +# "edge_flat_contact"] tests = ["TaylorAnvil", "TaylorAnvil_rz", "Compaction", "Compaction_rz", \ "Sedov", "Sod_X", "Sod_Y", "Sod_Z", "Sedov_Erosion", \ - "Sedov_Read_Ensight", "Sedov_rz_polar", "Abaqus_read", \ + "Sedov_rz_polar", \ "Pressure_bc_box","vtu_read","SGTM_cooling_cube", \ "lin_vol_frac_two_mat", "Bending-3D-plate", "Vel_bc_box", \ "slanted_bounce_contact", "slanted_impact_contact", \ diff --git a/single-node-refactor/scripts/build-fierro.sh b/single-node-refactor/scripts/build-fierro.sh index 6d6c395c2..6d3d43ed5 100755 --- a/single-node-refactor/scripts/build-fierro.sh +++ b/single-node-refactor/scripts/build-fierro.sh @@ -211,7 +211,6 @@ if [ "$build_action" = "full-app" ]; then elif [ "$trilinos" = "enabled" ]; then source trilinos-install.sh ${kokkos_build_type} ${intel_mkl} ${debug} fi - source matar-install.sh ${kokkos_build_type} ${debug} ${trilinos} source cmake_build.sh ${solver} ${debug} ${trilinos} elif [ "$build_action" = "install-kokkos" ]; then source kokkos-install.sh ${kokkos_build_type} diff --git a/single-node-refactor/scripts/cmake_build.sh b/single-node-refactor/scripts/cmake_build.sh index ccc7e278d..09b724078 100755 --- a/single-node-refactor/scripts/cmake_build.sh +++ b/single-node-refactor/scripts/cmake_build.sh @@ -16,7 +16,6 @@ if [ "$trilinos" = "enabled" ]; then Trilinos_DIR=${TRILINOS_INSTALL_DIR}/lib/cmake/Trilinos fi cmake_options+=( - -D CMAKE_PREFIX_PATH="${MATAR_INSTALL_DIR}" -D Trilinos_DIR="$Trilinos_DIR" -D FIERRO_ENABLE_TRILINOS=ON ) @@ -49,6 +48,6 @@ echo "CMake Options: ${cmake_options[@]}" cmake "${cmake_options[@]}" -B "${SGH_BUILD_DIR}" -S "${SGH_BASE_DIR}" # Build SGH -make -C "${SGH_BUILD_DIR}" -j${SGH_BUILD_CORES} +make -C "${SGH_BUILD_DIR}" -j${FIERRO_BUILD_CORES} cd $basedir diff --git a/single-node-refactor/scripts/trilinos-install.sh b/single-node-refactor/scripts/trilinos-install.sh index fb32c9005..97960e8ff 100644 --- a/single-node-refactor/scripts/trilinos-install.sh +++ b/single-node-refactor/scripts/trilinos-install.sh @@ -13,7 +13,7 @@ echo "Trilinos Kokkos Build Type: $kokkos_build_type" if [ ! -d "${TRILINOS_SOURCE_DIR}" ] then echo "Directory Trilinos does not exist, downloading Trilinos...." - git clone --depth 1 https://github.com/trilinos/Trilinos.git ${TRILINOS_SOURCE_DIR} + git clone --depth 1 --branch trilinos-release-16-0-branch https://github.com/trilinos/Trilinos.git ${TRILINOS_SOURCE_DIR} fi #check if Trilinos build directory exists, create Trilinos/build if it doesn't @@ -117,6 +117,7 @@ ${ADDITIONS[@]} -D Trilinos_ENABLE_Ifpack2=OFF -D Trilinos_ENABLE_Zoltan2=ON -D Trilinos_ENABLE_Anasazi=OFF +-D Trilinos_ENABLE_EXPLICIT_INSTANTIATION=OFF -D MueLu_ENABLE_TESTS=OFF -D Trilinos_ENABLE_ALL_PACKAGES=OFF -D Trilinos_ENABLE_ALL_OPTIONAL_PACKAGES=OFF @@ -167,7 +168,7 @@ cmake "${cmake_options[@]}" -B "${TRILINOS_BUILD_DIR}" -S "${TRILINOS_SOURCE_DIR # Build Trilinos echo "Building Trilinos..." -make -C "${TRILINOS_BUILD_DIR}" -j${MATAR_BUILD_CORES} +make -C "${TRILINOS_BUILD_DIR}" -j${FIERRO_BUILD_CORES} # Install Trilinos echo "Installing Trilinos..." diff --git a/single-node-refactor/src/CMakeLists.txt b/single-node-refactor/src/CMakeLists.txt index 84f284c27..b75ac1d85 100755 --- a/single-node-refactor/src/CMakeLists.txt +++ b/single-node-refactor/src/CMakeLists.txt @@ -38,8 +38,6 @@ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${VECTOR_CXX_FLAGS}") message("\n ****** ADDING FIERRO EXECUTABLE ******** \n ") -add_definitions(-DHAVE_KOKKOS=1) - if (CUDA) add_definitions(-DHAVE_CUDA=1) elseif (HIP) @@ -71,4 +69,4 @@ include_directories(Solvers/level_set_solver/include) add_subdirectory(Solvers/level_set_solver) add_executable(Fierro main.cpp driver.cpp solver.cpp ${COMMON_Files} ${EQUILIBRATION_Files} ${YAML_SRC_Files} ${SGH_3D_SRC_Files} ${SGH_RZ_SRC_Files} ${SGTM_3D_SRC_Files} ${LVLSET_SRC_Files}) -target_link_libraries(Fierro PRIVATE matar Kokkos::kokkos) +target_link_libraries(Fierro ${LINKING_LIBRARIES}) diff --git a/single-node-refactor/src/Solvers/SGH_solver_3D/include/contact.h b/single-node-refactor/src/Solvers/SGH_solver_3D/include/contact.h index d89863593..a4fdeeda0 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_3D/include/contact.h +++ b/single-node-refactor/src/Solvers/SGH_solver_3D/include/contact.h @@ -49,9 +49,9 @@ struct contact_state_t //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void initialize(size_t num_dims, size_t num_nodes_in_patch, const CArrayKokkos &bdy_patches, size_t num_bdy_nodes, size_t num_bdy_patches, CArrayKokkos &patches_in_elem, - CArrayKokkos &elems_in_patch, DCArrayKokkos &nodes_in_elem, + CArrayKokkos &elems_in_patch, DistributedDCArray &nodes_in_elem, CArrayKokkos &nodes_in_patch, CArrayKokkos &bdy_nodes, size_t num_patches, - size_t num_nodes, DCArrayKokkos &coords); + size_t num_nodes, DistributedDCArray &coords); /* * Here is a description of each array below: @@ -186,7 +186,7 @@ KOKKOS_FUNCTION void capture_box(double &vx_max, double &vy_max, double &vz_max, double &ax_max, double &ay_max, double &az_max, double bounding_box[], - const DCArrayKokkos &coords, const CArrayKokkos bdy_patches, + const DistributedDCArray &coords, const CArrayKokkos bdy_patches, const CArrayKokkos nodes_in_patch, int surf_lid, const double &dt); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -208,7 +208,7 @@ void capture_box(double &vx_max, double &vy_max, double &vz_max, //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION void penetration_capture_box(double depth_cap, double bounding_box[], size_t nodes_gid[4], - const DCArrayKokkos &coords); + const DistributedDCArray &coords); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// \fn construct_basis @@ -231,9 +231,9 @@ KOKKOS_FUNCTION void construct_basis(CArrayKokkos nodes_in_patch, CArrayKokkos bdy_patches, const CArrayKokkos &contact_forces, const CArrayKokkos &contact_surface_map, const DCArrayKokkos &corner_force, RaggedRightArrayKokkos corners_in_node, - const DCArrayKokkos &mass, const DCArrayKokkos &coords, + const DistributedDCArray &mass, const DistributedDCArray &coords, CArrayKokkos &num_corners_in_node, - const DCArrayKokkos &vel, double A[3][4], int &surf_lid, const double &del_t); + const DistributedDCArray &vel, double A[3][4], int &surf_lid, const double &del_t); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// \fn construct_penetration_basis @@ -322,9 +322,9 @@ KOKKOS_FUNCTION void get_normal(CArrayKokkos nodes_in_patch, CArrayKokkos bdy_patches, const CArrayKokkos &contact_forces, const CArrayKokkos &contact_surface_map, const DCArrayKokkos &corner_force, RaggedRightArrayKokkos corners_in_node, - const DCArrayKokkos &mass, const DCArrayKokkos &coords, + const DistributedDCArray &mass, const DistributedDCArray &coords, CArrayKokkos num_corners_in_node, - const DCArrayKokkos vel, double &xi_val, double &eta_val, const double &del_t, + const DistributedDCArray vel, double &xi_val, double &eta_val, const double &del_t, double normal[3], const CArrayKokkos &xi, const CArrayKokkos &eta, int &surf_lid); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -338,7 +338,7 @@ void get_normal(CArrayKokkos nodes_in_patch, CArrayKokkos bdy_ /// \param normal kokkos view that will be changed in place //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION -void get_penetration_normal(const DCArrayKokkos &coords, const double &xi_val, const double &eta_val, +void get_penetration_normal(const DistributedDCArray &coords, const double &xi_val, const double &eta_val, double normal[3], const CArrayKokkos &xi, const CArrayKokkos &eta, size_t node_gids[4]); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -362,8 +362,8 @@ KOKKOS_FUNCTION // will be called inside a macro bool get_contact_point(CArrayKokkos nodes_in_patch, CArrayKokkos bdy_patches, CArrayKokkos &contact_forces, CArrayKokkos &contact_surface_map, DCArrayKokkos &corner_force, RaggedRightArrayKokkos corners_in_node, - DCArrayKokkos &mass, DCArrayKokkos &coords, CArrayKokkos bdy_nodes, - CArrayKokkos num_corners_in_node, DCArrayKokkos &vel, + DistributedDCArray &mass, DistributedDCArray &coords, CArrayKokkos bdy_nodes, + CArrayKokkos num_corners_in_node, DistributedDCArray &vel, size_t &node_gid, size_t &node_lid, int &surf_lid, double &xi_val, double &eta_val, double &del_tc, CArrayKokkos &xi, CArrayKokkos &eta); @@ -397,8 +397,8 @@ KOKKOS_FUNCTION bool contact_check(CArrayKokkos nodes_in_patch, CArrayKokkos bdy_patches, CArrayKokkos &contact_forces, CArrayKokkos &contact_surface_map, DCArrayKokkos &corner_force, RaggedRightArrayKokkos corners_in_node, - DCArrayKokkos &mass, DCArrayKokkos &coords, CArrayKokkos bdy_nodes, - CArrayKokkos num_corners_in_node, DCArrayKokkos &vel, + DistributedDCArray &mass, DistributedDCArray &coords, CArrayKokkos bdy_nodes, + CArrayKokkos num_corners_in_node, DistributedDCArray &vel, size_t &node_gid, size_t &node_lid, int &surf_lid, double &xi_val, double &eta_val, const double &del_t, CArrayKokkos &xi, CArrayKokkos &eta, double &del_tc); @@ -421,9 +421,9 @@ bool contact_check(CArrayKokkos nodes_in_patch, CArrayKokkos b //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION void frictionless_increment(ViewCArrayKokkos &pair_vars, size_t &contact_id, const CArrayKokkos &xi, const CArrayKokkos &eta, const double &del_t, - DCArrayKokkos coords, CArrayKokkos bdy_nodes, ViewCArrayKokkos &contact_surface_map, - DCArrayKokkos mass, CArrayKokkos contact_forces, DCArrayKokkos corner_force, - DCArrayKokkos vel, RaggedRightArrayKokkos corners_in_node, + DistributedDCArray coords, CArrayKokkos bdy_nodes, ViewCArrayKokkos &contact_surface_map, + DistributedDCArray mass, CArrayKokkos contact_forces, DCArrayKokkos corner_force, + DistributedDCArray vel, RaggedRightArrayKokkos corners_in_node, CArrayKokkos num_corners_in_node); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -467,9 +467,9 @@ bool should_remove(ViewCArrayKokkos &pair_vars, CArrayKokkos nodes_in_patch, CArrayKokkos bdy_patches, const CArrayKokkos &contact_forces, const CArrayKokkos &contact_surface_map, const DCArrayKokkos &corner_force, RaggedRightArrayKokkos corners_in_node, - const DCArrayKokkos &mass, const DCArrayKokkos &coords, + const DistributedDCArray &mass, const DistributedDCArray &coords, CArrayKokkos num_corners_in_node, CArrayKokkos bdy_nodes, - DCArrayKokkos vel, const double &del_t, + DistributedDCArray vel, const double &del_t, const CArrayKokkos &xi, const CArrayKokkos &eta, int &surf_lid); /// end of pair specific functions ********************************************************************************* @@ -486,7 +486,7 @@ bool should_remove(ViewCArrayKokkos &pair_vars, /// \param num_nodes_found number of nodes that could potentially contact the patch (used to access possible_nodes) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void find_nodes(double &vx_max, double &vy_max, double &vz_max, double &ax_max, double &ay_max, double &az_max, - DCArrayKokkos &coords, CArrayKokkos bdy_patches, CArrayKokkos nodes_in_patch, + DistributedDCArray &coords, CArrayKokkos bdy_patches, CArrayKokkos nodes_in_patch, int &surf_lid, const double &del_t, size_t &Sx, size_t &Sy, size_t &Sz, double bounding_box[], double &x_min, double &y_min, double &z_min, double &bucket_size, CArrayKokkos &buckets, CArrayKokkos &possible_nodes, CArrayKokkos &contact_surface_map, size_t &num_nodes_found, @@ -517,8 +517,8 @@ bool get_edge_pair(double normal1[3], double normal2[3], size_t &node_gid, const CArrayKokkos nodes_in_patch, CArrayKokkos &contact_surface_map, CArrayKokkos bdy_nodes, CArrayKokkos &contact_forces, DCArrayKokkos &corner_force, RaggedRightArrayKokkos corners_in_node, - DCArrayKokkos &mass, DCArrayKokkos &coords, - CArrayKokkos num_corners_in_node, DCArrayKokkos &vel, + DistributedDCArray &mass, DistributedDCArray &coords, + CArrayKokkos num_corners_in_node, DistributedDCArray &vel, CArrayKokkos &xi, CArrayKokkos &eta, CArrayKokkos &num_surfs_in_node, RaggedRightArrayKokkos &surfs_in_node); @@ -548,7 +548,7 @@ void remove_pair(size_t &contact_id, const CArrayKokkos &node_patch_pai /// \return true if the node is penetrating the element; false otherwise //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION -bool penetration_check(size_t node_gid, ViewCArrayKokkos &surfaces, const DCArrayKokkos &coords, +bool penetration_check(size_t node_gid, ViewCArrayKokkos &surfaces, const DistributedDCArray &coords, const ViewCArrayKokkos &xi, const ViewCArrayKokkos &eta); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -578,13 +578,13 @@ void isoparametric_inverse(const double pos[3], const double elem_pos[3][8], dou /// \param elem_pos element nodal coordinates /// \param iso_pos isoparametric position (xi,eta,zeta) output //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -void find_penetrating_nodes(double depth_cap, DCArrayKokkos &coords, +void find_penetrating_nodes(double depth_cap, DistributedDCArray &coords, double num_bdy_patches, CArrayKokkos &penetration_surfaces, CArrayKokkos bdy_patches, double Sx, double Sy, double Sz, double x_min, double y_min, double z_min, double bucket_size, CArrayKokkos &buckets, CArrayKokkos &node_penetrations, CArrayKokkos &npoint, size_t num_patches, CArrayKokkos &nbox, CArrayKokkos &nsort, - DCArrayKokkos nodes_in_elem, CArrayKokkos elems_in_patch, + DistributedDCArray nodes_in_elem, CArrayKokkos elems_in_patch, size_t num_bdy_nodes, CArrayKokkos nodes_in_patch, CArrayKokkos xi, CArrayKokkos eta); @@ -600,9 +600,9 @@ void find_penetrating_nodes(double depth_cap, DCArrayKokkos &coords, /// \param State State object /// \param mesh mesh object //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -void sort(DCArrayKokkos &coords, size_t num_bdy_nodes, CArrayKokkos bdy_nodes, - DCArrayKokkos &vel, CArrayKokkos num_corners_in_node, RaggedRightArrayKokkos corners_in_node, - DCArrayKokkos &corner_force, CArrayKokkos &contact_forces, DCArrayKokkos &mass, +void sort(DistributedDCArray &coords, size_t num_bdy_nodes, CArrayKokkos bdy_nodes, + DistributedDCArray &vel, CArrayKokkos num_corners_in_node, RaggedRightArrayKokkos corners_in_node, + DCArrayKokkos &corner_force, CArrayKokkos &contact_forces, DistributedDCArray &mass, double &x_max, double &y_max, double &z_max, double &x_min, double &y_min, double &z_min, double &vx_max, double &vy_max, double &vz_max, double &ax_max, double &ay_max, double &az_max, size_t &Sx, size_t &Sy, size_t &Sz, double &bucket_size, CArrayKokkos &nbox, CArrayKokkos &lbox, CArrayKokkos &nsort, CArrayKokkos &npoint); @@ -618,11 +618,11 @@ void sort(DCArrayKokkos &coords, size_t num_bdy_nodes, CArrayKokkos &coords, +void penetration_sweep(double x_min, double y_min, double z_min, double bounding_box[], DistributedDCArray &coords, double num_bdy_patches, CArrayKokkos &penetration_surfaces, CArrayKokkos bdy_patches, double Sx, double Sy, double Sz, double bucket_size, CArrayKokkos &buckets, CArrayKokkos &node_penetrations, CArrayKokkos &npoint, size_t num_patches, - CArrayKokkos &nbox, CArrayKokkos &nsort, DCArrayKokkos nodes_in_elem, + CArrayKokkos &nbox, CArrayKokkos &nsort, DistributedDCArray nodes_in_elem, CArrayKokkos elems_in_patch, size_t num_bdy_nodes, CArrayKokkos nodes_in_patch, const CArrayKokkos &xi, const CArrayKokkos &eta, double x_max, double y_max, double z_max, DCArrayKokkos &num_active, RaggedRightArrayKokkos elems_in_node, CArrayKokkos num_nodes_in_elem, @@ -640,8 +640,8 @@ void penetration_sweep(double x_min, double y_min, double z_min, double bounding //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void force_resolution(CArrayKokkos &f_c_incs, DCArrayKokkos num_active, CArrayKokkos &active_set, CArrayKokkos &node_patch_pairs, CArrayKokkos &pair_vars, CArrayKokkos &contact_surface_map, - DCArrayKokkos &coords, CArrayKokkos bdy_nodes, DCArrayKokkos &mass, - CArrayKokkos &contact_forces, DCArrayKokkos &corner_force, DCArrayKokkos &vel, + DistributedDCArray &coords, CArrayKokkos bdy_nodes, DistributedDCArray &mass, + CArrayKokkos &contact_forces, DCArrayKokkos &corner_force, DistributedDCArray &vel, RaggedRightArrayKokkos corners_in_node, CArrayKokkos num_corners_in_node, const CArrayKokkos &xi, const CArrayKokkos &eta, const double &del_t, CArrayKokkos &contact_force, size_t num_bdy_nodes); @@ -659,9 +659,9 @@ void remove_pairs(DCArrayKokkos num_active, CArrayKokkos &acti CArrayKokkos &node_patch_pairs, CArrayKokkos nodes_in_patch, CArrayKokkos bdy_patches, CArrayKokkos &contact_forces, CArrayKokkos &contact_surface_map, DCArrayKokkos &corner_force, RaggedRightArrayKokkos corners_in_node, - DCArrayKokkos &mass, DCArrayKokkos &coords, + DistributedDCArray &mass, DistributedDCArray &coords, CArrayKokkos num_corners_in_node, CArrayKokkos bdy_nodes, - DCArrayKokkos &vel, const double &del_t, + DistributedDCArray &vel, const double &del_t, const CArrayKokkos &xi, const CArrayKokkos &eta, size_t num_bdy_patches); /// end of functions called in boundary.cpp ************************************************************************ diff --git a/single-node-refactor/src/Solvers/SGH_solver_3D/include/sgh_solver_3D.h b/single-node-refactor/src/Solvers/SGH_solver_3D/include/sgh_solver_3D.h index 1dab3d579..90223f782 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_3D/include/sgh_solver_3D.h +++ b/single-node-refactor/src/Solvers/SGH_solver_3D/include/sgh_solver_3D.h @@ -150,6 +150,10 @@ class SGH3D : public Solver ~SGH3D() = default; + //member variables + CommPlan node_velocity_comms; + CommPlan node_mass_comms; + ///////////////////////////////////////////////////////////////////////////// /// /// \fn Initialize @@ -157,14 +161,14 @@ class SGH3D : public Solver /// \brief Initializes data associated with the SGH3D solver /// ///////////////////////////////////////////////////////////////////////////// - void initialize(SimulationParameters_t& SimulationParamaters, + void initialize(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, - State_t& State) const override; + State_t& State) override; - void initialize_material_state(SimulationParameters_t& SimulationParamaters, + void initialize_material_state(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, @@ -177,7 +181,7 @@ class SGH3D : public Solver /// \brief Calls setup_sgh, which initializes state and material data /// ///////////////////////////////////////////////////////////////////////////// - void setup(SimulationParameters_t& SimulationParamaters, + void setup(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, @@ -191,7 +195,7 @@ class SGH3D : public Solver /// /// ///////////////////////////////////////////////////////////////////////////// - void execute(SimulationParameters_t& SimulationParamaters, + void execute(SimulationParameters_t& SimulationParameters, Material_t& Materials, BoundaryCondition_t& Boundary, Mesh_t& mesh, @@ -213,7 +217,7 @@ class SGH3D : public Solver /// \return /// ///////////////////////////////////////////////////////////////////////////// - void finalize(SimulationParameters_t& SimulationParamaters, + void finalize(SimulationParameters_t& SimulationParameters, Material_t& Materials, BoundaryCondition_t& Boundary) const override { @@ -226,21 +230,21 @@ class SGH3D : public Solver void boundary_velocity( const Mesh_t& mesh, const BoundaryCondition_t& Boundary, - DCArrayKokkos& node_vel, + DistributedDCArray& node_vel, const double time_value) const; void boundary_contact( const Mesh_t& mesh, const BoundaryCondition_t& Boundary, - DCArrayKokkos& node_vel, + DistributedDCArray& node_vel, const double time_value) const; void boundary_contact_force(State_t& State, const Mesh_t &mesh, const double &del_t, contact_state_t &Contact_State); void boundary_stress(const Mesh_t& mesh, const BoundaryCondition_t& BoundaryConditions, - DCArrayKokkos& node_bdy_force, - DCArrayKokkos& node_coords, + DistributedDCArray& node_bdy_force, + DistributedDCArray& node_coords, const double time_value) const; // **** Functions defined in energy_sgh.cpp **** // @@ -248,8 +252,8 @@ class SGH3D : public Solver const double rk_alpha, const double dt, const Mesh_t& mesh, - const DCArrayKokkos& node_vel, - const DCArrayKokkos& node_vel_n0, + const DistributedDCArray& node_vel, + const DistributedDCArray& node_vel_n0, const DRaggedRightArrayKokkos& MaterialPoints_sie, const DRaggedRightArrayKokkos& MaterialPoints_sie_n0, const DRaggedRightArrayKokkos& MaterialPoints_mass, @@ -267,8 +271,8 @@ class SGH3D : public Solver const DCArrayKokkos& GaussPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_eroded, const DCArrayKokkos& corner_force, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_sie, const DRaggedRightArrayKokkos& MaterialPoints_pres, @@ -292,20 +296,20 @@ class SGH3D : public Solver double dt, const size_t num_dims, const size_t num_nodes, - DCArrayKokkos& node_coords, - DCArrayKokkos& node_coords_n0, - const DCArrayKokkos& node_vel, - const DCArrayKokkos& node_vel_n0) const; + DistributedDCArray& node_coords, + DistributedDCArray& node_coords_n0, + const DistributedDCArray& node_vel, + const DistributedDCArray& node_vel_n0) const; // **** Functions defined in momentum.cpp **** // void update_velocity( double rk_alpha, double dt, const Mesh_t& mesh, - DCArrayKokkos& node_vel, - DCArrayKokkos& node_vel_n0, - const DCArrayKokkos& node_mass, - const DCArrayKokkos& node_force, + DistributedDCArray& node_vel, + DistributedDCArray& node_vel_n0, + const DistributedDCArray& node_mass, + const DistributedDCArray& node_force, const DCArrayKokkos& corner_force, const CArrayKokkos & contact_force, bool doing_contact) const; @@ -313,15 +317,15 @@ class SGH3D : public Solver void get_velgrad( DCArrayKokkos& vel_grad, const Mesh_t mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DCArrayKokkos& elem_vol) const; void get_divergence( DCArrayKokkos& GaussPoints_div, const Mesh_t mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DCArrayKokkos& GaussPoints_vol) const; KOKKOS_FUNCTION @@ -334,8 +338,8 @@ class SGH3D : public Solver void update_state( const Material_t& Materials, const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DCArrayKokkos& GaussPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_pres, @@ -363,8 +367,8 @@ class SGH3D : public Solver const Material_t& Materials, const Mesh_t& mesh, const DCArrayKokkos& GaussPoints_vol, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DCArrayKokkos& GaussPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_sie, @@ -388,10 +392,10 @@ class SGH3D : public Solver // **** Functions defined in time_integration.cpp **** // // NOTE: Consider pulling up void rk_init( - DCArrayKokkos& node_coords, - DCArrayKokkos& node_coords_n0, - DCArrayKokkos& node_vel, - DCArrayKokkos& node_vel_n0, + DistributedDCArray& node_coords, + DistributedDCArray& node_coords_n0, + DistributedDCArray& node_vel, + DistributedDCArray& node_vel_n0, DRaggedRightArrayKokkos& MaterialPoints_sie, DRaggedRightArrayKokkos& MaterialPoints_sie_n0, DRaggedRightArrayKokkos& MaterialPoints_stress, @@ -404,8 +408,8 @@ class SGH3D : public Solver void get_timestep( Mesh_t& mesh, - DCArrayKokkos& node_coords, - DCArrayKokkos& node_vel, + DistributedDCArray& node_coords, + DistributedDCArray& node_vel, DCArrayKokkos& GaussPoints_vol, DRaggedRightArrayKokkos& MaterialPoints_sspd, DRaggedRightArrayKokkos& MaterialPoints_eroded, @@ -447,8 +451,8 @@ class SGH3D : public Solver const double sie, const DCArrayKokkos& GaussPoints_vel_grad, const ViewCArrayKokkos& elem_node_gids, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const double vol, const double dt, const double rk_alpha); @@ -462,9 +466,8 @@ double sum_domain_internal_energy( double sum_domain_kinetic_energy( const Mesh_t& mesh, - const DCArrayKokkos& node_vel, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_mass); + const DistributedDCArray& node_vel, + const DistributedDCArray& node_mass); double sum_domain_material_mass( const DRaggedRightArrayKokkos& MaterialPoints_mass, @@ -472,8 +475,8 @@ double sum_domain_material_mass( const size_t mat_id); double sum_domain_node_mass(const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_mass); + const DistributedDCArray& node_coords, + const DistributedDCArray& node_mass); void set_corner_force_zero(const Mesh_t& mesh, const DCArrayKokkos& corner_force); diff --git a/single-node-refactor/src/Solvers/SGH_solver_3D/src/boundary.cpp b/single-node-refactor/src/Solvers/SGH_solver_3D/src/boundary.cpp index 019dcccea..52417427a 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_3D/src/boundary.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_3D/src/boundary.cpp @@ -50,7 +50,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ///////////////////////////////////////////////////////////////////////////// void SGH3D::boundary_velocity(const Mesh_t& mesh, const BoundaryCondition_t& BoundaryConditions, - DCArrayKokkos& node_vel, + DistributedDCArray& node_vel, const double time_value) const { size_t num_vel_bdy_sets = BoundaryConditions.num_vel_bdy_sets_in_solver.host(this->solver_id); @@ -96,7 +96,7 @@ void SGH3D::boundary_velocity(const Mesh_t& mesh, ///////////////////////////////////////////////////////////////////////////// void SGH3D::boundary_contact(const Mesh_t& mesh, const BoundaryCondition_t& BoundaryConditions, - DCArrayKokkos& node_vel, + DistributedDCArray& node_vel, const double time_value) const { return; @@ -165,8 +165,8 @@ void SGH3D:: boundary_contact_force(State_t& State, const Mesh_t &mesh, const do ///////////////////////////////////////////////////////////////////////////// void SGH3D::boundary_stress(const Mesh_t& mesh, const BoundaryCondition_t& BoundaryConditions, - DCArrayKokkos& node_bdy_force, - DCArrayKokkos& node_coords, + DistributedDCArray& node_bdy_force, + DistributedDCArray& node_coords, const double time_value) const { diff --git a/single-node-refactor/src/Solvers/SGH_solver_3D/src/contact.cpp b/single-node-refactor/src/Solvers/SGH_solver_3D/src/contact.cpp index f34854ae1..fb989ac3f 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_3D/src/contact.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_3D/src/contact.cpp @@ -9,7 +9,7 @@ KOKKOS_FUNCTION void capture_box(double &vx_max, double &vy_max, double &vz_max, double &ax_max, double &ay_max, double &az_max, double bounding_box[], - const DCArrayKokkos &coords, const CArrayKokkos bdy_patches, + const DistributedDCArray &coords, const CArrayKokkos bdy_patches, const CArrayKokkos nodes_in_patch, int surf_lid, const double &dt) { // WARNING: specific to first order hex elements @@ -52,7 +52,7 @@ void capture_box(double &vx_max, double &vy_max, double &vz_max, KOKKOS_FUNCTION void penetration_capture_box(double depth_cap, double bounding_box[], size_t nodes_gid[4], - const DCArrayKokkos &coords) + const DistributedDCArray &coords) { // WARNING: specific to first order hex elements // [- or +][x y z][node0 node1 node2 node3] @@ -88,9 +88,9 @@ KOKKOS_FUNCTION void construct_basis(CArrayKokkos nodes_in_patch, CArrayKokkos bdy_patches, const CArrayKokkos &contact_forces, const CArrayKokkos &contact_surface_map, const DCArrayKokkos &corner_force, RaggedRightArrayKokkos corners_in_node, - const DCArrayKokkos &mass, const DCArrayKokkos &coords, + const DistributedDCArray &mass, const DistributedDCArray &coords, CArrayKokkos &num_corners_in_node, - const DCArrayKokkos &vel, double A[3][4], int &surf_lid, const double &del_t) + const DistributedDCArray &vel, double A[3][4], int &surf_lid, const double &del_t) { for (int i = 0; i < 3; i++) { @@ -109,7 +109,7 @@ void construct_basis(CArrayKokkos nodes_in_patch, CArrayKokkos } // end construct_basis KOKKOS_FUNCTION -void construct_penetration_basis(size_t node_gids[4], const DCArrayKokkos &coords, double A[3][4]) +void construct_penetration_basis(size_t node_gids[4], const DistributedDCArray &coords, double A[3][4]) { for (int i = 0; i < 3; i++) { @@ -165,9 +165,9 @@ KOKKOS_FUNCTION void get_normal(CArrayKokkos nodes_in_patch, CArrayKokkos bdy_patches, const CArrayKokkos &contact_forces, const CArrayKokkos &contact_surface_map, const DCArrayKokkos &corner_force, RaggedRightArrayKokkos corners_in_node, - const DCArrayKokkos &mass, const DCArrayKokkos &coords, + const DistributedDCArray &mass, const DistributedDCArray &coords, CArrayKokkos num_corners_in_node, - const DCArrayKokkos vel, double &xi_val, double &eta_val, const double &del_t, + const DistributedDCArray vel, double &xi_val, double &eta_val, const double &del_t, double normal[3], const CArrayKokkos &xi, const CArrayKokkos &eta, int &surf_lid) { // Get the derivative arrays @@ -218,7 +218,7 @@ void get_normal(CArrayKokkos nodes_in_patch, CArrayKokkos bdy_ } KOKKOS_FUNCTION -void get_penetration_normal(const DCArrayKokkos &coords, const double &xi_val, const double &eta_val, +void get_penetration_normal(const DistributedDCArray &coords, const double &xi_val, const double &eta_val, double normal[3], const CArrayKokkos &xi, const CArrayKokkos &eta, size_t node_gids[4]) { // Get the derivative arrays @@ -269,8 +269,8 @@ KOKKOS_FUNCTION // will be called inside a macro bool get_contact_point(CArrayKokkos nodes_in_patch, CArrayKokkos bdy_patches, CArrayKokkos &contact_forces, CArrayKokkos &contact_surface_map, DCArrayKokkos &corner_force, RaggedRightArrayKokkos corners_in_node, - DCArrayKokkos &mass, DCArrayKokkos &coords, CArrayKokkos bdy_nodes, - CArrayKokkos num_corners_in_node, DCArrayKokkos &vel, + DistributedDCArray &mass, DistributedDCArray &coords, CArrayKokkos bdy_nodes, + CArrayKokkos num_corners_in_node, DistributedDCArray &vel, size_t &node_gid, size_t &node_lid, int &surf_lid, double &xi_val, double &eta_val, double &del_tc, CArrayKokkos &xi, CArrayKokkos &eta) { @@ -469,8 +469,8 @@ KOKKOS_FUNCTION bool contact_check(CArrayKokkos nodes_in_patch, CArrayKokkos bdy_patches, CArrayKokkos &contact_forces, CArrayKokkos &contact_surface_map, DCArrayKokkos &corner_force, RaggedRightArrayKokkos corners_in_node, - DCArrayKokkos &mass, DCArrayKokkos &coords, CArrayKokkos bdy_nodes, - CArrayKokkos num_corners_in_node, DCArrayKokkos &vel, + DistributedDCArray &mass, DistributedDCArray &coords, CArrayKokkos bdy_nodes, + CArrayKokkos num_corners_in_node, DistributedDCArray &vel, size_t &node_gid, size_t &node_lid, int &surf_lid, double &xi_val, double &eta_val, const double &del_t, CArrayKokkos &xi, CArrayKokkos &eta, double &del_tc) { @@ -595,9 +595,9 @@ bool contact_check(CArrayKokkos nodes_in_patch, CArrayKokkos b KOKKOS_FUNCTION void frictionless_increment(ViewCArrayKokkos &pair_vars, size_t &contact_id, const CArrayKokkos &xi, const CArrayKokkos &eta, const double &del_t, - DCArrayKokkos coords, CArrayKokkos bdy_nodes, ViewCArrayKokkos &contact_surface_map, - DCArrayKokkos mass, CArrayKokkos contact_forces, DCArrayKokkos corner_force, - DCArrayKokkos vel, RaggedRightArrayKokkos corners_in_node, + DistributedDCArray coords, CArrayKokkos bdy_nodes, ViewCArrayKokkos &contact_surface_map, + DistributedDCArray mass, CArrayKokkos contact_forces, DCArrayKokkos corner_force, + DistributedDCArray vel, RaggedRightArrayKokkos corners_in_node, CArrayKokkos num_corners_in_node) { // In order to understand this, just see this PDF: @@ -937,9 +937,9 @@ bool should_remove(ViewCArrayKokkos &pair_vars, CArrayKokkos nodes_in_patch, CArrayKokkos bdy_patches, const CArrayKokkos &contact_forces, const CArrayKokkos &contact_surface_map, const DCArrayKokkos &corner_force, RaggedRightArrayKokkos corners_in_node, - const DCArrayKokkos &mass, const DCArrayKokkos &coords, + const DistributedDCArray &mass, const DistributedDCArray &coords, CArrayKokkos num_corners_in_node, CArrayKokkos bdy_nodes, - DCArrayKokkos vel, const double &del_t, + DistributedDCArray vel, const double &del_t, const CArrayKokkos &xi, const CArrayKokkos &eta, int &surf_lid) { if (pair_vars(7) == 0.0 || fabs(pair_vars(0)) > 1.0 + edge_tol || fabs(pair_vars(1)) > 1.0 + edge_tol) @@ -968,7 +968,7 @@ bool should_remove(ViewCArrayKokkos &pair_vars, /// start of contact state functions ******************************************************************************* void find_nodes(double &vx_max, double &vy_max, double &vz_max, double &ax_max, double &ay_max, double &az_max, - DCArrayKokkos &coords, CArrayKokkos bdy_patches, CArrayKokkos nodes_in_patch, + DistributedDCArray &coords, CArrayKokkos bdy_patches, CArrayKokkos nodes_in_patch, int &surf_lid, const double &del_t, size_t &Sx, size_t &Sy, size_t &Sz, double bounding_box[], double &x_min, double &y_min, double &z_min, double &bucket_size, CArrayKokkos &buckets, CArrayKokkos &possible_nodes, CArrayKokkos &contact_surface_map, size_t &num_nodes_found, @@ -1045,8 +1045,8 @@ bool get_edge_pair(double normal1[3], double normal2[3], size_t &node_gid, const CArrayKokkos nodes_in_patch, CArrayKokkos &contact_surface_map, CArrayKokkos bdy_nodes, CArrayKokkos &contact_forces, DCArrayKokkos &corner_force, RaggedRightArrayKokkos corners_in_node, - DCArrayKokkos &mass, DCArrayKokkos &coords, - CArrayKokkos num_corners_in_node, DCArrayKokkos &vel, + DistributedDCArray &mass, DistributedDCArray &coords, + CArrayKokkos num_corners_in_node, DistributedDCArray &vel, CArrayKokkos &xi, CArrayKokkos &eta, CArrayKokkos &num_surfs_in_node, RaggedRightArrayKokkos &surfs_in_node) { @@ -1143,7 +1143,7 @@ void remove_pair(size_t &contact_id, const CArrayKokkos &node_patch_pai } // end remove_pair KOKKOS_FUNCTION -bool penetration_check(size_t node_gid, ViewCArrayKokkos &surfaces, const DCArrayKokkos &coords, const CArrayKokkos &xi, +bool penetration_check(size_t node_gid, ViewCArrayKokkos &surfaces, const DistributedDCArray &coords, const CArrayKokkos &xi, const CArrayKokkos &eta) { // define output boolean @@ -1342,13 +1342,13 @@ void isoparametric_inverse(const double pos[3], const double elem_pos[3][8], dou } // end isoparametric_inverse -void find_penetrating_nodes(double depth_cap, DCArrayKokkos &coords, +void find_penetrating_nodes(double depth_cap, DistributedDCArray &coords, double num_bdy_patches, CArrayKokkos &penetration_surfaces, CArrayKokkos bdy_patches, double Sx, double Sy, double Sz, double x_min, double y_min, double z_min, double bucket_size, CArrayKokkos &buckets, CArrayKokkos &node_penetrations, CArrayKokkos &npoint, size_t num_patches, CArrayKokkos &nbox, CArrayKokkos &nsort, - DCArrayKokkos nodes_in_elem, CArrayKokkos elems_in_patch, + DistributedDCArray nodes_in_elem, CArrayKokkos elems_in_patch, size_t num_bdy_nodes, CArrayKokkos nodes_in_patch, CArrayKokkos xi, CArrayKokkos eta) { @@ -1430,9 +1430,9 @@ void find_penetrating_nodes(double depth_cap, DCArrayKokkos &coords, /// start of functions called in boundary.cpp ********************************************************************** -void sort(DCArrayKokkos &coords, size_t num_bdy_nodes, CArrayKokkos bdy_nodes, - DCArrayKokkos &vel, CArrayKokkos num_corners_in_node, RaggedRightArrayKokkos corners_in_node, - DCArrayKokkos &corner_force, CArrayKokkos &contact_forces, DCArrayKokkos &mass, +void sort(DistributedDCArray &coords, size_t num_bdy_nodes, CArrayKokkos bdy_nodes, + DistributedDCArray &vel, CArrayKokkos num_corners_in_node, RaggedRightArrayKokkos corners_in_node, + DCArrayKokkos &corner_force, CArrayKokkos &contact_forces, DistributedDCArray &mass, double &x_max, double &y_max, double &z_max, double &x_min, double &y_min, double &z_min, double &vx_max, double &vy_max, double &vz_max, double &ax_max, double &ay_max, double &az_max, size_t &Sx, size_t &Sy, size_t &Sz, double &bucket_size, CArrayKokkos &nbox, CArrayKokkos &lbox, CArrayKokkos &nsort, CArrayKokkos &npoint) @@ -1663,11 +1663,11 @@ void sort(DCArrayKokkos &coords, size_t num_bdy_nodes, CArrayKokkos &coords, +void penetration_sweep(double x_min, double y_min, double z_min, double bounding_box[], DistributedDCArray &coords, double num_bdy_patches, CArrayKokkos &penetration_surfaces, CArrayKokkos bdy_patches, double Sx, double Sy, double Sz, double bucket_size, CArrayKokkos &buckets, CArrayKokkos &node_penetrations, CArrayKokkos &npoint, size_t num_patches, - CArrayKokkos &nbox, CArrayKokkos &nsort, DCArrayKokkos nodes_in_elem, + CArrayKokkos &nbox, CArrayKokkos &nsort, DistributedDCArray nodes_in_elem, CArrayKokkos elems_in_patch, size_t num_bdy_nodes, CArrayKokkos nodes_in_patch, const CArrayKokkos &xi, const CArrayKokkos &eta, double x_max, double y_max, double z_max, DCArrayKokkos &num_active, RaggedRightArrayKokkos elems_in_node, CArrayKokkos num_nodes_in_elem, @@ -1879,8 +1879,8 @@ void penetration_sweep(double x_min, double y_min, double z_min, double bounding void force_resolution(CArrayKokkos &f_c_incs, DCArrayKokkos num_active, CArrayKokkos &active_set, CArrayKokkos &node_patch_pairs, CArrayKokkos &pair_vars, CArrayKokkos &contact_surface_map, - DCArrayKokkos &coords, CArrayKokkos bdy_nodes, DCArrayKokkos &mass, - CArrayKokkos &contact_forces, DCArrayKokkos &corner_force, DCArrayKokkos &vel, + DistributedDCArray &coords, CArrayKokkos bdy_nodes, DistributedDCArray &mass, + CArrayKokkos &contact_forces, DCArrayKokkos &corner_force, DistributedDCArray &vel, RaggedRightArrayKokkos corners_in_node, CArrayKokkos num_corners_in_node, const CArrayKokkos &xi, const CArrayKokkos &eta, const double &del_t, CArrayKokkos &contact_force, size_t num_bdy_nodes) { @@ -1955,9 +1955,9 @@ void remove_pairs(DCArrayKokkos num_active, CArrayKokkos &acti CArrayKokkos &node_patch_pairs, CArrayKokkos nodes_in_patch, CArrayKokkos bdy_patches, CArrayKokkos &contact_forces, CArrayKokkos &contact_surface_map, DCArrayKokkos &corner_force, RaggedRightArrayKokkos corners_in_node, - DCArrayKokkos &mass, DCArrayKokkos &coords, + DistributedDCArray &mass, DistributedDCArray &coords, CArrayKokkos num_corners_in_node, CArrayKokkos bdy_nodes, - DCArrayKokkos &vel, const double &del_t, + DistributedDCArray &vel, const double &del_t, const CArrayKokkos &xi, const CArrayKokkos &eta, size_t num_bdy_patches) { RUN({ @@ -2108,9 +2108,9 @@ bool any(const ViewCArrayKokkos &a, const size_t &size) /// beginning of contact_state_t member functions //////////////////////////////////////////////////////////////////// void contact_state_t::initialize(size_t num_dims, size_t num_nodes_in_patch, const CArrayKokkos &bdy_patches, size_t num_bdy_nodes, size_t num_bdy_patches, CArrayKokkos &patches_in_elem, - CArrayKokkos &elems_in_patch, DCArrayKokkos &nodes_in_elem, + CArrayKokkos &elems_in_patch, DistributedDCArray &nodes_in_elem, CArrayKokkos &nodes_in_patch, CArrayKokkos &bdy_nodes, size_t num_patches, - size_t num_nodes, DCArrayKokkos &coords) + size_t num_nodes, DistributedDCArray &coords) { // Contact is only supported in 3D if (num_dims != 3) diff --git a/single-node-refactor/src/Solvers/SGH_solver_3D/src/energy_sgh.cpp b/single-node-refactor/src/Solvers/SGH_solver_3D/src/energy_sgh.cpp index f87a41cd7..7af90d3ad 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_3D/src/energy_sgh.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_3D/src/energy_sgh.cpp @@ -55,8 +55,8 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. void SGH3D::update_energy(const double rk_alpha, const double dt, const Mesh_t& mesh, - const DCArrayKokkos& node_vel, - const DCArrayKokkos& node_vel_n0, + const DistributedDCArray& node_vel, + const DistributedDCArray& node_vel_n0, const DRaggedRightArrayKokkos& MaterialPoints_sie, const DRaggedRightArrayKokkos& MaterialPoints_sie_n0, const DRaggedRightArrayKokkos& MaterialPoints_mass, diff --git a/single-node-refactor/src/Solvers/SGH_solver_3D/src/force_sgh.cpp b/single-node-refactor/src/Solvers/SGH_solver_3D/src/force_sgh.cpp index b892b92aa..b764bb34d 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_3D/src/force_sgh.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_3D/src/force_sgh.cpp @@ -72,8 +72,8 @@ void SGH3D::get_force(const Material_t& Materials, const DCArrayKokkos& GaussPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_eroded, const DCArrayKokkos& corner_force, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_sie, const DRaggedRightArrayKokkos& MaterialPoints_pres, diff --git a/single-node-refactor/src/Solvers/SGH_solver_3D/src/momentum.cpp b/single-node-refactor/src/Solvers/SGH_solver_3D/src/momentum.cpp index 742f40515..7b5415b75 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_3D/src/momentum.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_3D/src/momentum.cpp @@ -52,10 +52,10 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. void SGH3D::update_velocity(double rk_alpha, double dt, const Mesh_t& mesh, - DCArrayKokkos& node_vel, - DCArrayKokkos& node_vel_n0, - const DCArrayKokkos& node_mass, - const DCArrayKokkos& node_force, + DistributedDCArray& node_vel, + DistributedDCArray& node_vel_n0, + const DistributedDCArray& node_mass, + const DistributedDCArray& node_force, const DCArrayKokkos& corner_force, const CArrayKokkos & contact_force, bool doing_contact) const @@ -63,7 +63,7 @@ void SGH3D::update_velocity(double rk_alpha, const size_t num_dims = mesh.num_dims; // walk over the nodes to update the velocity - FOR_ALL(node_gid, 0, mesh.num_nodes, { + FOR_ALL(node_gid, 0, mesh.num_local_nodes, { // adding in contact force if (doing_contact) @@ -114,8 +114,8 @@ void SGH3D::update_velocity(double rk_alpha, ///////////////////////////////////////////////////////////////////////////// void SGH3D::get_velgrad(DCArrayKokkos& vel_grad, const Mesh_t mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DCArrayKokkos& elem_vol) const { const size_t num_nodes_in_elem = 8; @@ -219,8 +219,8 @@ void SGH3D::get_velgrad(DCArrayKokkos& vel_grad, ///////////////////////////////////////////////////////////////////////////// void SGH3D::get_divergence(DCArrayKokkos& elem_div, const Mesh_t mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DCArrayKokkos& elem_vol) const { // --- calculate the forces acting on the nodes from the element --- diff --git a/single-node-refactor/src/Solvers/SGH_solver_3D/src/position.cpp b/single-node-refactor/src/Solvers/SGH_solver_3D/src/position.cpp index e652a4bd8..2aa6b4413 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_3D/src/position.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_3D/src/position.cpp @@ -51,10 +51,10 @@ void SGH3D::update_position(double rk_alpha, double dt, const size_t num_dims, const size_t num_nodes, - DCArrayKokkos& node_coords, - DCArrayKokkos& node_coords_n0, - const DCArrayKokkos& node_vel, - const DCArrayKokkos& node_vel_n0) const + DistributedDCArray& node_coords, + DistributedDCArray& node_coords_n0, + const DistributedDCArray& node_vel, + const DistributedDCArray& node_vel_n0) const { // loop over all the nodes in the mesh FOR_ALL(node_gid, 0, num_nodes, { diff --git a/single-node-refactor/src/Solvers/SGH_solver_3D/src/properties.cpp b/single-node-refactor/src/Solvers/SGH_solver_3D/src/properties.cpp index a65741ee2..4d7457929 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_3D/src/properties.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_3D/src/properties.cpp @@ -67,8 +67,8 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. void SGH3D::update_state( const Material_t& Materials, const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DCArrayKokkos& GaussPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_pres, @@ -293,8 +293,8 @@ void SGH3D::update_stress( const Material_t& Materials, const Mesh_t& mesh, const DCArrayKokkos& GaussPoints_vol, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DCArrayKokkos& GaussPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_sie, diff --git a/single-node-refactor/src/Solvers/SGH_solver_3D/src/sgh_execute.cpp b/single-node-refactor/src/Solvers/SGH_solver_3D/src/sgh_execute.cpp index 7f2507263..5d8fbea2a 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_3D/src/sgh_execute.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_3D/src/sgh_execute.cpp @@ -51,7 +51,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /// Evolve the state according to the SGH method /// ///////////////////////////////////////////////////////////////////////////// -void SGH3D::execute(SimulationParameters_t& SimulationParamaters, +void SGH3D::execute(SimulationParameters_t& SimulationParameters, Material_t& Materials, BoundaryCondition_t& BoundaryConditions, Mesh_t& mesh, @@ -65,22 +65,22 @@ void SGH3D::execute(SimulationParameters_t& SimulationParamaters, mesh.bdy_nodes, mesh.num_patches, mesh.num_nodes, State.node.coords); } - double fuzz = SimulationParamaters.dynamic_options.fuzz; // 1.e-16 - double tiny = SimulationParamaters.dynamic_options.tiny; // 1.e-12 - double small = SimulationParamaters.dynamic_options.small; // 1.e-8 + double fuzz = SimulationParameters.dynamic_options.fuzz; // 1.e-16 + double tiny = SimulationParameters.dynamic_options.tiny; // 1.e-12 + double small = SimulationParameters.dynamic_options.small; // 1.e-8 - double graphics_dt_ival = SimulationParamaters.output_options.graphics_time_step; - int graphics_cyc_ival = SimulationParamaters.output_options.graphics_iteration_step; + double graphics_dt_ival = SimulationParameters.output_options.graphics_time_step; + int graphics_cyc_ival = SimulationParameters.output_options.graphics_iteration_step; - // double time_initial = SimulationParamaters.dynamic_options.time_initial; - double time_final = this->time_end; //SimulationParamaters.dynamic_options.time_final; - double dt_min = SimulationParamaters.dynamic_options.dt_min; - double dt_max = SimulationParamaters.dynamic_options.dt_max; - double dt_start = SimulationParamaters.dynamic_options.dt_start; - double dt_cfl = SimulationParamaters.dynamic_options.dt_cfl; + // double time_initial = SimulationParameters.dynamic_options.time_initial; + double time_final = this->time_end; //SimulationParameters.dynamic_options.time_final; + double dt_min = SimulationParameters.dynamic_options.dt_min; + double dt_max = SimulationParameters.dynamic_options.dt_max; + double dt_start = SimulationParameters.dynamic_options.dt_start; + double dt_cfl = SimulationParameters.dynamic_options.dt_cfl; - int rk_num_stages = SimulationParamaters.dynamic_options.rk_num_stages; - int cycle_stop = SimulationParamaters.dynamic_options.cycle_stop; + int rk_num_stages = SimulationParameters.dynamic_options.rk_num_stages; + int cycle_stop = SimulationParameters.dynamic_options.cycle_stop; // initialize time, time_step, and cycles double time_value = this->time_start; // was 0.0 @@ -91,6 +91,10 @@ void SGH3D::execute(SimulationParameters_t& SimulationParamaters, CArrayKokkos GaussPoint_pres_denominator(mesh.num_elems*mesh.num_gauss_in_elem); CArrayKokkos GaussPoint_volfrac_min(mesh.num_elems*mesh.num_gauss_in_elem); CArrayKokkos GaussPoint_volfrac_limiter(mesh.num_elems*mesh.num_gauss_in_elem); + + int myrank, nranks; + MPI_Comm_rank(MPI_COMM_WORLD,&myrank); + MPI_Comm_size(MPI_COMM_WORLD,&nranks); // Create mesh writer @@ -101,14 +105,17 @@ void SGH3D::execute(SimulationParameters_t& SimulationParamaters, graphics_times(0) = this->time_start; // was zero double graphics_time = this->time_start; // the times for writing graphics dump, was started at 0.0 size_t output_id=0; // the id for the outputs written - - std::cout << "Applying initial boundary conditions" << std::endl; + if(myrank==0){ + std::cout << "Applying initial boundary conditions" << std::endl; + } boundary_velocity(mesh, BoundaryConditions, State.node.vel, time_value); // Time value = 0.0; // extensive energy tallies over the entire mesh double IE_t0 = 0.0; double KE_t0 = 0.0; double TE_t0 = 0.0; + double local_IE_t0 = 0.0; + double local_KE_t0 = 0.0; double cached_pregraphics_dt = fuzz; @@ -118,40 +125,59 @@ void SGH3D::execute(SimulationParameters_t& SimulationParamaters, // extensive IE for (size_t mat_id = 0; mat_id < num_mats; mat_id++) { - IE_t0 += sum_domain_internal_energy(State.MaterialPoints.mass, + local_IE_t0 += sum_domain_internal_energy(State.MaterialPoints.mass, State.MaterialPoints.sie, - State.MaterialPoints.num_material_points.host(mat_id), + State.MaterialPoints.num_material_local_points.host(mat_id), mat_id); } // end loop over mat_id // extensive KE - KE_t0 = sum_domain_kinetic_energy(mesh, + local_KE_t0 = sum_domain_kinetic_energy(mesh, State.node.vel, - State.node.coords, State.node.mass); + + //collect KE and TE sums across all processes + MPI_Allreduce(&local_IE_t0, &IE_t0, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + MPI_Allreduce(&local_KE_t0, &KE_t0, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + // extensive TE TE_t0 = IE_t0 + KE_t0; // domain mass for each material (they are at material points) double mass_domain_all_mats_t0 = 0.0; double mass_domain_nodes_t0 = 0.0; + double global_mass_domain_nodes_t0; + double global_mass_domain_all_mats_t0; - for (size_t mat_id = 0; mat_id < num_mats; mat_id++) { + //debug print of mass + // for(int ielem = 0; ielem < mesh.num_local_elems; ielem++){ + // std::cout << State.MaterialPoints.mass(0,ielem) << " " << std::endl; + // } + for (size_t mat_id = 0; mat_id < num_mats; mat_id++) { + double global_mass_domain_mat; + //std::cout << " local element count for mass loop " << State.MaterialPoints.num_material_local_points.host(mat_id) << std::endl; double mass_domain_mat = sum_domain_material_mass(State.MaterialPoints.mass, - State.MaterialPoints.num_material_points.host(mat_id), + State.MaterialPoints.num_material_local_points.host(mat_id), mat_id); mass_domain_all_mats_t0 += mass_domain_mat; - printf("material %zu mass in domain = %f \n", mat_id, mass_domain_mat); + + MPI_Allreduce(&mass_domain_mat, &global_mass_domain_mat, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + if(myrank==0){ + printf("material %zu mass in domain = %f \n", mat_id, global_mass_domain_mat); + } } // end for // node mass of the domain mass_domain_nodes_t0 = sum_domain_node_mass(mesh, State.node.coords, State.node.mass); - - printf("nodal mass domain = %f \n", mass_domain_nodes_t0); + MPI_Allreduce(&mass_domain_nodes_t0, &global_mass_domain_nodes_t0, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + MPI_Allreduce(&mass_domain_all_mats_t0, &global_mass_domain_all_mats_t0, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + if(myrank==0){ + printf("nodal mass domain = %f \n", global_mass_domain_nodes_t0); + } // a flag to exit the calculation size_t stop_calc = 0; @@ -159,11 +185,13 @@ void SGH3D::execute(SimulationParameters_t& SimulationParamaters, auto time_1 = std::chrono::high_resolution_clock::now(); // Write initial state at t=0 - printf("Writing outputs to file at %f \n", graphics_time); + if(myrank==0){ + printf("Writing outputs to file at %f \n", graphics_time); + } mesh_writer.write_mesh( mesh, State, - SimulationParamaters, + SimulationParameters, dt, time_value, graphics_times, @@ -203,7 +231,7 @@ void SGH3D::execute(SimulationParameters_t& SimulationParamaters, State.MaterialPoints.sspd, State.MaterialPoints.eroded, State.MaterialToMeshMaps.elem_in_mat_elem, - State.MaterialToMeshMaps.num_mat_elems.host(mat_id), + State.MaterialToMeshMaps.num_mat_local_elems.host(mat_id), time_value, graphics_time, time_final, @@ -219,15 +247,17 @@ void SGH3D::execute(SimulationParameters_t& SimulationParamaters, min_dt_calc = fmin(dt_mat, min_dt_calc); } // end for loop over all mats - dt = min_dt_calc; // save this dt time step - - if (cycle == 0) { - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_value, dt); + //Find the minimum timestep across all MPI processes + MPI_Allreduce(&min_dt_calc, &dt, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); + if(myrank==0){ + if (cycle == 0) { + printf("cycle = %lu, time = %.12f, time step = %.12f \n", cycle, time_value, dt); + } + // print time step every 10 cycles + else if (cycle % 20 == 0) { + printf("cycle = %lu, time = %.12f, time step = %.12f \n", cycle, time_value, dt); + } // end if } - // print time step every 10 cycles - else if (cycle % 20 == 0) { - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_value, dt); - } // end if // --------------------------------------------------------------------- @@ -378,6 +408,9 @@ void SGH3D::execute(SimulationParameters_t& SimulationParamaters, // ---- apply contact boundary conditions to the boundary patches---- boundary_contact(mesh, BoundaryConditions, State.node.vel, time_value); + //update node velocity on ghosts + node_velocity_comms.execute_comms(); + // mpi_coms(); for (size_t mat_id = 0; mat_id < num_mats; mat_id++) { @@ -503,10 +536,12 @@ void SGH3D::execute(SimulationParameters_t& SimulationParamaters, // write outputs if (write == 1) { - printf("Writing outputs to file at %f \n", graphics_time); + if(myrank==0){ + printf("Writing outputs to file at %f \n", graphics_time); + } mesh_writer.write_mesh(mesh, State, - SimulationParamaters, + SimulationParameters, dt, time_value, graphics_times, @@ -528,56 +563,69 @@ void SGH3D::execute(SimulationParameters_t& SimulationParamaters, auto time_2 = std::chrono::high_resolution_clock::now(); auto calc_time = std::chrono::duration_cast(time_2 - time_1).count(); - - printf("\nCalculation time in seconds: %f \n", calc_time * 1e-9); + if(myrank==0){ + printf("\nCalculation time in seconds: %f \n", calc_time * 1e-9); + } // ---- Calculate energy tallies ---- double IE_tend = 0.0; double KE_tend = 0.0; double TE_tend = 0.0; + double local_IE_tend = 0.0; + double local_KE_tend = 0.0; // extensive IE for(size_t mat_id = 0; mat_id < num_mats; mat_id++){ - IE_tend += sum_domain_internal_energy(State.MaterialPoints.mass, + local_IE_tend += sum_domain_internal_energy(State.MaterialPoints.mass, State.MaterialPoints.sie, - State.MaterialPoints.num_material_points.host(mat_id), + State.MaterialPoints.num_material_local_points.host(mat_id), mat_id); } // end loop over mat_id // extensive KE - KE_tend = sum_domain_kinetic_energy(mesh, + local_KE_tend = sum_domain_kinetic_energy(mesh, State.node.vel, - State.node.coords, State.node.mass); + + MPI_Allreduce(&local_IE_tend, &IE_tend, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + MPI_Allreduce(&local_KE_tend, &KE_tend, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + // extensive TE TE_tend = IE_tend + KE_tend; - - printf("Time=0: KE = %.14f, IE = %.14f, TE = %.14f \n", KE_t0, IE_t0, TE_t0); - printf("Time=End: KE = %.14f, IE = %.14f, TE = %.14f \n", KE_tend, IE_tend, TE_tend); - printf("total energy change = %.15e \n\n", TE_tend - TE_t0); + if(myrank==0){ + printf("Time=0: KE = %.14f, IE = %.14f, TE = %.14f \n", KE_t0, IE_t0, TE_t0); + printf("Time=End: KE = %.14f, IE = %.14f, TE = %.14f \n", KE_tend, IE_tend, TE_tend); + printf("total energy change = %.15e \n\n", TE_tend - TE_t0); + } // domain mass for each material (they are at material points) double mass_domain_all_mats_tend = 0.0; double mass_domain_nodes_tend = 0.0; + double global_mass_domain_all_mats_tend; + double global_mass_domain_nodes_tend; for(size_t mat_id = 0; mat_id < num_mats; mat_id++){ - double mass_domain_mat = sum_domain_material_mass(State.MaterialPoints.mass, - State.MaterialPoints.num_material_points.host(mat_id), + State.MaterialPoints.num_material_local_points.host(mat_id), mat_id); mass_domain_all_mats_tend += mass_domain_mat; } // end for + MPI_Allreduce(&mass_domain_all_mats_tend, &global_mass_domain_all_mats_tend, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); // node mass of the domain mass_domain_nodes_tend = sum_domain_node_mass(mesh, State.node.coords, State.node.mass); - printf("material mass conservation error = %f \n", mass_domain_all_mats_tend - mass_domain_all_mats_t0); - printf("nodal mass conservation error = %f \n", mass_domain_nodes_tend - mass_domain_nodes_t0); - printf("nodal and material mass error = %f \n\n", mass_domain_nodes_tend - mass_domain_all_mats_tend); + MPI_Allreduce(&mass_domain_nodes_tend, &global_mass_domain_nodes_tend, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + MPI_Allreduce(&mass_domain_all_mats_tend, &global_mass_domain_all_mats_tend, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + if(myrank==0){ + printf("material mass conservation error = %f \n", global_mass_domain_all_mats_tend - global_mass_domain_all_mats_t0); + printf("nodal mass conservation error = %f \n", global_mass_domain_nodes_tend - global_mass_domain_nodes_t0); + printf("nodal and material mass error = %f \n\n", global_mass_domain_nodes_tend - global_mass_domain_all_mats_tend); + } } // end of SGH execute ///////////////////////////////////////////////////////////////////////////// @@ -695,18 +743,19 @@ double sum_domain_internal_energy( ///////////////////////////////////////////////////////////////////////////// double sum_domain_kinetic_energy( const Mesh_t& mesh, - const DCArrayKokkos& node_vel, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_mass) + const DistributedDCArray& node_vel, + const DistributedDCArray& node_mass) { // extensive KE double KE_sum = 0.0; double KE_loc_sum; + int num_dims = mesh.num_dims; + int num_local_nodes = mesh.num_local_nodes; - FOR_REDUCE_SUM(node_gid, 0, mesh.num_nodes, KE_loc_sum, { + FOR_REDUCE_SUM(node_gid, 0, num_local_nodes, KE_loc_sum, { double ke = 0; - for (size_t dim = 0; dim < mesh.num_dims; dim++) { + for (size_t dim = 0; dim < num_dims; dim++) { ke += node_vel(node_gid, dim) * node_vel(node_gid, dim); // 1/2 at end } // end for @@ -752,13 +801,13 @@ double sum_domain_material_mass( /// ///////////////////////////////////////////////////////////////////////////// double sum_domain_node_mass(const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_mass) + const DistributedDCArray& node_coords, + const DistributedDCArray& node_mass) { double mass_domain = 0.0; double mass_loc_domain; - FOR_REDUCE_SUM(node_gid, 0, mesh.num_nodes, mass_loc_domain, { + FOR_REDUCE_SUM(node_gid, 0, mesh.num_local_nodes, mass_loc_domain, { if (mesh.num_dims == 2) { mass_loc_domain += node_mass(node_gid) * node_coords(node_gid, 1); } diff --git a/single-node-refactor/src/Solvers/SGH_solver_3D/src/sgh_initialize.cpp b/single-node-refactor/src/Solvers/SGH_solver_3D/src/sgh_initialize.cpp index 1cb41db47..5072f7f8b 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_3D/src/sgh_initialize.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_3D/src/sgh_initialize.cpp @@ -38,11 +38,11 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "simulation_parameters.h" -void SGH3D::initialize(SimulationParameters_t& SimulationParamaters, +void SGH3D::initialize(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, - State_t& State) const + State_t& State) { const size_t num_nodes = mesh.num_nodes; const size_t num_gauss_pts = mesh.num_elems; @@ -55,14 +55,18 @@ void SGH3D::initialize(SimulationParameters_t& SimulationParamaters, } // mesh state - State.node.initialize(num_nodes, num_dims, SGH3D_State::required_node_state); + State.node.initialize(mesh.all_node_map, num_dims, SGH3D_State::required_node_state, mesh.node_map); //allocate shared nlocal+nghost contiguous array State.GaussPoints.initialize(num_gauss_pts, num_dims, SGH3D_State::required_gauss_pt_state); State.corner.initialize(num_corners, num_dims, SGH3D_State::required_corner_state); + //comms objects + node_velocity_comms = CommPlan(State.node.vel, State.node.local_vel, mesh.node_coords_comms); //copies MPI setup from coordinate comms since the node maps are the same + node_mass_comms = CommPlan(State.node.mass, State.node.local_mass, mesh.node_coords_comms); //copies MPI setup from coordinate comms since the node maps are the same + // check that the fills specify the required nodal fields bool filled_nodal_state = check_fill_node_states(SGH3D_State::required_fill_node_state, - SimulationParamaters.region_setups.fill_node_states); + SimulationParameters.region_setups.fill_node_states); if (filled_nodal_state == false){ std::cout <<" Missing required nodal state in the fill instructions for the dynx_FE solver \n"; @@ -75,7 +79,7 @@ void SGH3D::initialize(SimulationParameters_t& SimulationParamaters, -void SGH3D::initialize_material_state(SimulationParameters_t& SimulationParamaters, +void SGH3D::initialize_material_state(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, @@ -96,15 +100,15 @@ void SGH3D::initialize_material_state(SimulationParameters_t& SimulationParamate // check that the fills specify the required material point state fields bool filled_material_state_A = check_fill_mat_states(SGH3D_State::required_optA_fill_material_pt_state, - SimulationParamaters.region_setups.fill_gauss_states); + SimulationParameters.region_setups.fill_gauss_states); bool filled_material_state_B = check_fill_mat_states(SGH3D_State::required_optB_fill_material_pt_state, - SimulationParamaters.region_setups.fill_gauss_states); + SimulationParameters.region_setups.fill_gauss_states); // --- full stress tensor is not yet supported in region_fill --- //bool filled_material_state_C = // check_fill_mat_states(SGH3D_State::required_optC_fill_material_pt_state, - // SimulationParamaters.region_setups.fill_gauss_states); + // SimulationParameters.region_setups.fill_gauss_states); if (filled_material_state_A == false && filled_material_state_B == false){ diff --git a/single-node-refactor/src/Solvers/SGH_solver_3D/src/sgh_setup.cpp b/single-node-refactor/src/Solvers/SGH_solver_3D/src/sgh_setup.cpp index 882b39a80..6a1c5e0eb 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_3D/src/sgh_setup.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_3D/src/sgh_setup.cpp @@ -51,7 +51,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /// \brief Allocate state, setup models, and fill mesh regions per the YAML input /// ///////////////////////////////////////////////////////////////////////////// -void SGH3D::setup(SimulationParameters_t& SimulationParamaters, +void SGH3D::setup(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, @@ -59,6 +59,9 @@ void SGH3D::setup(SimulationParameters_t& SimulationParamaters, { // add a flag on whether SGH was set up, if(SGH_setup_already==false) + //update node velocity on ghosts + node_velocity_comms.execute_comms(); + const size_t num_mats = Materials.num_mats; // the number of materials on the mesh // calculate pressure, sound speed, and stress for each material @@ -126,5 +129,7 @@ void SGH3D::setup(SimulationParameters_t& SimulationParamaters, } } + //communicate node masses to ghosts + node_mass_comms.execute_comms(); } // end SGH setup diff --git a/single-node-refactor/src/Solvers/SGH_solver_3D/src/time_integration.cpp b/single-node-refactor/src/Solvers/SGH_solver_3D/src/time_integration.cpp index c62743048..157751f93 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_3D/src/time_integration.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_3D/src/time_integration.cpp @@ -51,10 +51,10 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /// ///////////////////////////////////////////////////////////////////////////// void SGH3D::rk_init( - DCArrayKokkos& node_coords, - DCArrayKokkos& node_coords_n0, - DCArrayKokkos& node_vel, - DCArrayKokkos& node_vel_n0, + DistributedDCArray& node_coords, + DistributedDCArray& node_coords_n0, + DistributedDCArray& node_vel, + DistributedDCArray& node_vel_n0, DRaggedRightArrayKokkos& MaterialPoints_sie, DRaggedRightArrayKokkos& MaterialPoints_sie_n0, DRaggedRightArrayKokkos& MaterialPoints_stress, @@ -108,8 +108,8 @@ void SGH3D::rk_init( /// ///////////////////////////////////////////////////////////////////////////// void SGH3D::get_timestep(Mesh_t& mesh, - DCArrayKokkos& node_coords, - DCArrayKokkos& node_vel, + DistributedDCArray& node_coords, + DistributedDCArray& node_vel, DCArrayKokkos& GaussPoints_vol, DRaggedRightArrayKokkos& MaterialPoints_sspd, DRaggedRightArrayKokkos& MaterialPoints_eroded, diff --git a/single-node-refactor/src/Solvers/SGH_solver_rz/include/sgh_solver_rz.h b/single-node-refactor/src/Solvers/SGH_solver_rz/include/sgh_solver_rz.h index e38f458c1..f06b81dd2 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_rz/include/sgh_solver_rz.h +++ b/single-node-refactor/src/Solvers/SGH_solver_rz/include/sgh_solver_rz.h @@ -117,6 +117,10 @@ class SGHRZ : public Solver } ~SGHRZ() = default; + + //member variables + CommPlan node_velocity_comms; + CommPlan node_mass_comms; ///////////////////////////////////////////////////////////////////////////// /// @@ -125,13 +129,13 @@ class SGHRZ : public Solver /// \brief Initializes data associated with the SGHRZ solver /// ///////////////////////////////////////////////////////////////////////////// - void initialize(SimulationParameters_t& SimulationParamaters, + void initialize(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, - State_t& State) const override; + State_t& State) override; - void initialize_material_state(SimulationParameters_t& SimulationParamaters, + void initialize_material_state(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, @@ -144,7 +148,7 @@ class SGHRZ : public Solver /// \brief Calls setup_sgh_rz, which initializes state and material data /// ///////////////////////////////////////////////////////////////////////////// - void setup(SimulationParameters_t& SimulationParamaters, + void setup(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, @@ -158,7 +162,7 @@ class SGHRZ : public Solver /// /// ///////////////////////////////////////////////////////////////////////////// - void execute(SimulationParameters_t& SimulationParamaters, + void execute(SimulationParameters_t& SimulationParameters, Material_t& Materials, BoundaryCondition_t& Boundary, Mesh_t& mesh, @@ -180,7 +184,7 @@ class SGHRZ : public Solver /// \return /// ///////////////////////////////////////////////////////////////////////////// - void finalize(SimulationParameters_t& SimulationParamaters, + void finalize(SimulationParameters_t& SimulationParameters, Material_t& Materials, BoundaryCondition_t& Boundary) const override { @@ -190,20 +194,20 @@ class SGHRZ : public Solver void init_corner_node_masses_zero_rz( const Mesh_t& mesh, - const DCArrayKokkos& node_mass, + const DistributedDCArray& node_mass, const DCArrayKokkos& corner_mass) const; // **** Functions defined in boundary.cpp **** // void boundary_velocity_rz( const Mesh_t& mesh, const BoundaryCondition_t& Boundary, - DCArrayKokkos& node_vel, + DistributedDCArray& node_vel, const double time_value) const; void boundary_contact_rz( const Mesh_t& mesh, const BoundaryCondition_t& Boundary, - DCArrayKokkos& node_vel, + DistributedDCArray& node_vel, const double time_value) const; // **** Functions defined in energy_SGHRZ.cpp **** // @@ -211,10 +215,10 @@ class SGHRZ : public Solver const double rk_alpha, const double dt, const Mesh_t& mesh, - const DCArrayKokkos& node_vel, - const DCArrayKokkos& node_vel_n0, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_coords_n0, + const DistributedDCArray& node_vel, + const DistributedDCArray& node_vel_n0, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_coords_n0, const DRaggedRightArrayKokkos& MaterialPoints_sie, const DRaggedRightArrayKokkos& MaterialPoints_sie_n0, const DRaggedRightArrayKokkos& MaterialPoints_mass, @@ -232,8 +236,8 @@ class SGHRZ : public Solver const DCArrayKokkos& GaussPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_eroded, const DCArrayKokkos& corner_force, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_sie, const DRaggedRightArrayKokkos& MaterialPoints_pres, @@ -258,26 +262,26 @@ class SGHRZ : public Solver double dt, const size_t num_dims, const size_t num_nodes, - DCArrayKokkos& node_coords, - const DCArrayKokkos& node_coords_n0, - const DCArrayKokkos& node_vel, - const DCArrayKokkos& node_vel_n0) const; + DistributedDCArray& node_coords, + const DistributedDCArray& node_coords_n0, + const DistributedDCArray& node_vel, + const DistributedDCArray& node_vel_n0) const; // **** Functions defined in momentum.cpp **** // void update_velocity_rz( double rk_alpha, double dt, const Mesh_t& mesh, - DCArrayKokkos& node_vel, - const DCArrayKokkos& node_vel_n0, - const DCArrayKokkos& node_mass, + DistributedDCArray& node_vel, + const DistributedDCArray& node_vel_n0, + const DistributedDCArray& node_mass, const DCArrayKokkos& corner_force) const; void get_velgrad_rz( DCArrayKokkos& elem_vel_grad, const Mesh_t mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DCArrayKokkos& elem_vol) const; KOKKOS_FUNCTION @@ -288,16 +292,16 @@ class SGHRZ : public Solver void get_divergence_rz( DCArrayKokkos& GaussPoints_div, const Mesh_t mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DCArrayKokkos& GaussPoints_vol) const; // **** Functions defined in properties.cpp **** // void update_state_rz( const Material_t& Materials, const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DCArrayKokkos& GaussPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_pres, @@ -325,8 +329,8 @@ class SGHRZ : public Solver const Material_t& Materials, const Mesh_t& mesh, const DCArrayKokkos& GaussPoints_vol, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DCArrayKokkos& GaussPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_sie, @@ -351,10 +355,10 @@ class SGHRZ : public Solver // **** Functions defined in time_integration.cpp **** // // NOTE: Consider pulling up void rk_init_rz( - DCArrayKokkos& node_coords, - DCArrayKokkos& node_coords_n0, - DCArrayKokkos& node_vel, - DCArrayKokkos& node_vel_n0, + DistributedDCArray& node_coords, + DistributedDCArray& node_coords_n0, + DistributedDCArray& node_vel, + DistributedDCArray& node_vel_n0, DRaggedRightArrayKokkos& MaterialPoints_sie, DRaggedRightArrayKokkos& MaterialPoints_sie_n0, DRaggedRightArrayKokkos& MaterialPoints_stress, @@ -368,8 +372,8 @@ class SGHRZ : public Solver void get_timestep_rz( Mesh_t& mesh, - DCArrayKokkos& node_coords, - DCArrayKokkos& node_vel, + DistributedDCArray& node_coords, + DistributedDCArray& node_vel, DCArrayKokkos& GaussPoints_vol, DRaggedRightArrayKokkos& MaterialPoints_sspd, DRaggedRightArrayKokkos& MaterialPoints_eroded, @@ -391,8 +395,8 @@ class SGHRZ : public Solver void calc_corner_mass_rz(const Material_t& Materials, const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_mass, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_mass, const DCArrayKokkos& corner_mass, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& elem_in_mat_elem, @@ -400,19 +404,19 @@ void calc_corner_mass_rz(const Material_t& Materials, const size_t mat_id); void calc_node_mass_rz(const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_mass, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_mass, const DCArrayKokkos& corner_mass); void calc_node_areal_mass_rz(const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_mass, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_mass, CArrayKokkos node_extensive_mass, double tiny); void calc_node_extensive_mass_rz(const CArrayKokkos& node_extensive_mass, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_mass, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_mass, double num_nodes); double sum_domain_internal_energy_rz(const DRaggedRightArrayKokkos& MaterialPoints_mass, @@ -421,7 +425,7 @@ double sum_domain_internal_energy_rz(const DRaggedRightArrayKokkos& Mate const size_t mat_id); double sum_domain_kinetic_energy_rz(const Mesh_t& mesh, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_vel, const CArrayKokkos& node_extensive_mass); double sum_domain_material_mass_rz(const DRaggedRightArrayKokkos& MaterialPoints_mass, diff --git a/single-node-refactor/src/Solvers/SGH_solver_rz/src/boundary_rz.cpp b/single-node-refactor/src/Solvers/SGH_solver_rz/src/boundary_rz.cpp index 2775e662e..7e8eab296 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_rz/src/boundary_rz.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_rz/src/boundary_rz.cpp @@ -51,7 +51,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ///////////////////////////////////////////////////////////////////////////// void SGHRZ::boundary_velocity_rz(const Mesh_t& mesh, const BoundaryCondition_t& BoundaryConditions, - DCArrayKokkos& node_vel, + DistributedDCArray& node_vel, const double time_value) const { @@ -102,7 +102,7 @@ void SGHRZ::boundary_velocity_rz(const Mesh_t& mesh, ///////////////////////////////////////////////////////////////////////////// void SGHRZ::boundary_contact_rz(const Mesh_t& mesh, const BoundaryCondition_t& BoundaryConditions, - DCArrayKokkos& node_vel, + DistributedDCArray& node_vel, const double time_value) const { return; diff --git a/single-node-refactor/src/Solvers/SGH_solver_rz/src/energy_sgh_rz.cpp b/single-node-refactor/src/Solvers/SGH_solver_rz/src/energy_sgh_rz.cpp index 262d1e6d7..6037bdab5 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_rz/src/energy_sgh_rz.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_rz/src/energy_sgh_rz.cpp @@ -56,10 +56,10 @@ void SGHRZ::update_energy_rz( const double rk_alpha, const double dt, const Mesh_t& mesh, - const DCArrayKokkos& node_vel, - const DCArrayKokkos& node_vel_n0, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_coords_n0, + const DistributedDCArray& node_vel, + const DistributedDCArray& node_vel_n0, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_coords_n0, const DRaggedRightArrayKokkos& MaterialPoints_sie, const DRaggedRightArrayKokkos& MaterialPoints_sie_n0, const DRaggedRightArrayKokkos& MaterialPoints_mass, diff --git a/single-node-refactor/src/Solvers/SGH_solver_rz/src/force_sgh_rz.cpp b/single-node-refactor/src/Solvers/SGH_solver_rz/src/force_sgh_rz.cpp index 9de92be96..a766c1e1b 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_rz/src/force_sgh_rz.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_rz/src/force_sgh_rz.cpp @@ -71,8 +71,8 @@ void SGHRZ::get_force_rz(const Material_t& Materials, const DCArrayKokkos& GaussPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_eroded, const DCArrayKokkos& corner_force, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_sie, const DRaggedRightArrayKokkos& MaterialPoints_pres, diff --git a/single-node-refactor/src/Solvers/SGH_solver_rz/src/momentum_rz.cpp b/single-node-refactor/src/Solvers/SGH_solver_rz/src/momentum_rz.cpp index 32830f3bb..7679efbe3 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_rz/src/momentum_rz.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_rz/src/momentum_rz.cpp @@ -53,15 +53,15 @@ void SGHRZ::update_velocity_rz( double rk_alpha, double dt, const Mesh_t& mesh, - DCArrayKokkos& node_vel, - const DCArrayKokkos& node_vel_n0, - const DCArrayKokkos& node_mass, + DistributedDCArray& node_vel, + const DistributedDCArray& node_vel_n0, + const DistributedDCArray& node_mass, const DCArrayKokkos& corner_force) const { const size_t num_dims = 2; // walk over the nodes to update the velocity - FOR_ALL(node_gid, 0, mesh.num_nodes, { + FOR_ALL(node_gid, 0, mesh.num_local_nodes, { double node_force[3]; for (size_t dim = 0; dim < num_dims; dim++) { node_force[dim] = 0.0; @@ -105,8 +105,8 @@ void SGHRZ::update_velocity_rz( void SGHRZ::get_velgrad_rz( DCArrayKokkos& elem_vel_grad, const Mesh_t mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DCArrayKokkos& elem_vol) const { // --- calculate the forces acting on the nodes from the element --- @@ -248,8 +248,8 @@ void SGHRZ::get_velgrad_rz(ViewCArrayKokkos& vel_grad, ///////////////////////////////////////////////////////////////////////////// void SGHRZ::get_divergence_rz(DCArrayKokkos& elem_div, const Mesh_t mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DCArrayKokkos& elem_vol) const { // --- calculate the forces acting on the nodes from the element --- diff --git a/single-node-refactor/src/Solvers/SGH_solver_rz/src/position_rz.cpp b/single-node-refactor/src/Solvers/SGH_solver_rz/src/position_rz.cpp index e60f620db..0d9427724 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_rz/src/position_rz.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_rz/src/position_rz.cpp @@ -52,10 +52,10 @@ void SGHRZ::update_position_rz( double dt, const size_t num_dims, const size_t num_nodes, - DCArrayKokkos& node_coords, - const DCArrayKokkos& node_coords_n0, - const DCArrayKokkos& node_vel, - const DCArrayKokkos& node_vel_n0) const + DistributedDCArray& node_coords, + const DistributedDCArray& node_coords_n0, + const DistributedDCArray& node_vel, + const DistributedDCArray& node_vel_n0) const { // loop over all the nodes in the mesh FOR_ALL(node_gid, 0, num_nodes, { diff --git a/single-node-refactor/src/Solvers/SGH_solver_rz/src/properties_rz.cpp b/single-node-refactor/src/Solvers/SGH_solver_rz/src/properties_rz.cpp index 1aca9a147..b46e24bc4 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_rz/src/properties_rz.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_rz/src/properties_rz.cpp @@ -67,8 +67,8 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. void SGHRZ::update_state_rz( const Material_t& Materials, const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DCArrayKokkos& GuassPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_pres, @@ -308,8 +308,8 @@ void SGHRZ::update_state_rz( void SGHRZ::update_stress(const Material_t& Materials, const Mesh_t& mesh, const DCArrayKokkos& GaussPoints_vol, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DCArrayKokkos& GuassPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_sie, diff --git a/single-node-refactor/src/Solvers/SGH_solver_rz/src/sgh_execute_rz.cpp b/single-node-refactor/src/Solvers/SGH_solver_rz/src/sgh_execute_rz.cpp index 89d4d17f6..17fbf5286 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_rz/src/sgh_execute_rz.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_rz/src/sgh_execute_rz.cpp @@ -49,29 +49,29 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /// Evolve the state according to the SGH method /// ///////////////////////////////////////////////////////////////////////////// -void SGHRZ::execute(SimulationParameters_t& SimulationParamaters, +void SGHRZ::execute(SimulationParameters_t& SimulationParameters, Material_t& Materials, BoundaryCondition_t& BoundaryConditions, Mesh_t& mesh, State_t& State) { - double fuzz = SimulationParamaters.dynamic_options.fuzz; - double tiny = SimulationParamaters.dynamic_options.tiny; - double small = SimulationParamaters.dynamic_options.small; + double fuzz = SimulationParameters.dynamic_options.fuzz; + double tiny = SimulationParameters.dynamic_options.tiny; + double small = SimulationParameters.dynamic_options.small; - double graphics_dt_ival = SimulationParamaters.output_options.graphics_time_step; - int graphics_cyc_ival = SimulationParamaters.output_options.graphics_iteration_step; + double graphics_dt_ival = SimulationParameters.output_options.graphics_time_step; + int graphics_cyc_ival = SimulationParameters.output_options.graphics_iteration_step; - // double time_initial = SimulationParamaters.dynamic_options.time_initial; - double time_final = this->time_end; //SimulationParamaters.dynamic_options.time_final; - double dt_min = SimulationParamaters.dynamic_options.dt_min; - double dt_max = SimulationParamaters.dynamic_options.dt_max; - double dt_start = SimulationParamaters.dynamic_options.dt_start; - double dt_cfl = SimulationParamaters.dynamic_options.dt_cfl; + // double time_initial = SimulationParameters.dynamic_options.time_initial; + double time_final = this->time_end; //SimulationParameters.dynamic_options.time_final; + double dt_min = SimulationParameters.dynamic_options.dt_min; + double dt_max = SimulationParameters.dynamic_options.dt_max; + double dt_start = SimulationParameters.dynamic_options.dt_start; + double dt_cfl = SimulationParameters.dynamic_options.dt_cfl; - int rk_num_stages = SimulationParamaters.dynamic_options.rk_num_stages; - int cycle_stop = SimulationParamaters.dynamic_options.cycle_stop; + int rk_num_stages = SimulationParameters.dynamic_options.rk_num_stages; + int cycle_stop = SimulationParameters.dynamic_options.cycle_stop; // initialize time, time_step, and cycles double time_value = this->time_start; // 0.0; @@ -80,6 +80,11 @@ void SGHRZ::execute(SimulationParameters_t& SimulationParamaters, // Create mesh writer MeshWriter mesh_writer; // Note: Pull to driver after refactoring evolution + //MPI data + int myrank, nranks; + MPI_Comm_rank(MPI_COMM_WORLD,&myrank); + MPI_Comm_size(MPI_COMM_WORLD,&nranks); + // --- graphics vars ---- CArray graphics_times = CArray(20000); graphics_times(0) = this->time_start; // was zero @@ -96,8 +101,9 @@ void SGHRZ::execute(SimulationParameters_t& SimulationParamaters, State.node.mass, mesh.num_nodes); - - std::cout << "Applying initial boundary conditions" << std::endl; + if(myrank==0){ + std::cout << "Applying initial boundary conditions" << std::endl; + } boundary_velocity_rz(mesh, BoundaryConditions, State.node.vel, time_value); // Time value = 0.0; @@ -106,6 +112,8 @@ void SGHRZ::execute(SimulationParameters_t& SimulationParamaters, double IE_t0 = 0.0; double KE_t0 = 0.0; double TE_t0 = 0.0; + double local_IE_t0 = 0.0; + double local_KE_t0 = 0.0; // the number of materials specified by the user input @@ -114,39 +122,55 @@ void SGHRZ::execute(SimulationParameters_t& SimulationParamaters, // extensive IE for(size_t mat_id=0; mat_id(time_2 - time_1).count(); - printf("\nCalculation time in seconds: %f \n", calc_time * 1e-9); + if(myrank==0){ + printf("\nCalculation time in seconds: %f \n", calc_time * 1e-9); + } // ---- Calculate energy tallies ---- double IE_tend = 0.0; double KE_tend = 0.0; double TE_tend = 0.0; + double local_IE_tend = 0.0; + double local_KE_tend = 0.0; // extensive IE for(size_t mat_id=0; mat_id& Mate } // end function double sum_domain_kinetic_energy_rz(const Mesh_t& mesh, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_vel, const CArrayKokkos& node_extensive_mass) { // extensive KE double KE_sum = 0.0; double KE_loc_sum; + int num_local_nodes = mesh.num_local_nodes; - FOR_REDUCE_SUM(node_gid, 0, mesh.num_nodes, KE_loc_sum, { + FOR_REDUCE_SUM(node_gid, 0, num_local_nodes, KE_loc_sum, { double ke = 0; for (size_t dim = 0; dim < mesh.num_dims; dim++) { @@ -626,8 +677,8 @@ double sum_domain_node_mass_rz(const CArrayKokkos& node_extensive_mass, /// ///////////////////////////////////////////////////////////////////////////// void calc_node_extensive_mass_rz(const CArrayKokkos& node_extensive_mass, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_mass, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_mass, double num_nodes) { // save the nodal mass @@ -655,8 +706,8 @@ void calc_node_extensive_mass_rz(const CArrayKokkos& node_extensive_mass /// ///////////////////////////////////////////////////////////////////////////// void calc_node_areal_mass_rz(const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_mass, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_mass, const CArrayKokkos node_extensive_mass, double tiny) { diff --git a/single-node-refactor/src/Solvers/SGH_solver_rz/src/sgh_initialize_rz.cpp b/single-node-refactor/src/Solvers/SGH_solver_rz/src/sgh_initialize_rz.cpp index b87b7c074..0f5fab4de 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_rz/src/sgh_initialize_rz.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_rz/src/sgh_initialize_rz.cpp @@ -37,11 +37,11 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "mesh.h" #include "simulation_parameters.h" -void SGHRZ::initialize(SimulationParameters_t& SimulationParamaters, +void SGHRZ::initialize(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, - State_t& State) const + State_t& State) { size_t num_nodes = mesh.num_nodes; size_t num_gauss_pts = mesh.num_elems; @@ -56,14 +56,18 @@ void SGHRZ::initialize(SimulationParameters_t& SimulationParamaters, // save the solver_id, which is a pravate class variable //this->solver_id = solver_id_inp; - State.node.initialize(num_nodes, num_dim, SGHRZ_State::required_node_state); + State.node.initialize(mesh.all_node_map, num_dim, SGHRZ_State::required_node_state, mesh.node_map); State.GaussPoints.initialize(num_gauss_pts, 3, SGHRZ_State::required_gauss_pt_state); // note: dims is always 3 State.corner.initialize(num_corners, num_dim, SGHRZ_State::required_corner_state); + + //comms objects + node_velocity_comms = CommPlan(State.node.vel, State.node.local_vel, mesh.node_coords_comms); //copies MPI setup from coordinate comms since the node maps are the same + node_mass_comms = CommPlan(State.node.mass, State.node.local_mass, mesh.node_coords_comms); //copies MPI setup from coordinate comms since the node maps are the same // NOTE: Material points and material corners are initialize in sgh_setup after calculating the material->mesh maps } -void SGHRZ::initialize_material_state(SimulationParameters_t& SimulationParamaters, +void SGHRZ::initialize_material_state(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, diff --git a/single-node-refactor/src/Solvers/SGH_solver_rz/src/sgh_setup_rz.cpp b/single-node-refactor/src/Solvers/SGH_solver_rz/src/sgh_setup_rz.cpp index 6077fdc0a..15d6bf00a 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_rz/src/sgh_setup_rz.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_rz/src/sgh_setup_rz.cpp @@ -53,7 +53,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /// ///////////////////////////////////////////////////////////////////////////// void SGHRZ::init_corner_node_masses_zero_rz(const Mesh_t& mesh, - const DCArrayKokkos& node_mass, + const DistributedDCArray& node_mass, const DCArrayKokkos& corner_mass) const { @@ -78,7 +78,7 @@ void SGHRZ::init_corner_node_masses_zero_rz(const Mesh_t& mesh, /// \brief Allocate state, setup models, and fill mesh regions per the YAML input /// ///////////////////////////////////////////////////////////////////////////// -void SGHRZ::setup(SimulationParameters_t& SimulationParamaters, +void SGHRZ::setup(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, @@ -86,6 +86,8 @@ void SGHRZ::setup(SimulationParameters_t& SimulationParamaters, { // add a flag on whether SGHRZ was set up, if(SGHRZ_setup_already==false) + //update node velocity on ghosts + node_velocity_comms.execute_comms(); const size_t num_mats = Materials.num_mats; // the number of materials on the mesh @@ -107,12 +109,9 @@ void SGHRZ::setup(SimulationParameters_t& SimulationParamaters, } // for loop over mat_id - // set corner and node masses to zero init_corner_node_masses_zero_rz(mesh, State.node.mass, State.corner.mass); - - // 2D RZ // calculate the corner massess if 2D @@ -134,6 +133,9 @@ void SGHRZ::setup(SimulationParameters_t& SimulationParamaters, State.node.mass, State.corner.mass); + //communicate node masses to ghosts + node_mass_comms.execute_comms(); + } // end SGHRZ setup @@ -154,8 +156,8 @@ void SGHRZ::setup(SimulationParameters_t& SimulationParamaters, ///////////////////////////////////////////////////////////////////////////// void calc_corner_mass_rz(const Material_t& Materials, const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_mass, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_mass, const DCArrayKokkos& corner_mass, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& elem_in_mat_elem, @@ -203,8 +205,8 @@ void calc_corner_mass_rz(const Material_t& Materials, /// ///////////////////////////////////////////////////////////////////////////// void calc_node_mass_rz(const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_mass, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_mass, const DCArrayKokkos& corner_mass) { diff --git a/single-node-refactor/src/Solvers/SGH_solver_rz/src/time_integration_rz.cpp b/single-node-refactor/src/Solvers/SGH_solver_rz/src/time_integration_rz.cpp index 459272c4f..e9f8ffb23 100644 --- a/single-node-refactor/src/Solvers/SGH_solver_rz/src/time_integration_rz.cpp +++ b/single-node-refactor/src/Solvers/SGH_solver_rz/src/time_integration_rz.cpp @@ -52,10 +52,10 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /// ///////////////////////////////////////////////////////////////////////////// void SGHRZ::rk_init_rz( - DCArrayKokkos& node_coords, - DCArrayKokkos& node_coords_n0, - DCArrayKokkos& node_vel, - DCArrayKokkos& node_vel_n0, + DistributedDCArray& node_coords, + DistributedDCArray& node_coords_n0, + DistributedDCArray& node_vel, + DistributedDCArray& node_vel_n0, DRaggedRightArrayKokkos& MaterialPoints_sie, DRaggedRightArrayKokkos& MaterialPoints_sie_n0, DRaggedRightArrayKokkos& MaterialPoints_stress, @@ -111,8 +111,8 @@ void SGHRZ::rk_init_rz( /// ///////////////////////////////////////////////////////////////////////////// void SGHRZ::get_timestep_rz(Mesh_t& mesh, - DCArrayKokkos& node_coords, - DCArrayKokkos& node_vel, + DistributedDCArray& node_coords, + DistributedDCArray& node_vel, DCArrayKokkos& GaussPoints_vol, DRaggedRightArrayKokkos& MaterialPoints_sspd, DRaggedRightArrayKokkos& MaterialPoints_eroded, diff --git a/single-node-refactor/src/Solvers/SGTM_solver_3D/include/sgtm_solver_3D.h b/single-node-refactor/src/Solvers/SGTM_solver_3D/include/sgtm_solver_3D.h index c0c97ca14..fae0204e3 100644 --- a/single-node-refactor/src/Solvers/SGTM_solver_3D/include/sgtm_solver_3D.h +++ b/single-node-refactor/src/Solvers/SGTM_solver_3D/include/sgtm_solver_3D.h @@ -164,13 +164,13 @@ class SGTM3D : public Solver /// \brief Initializes data associated with the SGTM solver /// ///////////////////////////////////////////////////////////////////////////// - void initialize(SimulationParameters_t& SimulationParamaters, + void initialize(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, - State_t& State) const override; + State_t& State) override; - void initialize_material_state(SimulationParameters_t& SimulationParamaters, + void initialize_material_state(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, @@ -183,7 +183,7 @@ class SGTM3D : public Solver /// \brief Calls setup_sgtm, which initializes state and material data /// ///////////////////////////////////////////////////////////////////////////// - void setup(SimulationParameters_t& SimulationParamaters, + void setup(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, @@ -197,7 +197,7 @@ class SGTM3D : public Solver /// /// ///////////////////////////////////////////////////////////////////////////// - void execute(SimulationParameters_t& SimulationParamaters, + void execute(SimulationParameters_t& SimulationParameters, Material_t& Materials, BoundaryCondition_t& Boundary, Mesh_t& mesh, @@ -219,7 +219,7 @@ class SGTM3D : public Solver /// \return /// ///////////////////////////////////////////////////////////////////////////// - void finalize(SimulationParameters_t& SimulationParamaters, + void finalize(SimulationParameters_t& SimulationParameters, Material_t& Materials, BoundaryCondition_t& Boundary) const override { @@ -230,30 +230,30 @@ class SGTM3D : public Solver void boundary_temperature( const Mesh_t& mesh, const BoundaryCondition_t& Boundary, - DCArrayKokkos& node_temp, + DistributedDCArray& node_temp, const double time_value) const; void boundary_convection( const Mesh_t& mesh, const BoundaryCondition_t& BoundaryConditions, - const DCArrayKokkos& node_temp, - const DCArrayKokkos& node_flux, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_temp, + const DistributedDCArray& node_flux, + const DistributedDCArray& node_coords, const double time_value) const; void boundary_radiation( const Mesh_t& mesh, const BoundaryCondition_t& BoundaryConditions, - const DCArrayKokkos& node_temp, - const DCArrayKokkos& node_flux, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_temp, + const DistributedDCArray& node_flux, + const DistributedDCArray& node_coords, const double time_value) const; void boundary_heat_flux( const Mesh_t& mesh, const BoundaryCondition_t& Boundary, - DCArrayKokkos& node_temp, + DistributedDCArray& node_temp, const double time_value) const; // **** Functions defined in energy_sgtm.cpp **** // @@ -277,10 +277,10 @@ class SGTM3D : public Solver void update_temperature( const Mesh_t& mesh, const DCArrayKokkos& corner_q_transfer, - const DCArrayKokkos& node_temp, - const DCArrayKokkos& node_temp_n0, - const DCArrayKokkos& node_mass, - const DCArrayKokkos& node_q_transfer, + const DistributedDCArray& node_temp, + const DistributedDCArray& node_temp_n0, + const DistributedDCArray& node_mass, + const DistributedDCArray& node_q_transfer, const DRaggedRightArrayKokkos& mat_pt_specific_heat, const double rk_alpha, const double dt) const; @@ -290,8 +290,8 @@ class SGTM3D : public Solver const Material_t& Materials, const Mesh_t& mesh, const DCArrayKokkos& GaussPoints_vol, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_temp, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_temp, const DRaggedRightArrayKokkos& MaterialPoints_q_flux, const DRaggedRightArrayKokkos& MaterialPoints_conductivity, const DRaggedRightArrayKokkos& MaterialPoints_temp_grad, @@ -310,7 +310,7 @@ class SGTM3D : public Solver const Material_t& Materials, const Mesh_t& mesh, const DCArrayKokkos& GaussPoints_vol, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const DCArrayKokkos& corner_q_flux, const DCArrayKokkos& sphere_position, const corners_in_mat_t corners_in_mat_elem, @@ -328,10 +328,10 @@ class SGTM3D : public Solver double dt, const size_t num_dims, const size_t num_nodes, - DCArrayKokkos& node_coords, - const DCArrayKokkos& node_coords_n0, - const DCArrayKokkos& node_vel, - const DCArrayKokkos& node_vel_n0) const; + DistributedDCArray& node_coords, + const DistributedDCArray& node_coords_n0, + const DistributedDCArray& node_vel, + const DistributedDCArray& node_vel_n0) const; // **** Functions defined in momentum.cpp **** // @@ -339,16 +339,16 @@ class SGTM3D : public Solver double rk_alpha, double dt, const Mesh_t& mesh, - DCArrayKokkos& node_vel, - const DCArrayKokkos& node_mass, + DistributedDCArray& node_vel, + const DistributedDCArray& node_mass, const DCArrayKokkos& corner_force) const; // **** Functions defined in properties.cpp **** // void update_state( const Material_t& Materials, const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_pres, const DRaggedRightArrayKokkos& MaterialPoints_stress, @@ -367,13 +367,13 @@ class SGTM3D : public Solver // **** Functions defined in time_integration.cpp **** // // NOTE: Consider pulling up void rk_init( - DCArrayKokkos& node_coords, - DCArrayKokkos& node_coords_n0, - DCArrayKokkos& node_vel, - DCArrayKokkos& node_vel_n0, - DCArrayKokkos& node_temp, - DCArrayKokkos& node_temp_n0, - DCArrayKokkos& node_q_flux, + DistributedDCArray& node_coords, + DistributedDCArray& node_coords_n0, + DistributedDCArray& node_vel, + DistributedDCArray& node_vel_n0, + DistributedDCArray& node_temp, + DistributedDCArray& node_temp_n0, + DistributedDCArray& node_q_flux, DRaggedRightArrayKokkos& MaterialPoints_stress, const size_t num_dims, const size_t num_elems, @@ -382,8 +382,8 @@ class SGTM3D : public Solver void get_timestep( Mesh_t& mesh, - DCArrayKokkos& node_coords, - DCArrayKokkos& node_vel, + DistributedDCArray& node_coords, + DistributedDCArray& node_vel, DCArrayKokkos& GaussPoints_vol, DRaggedRightArrayKokkos& MaterialPoints_sspd, DRaggedRightArrayKokkos& MaterialPoints_conductivity, diff --git a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/boundary.cpp b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/boundary.cpp index 0556a1dfc..c742191c6 100644 --- a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/boundary.cpp +++ b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/boundary.cpp @@ -50,7 +50,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ///////////////////////////////////////////////////////////////////////////// void SGTM3D::boundary_temperature(const Mesh_t& mesh, const BoundaryCondition_t& BoundaryConditions, - DCArrayKokkos& node_temp, + DistributedDCArray& node_temp, const double time_value) const { // ---- Loop over boundary sets ---- // @@ -100,9 +100,9 @@ void SGTM3D::boundary_temperature(const Mesh_t& mesh, ///////////////////////////////////////////////////////////////////////////// void SGTM3D::boundary_convection(const Mesh_t& mesh, const BoundaryCondition_t& BoundaryConditions, - const DCArrayKokkos& node_temp, - const DCArrayKokkos& node_flux, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_temp, + const DistributedDCArray& node_flux, + const DistributedDCArray& node_coords, const double time_value) const { // ---- Loop over boundary sets ---- // @@ -240,9 +240,9 @@ void SGTM3D::boundary_convection(const Mesh_t& mesh, ///////////////////////////////////////////////////////////////////////////// void SGTM3D::boundary_radiation(const Mesh_t& mesh, const BoundaryCondition_t& BoundaryConditions, - const DCArrayKokkos& node_temp, - const DCArrayKokkos& node_flux, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_temp, + const DistributedDCArray& node_flux, + const DistributedDCArray& node_coords, const double time_value) const { // ---- Loop over boundary sets ---- // @@ -388,7 +388,7 @@ void SGTM3D::boundary_radiation(const Mesh_t& mesh, ///////////////////////////////////////////////////////////////////////////// void SGTM3D::boundary_heat_flux(const Mesh_t& mesh, const BoundaryCondition_t& BoundaryConditions, - DCArrayKokkos& node_temp, + DistributedDCArray& node_temp, const double time_value) const { // // Loop over boundary sets diff --git a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/energy_sgtm.cpp b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/energy_sgtm.cpp index ff0604487..12a22937c 100644 --- a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/energy_sgtm.cpp +++ b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/energy_sgtm.cpp @@ -56,10 +56,10 @@ void SGTM3D::update_temperature( const double rk_alpha, const double dt, const Mesh_t& mesh, - const DCArrayKokkos& node_vel, - const DCArrayKokkos& node_vel_n0, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_coords_n0, + const DistributedDCArray& node_vel, + const DistributedDCArray& node_vel_n0, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_coords_n0, const DRaggedRightArrayKokkos& MaterialPoints_sie, const DRaggedRightArrayKokkos& MaterialPoints_sie_n0, const DRaggedRightArrayKokkos& MaterialPoints_mass, diff --git a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/heat_flux.cpp b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/heat_flux.cpp index 2aaa086e6..32840a62f 100644 --- a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/heat_flux.cpp +++ b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/heat_flux.cpp @@ -71,8 +71,8 @@ void SGTM3D::get_heat_flux( const Material_t& Materials, const Mesh_t& mesh, const DCArrayKokkos& GaussPoints_vol, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_temp, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_temp, const DRaggedRightArrayKokkos& MaterialPoints_q_flux, const DRaggedRightArrayKokkos& MaterialPoints_conductivity, const DRaggedRightArrayKokkos& MaterialPoints_temp_grad, @@ -221,7 +221,7 @@ void SGTM3D::moving_flux( const Material_t& Materials, const Mesh_t& mesh, const DCArrayKokkos& GaussPoints_vol, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const DCArrayKokkos& corner_q_flux, const DCArrayKokkos& sphere_position, const corners_in_mat_t corners_in_mat_elem, diff --git a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/momentum.cpp b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/momentum.cpp index 12879f960..17890246a 100644 --- a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/momentum.cpp +++ b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/momentum.cpp @@ -52,8 +52,8 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. void SGTM3D::update_velocity(double rk_alpha, double dt, const Mesh_t& mesh, - DCArrayKokkos& node_vel, - const DCArrayKokkos& node_mass, + DistributedDCArray& node_vel, + const DistributedDCArray& node_mass, const DCArrayKokkos& corner_force) const { // const size_t num_dims = mesh.num_dims; diff --git a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/position.cpp b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/position.cpp index d6bf2d67b..bbbe715f1 100644 --- a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/position.cpp +++ b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/position.cpp @@ -52,10 +52,10 @@ void SGTM3D::update_position( double dt, const size_t num_dims, const size_t num_nodes, - DCArrayKokkos& node_coords, - const DCArrayKokkos& node_coords_n0, - const DCArrayKokkos& node_vel, - const DCArrayKokkos& node_vel_n0) const + DistributedDCArray& node_coords, + const DistributedDCArray& node_coords_n0, + const DistributedDCArray& node_vel, + const DistributedDCArray& node_vel_n0) const { // loop over all the nodes in the mesh FOR_ALL(node_gid, 0, num_nodes, { diff --git a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/properties.cpp b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/properties.cpp index cc8c7628e..57ee70457 100644 --- a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/properties.cpp +++ b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/properties.cpp @@ -63,8 +63,8 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. void SGTM3D::update_state( const Material_t& Materials, const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_vel, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_pres, const DRaggedRightArrayKokkos& MaterialPoints_stress, diff --git a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/sgtm_execute.cpp b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/sgtm_execute.cpp index 1d6aacad8..47b9b3b96 100644 --- a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/sgtm_execute.cpp +++ b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/sgtm_execute.cpp @@ -49,29 +49,29 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /// Evolve the state according to the SGH method /// ///////////////////////////////////////////////////////////////////////////// -void SGTM3D::execute(SimulationParameters_t& SimulationParamaters, +void SGTM3D::execute(SimulationParameters_t& SimulationParameters, Material_t& Materials, BoundaryCondition_t& BoundaryConditions, Mesh_t& mesh, State_t& State) { - double fuzz = SimulationParamaters.dynamic_options.fuzz; - double tiny = SimulationParamaters.dynamic_options.tiny; - double small = SimulationParamaters.dynamic_options.small; + double fuzz = SimulationParameters.dynamic_options.fuzz; + double tiny = SimulationParameters.dynamic_options.tiny; + double small = SimulationParameters.dynamic_options.small; - double graphics_dt_ival = SimulationParamaters.output_options.graphics_time_step; - int graphics_cyc_ival = SimulationParamaters.output_options.graphics_iteration_step; + double graphics_dt_ival = SimulationParameters.output_options.graphics_time_step; + int graphics_cyc_ival = SimulationParameters.output_options.graphics_iteration_step; - // double time_initial = SimulationParamaters.dynamic_options.time_initial; - double time_final = this->time_end; // SimulationParamaters.dynamic_options.time_final; - double dt_min = SimulationParamaters.dynamic_options.dt_min; - double dt_max = SimulationParamaters.dynamic_options.dt_max; - double dt_start = SimulationParamaters.dynamic_options.dt_start; - double dt_cfl = SimulationParamaters.dynamic_options.dt_cfl; + // double time_initial = SimulationParameters.dynamic_options.time_initial; + double time_final = this->time_end; // SimulationParameters.dynamic_options.time_final; + double dt_min = SimulationParameters.dynamic_options.dt_min; + double dt_max = SimulationParameters.dynamic_options.dt_max; + double dt_start = SimulationParameters.dynamic_options.dt_start; + double dt_cfl = SimulationParameters.dynamic_options.dt_cfl; - int rk_num_stages = SimulationParamaters.dynamic_options.rk_num_stages; - int cycle_stop = SimulationParamaters.dynamic_options.cycle_stop; + int rk_num_stages = SimulationParameters.dynamic_options.rk_num_stages; + int cycle_stop = SimulationParameters.dynamic_options.cycle_stop; // initialize time, time_step, and cycles double time_value = this->time_start; // 0.0; @@ -120,7 +120,7 @@ void SGTM3D::execute(SimulationParameters_t& SimulationParamaters, mesh_writer.write_mesh( mesh, State, - SimulationParamaters, + SimulationParameters, dt, time_value, graphics_times, @@ -385,7 +385,7 @@ std::cout << "update temperature \n"; printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_value, dt); mesh_writer.write_mesh(mesh, State, - SimulationParamaters, + SimulationParameters, dt, time_value, graphics_times, diff --git a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/sgtm_initialize.cpp b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/sgtm_initialize.cpp index fd098e316..db0c465bc 100644 --- a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/sgtm_initialize.cpp +++ b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/sgtm_initialize.cpp @@ -37,11 +37,11 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "mesh.h" #include "simulation_parameters.h" -void SGTM3D::initialize(SimulationParameters_t& SimulationParamaters, +void SGTM3D::initialize(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, - State_t& State) const + State_t& State) { int num_nodes = mesh.num_nodes; int num_gauss_pts = mesh.num_elems; @@ -58,7 +58,7 @@ void SGTM3D::initialize(SimulationParameters_t& SimulationParamaters, // check that the fills specify the required nodal fields bool filled_nodal_state = check_fill_node_states(SGTM3D_State::required_fill_node_state, - SimulationParamaters.region_setups.fill_node_states); + SimulationParameters.region_setups.fill_node_states); if (filled_nodal_state == false){ std::cout <<" Missing required nodal state in the fill instructions for the thrmex_FE solver \n"; @@ -70,7 +70,7 @@ void SGTM3D::initialize(SimulationParameters_t& SimulationParamaters, } // end solver initialization -void SGTM3D::initialize_material_state(SimulationParameters_t& SimulationParamaters, +void SGTM3D::initialize_material_state(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, @@ -91,15 +91,15 @@ void SGTM3D::initialize_material_state(SimulationParameters_t& SimulationParamat // check that the fills specify the required material point state fields bool filled_material_state_A = check_fill_mat_states(SGTM3D_State::required_optA_fill_material_pt_state, - SimulationParamaters.region_setups.fill_gauss_states); + SimulationParameters.region_setups.fill_gauss_states); bool filled_material_state_B = check_fill_mat_states(SGTM3D_State::required_optB_fill_material_pt_state, - SimulationParamaters.region_setups.fill_gauss_states); + SimulationParameters.region_setups.fill_gauss_states); // --- full stress tensor is not yet supported in region_fill --- //bool filled_material_state_C = // check_fill_mat_states(SGTM3D_State::required_optC_fill_material_pt_state, - // SimulationParamaters.region_setups.fill_gauss_states); + // SimulationParameters.region_setups.fill_gauss_states); if (filled_material_state_A == false && filled_material_state_B == false){ diff --git a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/sgtm_setup.cpp b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/sgtm_setup.cpp index 4b3e9d090..f4d8ab542 100644 --- a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/sgtm_setup.cpp +++ b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/sgtm_setup.cpp @@ -50,7 +50,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /// \brief Calls setup_sgtm to unpack SimulationParameters for GPU access /// ///////////////////////////////////////////////////////////////////////////// -void SGTM3D::setup(SimulationParameters_t& SimulationParamaters, +void SGTM3D::setup(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, @@ -60,7 +60,6 @@ void SGTM3D::setup(SimulationParameters_t& SimulationParamaters, const size_t num_mats = Materials.num_mats; // the number of materials on the mesh - std::cout << "Calculating pressure, sound speed, and stress" << std::endl; // calculate pressure, sound speed, and stress for each material for (int mat_id = 0; mat_id < num_mats; mat_id++) { diff --git a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/temperature_sgtm.cpp b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/temperature_sgtm.cpp index 3cf8dcab3..1c667315d 100644 --- a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/temperature_sgtm.cpp +++ b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/temperature_sgtm.cpp @@ -54,10 +54,10 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. void SGTM3D::update_temperature( const Mesh_t& mesh, const DCArrayKokkos& corner_q_transfer, - const DCArrayKokkos& node_temp, - const DCArrayKokkos& node_temp_n0, - const DCArrayKokkos& node_mass, - const DCArrayKokkos& node_q_transfer, + const DistributedDCArray& node_temp, + const DistributedDCArray& node_temp_n0, + const DistributedDCArray& node_mass, + const DistributedDCArray& node_q_transfer, const DRaggedRightArrayKokkos& mat_pt_specific_heat, const double rk_alpha, const double dt) const diff --git a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/time_integration.cpp b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/time_integration.cpp index 431a79096..9820b54c2 100644 --- a/single-node-refactor/src/Solvers/SGTM_solver_3D/src/time_integration.cpp +++ b/single-node-refactor/src/Solvers/SGTM_solver_3D/src/time_integration.cpp @@ -51,13 +51,13 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /// ///////////////////////////////////////////////////////////////////////////// void SGTM3D::rk_init( - DCArrayKokkos& node_coords, - DCArrayKokkos& node_coords_n0, - DCArrayKokkos& node_vel, - DCArrayKokkos& node_vel_n0, - DCArrayKokkos& node_temp, - DCArrayKokkos& node_temp_n0, - DCArrayKokkos& node_q_flux, + DistributedDCArray& node_coords, + DistributedDCArray& node_coords_n0, + DistributedDCArray& node_vel, + DistributedDCArray& node_vel_n0, + DistributedDCArray& node_temp, + DistributedDCArray& node_temp_n0, + DistributedDCArray& node_q_flux, DRaggedRightArrayKokkos& MaterialPoints_stress, const size_t num_dims, const size_t num_elems, @@ -100,8 +100,8 @@ std::cout << "done with rk_int \n "; /// ///////////////////////////////////////////////////////////////////////////// void SGTM3D::get_timestep(Mesh_t& mesh, - DCArrayKokkos& node_coords, - DCArrayKokkos& node_vel, + DistributedDCArray& node_coords, + DistributedDCArray& node_vel, DCArrayKokkos& GaussPoints_vol, DRaggedRightArrayKokkos& MaterialPoints_sspd, DRaggedRightArrayKokkos& MaterialPoints_conductivity, diff --git a/single-node-refactor/src/Solvers/level_set_solver/include/level_set_solver.h b/single-node-refactor/src/Solvers/level_set_solver/include/level_set_solver.h index 73c2ecc6c..ef6275013 100644 --- a/single-node-refactor/src/Solvers/level_set_solver/include/level_set_solver.h +++ b/single-node-refactor/src/Solvers/level_set_solver/include/level_set_solver.h @@ -115,7 +115,7 @@ class LevelSet : public Solver Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, - State_t& State) const override; + State_t& State) override; void initialize_material_state(SimulationParameters_t& SimulationParamaters, Material_t& Materials, @@ -176,9 +176,9 @@ class LevelSet : public Solver // **** Functions defined in solver_functions.cpp **** // void nodal_gradient( const Mesh_t mesh, - const DCArrayKokkos& Node_coords, - const DCArrayKokkos& node_level_set_vel, - const DCArrayKokkos& Node_grad_level_set, + const DistributedDCArray& Node_coords, + const DistributedDCArray& node_level_set_vel, + const DistributedDCArray& Node_grad_level_set, const DCArrayKokkos& Corner_normal, const DCArrayKokkos& Corner_volume, const DCArrayKokkos& GaussPoints_level_set, @@ -189,8 +189,8 @@ class LevelSet : public Solver void update_level_set( const Mesh_t& mesh, const Material_t& Materials, - const DCArrayKokkos& node_level_set_vel, - const DCArrayKokkos& Node_grad_level_set, + const DistributedDCArray& node_level_set_vel, + const DistributedDCArray& Node_grad_level_set, const DCArrayKokkos& GaussPoints_level_set, const DCArrayKokkos& GaussPoints_level_set_n, const DCArrayKokkos& GaussPoints_vol, @@ -219,7 +219,7 @@ class LevelSet : public Solver void get_timestep( const Mesh_t& mesh, const Material_t& Materials, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const DCArrayKokkos& GaussPoints_vol, const DRaggedRightArrayKokkos& elem_in_mat_elem, const size_t num_mat_elems, @@ -237,7 +237,7 @@ class LevelSet : public Solver void get_timestep_2D( const Mesh_t& mesh, const Material_t& Materials, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const DCArrayKokkos& GaussPoints_vol, const DRaggedRightArrayKokkos& elem_in_mat_elem, const size_t num_mat_elems, @@ -258,7 +258,7 @@ class LevelSet : public Solver void boundary_velocity( const Mesh_t& mesh, const BoundaryCondition_t& BoundaryConditions, - DCArrayKokkos& node_vel, + DistributedDCArray& node_level_set_vel, const double time_value, const double small) const; diff --git a/single-node-refactor/src/Solvers/level_set_solver/src/level_set_boundary.cpp b/single-node-refactor/src/Solvers/level_set_solver/src/level_set_boundary.cpp index 9435bf1b0..36f893d01 100644 --- a/single-node-refactor/src/Solvers/level_set_solver/src/level_set_boundary.cpp +++ b/single-node-refactor/src/Solvers/level_set_solver/src/level_set_boundary.cpp @@ -50,7 +50,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ///////////////////////////////////////////////////////////////////////////// void LevelSet::boundary_velocity(const Mesh_t& mesh, const BoundaryCondition_t& BoundaryConditions, - DCArrayKokkos& node_level_set_vel, + DistributedDCArray& node_level_set_vel, const double time_value, const double small) const { diff --git a/single-node-refactor/src/Solvers/level_set_solver/src/level_set_execute.cpp b/single-node-refactor/src/Solvers/level_set_solver/src/level_set_execute.cpp index 6e2ed366c..86f653948 100644 --- a/single-node-refactor/src/Solvers/level_set_solver/src/level_set_execute.cpp +++ b/single-node-refactor/src/Solvers/level_set_solver/src/level_set_execute.cpp @@ -57,7 +57,7 @@ void LevelSet::execute(SimulationParameters_t& SimulationParamaters, { // arrays local to this solver - DCArrayKokkos node_level_set_vel(mesh.num_nodes, mesh.num_dims); + DistributedDCArray node_level_set_vel(mesh.global_num_nodes, mesh.num_dims); double fuzz = SimulationParamaters.dynamic_options.fuzz; double tiny = SimulationParamaters.dynamic_options.tiny; diff --git a/single-node-refactor/src/Solvers/level_set_solver/src/level_set_initialize.cpp b/single-node-refactor/src/Solvers/level_set_solver/src/level_set_initialize.cpp index 84a39ff52..99def98c3 100644 --- a/single-node-refactor/src/Solvers/level_set_solver/src/level_set_initialize.cpp +++ b/single-node-refactor/src/Solvers/level_set_solver/src/level_set_initialize.cpp @@ -42,7 +42,7 @@ Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, - State_t& State) const + State_t& State) { const size_t num_nodes = mesh.num_nodes; const size_t num_gauss_pts = mesh.num_elems; diff --git a/single-node-refactor/src/Solvers/level_set_solver/src/solver_functions.cpp b/single-node-refactor/src/Solvers/level_set_solver/src/solver_functions.cpp index fc6f39952..09569e2dd 100644 --- a/single-node-refactor/src/Solvers/level_set_solver/src/solver_functions.cpp +++ b/single-node-refactor/src/Solvers/level_set_solver/src/solver_functions.cpp @@ -53,9 +53,9 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ///////////////////////////////////////////////////////////////////////////// void LevelSet::nodal_gradient( const Mesh_t mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_level_set_vel, - const DCArrayKokkos& node_grad_level_set, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_level_set_vel, + const DistributedDCArray& node_grad_level_set, const DCArrayKokkos& corner_normal, const DCArrayKokkos& corner_volume, const DCArrayKokkos& GaussPoints_level_set, @@ -201,8 +201,8 @@ void LevelSet::nodal_gradient( void LevelSet::update_level_set( const Mesh_t& mesh, const Material_t& Materials, - const DCArrayKokkos& node_level_set_vel, - const DCArrayKokkos& node_grad_level_set, + const DistributedDCArray& node_level_set_vel, + const DistributedDCArray& node_grad_level_set, const DCArrayKokkos& GaussPoints_level_set, const DCArrayKokkos& GaussPoints_level_set_n0, const DCArrayKokkos& GaussPoints_vol, diff --git a/single-node-refactor/src/Solvers/level_set_solver/src/time_integration.cpp b/single-node-refactor/src/Solvers/level_set_solver/src/time_integration.cpp index bb778985d..4fd74edf8 100644 --- a/single-node-refactor/src/Solvers/level_set_solver/src/time_integration.cpp +++ b/single-node-refactor/src/Solvers/level_set_solver/src/time_integration.cpp @@ -94,7 +94,7 @@ void LevelSet::rk_init( void LevelSet::get_timestep( const Mesh_t& mesh, const Material_t& Materials, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const DCArrayKokkos& GaussPoints_vol, const DRaggedRightArrayKokkos& elem_mat_elem, const size_t num_mat_elems, @@ -221,7 +221,7 @@ void LevelSet::get_timestep( void LevelSet::get_timestep_2D( const Mesh_t& mesh, const Material_t& Materials, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const DCArrayKokkos& GaussPoints_vol, const DRaggedRightArrayKokkos& elem_mat_elem, const size_t num_mat_elems, diff --git a/single-node-refactor/src/boundary_conditions/temperature/constant_temp_bc.h b/single-node-refactor/src/boundary_conditions/temperature/constant_temp_bc.h index bcf390daa..3453ddb2a 100644 --- a/single-node-refactor/src/boundary_conditions/temperature/constant_temp_bc.h +++ b/single-node-refactor/src/boundary_conditions/temperature/constant_temp_bc.h @@ -63,7 +63,7 @@ static void temperature(const Mesh_t& mesh, const DCArrayKokkos& BoundaryConditionEnums, const RaggedRightArrayKokkos& temp_bc_global_vars, const DCArrayKokkos& bc_state_vars, - const DCArrayKokkos& node_temp, + const DistributedDCArray& node_temp, const double time_value, const size_t rk_stage, const size_t bdy_node_gid, diff --git a/single-node-refactor/src/boundary_conditions/velocity/constant_velocity_bc.h b/single-node-refactor/src/boundary_conditions/velocity/constant_velocity_bc.h index a1d487c61..063a47225 100644 --- a/single-node-refactor/src/boundary_conditions/velocity/constant_velocity_bc.h +++ b/single-node-refactor/src/boundary_conditions/velocity/constant_velocity_bc.h @@ -62,7 +62,7 @@ static void velocity(const Mesh_t& mesh, const DCArrayKokkos& BoundaryConditionEnums, const RaggedRightArrayKokkos& vel_bc_global_vars, const DCArrayKokkos& bc_state_vars, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_vel, const double time_value, const size_t rk_stage, const size_t bdy_node_gid, diff --git a/single-node-refactor/src/boundary_conditions/velocity/no_velocity_bc.h b/single-node-refactor/src/boundary_conditions/velocity/no_velocity_bc.h index 9a0a4e1c7..023e9bba2 100644 --- a/single-node-refactor/src/boundary_conditions/velocity/no_velocity_bc.h +++ b/single-node-refactor/src/boundary_conditions/velocity/no_velocity_bc.h @@ -62,7 +62,7 @@ static void velocity(const Mesh_t& mesh, const DCArrayKokkos& BoundaryConditionEnums, const RaggedRightArrayKokkos& vel_bc_global_vars, const DCArrayKokkos& bc_state_vars, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_vel, const double time_value, const size_t rk_stage, const size_t bdy_node_gid, diff --git a/single-node-refactor/src/boundary_conditions/velocity/piston_velocity_bc.h b/single-node-refactor/src/boundary_conditions/velocity/piston_velocity_bc.h index e11303f1a..6d97b8de6 100644 --- a/single-node-refactor/src/boundary_conditions/velocity/piston_velocity_bc.h +++ b/single-node-refactor/src/boundary_conditions/velocity/piston_velocity_bc.h @@ -77,7 +77,7 @@ static void velocity(const Mesh_t& mesh, const DCArrayKokkos& BoundaryConditionEnums, const RaggedRightArrayKokkos& vel_bc_global_vars, const DCArrayKokkos& bc_state_vars, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_vel, const double time_value, const size_t rk_stage, const size_t bdy_node_gid, diff --git a/single-node-refactor/src/boundary_conditions/velocity/reflected_velocity_bc.h b/single-node-refactor/src/boundary_conditions/velocity/reflected_velocity_bc.h index 3cb14a997..5bdb1089b 100644 --- a/single-node-refactor/src/boundary_conditions/velocity/reflected_velocity_bc.h +++ b/single-node-refactor/src/boundary_conditions/velocity/reflected_velocity_bc.h @@ -64,7 +64,7 @@ static void velocity(const Mesh_t& mesh, const DCArrayKokkos& BoundaryConditionEnums, const RaggedRightArrayKokkos& vel_bc_global_vars, const DCArrayKokkos& bc_state_vars, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_vel, const double time_value, const size_t rk_stage, const size_t bdy_node_gid, diff --git a/single-node-refactor/src/boundary_conditions/velocity/time_varying_velocity_bc.h b/single-node-refactor/src/boundary_conditions/velocity/time_varying_velocity_bc.h index 09a78a216..bbc9e48b1 100644 --- a/single-node-refactor/src/boundary_conditions/velocity/time_varying_velocity_bc.h +++ b/single-node-refactor/src/boundary_conditions/velocity/time_varying_velocity_bc.h @@ -78,7 +78,7 @@ static void velocity(const Mesh_t& mesh, const DCArrayKokkos& BoundaryConditionEnums, const RaggedRightArrayKokkos& vel_bc_global_vars, const DCArrayKokkos& bc_state_vars, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_vel, const double time_value, const size_t rk_stage, const size_t bdy_node_gid, diff --git a/single-node-refactor/src/boundary_conditions/velocity/user_defined_velocity_bc.h b/single-node-refactor/src/boundary_conditions/velocity/user_defined_velocity_bc.h index 0024ef024..92b02aec5 100644 --- a/single-node-refactor/src/boundary_conditions/velocity/user_defined_velocity_bc.h +++ b/single-node-refactor/src/boundary_conditions/velocity/user_defined_velocity_bc.h @@ -64,7 +64,7 @@ static void velocity(const Mesh_t& mesh, const DCArrayKokkos& BoundaryConditionEnums, const RaggedRightArrayKokkos& vel_bc_global_vars, const DCArrayKokkos& bc_state_vars, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_vel, const double time_value, const size_t rk_stage, const size_t bdy_node_gid, diff --git a/single-node-refactor/src/boundary_conditions/velocity/zero_velocity_bc.h b/single-node-refactor/src/boundary_conditions/velocity/zero_velocity_bc.h index 8334eb99c..4daacb273 100644 --- a/single-node-refactor/src/boundary_conditions/velocity/zero_velocity_bc.h +++ b/single-node-refactor/src/boundary_conditions/velocity/zero_velocity_bc.h @@ -64,7 +64,7 @@ static void velocity(const Mesh_t& mesh, const DCArrayKokkos& BoundaryConditionEnums, const RaggedRightArrayKokkos& vel_bc_global_vars, const DCArrayKokkos& bc_state_vars, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_vel, const double time_value, const size_t rk_stage, const size_t bdy_node_gid, diff --git a/single-node-refactor/src/common/include/boundary_conditions.h b/single-node-refactor/src/common/include/boundary_conditions.h index d58091ac9..2fcb128cb 100644 --- a/single-node-refactor/src/common/include/boundary_conditions.h +++ b/single-node-refactor/src/common/include/boundary_conditions.h @@ -214,7 +214,7 @@ struct BoundaryConditionFunctions_t const DCArrayKokkos& BoundaryConditionEnums, const RaggedRightArrayKokkos& vel_bc_global_vars, const DCArrayKokkos& bc_state_vars, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_vel, const double time_value, const size_t rk_stage, const size_t bdy_node_gid, @@ -225,7 +225,7 @@ struct BoundaryConditionFunctions_t const DCArrayKokkos& BoundaryConditionEnums, const RaggedRightArrayKokkos& temp_bc_global_vars, const DCArrayKokkos& bc_state_vars, - const DCArrayKokkos& node_temp, + const DistributedDCArray& node_temp, const double time_value, const size_t rk_stage, const size_t bdy_node_gid, @@ -236,7 +236,7 @@ struct BoundaryConditionFunctions_t const DCArrayKokkos& BoundaryConditionEnums, const RaggedRightArrayKokkos& heat_flux_bc_global_vars, const DCArrayKokkos& bc_state_vars, - const DCArrayKokkos& node_temp, + const DistributedDCArray& node_temp, const double time_value, const size_t rk_stage, const size_t bdy_node_gid, diff --git a/single-node-refactor/src/common/include/geometry_new.h b/single-node-refactor/src/common/include/geometry_new.h index 54153823e..b71a4be8d 100644 --- a/single-node-refactor/src/common/include/geometry_new.h +++ b/single-node-refactor/src/common/include/geometry_new.h @@ -70,7 +70,7 @@ namespace geometry KOKKOS_FUNCTION void get_bmatrix(const ViewCArrayKokkos& B_matrix, const size_t elem_gid, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const ViewCArrayKokkos& elem_node_gids); ///////////////////////////////////////////////////////////////////////////// @@ -88,7 +88,7 @@ void get_bmatrix(const ViewCArrayKokkos& B_matrix, KOKKOS_FUNCTION void get_vol_quad(const DCArrayKokkos& elem_vol, const size_t elem_gid, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const ViewCArrayKokkos& elem_node_gids); ///////////////////////////////////////////////////////////////////////////// @@ -106,7 +106,7 @@ void get_vol_quad(const DCArrayKokkos& elem_vol, KOKKOS_FUNCTION void get_vol_hex(const DCArrayKokkos& elem_vol, const size_t elem_gid, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const ViewCArrayKokkos& elem_node_gids); ///////////////////////////////////////////////////////////////////////////// @@ -117,7 +117,7 @@ void get_vol_hex(const DCArrayKokkos& elem_vol, /// ///////////////////////////////////////////////////////////////////////////// void get_vol(const DCArrayKokkos& elem_vol, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const Mesh_t& mesh); ///////////////////////////////////////////////////////////////////////////// @@ -135,7 +135,7 @@ void get_vol(const DCArrayKokkos& elem_vol, KOKKOS_FUNCTION void get_bmatrix2D(const ViewCArrayKokkos& B_matrix, const size_t elem_gid, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const ViewCArrayKokkos& elem_node_gids); ///////////////////////////////////////////////////////////////////////////// @@ -153,7 +153,7 @@ void get_bmatrix2D(const ViewCArrayKokkos& B_matrix, ///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION double get_area_quad(const size_t elem_gid, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const ViewCArrayKokkos& elem_node_gids); ///////////////////////////////////////////////////////////////////////////// @@ -195,7 +195,7 @@ double heron(const double x1, KOKKOS_FUNCTION void get_area_weights2D(const ViewCArrayKokkos& corner_areas, const size_t elem_gid, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const ViewCArrayKokkos& elem_node_gids); } // end namespace @@ -222,7 +222,7 @@ size_t check_bdy(const size_t patch_gid, const double orig_y, const double orig_z, const Mesh_t& mesh, - const DCArrayKokkos& node_coords); + const DistributedDCArray& node_coords); ///////////////////////////////////////////////////////////////////////////// /// @@ -237,7 +237,7 @@ size_t check_bdy(const size_t patch_gid, ///////////////////////////////////////////////////////////////////////////// void tag_bdys(const BoundaryCondition_t& boundary, Mesh_t& mesh, - const DCArrayKokkos& node_coords); + const DistributedDCArray& node_coords); ///////////////////////////////////////////////////////////////////////////// diff --git a/single-node-refactor/src/common/include/material.h b/single-node-refactor/src/common/include/material.h index ebd364478..138fff304 100644 --- a/single-node-refactor/src/common/include/material.h +++ b/single-node-refactor/src/common/include/material.h @@ -40,6 +40,8 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. using namespace mtr; +template +using DistributedDCArray = TpetraDCArray; namespace model { // strength model types @@ -330,9 +332,9 @@ struct MaterialFunctions_t // Material strength model function pointers void (*calc_stress)( const DCArrayKokkos &GaussPoints_vel_grad, - const DCArrayKokkos &node_coords, - const DCArrayKokkos &node_vel, - const DCArrayKokkos &nodes_in_elem, + const DistributedDCArray &node_coords, + const DistributedDCArray &node_vel, + const DistributedDCArray &nodes_in_elem, const DRaggedRightArrayKokkos &MaterialPoints_pres, const DRaggedRightArrayKokkos &MaterialPoints_stress, const DRaggedRightArrayKokkos &MaterialPoints_stress_n0, @@ -390,7 +392,7 @@ struct MaterialFunctions_t const RaggedRightArrayKokkos & dissipation_global_vars, const DCArrayKokkos& GaussPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_eroded, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_vel, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_sspd, const ViewCArrayKokkos& disp_corner_forces, diff --git a/single-node-refactor/src/common/include/mesh.h b/single-node-refactor/src/common/include/mesh.h index 599cb77da..8552af0f3 100644 --- a/single-node-refactor/src/common/include/mesh.h +++ b/single-node-refactor/src/common/include/mesh.h @@ -37,6 +37,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "matar.h" #include "state.h" #include "ref_elem.h" +#include "mesh_inputs.h" #include #define PI 3.141592653589793 @@ -229,7 +230,9 @@ struct Mesh_t size_t num_dims = 3; ///< Number of spatial dimension // ---- Element Data Definitions ---- // - size_t num_elems; ///< Number of elements in the mesh + size_t global_num_elems; ///< Global number of elements in the mesh + size_t num_elems; ///< number of local+shared elements on this process (forces usually employ this) + size_t num_local_elems; ///< number of local elements on this process (output and reductions for energy usually employ this) size_t num_nodes_in_elem; ///< Number of nodes in an element size_t num_patches_in_elem; ///< Number of patches in an element size_t num_surfs_in_elem; ///< Number of surfaces in an element @@ -238,7 +241,8 @@ struct Mesh_t size_t num_gauss_in_elem; ///< Number of Gauss points in an element size_t num_lobatto_in_elem; ///< Number of Gauss Lobatto points in an element - DCArrayKokkos nodes_in_elem; ///< Nodes in an element + DistributedDCArray nodes_in_elem; ///< Nodes in an element + DistributedDCArray local_nodes_in_elem; ///< Nodes in uniquely distributed element (subview of above) CArrayKokkos corners_in_elem; ///< Corners in an element -- this can just be a functor RaggedRightArrayKokkos elems_in_elem; ///< Elements connected to an element @@ -253,7 +257,21 @@ struct Mesh_t gauss_in_elem_t gauss_in_elem; ///< Gauss points in an element // ---- Node Data Definitions ---- // - size_t num_nodes; ///< Number of nodes in the mesh + size_t global_num_nodes; ///< Global Number of nodes in the mesh + size_t num_nodes; ///< number of local + ghost nodes on this process + size_t num_local_nodes; ///< number of nodes local to this process + size_t num_ghost_nodes; ///< number of ghost nodes on this process + + //distributed map definitions + DistributedMap node_map; ///< partition of local nodes (stores global node IDs on each process) + DistributedMap all_node_map; ///< partition of local + ghost nodes (stores global node IDs on each process) + DistributedMap ghost_node_map; ///< partition of local + ghost nodes (stores global node IDs on each process) + DistributedMap local_element_map; ///< partition of uniquely owned elements (stores global node IDs on each process) + DistributedMap element_map; ///< partition of uniquely owned + shared elements (stores global node IDs on each process) + DistributedMap nonoverlap_element_node_map; // map of node indices belonging to unique element map + + //communication plans + CommPlan node_coords_comms; RaggedRightArrayKokkos corners_in_node; ///< Corners connected to a node CArrayKokkos num_corners_in_node; ///< Number of corners connected to a node @@ -307,22 +325,21 @@ struct Mesh_t // initialization methods void initialize_nodes(const size_t num_nodes_inp) { - num_nodes = num_nodes_inp; + global_num_nodes = num_nodes_inp; return; }; // end method // initialization methods - void initialize_elems(const size_t num_elems_inp, const size_t num_dims_inp) + void initialize_elems(const size_t num_elems_inp, const size_t input_num_nodes_in_elem, const DistributedMap input_element_map) { - num_dims = num_dims_inp; - num_nodes_in_elem = 1; - - for (int dim = 0; dim < num_dims; dim++) { - num_nodes_in_elem *= 2; - } num_elems = num_elems_inp; - nodes_in_elem = DCArrayKokkos(num_elems, num_nodes_in_elem, "mesh.nodes_in_elem"); + element_map = input_element_map; + + //number of nodes per element + num_nodes_in_elem = input_num_nodes_in_elem; + + nodes_in_elem = DistributedDCArray(element_map, num_nodes_in_elem, "mesh.nodes_in_elem"); corners_in_elem = CArrayKokkos(num_elems, num_nodes_in_elem, "mesh.corners_in_elem"); // 1 Gauss point per element @@ -343,7 +360,8 @@ struct Mesh_t const size_t num_zones_in_elem_inp, const size_t num_nodes_in_zone_inp, const size_t num_surfs_in_elem_inp, - const size_t num_dims_inp) + const size_t num_dims_inp, + const DistributedMap input_element_map) { num_dims = num_dims_inp; num_elems = num_elems_inp; @@ -355,8 +373,9 @@ struct Mesh_t num_surfs_in_elem = num_surfs_in_elem_inp; num_zones = num_zones_in_elem * num_elems; + element_map = input_element_map; - nodes_in_elem = DCArrayKokkos(num_elems, num_nodes_in_elem, "mesh.nodes_in_elem"); + nodes_in_elem = DistributedDCArray(element_map, num_nodes_in_elem, "mesh.nodes_in_elem"); corners_in_elem = CArrayKokkos(num_elems, num_nodes_in_elem, "mesh.corners_in_elem"); zones_in_elem = zones_in_elem_t(num_zones_in_elem); surfs_in_elem = CArrayKokkos(num_elems, num_surfs_in_elem, "mesh.surfs_in_zone"); @@ -374,6 +393,335 @@ struct Mesh_t return; }; // end method + /* ---------------------------------------------------------------------- + Initialize Ghost and Non-Overlapping Element Maps + ------------------------------------------------------------------------- */ + void init_maps(node_t& node, mesh_input_t& mesh_inps) + { + int local_node_index, current_column_index; + long long int node_gid; + int myrank, nranks; + MPI_Comm_rank(MPI_COMM_WORLD,&myrank); + MPI_Comm_size(MPI_COMM_WORLD,&nranks); + + num_ghost_nodes=0; + DCArrayKokkos ghost_nodes; + DCArrayKokkos ghost_node_ranks; + if (num_elems >= 1) + { + // Construct set of ghost nodes; start with a buffer with upper limit + std::set ghost_node_set; + + for (int cell_rid = 0; cell_rid < num_elems; cell_rid++) + { + // set nodes per element + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + node_gid = nodes_in_elem.host(cell_rid, node_lid); //nodes in elem still stores global indices + if (!node_map.isProcessGlobalIndex(node_gid)) + { + ghost_node_set.insert(node_gid); + } + } + } + + // by now the set contains, with no repeats, all the global node indices that are ghosts for this rank + // now pass the contents of the set over to a CArrayKokkos, then create a map to find local ghost indices from global ghost indices + + num_ghost_nodes = ghost_node_set.size(); + if(num_ghost_nodes){ + int ighost = 0; + auto it = ghost_node_set.begin(); + + // create a Map for ghost node indices + ghost_nodes = DCArrayKokkos(num_ghost_nodes, "ghost_nodes"); //pass this into map object + while (it != ghost_node_set.end()) { + ghost_nodes.host(ighost++) = *it; + it++; + } + ghost_nodes.update_device(); + + //Use the ranks to break ties in shared element assignment for a unique element map used in elem set reductions later + //this wont be that great at load balancing element counts but its simple and works for now + ghost_node_ranks = DCArrayKokkos(num_ghost_nodes, "ghost_nodes_ranks"); + } + else{ + //ensure a size of at least 1 with bogus index to prevent segfault + ghost_nodes = DCArrayKokkos(1, "ghost_nodes"); //pass this into map object + ghost_node_ranks = DCArrayKokkos(1, "ghost_nodes_ranks"); + + } + // debug print of ghost nodes + // std::cout << " GHOST NODE SET ON TASK " << myrank << std::endl; + // for(int i = 0; i < num_ghost_nodes; i++) + // std::cout << "{" << i + 1 << "," << ghost_nodes(i) + 1 << "}" << std::endl; + + // find which mpi rank each ghost node belongs to and store the information in a CArrayKokkos + // // allocate Teuchos Views since they are the only input available at the moment in the Tpetra map definitions + // Teuchos::ArrayView ghost_nodes_pass(ghost_nodes.h_view.data(), num_ghost_nodes); + + // Teuchos::ArrayView ghost_node_ranks_pass(ghost_node_ranks.h_view.data(), num_ghost_nodes); + //node_map.print(); + node_map.getRemoteIndexList(ghost_nodes, ghost_node_ranks); + if(num_ghost_nodes){ + ghost_node_ranks.update_device(); + } + + } + + ghost_node_map = DistributedMap(ghost_nodes); + + // construct array for all indices (ghost + local) + num_nodes = num_local_nodes + num_ghost_nodes; + DCArrayKokkos all_nodes; + if(num_nodes){ + // CArrayKokkos all_node_indices(num_nodes, "all_node_indices"); + all_nodes = DCArrayKokkos(num_nodes, "num_nodes"); + for (int i = 0; i < num_nodes; i++) + { + if (i < num_local_nodes) + { + all_nodes.host(i) = node_map.getGlobalIndex(i); + } + else + { + all_nodes.host(i) = ghost_nodes.host(i - num_local_nodes); + } + } + all_nodes.update_device(); + // debug print of node indices + // for(int inode=0; inode < index_counter; inode++) + // std::cout << " my_reduced_global_indices " << my_reduced_global_indices(inode) < Initial_Element_Global_Indices(num_elems, "Initial_Element_Global_Indices"); + + size_t nonoverlapping_count = 0; + int my_element_flag; + for (int ielem = 0; ielem < num_elems; ielem++) + { + my_element_flag = 1; + + for (int lnode = 0; lnode < num_nodes_in_elem; lnode++) + { + node_gid = nodes_in_elem.host(ielem, lnode); + if (ghost_node_map.isProcessGlobalIndex(node_gid)) + { + local_node_index = ghost_node_map.getLocalIndex(node_gid); + if (ghost_node_ranks.host(local_node_index) < myrank) + { + my_element_flag = 0; + } + } + } + if (my_element_flag) + { + Initial_Element_Global_Indices.host(nonoverlapping_count++) = element_map.getGlobalIndex(ielem); + } + } + + // copy over from buffer to compressed storage + DCArrayKokkos Local_Element_Global_Indices(nonoverlapping_count, "Local_Element_Global_Indices"); + for (int ibuffer = 0; ibuffer < nonoverlapping_count; ibuffer++) + { + Local_Element_Global_Indices.host(ibuffer) = Initial_Element_Global_Indices.host(ibuffer); + } + num_local_elems = nonoverlapping_count; + Local_Element_Global_Indices.update_device(); + + // create nonoverlapping element map + local_element_map = DistributedMap(Local_Element_Global_Indices); + + // sort element connectivity so nonoverlaps are sequentially found first + // define initial sorting of global indices + + // element_map->describe(*fos,Teuchos::VERB_EXTREME); + + for (int ielem = 0; ielem < num_elems; ielem++) + { + Initial_Element_Global_Indices.host(ielem) = element_map.getGlobalIndex(ielem); + } + Initial_Element_Global_Indices.update_device(); + + // re-sort so local elements in the nonoverlapping map are first in storage + CArrayKokkos Temp_Nodes(num_nodes_in_elem); + + long long int temp_element_gid, current_element_gid; + int last_storage_index = num_elems - 1; + + //nodes_in_elem.print(); + + for (int ielem = 0; ielem < num_local_elems; ielem++) + { + current_element_gid = Initial_Element_Global_Indices.host(ielem); + // if this element is not part of the non overlap list then send it to the end of the storage and swap the element at the end + if (!local_element_map.isProcessGlobalIndex(current_element_gid)) + { + temp_element_gid = current_element_gid; + for (int lnode = 0; lnode < num_nodes_in_elem; lnode++) + { + Temp_Nodes(lnode) = nodes_in_elem.host(ielem, lnode); + } + Initial_Element_Global_Indices.host(ielem) = Initial_Element_Global_Indices.host(last_storage_index); + Initial_Element_Global_Indices.host(last_storage_index) = temp_element_gid; + for (int lnode = 0; lnode < num_nodes_in_elem; lnode++) + { + nodes_in_elem.host(ielem, lnode) = nodes_in_elem.host(last_storage_index, lnode); + nodes_in_elem.host(last_storage_index, lnode) = Temp_Nodes(lnode); + } + //resort element data that may have been read in such as vtu types and object ids + // if(mesh_inps.input_elem_types){ + // int temp; + // temp = elem_types.host(ielem); + // elem_types.host(ielem) = elem_types.host(last_storage_index); + // elem_types.host(last_storage_index) = temp; + // } + if(mesh_inps.input_elem_objectids){ + int temp; + temp = mesh_inps.object_ids.host(ielem); + mesh_inps.object_ids.host(ielem) = mesh_inps.object_ids.host(last_storage_index); + mesh_inps.object_ids.host(last_storage_index) = temp; + } + last_storage_index--; + + // test if swapped element is also not part of the non overlap map; if so lower loop counter to repeat the above + temp_element_gid = Initial_Element_Global_Indices.host(ielem); + if (!local_element_map.isProcessGlobalIndex(temp_element_gid)) + { + ielem--; + } + } + } + // reset all element map to its re-sorted version + Initial_Element_Global_Indices.update_device(); + nodes_in_elem.update_device(); + + element_map = DistributedMap(Initial_Element_Global_Indices); + //redefine nodes_in_elem so partition map of the distributed array is synchronized with permuted dual view contents + DistributedDCArray nodes_in_elem_temp(element_map, num_nodes_in_elem, "nodes_in_elem"); + //nodes_in_elem_temp.replace_kokkos_dual_view(nodes_in_elem.get_kokkos_dual_view()); + //nodes_in_elem.print(); + std::cout << "NUM ELEMS " << num_elems << " NUM NODES IN ELEM " << num_nodes_in_elem << std::endl; + for(int ielem= 0; ielem < num_elems; ielem++) { + for(int inode = 0; inode < num_nodes_in_elem; inode++){ + nodes_in_elem_temp.host(ielem, inode) = nodes_in_elem.host(ielem, inode); + } + } + //nodes_in_elem_temp.update_device(); + nodes_in_elem = nodes_in_elem_temp; + //nodes_in_elem.print(); + + //convert global ids stored in nodes_in_elem to local node ids spanning 0:num_nodes on this process + for(int ielem= 0; ielem < num_elems; ielem++) { + for(int inode = 0; inode < num_nodes_in_elem; inode++){ + nodes_in_elem.host(ielem, inode) = all_node_map.getLocalIndex(nodes_in_elem.host(ielem, inode)); + } + } + + nodes_in_elem.update_device(); + + //local element map may need resorting after above permuting; update the order + for (int ielem = 0; ielem < num_local_elems; ielem++) + { + Local_Element_Global_Indices.host(ielem) = element_map.getGlobalIndex(ielem); + } + Local_Element_Global_Indices.update_device(); + local_element_map = DistributedMap(Local_Element_Global_Indices); + + //nodes_in_elem.print(); + + /*connectivity data for uniquely assigned elements (used mostly to simplify file output comms) + constructed as a subview of nodes_in_elem*/ + local_nodes_in_elem = DistributedDCArray(nodes_in_elem,local_element_map); + + // element_map->describe(*fos,Teuchos::VERB_EXTREME); + // element_map->describe(*fos,Teuchos::VERB_EXTREME); + // create distributed multivector of the local node data and all (local + ghost) node storage + std::vector required_node_state = { node_state::coords }; + //constructs local + ghost coords array with local coords as a subview for first nlocal entrie + node.initialize(all_node_map, num_dims, required_node_state, node_map); + + /* create forward comms objects; setup for new map pairs should only be done here, construct using these existing comm plans + for any new pair of vectors requiring the same map pairs and comm mode afterwards*/ + forward_comms_setup(node); + node_coords_comms.execute_comms(); + + // create reverse comms + //reverse_comms_setup(node); + + // construct map of nodes that belong to the non-overlapping element set (contained by ghost + local node set but not all of them) + std::set nonoverlap_elem_node_set; + if (num_local_elems) + { + for (int cell_rid = 0; cell_rid < num_local_elems; cell_rid++) + { + // set nodes per element + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + node_gid = all_node_map.getGlobalIndex(nodes_in_elem.host(cell_rid, node_lid)); + nonoverlap_elem_node_set.insert(node_gid); + } + } + } + + // by now the set contains, with no repeats, all the global node indices belonging to the non overlapping element list on this MPI rank + // now pass the contents of the set over to a CArrayKokkos, then create a map to find local ghost indices from global ghost indices + size_t nnonoverlap_elem_nodes = nonoverlap_elem_node_set.size(); + DCArrayKokkos nonoverlap_elem_nodes(nnonoverlap_elem_nodes, "nonoverlap_elem_nodes"); + if(nnonoverlap_elem_nodes){ + int inonoverlap_elem_node = 0; + auto it = nonoverlap_elem_node_set.begin(); + while (it != nonoverlap_elem_node_set.end()) { + nonoverlap_elem_nodes.host(inonoverlap_elem_node++) = *it; + it++; + } + nonoverlap_elem_nodes.update_device(); + } + + // create a Map for node indices belonging to the non-overlapping set of elements + nonoverlap_element_node_map = DistributedMap(nonoverlap_elem_nodes); + + // std::cout << "number of patches = " << mesh->num_patches() << std::endl; + if (myrank == 0) + { + std::cout << "End of map setup " << std::endl; + } + } + + /* ---------------------------------------------------------------------- + Setup Tpetra importers for comms + ------------------------------------------------------------------------- */ + + void forward_comms_setup(node_t& node) + { + // create import object using local node indices map and ghost indices map + node_coords_comms = CommPlan(node.coords, node.local_coords); + + // output map and importers + //sorted_map = Teuchos::rcp(new Tpetra::Map(num_nodes, 0, comm)); + //node_sorting_importer = Teuchos::rcp(new Tpetra::Import(map, sorted_map)); + // sorted element mapping + //sorted_element_map = Teuchos::rcp(new Tpetra::Map(num_elem, 0, comm)); + //element_sorting_importer = Teuchos::rcp(new Tpetra::Import(element_map, sorted_element_map));; + } + + /* ---------------------------------------------------------------------- + Setup Tpetra exporters for reverse comms + ------------------------------------------------------------------------- */ + + void reverse_comms_setup(node_t& node) + { + //currently don't use anything like force tallies from ghost nodes + //only use in TO solver was a BC flag + // create import object using local node indices map and ghost indices map + //exporter = Teuchos::rcp(new Tpetra::Export(all_node_map, map)); + } + // build the corner mesh connectivity arrays void build_corner_connectivity() { diff --git a/single-node-refactor/src/common/include/mesh_inputs.h b/single-node-refactor/src/common/include/mesh_inputs.h index e76197488..47f0f5c07 100644 --- a/single-node-refactor/src/common/include/mesh_inputs.h +++ b/single-node-refactor/src/common/include/mesh_inputs.h @@ -101,7 +101,9 @@ struct mesh_input_t double scale_y = 1.0; ///< Scales mesh y coordinate dimensions double scale_z = 1.0; ///< Scales mesh z coordinate dimensions - DCArrayKokkos object_ids; ///< the object_ids in the vtu full mesh file (from exodus mesh) + DCArrayKokkos object_ids; ///< the object_ids in the vtu full mesh file (from exodus mesh) + bool input_elem_types = false; + bool input_elem_objectids = false; }; // mesh_input_t diff --git a/single-node-refactor/src/common/include/mesh_io.h b/single-node-refactor/src/common/include/mesh_io.h index 03fee6760..d4feb4f20 100644 --- a/single-node-refactor/src/common/include/mesh_io.h +++ b/single-node-refactor/src/common/include/mesh_io.h @@ -52,7 +52,8 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include - +#define BUFFER_SIZE 100000 +#define MAX_WORD 30 ///////////////////////////////////////////////////////////////////////////// /// @@ -212,8 +213,8 @@ inline bool extract_values_xml(T *values_xml, // find the number of points and number of cells in the mesh -inline bool extract_num_points_and_cells_xml(int& numberOfPoints, - int& numberOfCells, +inline bool extract_num_points_and_cells_xml(size_t& numberOfPoints, + size_t& numberOfCells, std::ifstream& in) { bool found = false; @@ -223,7 +224,7 @@ inline bool extract_num_points_and_cells_xml(int& numberOfPoints, // Read the file line by line looking for NumberOfPoints while (std::getline(in, line)) { - + //std::cout << line << std::endl; std::string word = "NumberOfPoints="; // A portion of a word if (line.find(word) != std::string::npos) { // Check if the portion of the word is in the line @@ -365,10 +366,10 @@ class MeshReader std::cout << "File extension is: " << extension << std::endl; if(extension == "geo"){ // Ensight meshfile extension - read_ensight_mesh(mesh, State.GaussPoints, State.node, State.corner, mesh_inps, num_dims); + //read_ensight_mesh(mesh, State.GaussPoints, State.node, State.corner, mesh_inps, num_dims); } else if(extension == "inp"){ // Abaqus meshfile extension - read_Abaqus_mesh(mesh, State, num_dims); + //read_Abaqus_mesh(mesh, State, num_dims); } else if(extension == "vtk"){ // vtk file format read_vtk_mesh(mesh, State.GaussPoints, State.node, State.corner, mesh_inps, num_dims); @@ -382,325 +383,542 @@ class MeshReader } + ///////////////////////////////////////////////////////////////////////////// /// - /// \fn read_ensight_mesh + /// \fn read_vtk_mesh /// - /// \brief Read .geo mesh file + /// \brief Read ASCII .vtk mesh file /// /// \param Simulation mesh - /// \param Element state struct + /// \param Simulation state /// \param Node state struct - /// \param Corner state struct /// \param Number of dimensions /// ///////////////////////////////////////////////////////////////////////////// - void read_ensight_mesh(Mesh_t& mesh, - GaussPoint_t& GaussPoints, - node_t& node, - corner_t& corner, - mesh_input_t& mesh_inps, - int num_dims) - { - FILE* in; - char ch; - size_t num_nodes_in_elem = 1; - for (int dim = 0; dim < num_dims; dim++) { - num_nodes_in_elem *= 2; - } + void read_vtk_mesh(Mesh_t& mesh, + GaussPoint_t& GaussPoints, + node_t& node, + corner_t& corner, + mesh_input_t& mesh_inps, + int num_dims) + { + char ch; + std::string skip_line, read_line, substring; + std::stringstream line_parse; + std::ifstream in; // FILE *in; - // read the mesh WARNING: assumes a .geo file - in = fopen(mesh_file_, "r"); + std::cout<<"Reading VTK mesh"< read_buffer; + + + // read the mesh + // --- Read the number of nodes in the mesh --- // + size_t global_num_nodes = 0; + if (myrank == 0) + { + std::cout << " NUM DIM is " << num_dims << std::endl; + in.open(mesh_file_); + bool found = false; + + while (found == false&&in.good()) { + std::getline(in, read_line); + line_parse.str(""); + line_parse.clear(); + line_parse << read_line; + line_parse >> substring; + + // looking for the following text: + // POINTS %d float + if (substring == "POINTS") + { + line_parse >> global_num_nodes; + std::cout << "declared node count: " << global_num_nodes << std::endl; + if (global_num_nodes <= 0) + { + throw std::runtime_error("ERROR, NO NODES IN MESH"); + } + found = true; + } // end if + } // end while - // skip 8 lines - for (int j = 1; j <= 8; j++) { - int i = 0; - while ((ch = (char)fgetc(in)) != '\n') { - i++; - } - } + if (!found){ + throw std::runtime_error("ERROR: Failed to find POINTS"); + } // end if - // --- Read in the nodes in the mesh --- + } // end if(myrank==0) - size_t num_nodes = 0; + // broadcast number of nodes + MPI_Bcast(&global_num_nodes, 1, MPI_LONG_LONG_INT, 0, MPI_COMM_WORLD); - fscanf(in, "%lu", &num_nodes); - printf("Number of nodes read in %lu\n", num_nodes); + // host_vec_array node_coords = dual_node_coords.view_host(); + // notify that the host view is going to be modified in the file readin + // dual_node_coords.modify_host(); - - mesh.initialize_nodes(num_nodes); + // old swage method + // mesh->init_nodes(local_nrows); // add 1 for index starting at 1 - // initialize node state variables, for now, we just need coordinates, the rest will be initialize by the respective solvers - std::vector required_node_state = { node_state::coords }; - node.initialize(num_nodes, num_dims, required_node_state); + //std::cout << "Num nodes assigned to task " << myrank << " = " << num_local_nodes << std::endl; // read the initial mesh coordinates // x-coords - for (int node_id = 0; node_id < mesh.num_nodes; node_id++) { - fscanf(in, "%le", &node.coords.host(node_id, 0)); - node.coords.host(node_id, 0)*= mesh_inps.scale_x; - } + /*only task 0 reads in nodes and elements from the input file + stores node data in a buffer and communicates once the buffer cap is reached + or the data ends*/ - // y-coords - for (int node_id = 0; node_id < mesh.num_nodes; node_id++) { - fscanf(in, "%le", &node.coords.host(node_id, 1)); - node.coords.host(node_id, 1)*= mesh_inps.scale_y; + size_t num_nodes_in_elem = 1; + for (int dim = 0; dim < num_dims; dim++) { + num_nodes_in_elem *= 2; } - // z-coords - for (int node_id = 0; node_id < mesh.num_nodes; node_id++) { - if (num_dims == 3) { - fscanf(in, "%le", &node.coords.host(node_id, 2)); - node.coords.host(node_id, 2)*= mesh_inps.scale_z; - } - else{ - double dummy; - fscanf(in, "%le", &dummy); - } - } // end for - - - // Update device nodal positions - node.coords.update_device(); + int words_per_line = num_dims; + int elem_words_per_line = num_nodes_in_elem; - ch = (char)fgetc(in); + // allocate read buffer + read_buffer = CArrayKokkos(BUFFER_SIZE, words_per_line, MAX_WORD); - // skip 1 line - for (int j = 1; j <= 1; j++) { - int i = 0; - while ((ch = (char)fgetc(in)) != '\n') { - i++; - } + dof_limit = global_num_nodes; + buffer_iterations = dof_limit / BUFFER_SIZE; + if (dof_limit % BUFFER_SIZE != 0) + { + buffer_iterations++; } - // --- read in the elements in the mesh --- - size_t num_elem = 0; - - fscanf(in, "%lu", &num_elem); - printf("Number of elements read in %lu\n", num_elem); + // read coords + read_index_start = 0; + size_t num_local_nodes; + DistributedMap node_map; + { //scoped so temp FArray data is auto deleted to save memory + //allocate pre-partition node coords using contiguous decomposition + //FArray type used since CArray type still doesnt support zoltan2 decomposition + DistributedDFArray node_coords_distributed(global_num_nodes, num_dims); + + // construct contiguous parallel row map now that we know the number of nodes + node_map = node_coords_distributed.pmap; + // map->describe(*fos,Teuchos::VERB_EXTREME); + + // set the vertices in the mesh read in + num_local_nodes = node_map.size(); + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_SIZE; buffer_loop++) + { + getline(in, read_line); + line_parse.clear(); + line_parse.str(read_line); + + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // debug print + // std::cout<<" "<< substring <> substring; + // debug print + // std::cout<<" "<< substring < required_node_state = { node_state::coords }; + node_map = node_coords_distributed.pmap; + node.initialize(node_map, num_dims, required_node_state); + //copy coordinate data from repartitioned FArray into CArray + FOR_ALL(node_id, 0, node_map.size(), { + for(int idim = 0; idim < num_dims; idim++){ + node.coords(node_id,idim) = node_coords_distributed(node_id,idim); + } + }); } + node.coords.update_host(); + //initialize some mesh data + mesh.initialize_nodes(global_num_nodes); + num_local_nodes = node_map.size(); + mesh.num_local_nodes = num_local_nodes; + mesh.node_map = node_map; + mesh.num_dims = num_dims; + + // debug print of nodal data - // Convert from ensight to IJK mesh - int convert_ensight_to_ijk[8]; - convert_ensight_to_ijk[0] = 0; - convert_ensight_to_ijk[1] = 1; - convert_ensight_to_ijk[2] = 3; - convert_ensight_to_ijk[3] = 2; - convert_ensight_to_ijk[4] = 4; - convert_ensight_to_ijk[5] = 5; - convert_ensight_to_ijk[6] = 7; - convert_ensight_to_ijk[7] = 6; - - int tmp_ijk_indx[8]; - - for (int elem_gid = 0; elem_gid < num_elem; elem_gid++) { - for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - tmp_ijk_indx[node_lid] = mesh.nodes_in_elem.host(elem_gid, convert_ensight_to_ijk[node_lid]); - } - - for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - mesh.nodes_in_elem.host(elem_gid, node_lid) = tmp_ijk_indx[node_lid]; - } + // debug print nodal positions and indices + /* + std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; + for (int istride = 0; istride < num_dims; istride++){ + std::cout << node_coords(inode,istride) << " , "; } - // update device side - mesh.nodes_in_elem.update_device(); - - // initialize corner variables - int num_corners = num_elem * mesh.num_nodes_in_elem; - mesh.initialize_corners(num_corners); - // corner.initialize(num_corners, num_dims); - - // Close mesh input file - fclose(in); + std::cout << " }"<< std::endl; + } + */ - // Build connectivity - mesh.build_connectivity(); + // check that local assignments match global total + + // read in element info (ensight file format is organized in element type sections) + // loop over this later for several element type sections + + size_t global_num_elems = 0; + size_t num_elems = 0; + CArrayKokkos node_store(elem_words_per_line); + + // --- read the number of cells in the mesh --- + // --- Read the number of vertices in the mesh --- // + if (myrank == 0) + { + bool found = false; + while (found == false&&in.good()) { + std::getline(in, read_line); + line_parse.str(""); + line_parse.clear(); + line_parse << read_line; + line_parse >> substring; + + // looking for the following text: + // CELLS num_cells size + if (substring == "CELLS") + { + line_parse >> global_num_elems; + std::cout << "declared element count: " << global_num_elems << std::endl; + if (global_num_elems <= 0) + { + throw std::runtime_error("ERROR, NO ELEMENTS IN MESH"); + } + found = true; + } // end if + } // end while - return; - } // end read ensight mesh + if (!found){ + throw std::runtime_error("ERROR: Failed to find CELLS"); + } // end if + } // end if(myrank==0) - ///////////////////////////////////////////////////////////////////////////// - /// - /// \fn read_Abaqus_mesh - /// - /// \brief Read .inp mesh file - /// - /// \param Simulation mesh - /// \param Simulation state - /// \param Node state struct - /// \param Number of dimensions - /// - ///////////////////////////////////////////////////////////////////////////// - void read_Abaqus_mesh(Mesh_t& mesh, - State_t& State, - int num_dims) - { + // broadcast number of elements + MPI_Bcast(&global_num_elems, 1, MPI_LONG_LONG_INT, 0, MPI_COMM_WORLD); - std::cout<<"Reading abaqus input file for mesh"< nodes; - std::vector elements; + // read in element connectivity + // we're gonna reallocate for the words per line expected for the element connectivity + read_buffer = CArrayKokkos(BUFFER_SIZE, elem_words_per_line, MAX_WORD); - std::string line; - bool readingNodes = false; - bool readingElements = false; + // calculate buffer iterations to read number of lines + buffer_iterations = global_num_elems / BUFFER_SIZE; + int assign_flag; - while (std::getline(inputFile, line)) { - if (line.find("*Node") != std::string::npos) { - readingNodes = true; - std::cout<<"Found *Node"< element_temp(BUFFER_SIZE * elem_words_per_line); + std::vector global_indices_temp(BUFFER_SIZE); + size_t buffer_max = BUFFER_SIZE * elem_words_per_line; + size_t indices_buffer_max = BUFFER_SIZE; - } - else if (readingNodes && !line.find("*") ) { // End of nodes - readingNodes = false; - } - else if (readingNodes) { - // std::cout<<"Reading Nodes"<> node.id && std::getline(iss, token, ',') && iss >> node.x && - std::getline(iss, token, ',') && iss >> node.y && - std::getline(iss, token, ',') && iss >> node.z)) { - std::cerr << "Failed to parse line: " << line << std::endl; - continue; // Skip this line if parsing failed + if (global_num_elems % BUFFER_SIZE != 0) + { + buffer_iterations++; + } + read_index_start = 0; + // std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_SIZE; buffer_loop++) + { + getline(in, read_line); + line_parse.clear(); + line_parse.str(read_line); + // disregard node count line since we're using one element type per mesh + line_parse >> substring; + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // debug print + // std::cout<<" "<< substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + // std::cout <> element.id)){ - std::cout << "Failed to parse line: " << line << std::endl; - continue; // Skip this line if parsing failed - } - - while ((std::getline(iss, token, ','))) { - // Now extract the integer, ignoring any trailing whitespace - int val; - iss >> val; - element.connectivity.push_back(val); + else if (myrank == 0) + { + buffer_loop = 0; + while (buffer_iteration * BUFFER_SIZE + buffer_loop < global_num_elems) { + getline(in, read_line); + line_parse.clear(); + line_parse.str(read_line); + line_parse >> substring; + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // debug print + // std::cout<<" "<< substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + // std::cout <= buffer_max) + { + element_temp.resize((num_elems - 1) * elem_words_per_line + inode + BUFFER_SIZE * elem_words_per_line); + buffer_max = (num_elems - 1) * elem_words_per_line + inode + BUFFER_SIZE * elem_words_per_line; + } + element_temp[(num_elems - 1) * elem_words_per_line + inode] = node_store(inode); + // std::cout << "VECTOR STORAGE FOR ELEM " << num_elems << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; + } + // assign global element id to temporary list + if (num_elems - 1 >= indices_buffer_max) + { + global_indices_temp.resize(num_elems - 1 + BUFFER_SIZE); + indices_buffer_max = num_elems - 1 + BUFFER_SIZE; + } + global_indices_temp[num_elems - 1] = elem_gid; } - - elements.push_back(element); } + read_index_start += BUFFER_SIZE; } - inputFile.close(); - - size_t num_nodes = nodes.size(); - - printf("Number of nodes read in %lu\n", num_nodes); - - // initialize node variables - mesh.initialize_nodes(num_nodes); - - // initialize node state, for now, we just need coordinates, the rest will be initialize by the respective solvers - std::vector required_node_state = { node_state::coords }; - - State.node.initialize(num_nodes, num_dims, required_node_state); + //set global and local shared element counts + mesh.global_num_elems = global_num_elems; + // construct partition mapping for shared elements on each process + DCArrayKokkos All_Element_Global_Indices(num_elems); + // copy temporary global indices storage to view storage + for (int ielem = 0; ielem < num_elems; ielem++) + { + All_Element_Global_Indices.host(ielem) = global_indices_temp[ielem]; + if (global_indices_temp[ielem] < 0) + { + negative_index_found = 1; + } + } - // Copy nodes to mesh - for(int node_gid = 0; node_gid < num_nodes; node_gid++){ - State.node.coords.host(node_gid, 0) = nodes[node_gid].x; - State.node.coords.host(node_gid, 1) = nodes[node_gid].y; - State.node.coords.host(node_gid, 2) = nodes[node_gid].z; + MPI_Allreduce(&negative_index_found, &global_negative_index_found, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + if (global_negative_index_found) + { + if (myrank == 0) + { + std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; + } + MPI_Barrier(MPI_COMM_WORLD); + MPI_Finalize(); + exit(0); } - // Update device nodal positions - State.node.coords.update_device(); + All_Element_Global_Indices.update_device(); + // construct global map of local and shared elements (since different ranks can own the same elements due to the local node map) + DistributedMap element_map = DistributedMap(All_Element_Global_Indices); - // --- read in the elements in the mesh --- - size_t num_elem = elements.size(); - printf("Number of elements read in %lu\n", num_elem); + //initialize elem data structures + mesh.initialize_elems(num_elems, num_nodes_in_elem, element_map); - // initialize elem variables - mesh.initialize_elems(num_elem, num_dims); + // copy temporary element storage to distributed storage + DistributedDCArray nodes_in_elem = mesh.nodes_in_elem; + for (int ielem = 0; ielem < num_elems; ielem++) + { + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { //assign local indices to element-node connectivity (stores global indices until ghost maps are made later) + nodes_in_elem.host(ielem, inode) = element_temp[ielem * num_nodes_in_elem + inode]; + } + } - // for each cell read the list of associated nodes - for (int elem_gid = 0; elem_gid < num_elem; elem_gid++) { - for (int node_lid = 0; node_lid < 8; node_lid++) { - mesh.nodes_in_elem.host(elem_gid, node_lid) = elements[elem_gid].connectivity[node_lid]; + // delete temporary element connectivity and index storage + std::vector().swap(element_temp); + std::vector().swap(global_indices_temp); + + // Convert ensight index system to the ijk finite element numbering convention + // for vertices in cell + CArrayKokkos convert_ensight_to_ijk(num_nodes_in_elem); + CArrayKokkos tmp_ijk_indx(num_nodes_in_elem); + convert_ensight_to_ijk(0) = 0; + convert_ensight_to_ijk(1) = 1; + convert_ensight_to_ijk(2) = 3; + convert_ensight_to_ijk(3) = 2; + convert_ensight_to_ijk(4) = 4; + convert_ensight_to_ijk(5) = 5; + convert_ensight_to_ijk(6) = 7; + convert_ensight_to_ijk(7) = 6; + + for (int cell_rid = 0; cell_rid < num_elems; cell_rid++) + { + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem.host(cell_rid, convert_ensight_to_ijk(node_lid)); + } - // shift to start node index space at 0 - mesh.nodes_in_elem.host(elem_gid, node_lid) -= 1; + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + nodes_in_elem.host(cell_rid, node_lid) = tmp_ijk_indx(node_lid); } } + - // update device side - mesh.nodes_in_elem.update_device(); + nodes_in_elem.update_device(); // initialize corner variables - int num_corners = num_elem * mesh.num_nodes_in_elem; + size_t num_corners = num_elems * num_nodes_in_elem; mesh.initialize_corners(num_corners); - // State.corner.initialize(num_corners, num_dims); - // Build connectivity - mesh.build_connectivity(); - } // end read abaqus mesh + // Close mesh input file + if (myrank == 0) + { + in.close(); + } + } // end read_mesh ///////////////////////////////////////////////////////////////////////////// /// - /// \fn read_vtk_mesh + /// \fn read_vtu_mesh /// - /// \brief Read ASCII .vtk mesh file + /// \brief Read ASCII .vtu mesh file /// /// \param Simulation mesh /// \param Simulation state @@ -708,386 +926,482 @@ class MeshReader /// \param Number of dimensions /// ///////////////////////////////////////////////////////////////////////////// - void read_vtk_mesh(Mesh_t& mesh, + void read_vtu_mesh(Mesh_t& mesh, GaussPoint_t& GaussPoints, node_t& node, corner_t& corner, mesh_input_t& mesh_inps, int num_dims) { + int myrank, nranks; + MPI_Comm_rank(MPI_COMM_WORLD,&myrank); + MPI_Comm_size(MPI_COMM_WORLD,&nranks); - std::cout<<"Reading VTK mesh"< read_buffer(BUFFER_SIZE*num_dims); + mesh_inps.input_elem_types = true; + mesh_inps.input_elem_objectids = true; + + + // read the mesh + + // ------------------------ + // Mesh file storage order: + // objectId + // Points + // connectivity + // offsets + // types + // ------------------------ + // --- Read the number of nodes in the mesh --- // + size_t global_num_nodes = 0; + size_t global_num_elems = 0; + size_t num_elems = 0; + std::streampos objectid_streampos; + + // dimensional scaling of the mesh + const double scl_x = mesh_inps.scale_x; + const double scl_y = mesh_inps.scale_y; + const double scl_z = mesh_inps.scale_z; + + + // + int Pn_order = mesh_inps.p_order; size_t num_nodes_in_elem = 1; for (int dim = 0; dim < num_dims; dim++) { - num_nodes_in_elem *= 2; + num_nodes_in_elem *= (Pn_order + 1); } - - std::string token; + if(myrank==0){ + std::cout<<"Reading VTU file in a multiblock VTK mesh"< v = split (str, delimiter); - - // looking for the following text: - // POINTS %d float - if(v[0] == "POINTS"){ - size_t num_nodes = std::stoi(v[1]); - printf("Number of nodes read in %zu\n", num_nodes); - mesh.initialize_nodes(num_nodes); - - std::vector required_node_state = { node_state::coords }; - node.initialize(num_nodes, num_dims, required_node_state); - - found=true; - } // end if - - - if (i>1000){ - std::cerr << "ERROR: Failed to find POINTS in file" << std::endl; - break; - } // end if - - i++; - } // end while - - // read the node coordinates - for (node_gid=0; node_gid v = split (str, delimiter); - - // save the nodal coordinates - node.coords.host(node_gid, 0) = mesh_inps.scale_x*std::stod(v[0]); // double - node.coords.host(node_gid, 1) = mesh_inps.scale_y*std::stod(v[1]); // double - if(num_dims==3){ - node.coords.host(node_gid, 2) = mesh_inps.scale_z*std::stod(v[2]); // double - } - - } // end for nodes + //allocate initial contiguously partitioned node coordinate vector and get map of node indices + DistributedDFArray node_coords_distributed(global_num_nodes, num_dims); + // construct contiguous parallel row map now that we know the number of nodes + DistributedMap node_map = node_coords_distributed.pmap; + // map->describe(*fos,Teuchos::VERB_EXTREME); - // Update device nodal positions - node.coords.update_device(); - + // set the vertices in the mesh read in + size_t num_local_nodes = node_map.size(); - found=false; - // look for CELLS - i = 0; - size_t num_elem = 0; - while (found==false) { - std::string str; - std::getline(in, str); - - std::string delimiter = " "; - std::vector v = split (str, delimiter); - std::cout << v[0] << std::endl; // printing - - // looking for the following text: - // CELLS num_elem size - if(v[0] == "CELLS"){ - num_elem = std::stoi(v[1]); - printf("Number of elements read in %zu\n", num_elem); - - // initialize elem variables - mesh.initialize_elems(num_elem, num_dims); - - found=true; - } // end if - - - if (i>1000){ - printf("ERROR: Failed to find CELLS \n"); - break; - } // end if - - i++; - } // end while - - - // read the node ids in the element - for (elem_gid=0; elem_gid v = split (str, delimiter); - num_nodes_in_elem = std::stoi(v[0]); - - for (size_t node_lid=0; node_lid v = split (str, delimiter); - - // looking for the following text: - // CELLS num_elem size - if(v[0] == "CELL_TYPES"){ + break; - std::getline(in, str); - elem_type = std::stoi(str); - - found=true; - } // end if - - - if (i>1000){ - printf("ERROR: Failed to find elem_TYPE \n"); - break; - } // end if - - i++; - } // end while - printf("Element type = %zu \n", elem_type); - // elem types: - // linear hex = 12, linear quad = 9 - found=false; - - - if(num_nodes_in_elem==8 & elem_type != 12) { - printf("Wrong element type of %zu \n", elem_type); - std::cerr << "ERROR: incorrect element type in VTK file" << std::endl; + } // end if found + + } // end while + + if(found==false){ + throw std::runtime_error("**** ERROR: mesh nodes were not found in the XML file! ****"); + //std::cout << "ERROR: mesh nodes were not found in the XML file!" << std::endl; + } } - - in.close(); - - } // end of VTKread function - ///////////////////////////////////////////////////////////////////////////// - /// - /// \fn read_vtu_mesh - /// - /// \brief Read ASCII .vtu mesh file - /// - /// \param Simulation mesh - /// \param Simulation state - /// \param Node state struct - /// \param Number of dimensions - /// - ///////////////////////////////////////////////////////////////////////////// - void read_vtu_mesh(Mesh_t& mesh, - GaussPoint_t& GaussPoints, - node_t& node, - corner_t& corner, - mesh_input_t& mesh_inps, - int num_dims) - { + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ + // pack buffer on rank 0 + size_t buffer_iteration_size; + if(buffer_iteration < buffer_iterations - 1){ + buffer_iteration_size = BUFFER_SIZE*num_dims; + } + else{ + buffer_iteration_size = remainder_size; + } + + if(myrank==0){ - std::cout<<"Reading VTU file in a multiblock VTK mesh"<> value; + if (value == stop) { // Check if the stop word is in the line + break; + } // end if + read_buffer(idata) = std::stod(value); + size++; + } // end for + } + // broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong + MPI_Bcast(read_buffer.pointer(), BUFFER_SIZE*num_dims, MPI_DOUBLE, 0, MPI_COMM_WORLD); + // broadcast how many nodes were read into this buffer iteration + MPI_Bcast(&buffer_iteration_size, 1, MPI_INT, 0, MPI_COMM_WORLD); + + // debug_print + // std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; + // for(int iprint=0; iprint < buffer_loop; iprint++) + // std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; + // return; + + // determine which data to store in the swage mesh members (the local node data) + // loop through read buffer + for (scan_loop = 0; scan_loop < buffer_iteration_size/num_dims; scan_loop++) + { + // set global node id (ensight specific order) + node_gid = read_index_start + scan_loop; + // let map decide if this node id belongs locally; if yes store data + if (node_map.isProcessGlobalIndex(node_gid)) + { + // set local node index in this mpi rank + node_rid = node_map.getLocalIndex(node_gid); + // extract nodal position from the read buffer + // for tecplot format this is the three coords in the same line + dof_value = read_buffer(scan_loop*num_dims); + node_coords_distributed.host(node_rid, 0) = dof_value * scl_x; + dof_value = read_buffer(scan_loop*num_dims + 1); + node_coords_distributed.host(node_rid, 1) = dof_value * scl_y; + if (num_dims == 3) + { + dof_value = read_buffer(scan_loop*num_dims + 2); + node_coords_distributed.host(node_rid, 2) = dof_value * scl_z; + } + } + } + read_index_start += buffer_iteration_size/num_dims; } - - bool found; - - std::ifstream in; // FILE *in; - in.open(mesh_file_); - - // --- extract the number of points and cells from the XML file --- - int num_nodes; - int num_elems; - found = extract_num_points_and_cells_xml(num_nodes, - num_elems, - in); - if(found==false){ - throw std::runtime_error("ERROR: number of points and/or cells not found in the XML file!"); - //std::cout << "ERROR: number of points and cells not found in the XML file!" << std::endl; + if(myrank==0){ + if (size!=global_num_nodes*num_dims){ + throw std::runtime_error("ERROR: failed to read all the mesh nodes!"); + //std::cout << "ERROR: failed to read all the mesh nodes!" << std::endl; + } } - std::cout << "Number of nodes in the mesh file: " << num_nodes << std::endl; - std::cout << "Number of elements in the mesh file: " << num_elems << std::endl; - - //------------------------------------ - // allocate mesh class nodes and elems - mesh.initialize_nodes(num_nodes); - mesh.initialize_elems(num_elems, num_dims); - //------------------------------------ - // allocate node coordinate state + node_coords_distributed.update_device(); + // repartition node distribution + node_coords_distributed.repartition_vector(); + //get map from repartitioned Farray and feed it into distributed CArray type; FArray data will be discared after scope std::vector required_node_state = { node_state::coords }; - node.initialize(num_nodes, num_dims, required_node_state); + node_map = node_coords_distributed.pmap; + node.initialize(node_map, num_dims, required_node_state); + //copy coordinate data from repartitioned FArray into CArray + FOR_ALL(node_id, 0, node_map.size(), { + for(int idim = 0; idim < num_dims; idim++){ + node.coords(node_id,idim) = node_coords_distributed(node_id,idim); + } + }); + node.coords.update_host(); + //initialize some mesh data + mesh.initialize_nodes(global_num_nodes); + num_local_nodes = node_map.size(); + mesh.num_local_nodes = num_local_nodes; + mesh.node_map = node_map; - //------------------------------------ - // allocate the elem object id array - mesh_inps.object_ids = DCArrayKokkos (num_elems, "ObjectIDs"); + /***Element data***/ + // broadcast number of elems + MPI_Bcast(&global_num_elems, 1, MPI_LONG_LONG_INT, 0, MPI_COMM_WORLD); + + // --- + // Nodes in the element + // --- + + dof_limit = global_num_elems*num_nodes_in_elem; + buffer_iterations = dof_limit / (BUFFER_SIZE*num_nodes_in_elem); + remainder_size = dof_limit % (BUFFER_SIZE*num_nodes_in_elem); + if (remainder_size != 0) + { + buffer_iterations++; + } - // ------------------------ - // Mesh file storage order: - // objectId - // Points - // connectivity - // offsets - // types - // ------------------------ - // temporary arrays - DCArrayKokkos node_coords(num_nodes,3, "node_coords_vtu_file"); // always 3 with vtu files - DCArrayKokkos connectivity(num_elems,num_nodes_in_elem, "connectivity_vtu_file"); - DCArrayKokkos elem_types(num_elems, "elem_types_vtu_file"); // element types + CArrayKokkos elem_read_buffer(BUFFER_SIZE*num_nodes_in_elem); + CArrayKokkos node_store(num_nodes_in_elem); + + std::vector element_temp(BUFFER_SIZE * num_nodes_in_elem); + std::vector global_indices_temp(BUFFER_SIZE); + size_t buffer_max = BUFFER_SIZE * num_nodes_in_elem; + size_t indices_buffer_max = BUFFER_SIZE; + int assign_flag; + read_index_start = 0; + size = 0; + //first find the block with element connectivity data data + if(myrank==0){ - // for all fields, we stop recording when we get to "<" - std::string stop = "<"; + bool found = false; - // the size of 1D storage from reading the mesh file - size_t size; + std::string line; + const std::string word = "\"connectivity\""; - // --- - // Object ids - // --- + // Read the file line by line looking for specified word + while (std::getline(in, line)) { + + if (line.find(word) != std::string::npos) { // Check if the portion of the word is in the line + found = true; + } + if(found) { + + break; + + } // end if found - // the object id in the element - // array dims are (num_elems) - found = extract_values_xml(mesh_inps.object_ids.host.pointer(), - "\"ObjectId\"", - stop, - in, - size); - if(found==false){ - throw std::runtime_error("ERROR: ObjectIDs were not found in the XML file!"); - //std::cout << "ERROR: ObjectIDs were not found in the XML file!" << std::endl; + } // end while + + if(found==false){ + throw std::runtime_error("ERROR: mesh connectivity was not found in the XML file!"); + //std::cout << "ERROR: mesh nodes were not found in the XML file!" << std::endl; + } } - mesh_inps.object_ids.update_device(); - // --- - // Nodal coordinates of mesh - // --- + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ + // pack buffer on rank 0 + size_t buffer_iteration_size; + if(buffer_iteration < buffer_iterations - 1){ + buffer_iteration_size = BUFFER_SIZE*num_nodes_in_elem; + } + else{ + buffer_iteration_size = remainder_size; + } + + if(myrank==0){ - // coordinates of the node - // array dims are (num_nodes,dims) - // must use the quotes around Points to read the point values - found = extract_values_xml(node_coords.host.pointer(), - "\"Points\"", - stop, - in, - size); - if(found==false){ - throw std::runtime_error("**** ERROR: mesh nodes were not found in the XML file! ****"); - //std::cout << "ERROR: mesh nodes were not found in the XML file!" << std::endl; + std::string line; + + // loop over the lines in the file until the buffer limit is reached + for(int idata = 0; idata < buffer_iteration_size; idata++){ + + // extract the individual values from the stream + std::string value; + in >> value; + if (value == stop) { // Check if the stop word is in the line + break; + } // end if + elem_read_buffer(idata) = std::stoll(value); + size++; + } // end for + } + // broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong + MPI_Bcast(elem_read_buffer.pointer(), BUFFER_SIZE*num_nodes_in_elem, MPI_LONG_LONG_INT, 0, MPI_COMM_WORLD); + // broadcast how many nodes were read into this buffer iteration + MPI_Bcast(&buffer_iteration_size, 1, MPI_INT, 0, MPI_COMM_WORLD); + + // debug_print + // std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; + // for(int iprint=0; iprint < buffer_loop; iprint++) + // std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; + // return; + + // determine which data to store in the swage mesh members (the local node data) + // loop through read buffer + for (scan_loop = 0; scan_loop < buffer_iteration_size/num_nodes_in_elem; scan_loop++) + { + // set global node id (ensight specific order) + elem_gid = read_index_start + scan_loop; + // add this element to the local list if any of its nodes belong to this rank according to the map + // get list of nodes for each element line and check if they belong to the map + assign_flag = 0; + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + // as we loop through the nodes belonging to this element we store them + // if any of these nodes belongs to this rank this list is used to store the element locally + node_gid = elem_read_buffer(scan_loop*num_nodes_in_elem + inode); + if (zero_index_base) + { + node_store(inode) = node_gid; // subtract 1 since file index start is 1 but code expects 0 + } + else + { + node_store(inode) = node_gid - 1; // subtract 1 since file index start is 1 but code expects 0 + } + if (node_store(inode) < 0) + { + negative_index_found = 1; + } + // first we add the elements to a dynamically allocated list + if (zero_index_base) + { + if (node_map.isProcessGlobalIndex(node_gid) && !assign_flag) + { + assign_flag = 1; + num_elems++; + } + } + else + { + if (node_map.isProcessGlobalIndex(node_gid - 1) && !assign_flag) + { + assign_flag = 1; + num_elems++; + } + } + } + + if (assign_flag) + { + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + if ((num_elems - 1) * num_nodes_in_elem + inode >= buffer_max) + { + element_temp.resize((num_elems - 1) * num_nodes_in_elem + inode + BUFFER_SIZE * num_nodes_in_elem); + buffer_max = (num_elems - 1) * num_nodes_in_elem + inode + BUFFER_SIZE * num_nodes_in_elem; + } + element_temp[(num_elems - 1) * num_nodes_in_elem + inode] = node_store(inode); + // std::cout << "VECTOR STORAGE FOR ELEM " << num_elems << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; + } + // assign global element id to temporary list + if (num_elems - 1 >= indices_buffer_max) + { + global_indices_temp.resize(num_elems - 1 + BUFFER_SIZE); + indices_buffer_max = num_elems - 1 + BUFFER_SIZE; + } + global_indices_temp[num_elems - 1] = elem_gid; + } + } + read_index_start += buffer_iteration_size/num_nodes_in_elem; } - if (size!=num_nodes*3){ - throw std::runtime_error("ERROR: failed to read all the mesh nodes!"); - //std::cout << "ERROR: failed to read all the mesh nodes!" << std::endl; + + if(myrank==0){ + if (size!=global_num_elems*num_nodes_in_elem){ + throw std::runtime_error("ERROR: failed to read all the mesh elements!"); + //std::cout << "ERROR: failed to read all the mesh nodes!" << std::endl; + } } - node_coords.update_device(); - // dimensional scaling of the mesh - const double scl_x = mesh_inps.scale_x; - const double scl_y = mesh_inps.scale_y; - const double scl_z = mesh_inps.scale_z; + //interface with permanent storage of nodes in elem and construct element maps + DCArrayKokkos All_Element_Global_Indices(num_elems); + // copy temporary global indices storage to view storage + for (int ielem = 0; ielem < num_elems; ielem++) + { + All_Element_Global_Indices.host(ielem) = global_indices_temp[ielem]; + if (global_indices_temp[ielem] < 0) + { + negative_index_found = 1; + } + } - // save the node coordinates to the state array - FOR_ALL(node_gid, 0, mesh.num_nodes, { - - // save the nodal coordinates - node.coords(node_gid, 0) = scl_x*node_coords(node_gid, 0); // double - node.coords(node_gid, 1) = scl_y*node_coords(node_gid, 1); // double - if(num_dims==3){ - node.coords(node_gid, 2) = scl_z*node_coords(node_gid, 2); // double + MPI_Allreduce(&negative_index_found, &global_negative_index_found, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + if (global_negative_index_found) + { + if (myrank == 0) + { + std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; } + MPI_Barrier(MPI_COMM_WORLD); + MPI_Finalize(); + exit(0); + } - }); // end for parallel nodes - node.coords.update_host(); + All_Element_Global_Indices.update_device(); + // construct global map of local and shared elements (since different ranks can own the same elements due to the local node map) + DistributedMap element_map = DistributedMap(All_Element_Global_Indices); - // --- - // Nodes in the element - // --- + //initialize elem data structures + mesh.initialize_elems(num_elems, num_nodes_in_elem, element_map); + + // copy temporary element storage to distributed storage + DistributedDCArray nodes_in_elem = mesh.nodes_in_elem; - // fill temporary nodes in the element array - // array dims are (num_elems,num_nodes_in_elem) - found = extract_values_xml(connectivity.host.pointer(), - "\"connectivity\"", - stop, - in, - size); - if(found==false){ - std::cout << "ERROR: mesh connectivity was not found in the XML file!" << std::endl; + for (int ielem = 0; ielem < num_elems; ielem++) + { + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { //assign local indices to element-node connectivity (stores global indices until ghost maps are made later) + nodes_in_elem.host(ielem, inode) = element_temp[ielem * num_nodes_in_elem + inode]; + } } - connectivity.update_device(); + nodes_in_elem.update_device(); + + //debug print + //for (int inode = 0; inode < num_nodes_in_elem; inode++) + //std::cout << "Element nodes " << inode << " " << nodes_in_elem(num_elems-1,inode) << std::endl; + + // delete temporary element connectivity and index storage + std::vector().swap(element_temp); + std::vector().swap(global_indices_temp); + + /****read in element types ****/ + DCArrayKokkos elem_types(num_elems, "elem_types_vtu_file"); // element types // array dims are the (num_elems) // 8 = pixal i,j,k linear quad format @@ -1095,18 +1409,114 @@ class MeshReader // 12 = linear ensight hex ordering // 72 = VTK_LAGRANGE_HEXAHEDRON // .... - found = extract_values_xml(elem_types.host.pointer(), - "\"types\"", - stop, - in, - size); - if(found==false){ - std::cout << "ERROR: element types were not found in the XML file!" << std::endl; + + dof_limit = global_num_elems; + buffer_iterations = dof_limit / (BUFFER_SIZE); + remainder_size = dof_limit % (BUFFER_SIZE); + if (remainder_size != 0) + { + buffer_iterations++; + } + + read_index_start = 0; + size = 0; + elem_read_buffer = CArrayKokkos(BUFFER_SIZE); + + //allocate types array + + //first find the block with element connectivity data data + if(myrank==0){ + + bool found = false; + + std::string line; + const std::string word = "\"types\""; + + // Read the file line by line looking for specified word + while (std::getline(in, line)) { + + if (line.find(word) != std::string::npos) { // Check if the portion of the word is in the line + found = true; + } + if(found) { + + break; + + } // end if found + + } // end while + + if(found==false){ + throw std::runtime_error("ERROR: mesh element types were not found in the XML file!"); + //std::cout << "ERROR: mesh nodes were not found in the XML file!" << std::endl; + } } + + + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ + // pack buffer on rank 0 + size_t buffer_iteration_size; + if(buffer_iteration < buffer_iterations - 1){ + buffer_iteration_size = BUFFER_SIZE; + } + else{ + buffer_iteration_size = remainder_size; + } + + if(myrank==0){ + + std::string line; + + // loop over the lines in the file until the buffer limit is reached + for(int idata = 0; idata < buffer_iteration_size; idata++){ + + // extract the individual values from the stream + std::string value; + in >> value; + if (value == stop) { // Check if the stop word is in the line + break; + } // end if + elem_read_buffer(idata) = std::stoi(value); + // if(elem_read_buffer(i)!=11){ + // std::cout << "FOUND TYPE ERROR " << std::endl; + // } + size++; + } // end for + } + // broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong + MPI_Bcast(elem_read_buffer.pointer(), BUFFER_SIZE, MPI_LONG_LONG_INT, 0, MPI_COMM_WORLD); + // broadcast how many nodes were read into this buffer iteration + MPI_Bcast(&buffer_iteration_size, 1, MPI_INT, 0, MPI_COMM_WORLD); + + // debug_print + // std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; + // for(int iprint=0; iprint < buffer_loop; iprint++) + // std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; + // return; + + // determine which data to store in the swage mesh members (the local node data) + // loop through read buffer + for (scan_loop = 0; scan_loop < buffer_iteration_size; scan_loop++) + { + // set global node id (ensight specific order) + elem_gid = read_index_start + scan_loop; + //add to the local type array if this elem gid belongs to this rank + if (element_map.isProcessGlobalIndex(elem_gid)){ + //std::cout << "elocal id " << element_map.getLocalIndex(elem_gid) << " " << elem_read_buffer(scan_loop) << std::endl; + elem_types.host(element_map.getLocalIndex(elem_gid)) = elem_read_buffer(scan_loop); + } + + + } + read_index_start += buffer_iteration_size; + } + elem_types.update_device(); + //std::cout << "Type read size " << size << std::endl; // check that the element type is supported by Fierro - FOR_ALL (elem_gid, 0, mesh.num_elems, { + FOR_ALL (elem_gid, 0, num_elems, { + //std::cout << "Element type is " << elem_types(elem_gid) << std::endl; if(elem_types(elem_gid) == element_types::linear_quad || elem_types(elem_gid) == element_types::linear_hex_ijk || elem_types(elem_gid) == element_types::linear_hex || @@ -1125,7 +1535,7 @@ class MeshReader CArrayKokkos convert_ensight_to_ijk(8, "convert_ensight_to_ijk"); // Convert the arbitrary order hex to a IJK mesh - DCArrayKokkos convert_pn_vtk_to_ijk(mesh.num_nodes_in_elem, "convert_pn_vtk_to_ijk"); + DCArrayKokkos convert_pn_vtk_to_ijk(num_nodes_in_elem, "convert_pn_vtk_to_ijk"); //build the connectivity for element type 12 // elem_types.host(0) @@ -1134,13 +1544,13 @@ class MeshReader case element_types::linear_quad: // the node order is correct, no changes required - FOR_ALL (elem_gid, 0, mesh.num_elems, { + // FOR_ALL (elem_gid, 0, mesh.num_elems, { - for (size_t node_lid=0; node_lid (num_elems, "ObjectIDs"); + + //reset file stream pointer to area where ObjectId is located before node coordinates + in.seekg(objectid_streampos); + + dof_limit = global_num_elems; + buffer_iterations = dof_limit / (BUFFER_SIZE); + remainder_size = dof_limit % (BUFFER_SIZE); + if (remainder_size != 0) + { + buffer_iterations++; + } + + read_index_start = 0; + size = 0; + + //allocate types array + + //first find the block with element connectivity data data + if(myrank==0){ + + bool found = false; + + std::string line; + const std::string word = "\"ObjectId\""; + // Read the file line by line looking for specified word + while (std::getline(in, line)) { - // Build connectivity - mesh.build_connectivity(); + if (line.find(word) != std::string::npos) { // Check if the portion of the word is in the line + found = true; + } + if(found) { + + break; + + } // end if found + + } // end while + + if(found==false){ + throw std::runtime_error("ERROR: ObjectIDs were not found in the XML file!"); + //std::cout << "ERROR: mesh nodes were not found in the XML file!" << std::endl; + } + } + + + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ + // pack buffer on rank 0 + size_t buffer_iteration_size; + if(buffer_iteration < buffer_iterations - 1){ + buffer_iteration_size = BUFFER_SIZE; + } + else{ + buffer_iteration_size = remainder_size; + } + + if(myrank==0){ + + std::string line; + + // loop over the lines in the file until the buffer limit is reached + for(int idata = 0; idata < buffer_iteration_size; idata++){ + // extract the individual values from the stream + std::string value; + in >> value; + if (value == stop) { // Check if the stop word is in the line + break; + } // end if + elem_read_buffer(idata) = std::stoi(value); + size++; + } // end for + } + // broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong + MPI_Bcast(elem_read_buffer.pointer(), BUFFER_SIZE, MPI_LONG_LONG_INT, 0, MPI_COMM_WORLD); + // broadcast how many nodes were read into this buffer iteration + MPI_Bcast(&buffer_iteration_size, 1, MPI_INT, 0, MPI_COMM_WORLD); + + // debug_print + // std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; + // for(int iprint=0; iprint < buffer_loop; iprint++) + // std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; + // return; + + // determine which data to store in the swage mesh members (the local node data) + // loop through read buffer + for (scan_loop = 0; scan_loop < buffer_iteration_size; scan_loop++) + { + // set global node id (ensight specific order) + elem_gid = read_index_start + scan_loop; + //add to the local type array if this elem gid belongs to this rank + if (element_map.isProcessGlobalIndex(elem_gid)){ + mesh_inps.object_ids.host(element_map.getLocalIndex(elem_gid)) = elem_read_buffer(scan_loop); + } + + + } + read_index_start += buffer_iteration_size; + } + + mesh_inps.object_ids.update_device(); in.close(); @@ -1279,14 +1792,14 @@ class MeshBuilder GaussPoint_t& GaussPoints, node_t& node, corner_t& corner, - SimulationParameters_t& SimulationParamaters) + SimulationParameters_t& SimulationParameters) { - if (SimulationParamaters.mesh_input.num_dims == 2) { - if (SimulationParamaters.mesh_input.type == mesh_input::Polar) { - build_2d_polar(mesh, GaussPoints, node, corner, SimulationParamaters); + if (SimulationParameters.mesh_input.num_dims == 2) { + if (SimulationParameters.mesh_input.type == mesh_input::Polar) { + build_2d_polar(mesh, GaussPoints, node, corner, SimulationParameters); } - else if (SimulationParamaters.mesh_input.type == mesh_input::Box) { - build_2d_box(mesh, GaussPoints, node, corner, SimulationParamaters); + else if (SimulationParameters.mesh_input.type == mesh_input::Box) { + build_2d_box(mesh, GaussPoints, node, corner, SimulationParameters); } else{ std::cout << "**** 2D MESH TYPE NOT SUPPORTED **** " << std::endl; @@ -1298,8 +1811,8 @@ class MeshBuilder throw std::runtime_error("**** 2D MESH TYPE NOT SUPPORTED ****"); } } - else if (SimulationParamaters.mesh_input.num_dims == 3) { - build_3d_box(mesh, GaussPoints, node, corner, SimulationParamaters); + else if (SimulationParameters.mesh_input.num_dims == 3) { + build_3d_box(mesh, GaussPoints, node, corner, SimulationParameters); } else{ throw std::runtime_error("**** ONLY 2D RZ OR 3D MESHES ARE SUPPORTED ****"); @@ -1319,117 +1832,418 @@ class MeshBuilder /// \param Simulation parameters /// ///////////////////////////////////////////////////////////////////////////// + void build_2d_box(Mesh_t& mesh, GaussPoint_t& GaussPoints, node_t& node, corner_t& corner, - SimulationParameters_t& SimulationParamaters) const + SimulationParameters_t& SimulationParameters) const { - printf("Creating a 2D box mesh \n"); + int myrank, nranks; + MPI_Comm_rank(MPI_COMM_WORLD,&myrank); + MPI_Comm_size(MPI_COMM_WORLD,&nranks); + /*currently we just build the global mesh data on rank 0 and then broadcast relevant data to each rank + before the global mesh data on rank 0 falls out of scope*/ + int global_num_nodes, global_num_elems; + CArrayKokkos read_buffer; + CArrayKokkos read_buffer_edof; + CArrayKokkos global_coords; + CArrayKokkos global_nodes_in_elem; + auto convert_point_number_in_quad = CArray(4); + convert_point_number_in_quad(0) = 0; + convert_point_number_in_quad(1) = 1; + convert_point_number_in_quad(2) = 3; + convert_point_number_in_quad(3) = 2; + + int local_node_index, current_column_index; + int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop; + int negative_index_found = 0; + int global_negative_index_found = 0; - const int num_dim = 2; + size_t read_index_start, node_rid, elem_gid; + long long int node_gid; + real_t dof_value; + real_t unit_scaling = 1; + + const int num_dims = 2; + size_t num_nodes_in_elem = 1; + for (int dim = 0; dim < num_dims; dim++) { + num_nodes_in_elem *= 2; + } + if(myrank==0){ + printf("Creating a 2D box mesh \n"); + } - const double lx = SimulationParamaters.mesh_input.length[0]; - const double ly = SimulationParamaters.mesh_input.length[1]; + // SimulationParameters.mesh_input.length.update_host(); + const double lx = SimulationParameters.mesh_input.length[0]; + const double ly = SimulationParameters.mesh_input.length[1]; - const int num_elems_i = SimulationParamaters.mesh_input.num_elems[0]; - const int num_elems_j = SimulationParamaters.mesh_input.num_elems[1]; + // SimulationParameters.mesh_input.num_elems.update_host(); + const int num_elems_i = SimulationParameters.mesh_input.num_elems[0]; + const int num_elems_j = SimulationParameters.mesh_input.num_elems[1]; const int num_points_i = num_elems_i + 1; // num points in x const int num_points_j = num_elems_j + 1; // num points in y - const int num_nodes = num_points_i * num_points_j; + global_num_nodes = num_points_i * num_points_j; const double dx = lx / ((double)num_elems_i); // len/(num_elems_i) const double dy = ly / ((double)num_elems_j); // len/(num_elems_j) - const int num_elems = num_elems_i * num_elems_j; + global_num_elems = num_elems_i * num_elems_j; + + if(myrank==0){ + std::vector origin(num_dims); + // SimulationParameters.mesh_input.origin.update_host(); + for (int i = 0; i < num_dims; i++) { origin[i] = SimulationParameters.mesh_input.origin[i]; } + + // --- 3D parameters --- + // const int num_faces_in_elem = 6; // number of faces in elem + // const int num_points_in_elem = 8; // number of points in elem + // const int num_points_in_face = 4; // number of points in a face + // const int num_edges_in_elem = 12; // number of edges in a elem + + // node coords data on rank 0 for all global nodes + global_coords = CArrayKokkos(global_num_nodes, num_dims, "global_mesh_build_node_coordinates"); - std::vector origin(num_dim); - // SimulationParamaters.mesh_input.origin.update_host(); - for (int i = 0; i < num_dim; i++) { origin[i] = SimulationParamaters.mesh_input.origin[i]; } + // --- Build nodes --- - // --- 2D parameters --- - // const int num_faces_in_elem = 4; // number of faces in elem - // const int num_points_in_elem = 4; // number of points in elem - // const int num_points_in_face = 2; // number of points in a face - // const int num_edges_in_elem = 4; // number of edges in a elem + // populate the point data structures + for (int j = 0; j < num_points_j; j++) { + for (int i = 0; i < num_points_i; i++) { + // global id for the point + int node_gid = get_id(i, j, 0, num_points_i, num_points_j); - // --- mesh node ordering --- - // Convert ijk index system to the finite element numbering convention - // for vertices in elem - auto convert_point_number_in_quad = CArray(4); - convert_point_number_in_quad(0) = 0; - convert_point_number_in_quad(1) = 1; - convert_point_number_in_quad(2) = 3; - convert_point_number_in_quad(3) = 2; + // store the point coordinates + global_coords(node_gid, 0) = origin[0] + (double)i * dx; + global_coords(node_gid, 1) = origin[1] + (double)j * dy; + } // end for i + } // end for j - // intialize node variables - mesh.initialize_nodes(num_nodes); - // initialize node state, for now, we just need coordinates, the rest will be initialize by the respective solvers - std::vector required_node_state = { node_state::coords }; - node.initialize(num_nodes, num_dim, required_node_state); + // initialize elem variables + global_nodes_in_elem = CArrayKokkos(global_num_elems, num_nodes_in_elem, "global_mesh_build_nodes_in_elem"); - // --- Build nodes --- + // --- Build elems --- - // populate the point data structures - for (int j = 0; j < num_points_j; j++) { - for (int i = 0; i < num_points_i; i++) { - // global id for the point - int node_gid = get_id(i, j, 0, num_points_i, num_points_j); + // populate the elem center data structures + for (int j = 0; j < num_elems_j; j++) { + for (int i = 0; i < num_elems_i; i++) { + // global id for the elem + int elem_gid = get_id(i, j, 0, num_elems_i, num_elems_j); - // store the point coordinates - node.coords.host(node_gid, 0) = origin[0] + (double)i * dx; - node.coords.host(node_gid, 1) = origin[1] + (double)j * dy; - } // end for i - } // end for j + // store the point IDs for this elem where the range is + // (i:i+1, j:j+1, k:k+1) for a linear hexahedron + int this_point = 0; + for (int jcount = j; jcount <= j + 1; jcount++) { + for (int icount = i; icount <= i + 1; icount++) { + // global id for the points + int node_gid = get_id(icount, jcount, 0, + num_points_i, num_points_j); + // convert this_point index to the FE index convention + int this_index = convert_point_number_in_quad(this_point); - node.coords.update_device(); + // store the points in this elem according the the finite + // element numbering convention + global_nodes_in_elem(elem_gid, this_index) = node_gid; + + // increment the point counting index + this_point = this_point + 1; + } // end for icount + } // end for jcount + } // end for i + } // end for j + } + + //distribute partitioned data from the global mesh build data on rank 0 + size_t num_local_nodes; + DistributedMap node_map; + // read coords + read_index_start = 0; + + buffer_iterations = global_num_nodes / BUFFER_SIZE; + if (global_num_nodes % BUFFER_SIZE != 0) + { + buffer_iterations++; + } + + read_buffer = CArrayKokkos(BUFFER_SIZE, num_dims); + + { //scoped so temp FArray data is auto deleted to save memory + //allocate pre-partition node coords using contiguous decomposition + //FArray type used since CArray type still doesnt support zoltan2 decomposition + DistributedDFArray node_coords_distributed(global_num_nodes, num_dims); + + // construct contiguous parallel row map now that we know the number of nodes + node_map = node_coords_distributed.pmap; + // map->describe(*fos,Teuchos::VERB_EXTREME); + + // set the vertices in the mesh read in + num_local_nodes = node_map.size(); + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_SIZE; buffer_loop++) + { + + for (int idim = 0; idim < num_dims; idim++) + { + // debug print + // std::cout<<" "<< substring < required_node_state = { node_state::coords }; + node_map = node_coords_distributed.pmap; + node.initialize(node_map, num_dims, required_node_state); + //copy coordinate data from repartitioned FArray into CArray + FOR_ALL(node_id, 0, node_map.size(), { + for(int idim = 0; idim < num_dims; idim++){ + node.coords(node_id,idim) = node_coords_distributed(node_id,idim); + } + }); + } + + //initialize some mesh data + mesh.initialize_nodes(global_num_nodes); + num_local_nodes = node_map.size(); + mesh.num_local_nodes = num_local_nodes; + mesh.node_map = node_map; + mesh.num_dims = num_dims; + //node.coords.print(); + + // debug print of nodal data + + // debug print nodal positions and indices + /* + std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; + for (int istride = 0; istride < num_dims; istride++){ + std::cout << node_coords(inode,istride) << " , "; + } + std::cout << " }"<< std::endl; + } + */ + + // check that local assignments match global total + + // read in element info (ensight file format is organized in element type sections) + // loop over this later for several element type sections + CArrayKokkos node_store(num_nodes_in_elem); + + // broadcast number of elements + MPI_Bcast(&global_num_elems, 1, MPI_LONG_LONG_INT, 0, MPI_COMM_WORLD); + + //initialize num elem in mesh struct + + if (myrank == 0) + { + std::cout << "before mesh initialization" << std::endl; + } + + // read in element connectivity + // we're gonna reallocate for the words per line expected for the element connectivity + read_buffer_edof = CArrayKokkos(BUFFER_SIZE, num_nodes_in_elem); + + // calculate buffer iterations to read number of lines + buffer_iterations = global_num_elems / BUFFER_SIZE; + int assign_flag; + + // dynamic buffer used to store elements before we know how many this rank needs + std::vector element_temp(BUFFER_SIZE * num_nodes_in_elem); + std::vector global_indices_temp(BUFFER_SIZE); + size_t buffer_max = BUFFER_SIZE * num_nodes_in_elem; + size_t indices_buffer_max = BUFFER_SIZE; + size_t num_elems = 0; + + if (global_num_elems % BUFFER_SIZE != 0) + { + buffer_iterations++; + } + read_index_start = 0; + // std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_SIZE; buffer_loop++) + { + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + read_buffer_edof(buffer_loop,inode) = global_nodes_in_elem(buffer_iteration * BUFFER_SIZE + buffer_loop, inode); + } + // std::cout <= buffer_max) + { + element_temp.resize((num_elems - 1) * num_nodes_in_elem + inode + BUFFER_SIZE * num_nodes_in_elem); + buffer_max = (num_elems - 1) * num_nodes_in_elem + inode + BUFFER_SIZE * num_nodes_in_elem; + } + element_temp[(num_elems - 1) * num_nodes_in_elem + inode] = node_store(inode); + // std::cout << "VECTOR STORAGE FOR ELEM " << num_elems << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; + } + // assign global element id to temporary list + if (num_elems - 1 >= indices_buffer_max) + { + global_indices_temp.resize(num_elems - 1 + BUFFER_SIZE); + indices_buffer_max = num_elems - 1 + BUFFER_SIZE; + } + global_indices_temp[num_elems - 1] = elem_gid; + } + } + read_index_start += BUFFER_SIZE; + } + //set global and local shared element counts + mesh.global_num_elems = global_num_elems; + + // construct partition mapping for shared elements on each process + DCArrayKokkos All_Element_Global_Indices(num_elems); + // copy temporary global indices storage to view storage + for (int ielem = 0; ielem < num_elems; ielem++) + { + All_Element_Global_Indices.host(ielem) = global_indices_temp[ielem]; + } - // store the point IDs for this elem where the range is - // (i:i+1, j:j+1 for a linear quad - int this_point = 0; + All_Element_Global_Indices.update_device(); - for (int jcount = j; jcount <= j + 1; jcount++) { - for (int icount = i; icount <= i + 1; icount++) { - // global id for the points - int node_gid = get_id(icount, jcount, 0, num_points_i, num_points_j); + // construct global map of local and shared elements (since different ranks can own the same elements due to the local node map) + DistributedMap element_map = DistributedMap(All_Element_Global_Indices); - // convert this_point index to the FE index convention - int this_index = convert_point_number_in_quad(this_point); + //initialize elem data structures + mesh.initialize_elems(num_elems, num_nodes_in_elem, element_map); - // store the points in this elem according the the finite - // element numbering convention - mesh.nodes_in_elem.host(elem_gid, this_index) = node_gid; + // copy temporary element storage to distributed storage + DistributedDCArray nodes_in_elem = mesh.nodes_in_elem; - // increment the point counting index - this_point = this_point + 1; - } // end for icount - } // end for jcount - } // end for i - } // end for j + for (int ielem = 0; ielem < num_elems; ielem++) + { + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { //assign local indices to element-node connectivity (stores global indices until ghost maps are made later) + nodes_in_elem.host(ielem, inode) = element_temp[ielem * num_nodes_in_elem + inode]; + } + } - // update device side - mesh.nodes_in_elem.update_device(); + nodes_in_elem.update_device(); + + // delete temporary element connectivity and index storage + //std::vector().swap(element_temp); + //std::vector().swap(global_indices_temp); - // intialize corner variables - int num_corners = num_elems * mesh.num_nodes_in_elem; + // initialize corner variables + size_t num_corners = num_elems * num_nodes_in_elem; mesh.initialize_corners(num_corners); - // corner.initialize(num_corners, num_dim); - // Build connectivity - mesh.build_connectivity(); } // end build_2d_box ///////////////////////////////////////////////////////////////////////////// @@ -1445,127 +2259,428 @@ class MeshBuilder /// \param Simulation parameters /// ///////////////////////////////////////////////////////////////////////////// + void build_2d_polar(Mesh_t& mesh, GaussPoint_t& GaussPoints, node_t& node, corner_t& corner, - SimulationParameters_t& SimulationParamaters) const + SimulationParameters_t& SimulationParameters) const { - printf("Creating a 2D polar mesh \n"); + int myrank, nranks; + MPI_Comm_rank(MPI_COMM_WORLD,&myrank); + MPI_Comm_size(MPI_COMM_WORLD,&nranks); + /*currently we just build the global mesh data on rank 0 and then broadcast relevant data to each rank + before the global mesh data on rank 0 falls out of scope*/ + int global_num_nodes, global_num_elems; + CArrayKokkos read_buffer; + CArrayKokkos read_buffer_edof; + CArrayKokkos global_coords; + CArrayKokkos global_nodes_in_elem; + auto convert_point_number_in_quad = CArray(4); + convert_point_number_in_quad(0) = 0; + convert_point_number_in_quad(1) = 1; + convert_point_number_in_quad(2) = 3; + convert_point_number_in_quad(3) = 2; + + int local_node_index, current_column_index; + int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop; + int negative_index_found = 0; + int global_negative_index_found = 0; + + size_t read_index_start, node_rid, elem_gid; + long long int node_gid; + real_t dof_value; + real_t unit_scaling = 1; - int num_dim = 2; + const int num_dims = 2; + size_t num_nodes_in_elem = 1; + for (int dim = 0; dim < num_dims; dim++) { + num_nodes_in_elem *= 2; + } + if(myrank==0){ + printf("Creating a 2D box mesh \n"); + } - const double inner_radius = SimulationParamaters.mesh_input.inner_radius; - const double outer_radius = SimulationParamaters.mesh_input.outer_radius; + // SimulationParameters.mesh_input.length.update_host(); + const double inner_radius = SimulationParameters.mesh_input.inner_radius; + const double outer_radius = SimulationParameters.mesh_input.outer_radius; - const double start_angle = PI / 180.0 * SimulationParamaters.mesh_input.starting_angle; - const double end_angle = PI / 180.0 * SimulationParamaters.mesh_input.ending_angle; + const double start_angle = PI / 180.0 * SimulationParameters.mesh_input.starting_angle; + const double end_angle = PI / 180.0 * SimulationParameters.mesh_input.ending_angle; - const int num_elems_i = SimulationParamaters.mesh_input.num_radial_elems; - const int num_elems_j = SimulationParamaters.mesh_input.num_angular_elems; + const int num_elems_i = SimulationParameters.mesh_input.num_radial_elems; + const int num_elems_j = SimulationParameters.mesh_input.num_angular_elems; const int num_points_i = num_elems_i + 1; // num points in x const int num_points_j = num_elems_j + 1; // num points in y - const int num_nodes = num_points_i * num_points_j; - const double dx = (outer_radius - inner_radius) / ((double)num_elems_i); // len/(elems) const double dy = (end_angle - start_angle) / ((double)num_elems_j); // len/(elems) - const int num_elems = num_elems_i * num_elems_j; + global_num_elems = num_elems_i * num_elems_j; - std::vector origin(num_dim); + global_num_nodes = num_points_i * num_points_j; + + if(myrank==0){ + std::vector origin(num_dims); + // SimulationParameters.mesh_input.origin.update_host(); + for (int i = 0; i < num_dims; i++) { origin[i] = SimulationParameters.mesh_input.origin[i]; } + + // --- 3D parameters --- + // const int num_faces_in_elem = 6; // number of faces in elem + // const int num_points_in_elem = 8; // number of points in elem + // const int num_points_in_face = 4; // number of points in a face + // const int num_edges_in_elem = 12; // number of edges in a elem + + // node coords data on rank 0 for all global nodes + global_coords = CArrayKokkos(global_num_nodes, num_dims, "global_mesh_build_node_coordinates"); - for (int i = 0; i < num_dim; i++) { origin[i] = SimulationParamaters.mesh_input.origin[i]; } + // --- Build nodes --- - // --- 2D parameters --- - // const int num_faces_in_elem = 4; // number of faces in elem - // const int num_points_in_elem = 4; // number of points in elem - // const int num_points_in_face = 2; // number of points in a face - // const int num_edges_in_elem = 4; // number of edges in a elem + // populate the point data structures + for (int j = 0; j < num_points_j; j++) { + for (int i = 0; i < num_points_i; i++) { + // global id for the point + int node_gid = get_id(i, j, 0, num_points_i, num_points_j); - // --- mesh node ordering --- - // Convert ijk index system to the finite element numbering convention - // for vertices in elem - auto convert_point_number_in_quad = CArray(4); - convert_point_number_in_quad(0) = 0; - convert_point_number_in_quad(1) = 1; - convert_point_number_in_quad(2) = 3; - convert_point_number_in_quad(3) = 2; + double r_i = inner_radius + (double)i * dx; + double theta_j = start_angle + (double)j * dy; - // intialize node variables - mesh.initialize_nodes(num_nodes); + // store the point coordinates + global_coords(node_gid, 0) = origin[0] + r_i * cos(theta_j); + global_coords(node_gid, 1) = origin[1] + r_i * sin(theta_j); - // initialize node state, for now, we just need coordinates, the rest will be initialize by the respective solvers - std::vector required_node_state = { node_state::coords }; - node.initialize(num_nodes, num_dim, required_node_state); + if(global_coords(node_gid, 0) < 0.0){ + throw std::runtime_error("**** NODE RADIUS FOR RZ MESH MUST BE POSITIVE ****"); + } + } // end for i + } // end for j + + + // initialize elem variables + global_nodes_in_elem = CArrayKokkos(global_num_elems, num_nodes_in_elem, "global_mesh_build_nodes_in_elem"); + + // --- Build elems --- + + // populate the elem center data structures + for (int j = 0; j < num_elems_j; j++) { + for (int i = 0; i < num_elems_i; i++) { + // global id for the elem + int elem_gid = get_id(i, j, 0, num_elems_i, num_elems_j); + + // store the point IDs for this elem where the range is + // (i:i+1, j:j+1, k:k+1) for a linear hexahedron + int this_point = 0; + for (int jcount = j; jcount <= j + 1; jcount++) { + for (int icount = i; icount <= i + 1; icount++) { + // global id for the points + int node_gid = get_id(icount, jcount, 0, + num_points_i, num_points_j); + + // convert this_point index to the FE index convention + int this_index = convert_point_number_in_quad(this_point); + + // store the points in this elem according the the finite + // element numbering convention + global_nodes_in_elem(elem_gid, this_index) = node_gid; + + // increment the point counting index + this_point = this_point + 1; + } // end for icount + } // end for jcount + } // end for i + } // end for j + } + + //distribute partitioned data from the global mesh build data on rank 0 + size_t num_local_nodes; + DistributedMap node_map; + // read coords + read_index_start = 0; + + buffer_iterations = global_num_nodes / BUFFER_SIZE; + if (global_num_nodes % BUFFER_SIZE != 0) + { + buffer_iterations++; + } + + read_buffer = CArrayKokkos(BUFFER_SIZE, num_dims); - // populate the point data structures - for (int j = 0; j < num_points_j; j++) { - for (int i = 0; i < num_points_i; i++) { - // global id for the point - int node_gid = get_id(i, j, 0, num_points_i, num_points_j); + { //scoped so temp FArray data is auto deleted to save memory + //allocate pre-partition node coords using contiguous decomposition + //FArray type used since CArray type still doesnt support zoltan2 decomposition + DistributedDFArray node_coords_distributed(global_num_nodes, num_dims); + + // construct contiguous parallel row map now that we know the number of nodes + node_map = node_coords_distributed.pmap; + // map->describe(*fos,Teuchos::VERB_EXTREME); + + // set the vertices in the mesh read in + num_local_nodes = node_map.size(); + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_SIZE; buffer_loop++) + { + + for (int idim = 0; idim < num_dims; idim++) + { + // debug print + // std::cout<<" "<< substring < required_node_state = { node_state::coords }; + node_map = node_coords_distributed.pmap; + node.initialize(node_map, num_dims, required_node_state); + //copy coordinate data from repartitioned FArray into CArray + FOR_ALL(node_id, 0, node_map.size(), { + for(int idim = 0; idim < num_dims; idim++){ + node.coords(node_id,idim) = node_coords_distributed(node_id,idim); } + }); + } + + //initialize some mesh data + mesh.initialize_nodes(global_num_nodes); + num_local_nodes = node_map.size(); + mesh.num_local_nodes = num_local_nodes; + mesh.node_map = node_map; + mesh.num_dims = num_dims; + //node.coords.print(); + + // debug print of nodal data - } // end for i - } // end for j + // debug print nodal positions and indices + /* + std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; + for (int istride = 0; istride < num_dims; istride++){ + std::cout << node_coords(inode,istride) << " , "; + } + std::cout << " }"<< std::endl; + } + */ + // check that local assignments match global total - node.coords.update_device(); + // read in element info (ensight file format is organized in element type sections) + // loop over this later for several element type sections + CArrayKokkos node_store(num_nodes_in_elem); - // initialize elem variables - mesh.initialize_elems(num_elems, num_dim); + // broadcast number of elements + MPI_Bcast(&global_num_elems, 1, MPI_LONG_LONG_INT, 0, MPI_COMM_WORLD); - // populate the elem center data structures - for (int j = 0; j < num_elems_j; j++) { - for (int i = 0; i < num_elems_i; i++) { - // global id for the elem - int elem_gid = get_id(i, j, 0, num_elems_i, num_elems_j); + //initialize num elem in mesh struct + + if (myrank == 0) + { + std::cout << "before mesh initialization" << std::endl; + } + + // read in element connectivity + // we're gonna reallocate for the words per line expected for the element connectivity + read_buffer_edof = CArrayKokkos(BUFFER_SIZE, num_nodes_in_elem); + + // calculate buffer iterations to read number of lines + buffer_iterations = global_num_elems / BUFFER_SIZE; + int assign_flag; + + // dynamic buffer used to store elements before we know how many this rank needs + std::vector element_temp(BUFFER_SIZE * num_nodes_in_elem); + std::vector global_indices_temp(BUFFER_SIZE); + size_t buffer_max = BUFFER_SIZE * num_nodes_in_elem; + size_t indices_buffer_max = BUFFER_SIZE; + size_t num_elems = 0; + + if (global_num_elems % BUFFER_SIZE != 0) + { + buffer_iterations++; + } + read_index_start = 0; + // std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_SIZE; buffer_loop++) + { + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + read_buffer_edof(buffer_loop,inode) = global_nodes_in_elem(buffer_iteration * BUFFER_SIZE + buffer_loop, inode); + } + // std::cout <= buffer_max) + { + element_temp.resize((num_elems - 1) * num_nodes_in_elem + inode + BUFFER_SIZE * num_nodes_in_elem); + buffer_max = (num_elems - 1) * num_nodes_in_elem + inode + BUFFER_SIZE * num_nodes_in_elem; + } + element_temp[(num_elems - 1) * num_nodes_in_elem + inode] = node_store(inode); + // std::cout << "VECTOR STORAGE FOR ELEM " << num_elems << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; + } + // assign global element id to temporary list + if (num_elems - 1 >= indices_buffer_max) + { + global_indices_temp.resize(num_elems - 1 + BUFFER_SIZE); + indices_buffer_max = num_elems - 1 + BUFFER_SIZE; + } + global_indices_temp[num_elems - 1] = elem_gid; + } + } + read_index_start += BUFFER_SIZE; + } + //set global and local shared element counts + mesh.global_num_elems = global_num_elems; + + // construct partition mapping for shared elements on each process + DCArrayKokkos All_Element_Global_Indices(num_elems); + // copy temporary global indices storage to view storage + for (int ielem = 0; ielem < num_elems; ielem++) + { + All_Element_Global_Indices.host(ielem) = global_indices_temp[ielem]; + } - // store the point IDs for this elem where the range is - // (i:i+1, j:j+1 for a linear quad - int this_point = 0; + All_Element_Global_Indices.update_device(); - for (int jcount = j; jcount <= j + 1; jcount++) { - for (int icount = i; icount <= i + 1; icount++) { - // global id for the points - int node_gid = get_id(icount, jcount, 0, num_points_i, num_points_j); + // construct global map of local and shared elements (since different ranks can own the same elements due to the local node map) + DistributedMap element_map = DistributedMap(All_Element_Global_Indices); - // convert this_point index to the FE index convention - int this_index = convert_point_number_in_quad(this_point); + //initialize elem data structures + mesh.initialize_elems(num_elems, num_nodes_in_elem, element_map); - // store the points in this elem according the the finite - // element numbering convention - mesh.nodes_in_elem.host(elem_gid, this_index) = node_gid; + // copy temporary element storage to distributed storage + DistributedDCArray nodes_in_elem = mesh.nodes_in_elem; - // increment the point counting index - this_point = this_point + 1; - } // end for icount - } // end for jcount - } // end for i - } // end for j + for (int ielem = 0; ielem < num_elems; ielem++) + { + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { //assign local indices to element-node connectivity (stores global indices until ghost maps are made later) + nodes_in_elem.host(ielem, inode) = element_temp[ielem * num_nodes_in_elem + inode]; + } + } - // update device side - mesh.nodes_in_elem.update_device(); + nodes_in_elem.update_device(); + + // delete temporary element connectivity and index storage + //std::vector().swap(element_temp); + //std::vector().swap(global_indices_temp); - // intialize corner variables - int num_corners = num_elems * mesh.num_nodes_in_elem; + // initialize corner variables + size_t num_corners = num_elems * num_nodes_in_elem; mesh.initialize_corners(num_corners); - // corner.initialize(num_corners, num_dim); - // Build connectivity - mesh.build_connectivity(); - } // end build_2d_box + } // end build_2d_polar ///////////////////////////////////////////////////////////////////////////// /// @@ -1584,120 +2699,417 @@ class MeshBuilder GaussPoint_t& GaussPoints, node_t& node, corner_t& corner, - SimulationParameters_t& SimulationParamaters) const - { - printf("Creating a 3D box mesh \n"); - - const int num_dim = 3; + SimulationParameters_t& SimulationParameters) const + { + int myrank, nranks; + MPI_Comm_rank(MPI_COMM_WORLD,&myrank); + MPI_Comm_size(MPI_COMM_WORLD,&nranks); + /*currently we just build the global mesh data on rank 0 and then broadcast relevant data to each rank + before the global mesh data on rank 0 falls out of scope*/ + int global_num_nodes, global_num_elems; + CArrayKokkos read_buffer; + CArrayKokkos read_buffer_edof; + CArrayKokkos global_coords; + CArrayKokkos global_nodes_in_elem; + + int local_node_index, current_column_index; + int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop; + int negative_index_found = 0; + int global_negative_index_found = 0; + + size_t read_index_start, node_rid, elem_gid; + long long int node_gid; + real_t dof_value; + real_t unit_scaling = 1; + + const int num_dims = 3; + size_t num_nodes_in_elem = 1; + for (int dim = 0; dim < num_dims; dim++) { + num_nodes_in_elem *= 2; + } + if(myrank==0){ + printf("Creating a 3D box mesh \n"); + } - // SimulationParamaters.mesh_input.length.update_host(); - const double lx = SimulationParamaters.mesh_input.length[0]; - const double ly = SimulationParamaters.mesh_input.length[1]; - const double lz = SimulationParamaters.mesh_input.length[2]; + // SimulationParameters.mesh_input.length.update_host(); + const double lx = SimulationParameters.mesh_input.length[0]; + const double ly = SimulationParameters.mesh_input.length[1]; + const double lz = SimulationParameters.mesh_input.length[2]; - // SimulationParamaters.mesh_input.num_elems.update_host(); - const int num_elems_i = SimulationParamaters.mesh_input.num_elems[0]; - const int num_elems_j = SimulationParamaters.mesh_input.num_elems[1]; - const int num_elems_k = SimulationParamaters.mesh_input.num_elems[2]; + // SimulationParameters.mesh_input.num_elems.update_host(); + const int num_elems_i = SimulationParameters.mesh_input.num_elems[0]; + const int num_elems_j = SimulationParameters.mesh_input.num_elems[1]; + const int num_elems_k = SimulationParameters.mesh_input.num_elems[2]; const int num_points_i = num_elems_i + 1; // num points in x const int num_points_j = num_elems_j + 1; // num points in y const int num_points_k = num_elems_k + 1; // num points in y - const int num_nodes = num_points_i * num_points_j * num_points_k; + global_num_nodes = num_points_i * num_points_j * num_points_k; const double dx = lx / ((double)num_elems_i); // len/(num_elems_i) const double dy = ly / ((double)num_elems_j); // len/(num_elems_j) const double dz = lz / ((double)num_elems_k); // len/(num_elems_k) - const int num_elems = num_elems_i * num_elems_j * num_elems_k; + global_num_elems = num_elems_i * num_elems_j * num_elems_k; + + if(myrank==0){ + std::vector origin(num_dims); + // SimulationParameters.mesh_input.origin.update_host(); + for (int i = 0; i < num_dims; i++) { origin[i] = SimulationParameters.mesh_input.origin[i]; } + + // --- 3D parameters --- + // const int num_faces_in_elem = 6; // number of faces in elem + // const int num_points_in_elem = 8; // number of points in elem + // const int num_points_in_face = 4; // number of points in a face + // const int num_edges_in_elem = 12; // number of edges in a elem + + // node coords data on rank 0 for all global nodes + global_coords = CArrayKokkos(global_num_nodes, num_dims, "global_mesh_build_node_coordinates"); + + // --- Build nodes --- + + // populate the point data structures + for (int k = 0; k < num_points_k; k++) { + for (int j = 0; j < num_points_j; j++) { + for (int i = 0; i < num_points_i; i++) { + // global id for the point + int node_gid = get_id(i, j, k, num_points_i, num_points_j); + + // store the point coordinates + global_coords(node_gid, 0) = origin[0] + (double)i * dx; + global_coords(node_gid, 1) = origin[1] + (double)j * dy; + global_coords(node_gid, 2) = origin[2] + (double)k * dz; + } // end for i + } // end for j + } // end for k + + + // initialize elem variables + global_nodes_in_elem = CArrayKokkos(global_num_elems, num_nodes_in_elem, "global_mesh_build_nodes_in_elem"); + + // --- Build elems --- + + // populate the elem center data structures + for (int k = 0; k < num_elems_k; k++) { + for (int j = 0; j < num_elems_j; j++) { + for (int i = 0; i < num_elems_i; i++) { + // global id for the elem + int elem_gid = get_id(i, j, k, num_elems_i, num_elems_j); + + // store the point IDs for this elem where the range is + // (i:i+1, j:j+1, k:k+1) for a linear hexahedron + int this_point = 0; + for (int kcount = k; kcount <= k + 1; kcount++) { + for (int jcount = j; jcount <= j + 1; jcount++) { + for (int icount = i; icount <= i + 1; icount++) { + // global id for the points + int node_gid = get_id(icount, jcount, kcount, + num_points_i, num_points_j); + + // convert this_point index to the FE index convention + int this_index = this_point; //convert_point_number_in_Hex(this_point); + + // store the points in this elem according the the finite + // element numbering convention + global_nodes_in_elem(elem_gid, this_index) = node_gid; + + // increment the point counting index + this_point = this_point + 1; + } // end for icount + } // end for jcount + } // end for kcount + } // end for i + } // end for j + } // end for k + } + + //distribute partitioned data from the global mesh build data on rank 0 + size_t num_local_nodes; + DistributedMap node_map; + // read coords + read_index_start = 0; + + buffer_iterations = global_num_nodes / BUFFER_SIZE; + if (global_num_nodes % BUFFER_SIZE != 0) + { + buffer_iterations++; + } - std::vector origin(num_dim); - // SimulationParamaters.mesh_input.origin.update_host(); - for (int i = 0; i < num_dim; i++) { origin[i] = SimulationParamaters.mesh_input.origin[i]; } + read_buffer = CArrayKokkos(BUFFER_SIZE, num_dims); - // --- 3D parameters --- - // const int num_faces_in_elem = 6; // number of faces in elem - // const int num_points_in_elem = 8; // number of points in elem - // const int num_points_in_face = 4; // number of points in a face - // const int num_edges_in_elem = 12; // number of edges in a elem + { //scoped so temp FArray data is auto deleted to save memory + //allocate pre-partition node coords using contiguous decomposition + //FArray type used since CArray type still doesnt support zoltan2 decomposition + DistributedDFArray node_coords_distributed(global_num_nodes, num_dims); + // construct contiguous parallel row map now that we know the number of nodes + node_map = node_coords_distributed.pmap; + // map->describe(*fos,Teuchos::VERB_EXTREME); + + // set the vertices in the mesh read in + num_local_nodes = node_map.size(); + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_SIZE; buffer_loop++) + { + + for (int idim = 0; idim < num_dims; idim++) + { + // debug print + // std::cout<<" "<< substring < required_node_state = { node_state::coords }; - node.initialize(num_nodes, num_dim, required_node_state); + // debug_print + // std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; + // for(int iprint=0; iprint < buffer_loop; iprint++) + // std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; + // return; - // --- Build nodes --- + // determine which data to store in the swage mesh members (the local node data) + // loop through read buffer + for (scan_loop = 0; scan_loop < buffer_loop; scan_loop++) + { + // set global node id (ensight specific order) + node_gid = read_index_start + scan_loop; + // let map decide if this node id belongs locally; if yes store data + if (node_map.isProcessGlobalIndex(node_gid)) + { + // set local node index in this mpi rank + node_rid = node_map.getLocalIndex(node_gid); + // extract nodal position from the read buffer + // for tecplot format this is the three coords in the same line + dof_value = read_buffer(scan_loop,0); + node_coords_distributed.host(node_rid, 0) = dof_value * unit_scaling; + dof_value = read_buffer(scan_loop,1); + node_coords_distributed.host(node_rid, 1) = dof_value * unit_scaling; + if (num_dims == 3) + { + dof_value = read_buffer(scan_loop,2); + node_coords_distributed.host(node_rid, 2) = dof_value * unit_scaling; + } + } + } + read_index_start += BUFFER_SIZE; + } + // end of coordinate readin + node_coords_distributed.update_device(); + // repartition node distribution + node_coords_distributed.repartition_vector(); + //get map from repartitioned Farray and feed it into distributed CArray type; FArray data will be discared after scope + std::vector required_node_state = { node_state::coords }; + node_map = node_coords_distributed.pmap; + node.initialize(node_map, num_dims, required_node_state); + //copy coordinate data from repartitioned FArray into CArray + FOR_ALL(node_id, 0, node_map.size(), { + for(int idim = 0; idim < num_dims; idim++){ + node.coords(node_id,idim) = node_coords_distributed(node_id,idim); + } + }); + } + + //initialize some mesh data + mesh.initialize_nodes(global_num_nodes); + num_local_nodes = node_map.size(); + mesh.num_local_nodes = num_local_nodes; + mesh.node_map = node_map; + //node.coords.print(); + + // debug print of nodal data + + // debug print nodal positions and indices + /* + std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; + for (int istride = 0; istride < num_dims; istride++){ + std::cout << node_coords(inode,istride) << " , "; + } + std::cout << " }"<< std::endl; + } + */ + + // check that local assignments match global total + + // read in element info (ensight file format is organized in element type sections) + // loop over this later for several element type sections + CArrayKokkos node_store(num_nodes_in_elem); + + // broadcast number of elements + MPI_Bcast(&global_num_elems, 1, MPI_LONG_LONG_INT, 0, MPI_COMM_WORLD); + + //initialize num elem in mesh struct + + if (myrank == 0) + { + std::cout << "before mesh initialization" << std::endl; + } + + // read in element connectivity + // we're gonna reallocate for the words per line expected for the element connectivity + read_buffer_edof = CArrayKokkos(BUFFER_SIZE, num_nodes_in_elem); + + // calculate buffer iterations to read number of lines + buffer_iterations = global_num_elems / BUFFER_SIZE; + int assign_flag; + + // dynamic buffer used to store elements before we know how many this rank needs + std::vector element_temp(BUFFER_SIZE * num_nodes_in_elem); + std::vector global_indices_temp(BUFFER_SIZE); + size_t buffer_max = BUFFER_SIZE * num_nodes_in_elem; + size_t indices_buffer_max = BUFFER_SIZE; + size_t num_elems = 0; + + if (global_num_elems % BUFFER_SIZE != 0) + { + buffer_iterations++; + } + read_index_start = 0; + // std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_SIZE; buffer_loop++) + { + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + read_buffer_edof(buffer_loop,inode) = global_nodes_in_elem(buffer_iteration * BUFFER_SIZE + buffer_loop, inode); + } + // std::cout <= buffer_max) + { + element_temp.resize((num_elems - 1) * num_nodes_in_elem + inode + BUFFER_SIZE * num_nodes_in_elem); + buffer_max = (num_elems - 1) * num_nodes_in_elem + inode + BUFFER_SIZE * num_nodes_in_elem; + } + element_temp[(num_elems - 1) * num_nodes_in_elem + inode] = node_store(inode); + // std::cout << "VECTOR STORAGE FOR ELEM " << num_elems << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; + } + // assign global element id to temporary list + if (num_elems - 1 >= indices_buffer_max) + { + global_indices_temp.resize(num_elems - 1 + BUFFER_SIZE); + indices_buffer_max = num_elems - 1 + BUFFER_SIZE; + } + global_indices_temp[num_elems - 1] = elem_gid; + } + } + read_index_start += BUFFER_SIZE; + } + //set global and local shared element counts + mesh.global_num_elems = global_num_elems; + + // construct partition mapping for shared elements on each process + DCArrayKokkos All_Element_Global_Indices(num_elems); + // copy temporary global indices storage to view storage + for (int ielem = 0; ielem < num_elems; ielem++) + { + All_Element_Global_Indices.host(ielem) = global_indices_temp[ielem]; + } - node.coords.update_device(); + All_Element_Global_Indices.update_device(); - // initialize elem variables - mesh.initialize_elems(num_elems, num_dim); + // construct global map of local and shared elements (since different ranks can own the same elements due to the local node map) + DistributedMap element_map = DistributedMap(All_Element_Global_Indices); - // --- Build elems --- + //initialize elem data structures + mesh.initialize_elems(num_elems, num_nodes_in_elem, element_map); - // populate the elem center data structures - for (int k = 0; k < num_elems_k; k++) { - for (int j = 0; j < num_elems_j; j++) { - for (int i = 0; i < num_elems_i; i++) { - // global id for the elem - int elem_gid = get_id(i, j, k, num_elems_i, num_elems_j); + // copy temporary element storage to distributed storage + DistributedDCArray nodes_in_elem = mesh.nodes_in_elem; - // store the point IDs for this elem where the range is - // (i:i+1, j:j+1, k:k+1) for a linear hexahedron - int this_point = 0; - for (int kcount = k; kcount <= k + 1; kcount++) { - for (int jcount = j; jcount <= j + 1; jcount++) { - for (int icount = i; icount <= i + 1; icount++) { - // global id for the points - int node_gid = get_id(icount, jcount, kcount, - num_points_i, num_points_j); - - // convert this_point index to the FE index convention - int this_index = this_point; //convert_point_number_in_Hex(this_point); - - // store the points in this elem according the the finite - // element numbering convention - mesh.nodes_in_elem.host(elem_gid, this_index) = node_gid; - - // increment the point counting index - this_point = this_point + 1; - } // end for icount - } // end for jcount - } // end for kcount - } // end for i - } // end for j - } // end for k + for (int ielem = 0; ielem < num_elems; ielem++) + { + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { //assign local indices to element-node connectivity (stores global indices until ghost maps are made later) + nodes_in_elem.host(ielem, inode) = element_temp[ielem * num_nodes_in_elem + inode]; + } + } - // update device side - mesh.nodes_in_elem.update_device(); + nodes_in_elem.update_device(); + + // delete temporary element connectivity and index storage + //std::vector().swap(element_temp); + //std::vector().swap(global_indices_temp); // initialize corner variables - int num_corners = num_elems * mesh.num_nodes_in_elem; + size_t num_corners = num_elems * num_nodes_in_elem; mesh.initialize_corners(num_corners); - // corner.initialize(num_corners, num_dim); - - // Build connectivity - mesh.build_connectivity(); } // end build_3d_box ///////////////////////////////////////////////////////////////////////////// @@ -1713,164 +3125,161 @@ class MeshBuilder /// \param Simulation parameters /// ///////////////////////////////////////////////////////////////////////////// - void build_3d_HexN_box(Mesh_t& mesh, - GaussPoint_t& GaussPoints, - node_t& node, - corner_t& corner, - SimulationParameters_t& SimulationParamaters) const - { - printf(" ***** WARNING:: build_3d_HexN_box not yet implemented\n"); - const int num_dim = 3; - - // SimulationParamaters.mesh_input.length.update_host(); - const double lx = SimulationParamaters.mesh_input.length[0]; - const double ly = SimulationParamaters.mesh_input.length[1]; - const double lz = SimulationParamaters.mesh_input.length[2]; - - // SimulationParamaters.mesh_input.num_elems.update_host(); - const int num_elems_i = SimulationParamaters.mesh_input.num_elems[0]; - const int num_elems_j = SimulationParamaters.mesh_input.num_elems[1]; - const int num_elems_k = SimulationParamaters.mesh_input.num_elems[2]; - - // creating zones for the Pn order - const int Pn_order = SimulationParamaters.mesh_input.p_order; + // void build_3d_HexN_box(Mesh_t& mesh, + // GaussPoint_t& GaussPoints, + // node_t& node, + // corner_t& corner, + // SimulationParameters_t& SimulationParameters) const + // { + // printf(" ***** WARNING:: build_3d_HexN_box not yet implemented\n"); + // const int num_dim = 3; + + // // SimulationParameters.mesh_input.length.update_host(); + // const double lx = SimulationParameters.mesh_input.length[0]; + // const double ly = SimulationParameters.mesh_input.length[1]; + // const double lz = SimulationParameters.mesh_input.length[2]; + + // // SimulationParameters.mesh_input.num_elems.update_host(); + // const int num_elems_i = SimulationParameters.mesh_input.num_elems[0]; + // const int num_elems_j = SimulationParameters.mesh_input.num_elems[1]; + // const int num_elems_k = SimulationParameters.mesh_input.num_elems[2]; + + // // creating zones for the Pn order + // const int Pn_order = SimulationParameters.mesh_input.p_order; - if (Pn_order > 19) { - printf("Fierro DG and RD solvers are only valid for elements up to Pn = 19 \n"); - return; - } - - const int num_zones_i = Pn_order*num_elems_i; - const int num_zones_j = Pn_order*num_elems_j; - const int num_zones_k = Pn_order*num_elems_k; + // if (Pn_order > 19) { + // printf("Fierro DG and RD solvers are only valid for elements up to Pn = 19 \n"); + // return; + // } + + // const int num_zones_i = Pn_order*num_elems_i; + // const int num_zones_j = Pn_order*num_elems_j; + // const int num_zones_k = Pn_order*num_elems_k; - const int num_points_i = num_zones_i+1; // num points in x accounting for Pn - const int num_points_j = num_zones_j+1; // num points in y accounting for Pn - const int num_points_k = num_zones_k+1; // num points in y accounting for Pn + // const int num_points_i = num_zones_i+1; // num points in x accounting for Pn + // const int num_points_j = num_zones_j+1; // num points in y accounting for Pn + // const int num_points_k = num_zones_k+1; // num points in y accounting for Pn - const double dx = lx/((double)num_zones_i); // len/(num_zones_i) - const double dy = ly/((double)num_zones_j); // len/(num_zones_j) - const double dz = lz/((double)num_zones_k); // len/(num_zones_k) + // const double dx = lx/((double)num_zones_i); // len/(num_zones_i) + // const double dy = ly/((double)num_zones_j); // len/(num_zones_j) + // const double dz = lz/((double)num_zones_k); // len/(num_zones_k) - const int num_elems = num_elems_i*num_elems_j*num_elems_k; - // const int num_zones = num_zones_i*num_zones_j*num_zones_k; // accounts for Pn + // const int num_elems = num_elems_i*num_elems_j*num_elems_k; + // // const int num_zones = num_zones_i*num_zones_j*num_zones_k; // accounts for Pn - std::vector origin(num_dim); - for (int i = 0; i < num_dim; i++) { origin[i] = SimulationParamaters.mesh_input.origin[i]; } + // std::vector origin(num_dim); + // for (int i = 0; i < num_dim; i++) { origin[i] = SimulationParameters.mesh_input.origin[i]; } - // --- 3D parameters --- - // const int num_faces_in_zone = 6; // number of faces in zone - // const int num_points_in_zone = 8; // number of points in zone - // const int num_points_in_face = 4; // number of points in a face + // // --- 3D parameters --- + // // const int num_faces_in_zone = 6; // number of faces in zone + // // const int num_points_in_zone = 8; // number of points in zone + // // const int num_points_in_face = 4; // number of points in a face - // p_order = 1, 2, 3, 4, 5 - // num_nodes = 2, 3, 4, 5, 6 - const int num_1D_points = Pn_order+1; - const int num_points_in_elem = num_1D_points*num_1D_points*num_1D_points; + // // p_order = 1, 2, 3, 4, 5 + // // num_nodes = 2, 3, 4, 5, 6 + // const int num_1D_points = Pn_order+1; + // const int num_points_in_elem = num_1D_points*num_1D_points*num_1D_points; - // --- elem --- - auto elem_coords = CArray (num_elems, num_dim); - auto elem_point_list = CArray (num_elems, num_points_in_elem); + // // --- elem --- + // auto elem_coords = CArray (num_elems, num_dim); + // auto elem_point_list = CArray (num_elems, num_points_in_elem); - // --- point --- - int num_points = num_points_i * num_points_j * num_points_k; - auto pt_coords = CArray (num_points, num_dim); + // // --- point --- + // int num_points = num_points_i * num_points_j * num_points_k; + // auto pt_coords = CArray (num_points, num_dim); - // --- Build nodes --- + // // --- Build nodes --- - // initialize node variables - mesh.initialize_nodes(num_points); + // // initialize node variables + // mesh.initialize_nodes(num_points); - // - std::vector required_node_state = { node_state::coords }; - node.initialize(num_points, num_dim, required_node_state); - // populate the point data structures - for (int k = 0; k < num_points_k; k++){ - for (int j = 0; j < num_points_j; j++){ - for (int i = 0; i < num_points_i; i++){ + // // + // std::vector required_node_state = { node_state::coords }; + // node.initialize(num_points, num_dim, required_node_state); + // // populate the point data structures + // for (int k = 0; k < num_points_k; k++){ + // for (int j = 0; j < num_points_j; j++){ + // for (int i = 0; i < num_points_i; i++){ - // global id for the point - int node_gid = get_id(i, j, k, num_points_i, num_points_j); + // // global id for the point + // int node_gid = get_id(i, j, k, num_points_i, num_points_j); - // store the point coordinates - node.coords.host(node_gid, 0) = origin[0] + (double)i * dx; - node.coords.host(node_gid, 1) = origin[1] + (double)j * dy; - node.coords.host(node_gid, 2) = origin[2] + (double)k * dz; + // // store the point coordinates + // node.coords.host(node_gid, 0) = origin[0] + (double)i * dx; + // node.coords.host(node_gid, 1) = origin[1] + (double)j * dy; + // node.coords.host(node_gid, 2) = origin[2] + (double)k * dz; - } // end for k - } // end for i - } // end for j + // } // end for k + // } // end for i + // } // end for j - node.coords.update_device(); + // node.coords.update_device(); - // initialize elem variables - mesh.initialize_elems(num_elems, num_dim); + // // initialize elem variables + // mesh.initialize_elems(num_elems, num_dim); - // --- Build elems --- + // // --- Build elems --- - // populate the elem center data structures accounting for Pn - for (int k=0; k graphics_times, @@ -1919,7 +3328,6 @@ class MeshWriter const size_t solver_id) { - // node_state is an enum for possible fields (e.g., coords, velocity, etc.), see state.h // gauss_pt_state is an enum for possible fields (e.g., vol, divergence, etc.) // material_pt_state is an enum for possible fields (e.g., den, pres, etc.) @@ -2054,7 +3462,7 @@ class MeshWriter size_t num_mat_pt_tensor_vars = 0; // count the number of material point state vars to write out - for (auto field : SimulationParamaters.output_options.output_mat_pt_state){ + for (auto field : SimulationParameters.output_options.output_mat_pt_state){ switch(field){ // scalar vars to write out case material_pt_state::density: @@ -2116,7 +3524,7 @@ class MeshWriter size_t num_elem_tensor_vars = 0; // count the number of element average fields to write out - for (auto field : SimulationParamaters.output_options.output_elem_state){ + for (auto field : SimulationParameters.output_options.output_elem_state){ switch(field){ // scalar vars to write out case material_pt_state::density: @@ -2173,7 +3581,7 @@ class MeshWriter size_t num_gauss_pt_tensor_vars = 0; // gauss point values to ouptput - for (auto field : SimulationParamaters.output_options.output_gauss_pt_state){ + for (auto field : SimulationParameters.output_options.output_gauss_pt_state){ switch(field){ // scalar vars to write out case gauss_pt_state::volume: @@ -2230,7 +3638,7 @@ class MeshWriter size_t tensor_var = 0; // material point state to output - for (auto field : SimulationParamaters.output_options.output_mat_pt_state){ + for (auto field : SimulationParameters.output_options.output_mat_pt_state){ switch(field){ // scalar vars case material_pt_state::density: @@ -2328,7 +3736,7 @@ class MeshWriter tensor_var = 0; // element state to output - for (auto field : SimulationParamaters.output_options.output_elem_state){ + for (auto field : SimulationParameters.output_options.output_elem_state){ switch(field){ // scalar vars case material_pt_state::density: @@ -2401,7 +3809,7 @@ class MeshWriter int vel_grad_id = -1; - for (auto field : SimulationParamaters.output_options.output_gauss_pt_state){ + for (auto field : SimulationParameters.output_options.output_gauss_pt_state){ switch(field){ // scalars case gauss_pt_state::volume: @@ -2438,7 +3846,7 @@ class MeshWriter size_t num_node_scalar_vars = 0; size_t num_node_vector_vars = 0; - for (auto field : SimulationParamaters.output_options.output_node_state){ + for (auto field : SimulationParameters.output_options.output_node_state){ switch(field){ // --- scalars case node_state::mass: @@ -2485,7 +3893,7 @@ class MeshWriter vector_var = 0; tensor_var = 0; - for (auto field : SimulationParamaters.output_options.output_node_state){ + for (auto field : SimulationParameters.output_options.output_node_state){ switch(field){ // scalars case node_state::mass: @@ -2536,7 +3944,165 @@ class MeshWriter } // end switch } // end for over + // ******************************** + // Write the collective nodal and elem fields + // ******************************** + + if (SimulationParameters.output_options.format == output_options::viz || + SimulationParameters.output_options.format == output_options::viz_and_state) { + + write_parallel_viz(mesh, + State, + SimulationParameters, + dt, + time_value, + graphics_times, + solver_id, + node_states, + gauss_pt_states, + material_pt_states, + num_elem_scalar_vars, + num_elem_tensor_vars, + num_node_scalar_vars, + num_node_vector_vars, + num_mat_pt_scalar_vars, + num_mat_pt_tensor_vars, + elem_scalar_var_names, + elem_tensor_var_names, + mat_elem_scalar_var_names, + mat_elem_tensor_var_names, + node_scalar_var_names, + node_vector_var_names, + den_id, + pres_id, + sie_id, + sspd_id, + mass_id, + stress_id, + vol_id, + div_id, + level_set_id, + vel_grad_id, + conductivity_id, + specific_heat_id, + node_mass_id, + node_vel_id, + node_accel_id, + node_coord_id, + node_grad_level_set_id, + node_temp_id, + mat_den_id, + mat_pres_id, + mat_sie_id, + mat_sspd_id, + mat_mass_id, + mat_volfrac_id, + mat_geo_volfrac_id, + mat_eroded_id, + mat_stress_id, + mat_conductivity_id, + mat_specific_heat_id); + } // end if viz paraview output is to be written + + + // STATE + if (SimulationParameters.output_options.format == output_options::state || + SimulationParameters.output_options.format == output_options::viz_and_state) { + + write_material_point_state(mesh, + State, + SimulationParameters, + time_value, + graphics_times, + node_states, + gauss_pt_states, + material_pt_states); + + } // end if state is to be written + + + // will drop ensight outputs in the near future + if (SimulationParameters.output_options.format == output_options::ensight){ + write_ensight(mesh, + State, + SimulationParameters, + dt, + time_value, + graphics_times, + node_states, + gauss_pt_states, + material_pt_states); + } + + return; + + } // end write_mesh + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn write_collective vtm + /// + /// \brief Writes an ensight output file + /// + /// \param Simulation mesh + /// \param State data + /// \param Simulation parameters + /// \param current time value + /// \param Vector of all graphics output times + /// + ///////////////////////////////////////////////////////////////////////////// + void write_parallel_viz(Mesh_t& mesh, + State_t& State, + SimulationParameters_t& SimulationParameters, + double dt, + double time_value, + CArray graphics_times, + const size_t solver_id, + std::vector node_states, + std::vector gauss_pt_states, + std::vector material_pt_states, + size_t num_elem_scalar_vars, + size_t num_elem_tensor_vars, + size_t num_node_scalar_vars, + size_t num_node_vector_vars, + size_t num_mat_pt_scalar_vars, + size_t num_mat_pt_tensor_vars, + std::vector elem_scalar_var_names, + std::vector elem_tensor_var_names, + std::vector mat_elem_scalar_var_names, + std::vector mat_elem_tensor_var_names, + std::vector node_scalar_var_names, + std::vector node_vector_var_names, + const int den_id, + const int pres_id, + const int sie_id, + const int sspd_id, + const int mass_id, + const int stress_id, + const int vol_id, + const int div_id, + const int level_set_id, + const int vel_grad_id, + const int conductivity_id, + const int specific_heat_id, + const int node_mass_id, + const int node_vel_id, + const int node_accel_id, + const int node_coord_id, + const int node_grad_level_set_id, + const int node_temp_id, + const int mat_den_id, + const int mat_pres_id, + const int mat_sie_id, + const int mat_sspd_id, + const int mat_mass_id, + const int mat_volfrac_id, + const int mat_geo_volfrac_id, + const int mat_eroded_id, + const int mat_stress_id, + const int mat_conductivity_id, + const int mat_specific_heat_id) + { // ************************************** // build and save element average fields // ************************************** @@ -2546,15 +4112,29 @@ class MeshWriter const size_t num_elems = mesh.num_elems; const size_t num_dims = mesh.num_dims; const size_t num_nodes_in_elem = mesh.num_nodes_in_elem; + const size_t num_local_elems = mesh.num_local_elems; + const size_t num_local_nodes = mesh.num_local_nodes; + DistributedMap local_element_map = mesh.local_element_map; + DistributedMap node_map = mesh.node_map; + DistributedMap all_node_map = mesh.all_node_map; + DistributedMap nonoverlap_elem_node_map = mesh.nonoverlap_element_node_map; const int Pn_order = mesh.Pn; - // save the elem state to an array for exporting to graphics files - DCArrayKokkos elem_scalar_fields(num_elem_scalar_vars, num_elems, "elem_scalars"); - DCArrayKokkos elem_tensor_fields(num_elem_tensor_vars, num_elems, 3, 3, "elem_tensors"); + const size_t num_mats = State.MaterialPoints.num_material_points.size(); + + int myrank, nranks; + MPI_Comm_rank(MPI_COMM_WORLD,&myrank); + MPI_Comm_size(MPI_COMM_WORLD,&nranks); + + /* save the elem state to an array for exporting to graphics files*/ + + //host version of local element map for argument compatibility + DistributedDFArray elem_scalar_fields(local_element_map, num_elem_scalar_vars, "elem_scalars"); + DistributedDFArray elem_tensor_fields(local_element_map, num_elem_tensor_vars, 3, 3, "elem_tensors"); + DistributedDCArray nonoverlap_nodes_in_elem(local_element_map, num_nodes_in_elem, "nonoverlap_nodes_in_elem"); elem_scalar_fields.set_values(0.0); elem_tensor_fields.set_values(0.0); - // ----------------------------------------------------------------------- // save the output fields to a single element average array for all state // ----------------------------------------------------------------------- @@ -2566,11 +4146,11 @@ class MeshWriter elem_scalar_fields, elem_tensor_fields, State.MaterialToMeshMaps.elem_in_mat_elem, - SimulationParamaters.output_options.output_elem_state, - SimulationParamaters.output_options.output_gauss_pt_state, - State.MaterialToMeshMaps.num_mat_elems.host(mat_id), + SimulationParameters.output_options.output_elem_state, + SimulationParameters.output_options.output_gauss_pt_state, + State.MaterialToMeshMaps.num_mat_local_elems.host(mat_id), mat_id, - num_elems, + num_local_elems, den_id, pres_id, sie_id, @@ -2587,9 +4167,9 @@ class MeshWriter // make specific fields for the element average if (sie_id>=0){ - FOR_ALL(elem_gid, 0, num_elems, { + FOR_ALL(elem_gid, 0, num_local_elems, { // get sie by dividing by the mass - elem_scalar_fields(sie_id, elem_gid) /= (elem_scalar_fields(mass_id, elem_gid)+1.e-20); + elem_scalar_fields(elem_gid, sie_id) /= (elem_scalar_fields(elem_gid, mass_id)+1.e-20); }); } // end if @@ -2603,36 +4183,56 @@ class MeshWriter // ************************ // save the nodal fields to an array for exporting to graphics files - DCArrayKokkos node_scalar_fields(num_node_scalar_vars, num_nodes, "node_scalars"); - DCArrayKokkos node_vector_fields(num_node_vector_vars, num_nodes, 3, "node_tenors"); - + DistributedDFArray node_scalar_fields(node_map, num_node_scalar_vars, "node_scalars"); + DistributedDFArray node_vector_fields(node_map, num_node_vector_vars, 3, "node_tenors"); + DistributedDFArray nonoverlap_node_scalar_fields(nonoverlap_elem_node_map, num_node_scalar_vars, "node_scalars"); + DistributedDFArray nonoverlap_node_vector_fields(nonoverlap_elem_node_map, num_node_vector_vars, 3, "node_tenors"); + concatenate_nodal_fields(State.node, - node_scalar_fields, - node_vector_fields, - SimulationParamaters.output_options.output_node_state, - dt, - num_nodes, - num_dims, - node_mass_id, - node_vel_id, - node_accel_id, - node_coord_id, - node_grad_level_set_id, - node_temp_id); - + node_scalar_fields, + node_vector_fields, + SimulationParameters.output_options.output_node_state, + dt, + num_local_nodes, + num_dims, + node_mass_id, + node_vel_id, + node_accel_id, + node_coord_id, + node_grad_level_set_id, + node_temp_id); + Kokkos::fence(); node_scalar_fields.update_host(); node_vector_fields.update_host(); + // *************************************************************************** + // Communications for node data from node map to nodes on unique element map + // *************************************************************************** - // ******************************** - // Write the nodal and elem fields - // ******************************** + //node data comms + OutputCommPlan node_scalars_comms(nonoverlap_node_scalar_fields, node_scalar_fields); + OutputCommPlan node_vectors_comms(nonoverlap_node_vector_fields, node_vector_fields, node_scalars_comms); + node_scalars_comms.execute_comms(); + node_vectors_comms.execute_comms(); + + //nodal coordinates comms + //convert nodes in elem back to global (convert back to local after we've collected global ids in collective vector) + DistributedDCArray nonoverlap_node_coords(nonoverlap_elem_node_map, mesh.num_dims); + CommPlan node_coords_comms(nonoverlap_node_coords, State.node.coords); + node_coords_comms.execute_comms(); + + //convert local node ids in nodes in elem so they correspond to nonoverlap map + for (size_t elem_id = 0; elem_id < num_local_elems; elem_id++) { + for (int node_lid = 0; node_lid < mesh.num_nodes_in_elem; node_lid++) { + nonoverlap_nodes_in_elem.host(elem_id, node_lid) = nonoverlap_elem_node_map.getLocalIndex(all_node_map.getGlobalIndex(mesh.nodes_in_elem.host(elem_id, node_lid))); + } + } // end for elem_gid - if (SimulationParamaters.output_options.format == output_options::viz || - SimulationParamaters.output_options.format == output_options::viz_and_state) { + //nonoverlap_nodes_in_elem.print(); + if(myrank==0){ // create the folder structure if it does not exist struct stat st; @@ -2668,222 +4268,814 @@ class MeshWriter } } } - - // call the .vtu writer for element fields - std::string elem_fields_name = "fields"; - - // make a view of node coords for passing into functions - ViewCArray node_coords_host(&State.node.coords.host(0,0), num_nodes, num_dims); - ViewCArray nodes_in_elem_host(&mesh.nodes_in_elem.host(0,0), num_elems, num_nodes_in_elem); - - - write_vtu(node_coords_host, - nodes_in_elem_host, - elem_scalar_fields, - elem_tensor_fields, - node_scalar_fields, - node_vector_fields, - elem_scalar_var_names, - elem_tensor_var_names, - node_scalar_var_names, - node_vector_var_names, - elem_fields_name, - graphics_id, - num_nodes, - num_elems, - num_nodes_in_elem, - Pn_order, - num_dims, - solver_id); - - - // ******************************** - // Build and write the mat fields - // ******************************** - - - // note: the file path and folder was created in the elem and node outputs - size_t num_mat_files_written = 0; - if(num_mat_pt_scalar_vars > 0 || num_mat_pt_tensor_vars >0){ - - for (int mat_id = 0; mat_id < num_mats; mat_id++) { - - const size_t num_mat_elems = State.MaterialToMeshMaps.num_mat_elems.host(mat_id); - - // only save material data if the mat lives on the mesh, ie. has state allocated - if (num_mat_elems>0){ - - // set the nodal vars to zero size, we don't write these fields again - node_scalar_var_names.clear(); - node_vector_var_names.clear(); - - // the arrays storing all the material field data - DCArrayKokkos mat_elem_scalar_fields(num_mat_pt_scalar_vars, num_mat_elems, "mat_pt_scalars"); - DCArrayKokkos mat_elem_tensor_fields(num_mat_pt_tensor_vars, num_mat_elems, 3, 3, "mat_pt_tensors"); - - - // concatenate material fields into a single array - concatenate_mat_fields(State.MaterialPoints, - mat_elem_scalar_fields, - mat_elem_tensor_fields, - State.MaterialToMeshMaps.elem_in_mat_elem, - SimulationParamaters.output_options.output_mat_pt_state, - num_mat_elems, - mat_id, - mat_den_id, - mat_pres_id, - mat_sie_id, - mat_sspd_id, - mat_mass_id, - mat_volfrac_id, - mat_geo_volfrac_id, - mat_eroded_id, - mat_stress_id, - mat_conductivity_id, - mat_specific_heat_id); - Kokkos::fence(); - mat_elem_scalar_fields.update_host(); - mat_elem_tensor_fields.update_host(); - - - std::string str_mat_val = std::to_string(mat_id); - std::string mat_fields_name = "mat"; - mat_fields_name += str_mat_val; // add the mat number - - // save the nodes belonging to this part (i.e., the material) - DCArrayKokkos mat_node_coords(num_nodes,num_dims, "mat_node_coords"); - DCArrayKokkos mat_nodes_in_mat_elem(num_mat_elems, num_nodes_in_elem, "mat_nodes_in_mat_elem"); - - // the number of actual nodes belonging to the part (i.e., the material) - size_t num_mat_nodes = 0; - - // build a unique mesh (element and nodes) for the material (i.e., the part) - build_material_elem_node_lists(mesh, - State.node.coords, - mat_node_coords, - mat_nodes_in_mat_elem, - State.MaterialToMeshMaps.elem_in_mat_elem, - mat_id, - num_mat_nodes, - num_mat_elems, - num_nodes_in_elem, - num_dims); - - ViewCArray mat_node_coords_host(&mat_node_coords.host(0,0), num_mat_nodes, num_dims); - ViewCArray mat_nodes_in_elem_host(&mat_nodes_in_mat_elem.host(0,0), num_mat_elems, num_nodes_in_elem); - - // write out a vtu file this - write_vtu(mat_node_coords_host, - mat_nodes_in_elem_host, - mat_elem_scalar_fields, - mat_elem_tensor_fields, - node_scalar_fields, - node_vector_fields, - mat_elem_scalar_var_names, - mat_elem_tensor_var_names, - node_scalar_var_names, - node_vector_var_names, - mat_fields_name, - graphics_id, - num_mat_nodes, - num_mat_elems, - num_nodes_in_elem, - Pn_order, - num_dims, - solver_id); + } + MPI_Barrier(MPI_COMM_WORLD); + // call the .vtu writer for element fields + std::string elem_fields_name = "fields"; + + write_vtu(nonoverlap_node_coords, + nonoverlap_nodes_in_elem, + elem_scalar_fields, + elem_tensor_fields, + nonoverlap_node_scalar_fields, + nonoverlap_node_vector_fields, + elem_scalar_var_names, + elem_tensor_var_names, + node_scalar_var_names, + node_vector_var_names, + elem_fields_name, + graphics_id, + nonoverlap_elem_node_map.size(), + mesh.num_local_elems, + num_nodes_in_elem, + Pn_order, + num_dims, + solver_id); - num_mat_files_written++; + // ******************************** + // Build and write the mat fields + // ******************************** - } // end for mat_id - } // end if material is on the mesh + // note: the file path and folder was created in the elem and node outputs + size_t num_mat_files_written = 0; + CArray local_mats_in_rank(num_mats); + int local_num_mats = 0; + if(num_mat_pt_scalar_vars > 0 || num_mat_pt_tensor_vars >0){ + for (int mat_id = 0; mat_id < num_mats; mat_id++) { - } // end if mat variables are to be written + const size_t num_mat_local_elems = State.MaterialToMeshMaps.num_mat_local_elems.host(mat_id); + if(num_mat_local_elems){ + local_mats_in_rank(local_num_mats) = mat_id; + local_num_mats++; + } + //set global element indices on this rank for this mat + DistributedMap element_map = mesh.element_map; + DCArrayKokkos global_indices_of_local_mat_elems(num_mat_local_elems, " global_indices_of_local_mat_elems"); + // FOR_ALL(ielem, 0, num_mat_local_elems,{ + // global_indices_of_local_mat_elems(ielem) = mesh.element_map(State.MaterialToMeshMaps.elem_in_mat_elem(mat_id, ielem)); + // }); + for(int ielem = 0; ielem < num_mat_local_elems; ielem++){ + global_indices_of_local_mat_elems.host(ielem) = mesh.element_map.getGlobalIndex(State.MaterialToMeshMaps.elem_in_mat_elem.host(mat_id, ielem)); + } + global_indices_of_local_mat_elems.update_device(); + DistributedMap mat_elem_map = DistributedMap(global_indices_of_local_mat_elems); + //mat_elem_map.print(); + + //allocate arrays for distributed mat elem data + DistributedDFArray mat_elem_scalar_fields(mat_elem_map, num_mat_pt_scalar_vars, "mat_elem_scalars"); + DistributedDFArray mat_elem_tensor_fields(mat_elem_map, num_mat_pt_tensor_vars, 3, 3, "mat_elem_tensors"); + + // set the nodal vars to zero size, we don't write these fields again + node_scalar_var_names.clear(); + node_vector_var_names.clear(); + + // concatenate material fields into a single array + concatenate_mat_fields(State.MaterialPoints, + mat_elem_scalar_fields, + mat_elem_tensor_fields, + State.MaterialToMeshMaps.elem_in_mat_elem, + SimulationParameters.output_options.output_mat_pt_state, + num_mat_local_elems, + mat_id, + mat_den_id, + mat_pres_id, + mat_sie_id, + mat_sspd_id, + mat_mass_id, + mat_volfrac_id, + mat_geo_volfrac_id, + mat_eroded_id, + mat_stress_id, + mat_conductivity_id, + mat_specific_heat_id); + + + std::string str_mat_val = std::to_string(mat_id); + std::string mat_fields_name = "mat"; + mat_fields_name += str_mat_val; // add the mat number + + // the number of actual nodes belonging to the part (i.e., the material) + size_t num_mat_nodes = 0; + + //communicate scalars, tensors, and nodes in elem to collective mat arrays on rank 0 + + //collect nodes in elem for this material on rank 0 + DistributedDCArray mat_nodes_in_mat_elem(mat_elem_map, num_nodes_in_elem, "mat_nodes_in_mat_elem"); + CommPlan mat_nodes_in_elem_comms(mat_nodes_in_mat_elem,mesh.nodes_in_elem); //shouldnt do comms since subview of map on this rank + mat_nodes_in_elem_comms.execute_comms(); + + //convert mesh.nodes_in_elem stores local indices and we communicated these in, convert to global + for (size_t elem_id = 0; elem_id < num_mat_local_elems; elem_id++) { + for (int node_lid = 0; node_lid < mesh.num_nodes_in_elem; node_lid++) { + mat_nodes_in_mat_elem.host(elem_id, node_lid) = mesh.all_node_map.getGlobalIndex(mat_nodes_in_mat_elem.host(elem_id, node_lid)); + } + } // end for elem_gid - // ************************************************* - // write Paraview files to open the graphics files - // ************************************************* + //define set of nodes for this mat, collect on rank 0, comms on coords, scalars, and vectors for nodes for this mat + + // build a unique mesh (element and nodes) for the material (i.e., the part) + DCArrayKokkos mat_node_indices; + build_material_node_list(mesh, + mat_node_indices, + mat_nodes_in_mat_elem, + State.MaterialToMeshMaps.elem_in_mat_elem, + mat_id, + num_mat_nodes, + num_mat_local_elems, + num_nodes_in_elem, + num_dims); + + //map object for mat node indices + DistributedMap mat_node_map = DistributedMap(mat_node_indices); - // save the graphics time - graphics_times(graphics_id) = time_value; + DistributedDCArray mat_node_coords(mat_node_map, num_dims, "mat_node_coords"); + CommPlan mat_node_coords_comms(mat_node_coords,State.node.coords); //shouldnt do comms since subview of map on this rank + mat_node_coords_comms.execute_comms(); - // check to see if an mesh state was written - bool write_mesh_state = false; - if( num_elem_scalar_vars > 0 || - num_elem_tensor_vars > 0 || - num_node_scalar_vars > 0 || - num_node_vector_vars > 0) - { - write_mesh_state = true; - } + DistributedDFArray mat_node_scalar_fields(mat_node_map, num_node_scalar_vars, "mat_node_scalars"); + OutputCommPlan mat_node_scalars_comms(mat_node_scalar_fields,node_scalar_fields); //shouldnt do comms since subview of map on this rank + mat_node_scalars_comms.execute_comms(); - // check to see if a mat state was written - bool write_mat_pt_state = false; - if( num_mat_pt_scalar_vars > 0 || - num_mat_pt_tensor_vars > 0) - { - write_mat_pt_state = true; - } + DistributedDFArray mat_node_vector_fields(mat_node_map, num_node_vector_vars, 3, "mat_node_vectors"); + OutputCommPlan mat_node_vectors_comms(mat_node_vector_fields,node_vector_fields); //shouldnt do comms since subview of map on this rank + mat_node_vectors_comms.execute_comms(); + + //convert mat_nodes_in_mat_elem so it uses contiguous local node ids for this mat portion of the mesh + for (size_t elem_id = 0; elem_id < num_mat_local_elems; elem_id++) { + for (int node_lid = 0; node_lid < mesh.num_nodes_in_elem; node_lid++) { + mat_nodes_in_mat_elem.host(elem_id, node_lid) = mat_node_map.getLocalIndex(mat_nodes_in_mat_elem.host(elem_id, node_lid)); + } + } // end for elem_gid + + // only write material data if the mat lives on the mesh, ie. has state allocated + if (num_mat_local_elems>0){ + // write out a vtu file this + write_vtu(mat_node_coords, + mat_nodes_in_mat_elem, + mat_elem_scalar_fields, + mat_elem_tensor_fields, + mat_node_scalar_fields, + mat_node_vector_fields, + mat_elem_scalar_var_names, + mat_elem_tensor_var_names, + node_scalar_var_names, + node_vector_var_names, + mat_fields_name, + graphics_id, + num_mat_nodes, + num_mat_local_elems, + num_nodes_in_elem, + Pn_order, + num_dims, + solver_id); + + + num_mat_files_written++; + + } // end for mat_id + + } // end if material is on the mesh + + } // end if mat variables are to be written + + + // ************************************************* + // write Paraview files to open the graphics files + // ************************************************* + + // save the graphics time + graphics_times(graphics_id) = time_value; - // call the vtm file writer - std::string mat_fields_name = "mat"; - write_vtm(graphics_times, - elem_fields_name, - mat_fields_name, - time_value, - graphics_id, - num_mat_files_written, - write_mesh_state, - write_mat_pt_state, - solver_id); + // check to see if an mesh state was written + bool write_mesh_state = false; + if( num_elem_scalar_vars > 0 || + num_elem_tensor_vars > 0 || + num_node_scalar_vars > 0 || + num_node_vector_vars > 0) + { + write_mesh_state = true; + } + // check to see if a mat state was written + bool write_mat_pt_state = false; + if( num_mat_pt_scalar_vars > 0 || + num_mat_pt_tensor_vars > 0) + { + write_mat_pt_state = true; + } + // call the vtm file writer + std::string mat_fields_name = "mat"; + //gather MPI ranks that are writing mat blocks + write_vtm(graphics_times, + elem_fields_name, + mat_fields_name, + time_value, + graphics_id, + num_mats, + local_num_mats, + local_mats_in_rank, + nranks, + myrank, + write_mesh_state, + write_mat_pt_state, + solver_id); + + if(myrank==0){ // call the pvd file writer write_pvd(graphics_times, - time_value, - graphics_id, - solver_id); - - - // increment graphics id counter - graphics_id++; // this is private variable in the class - - } // end if viz paraview output is to be written - + time_value, + graphics_id, + solver_id); + } - // STATE - if (SimulationParamaters.output_options.format == output_options::state || - SimulationParamaters.output_options.format == output_options::viz_and_state) { + // increment graphics id counter + graphics_id++; // this is private variable in the class + } - write_material_point_state(mesh, - State, - SimulationParamaters, - time_value, - graphics_times, - node_states, - gauss_pt_states, - material_pt_states); + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn write_collective vtm + /// + /// \brief Writes an ensight output file + /// + /// \param Simulation mesh + /// \param State data + /// \param Simulation parameters + /// \param current time value + /// \param Vector of all graphics output times + /// + ///////////////////////////////////////////////////////////////////////////// + void write_collective_viz(Mesh_t& mesh, + State_t& State, + SimulationParameters_t& SimulationParameters, + double dt, + double time_value, + CArray graphics_times, + const size_t solver_id, + std::vector node_states, + std::vector gauss_pt_states, + std::vector material_pt_states, + size_t num_elem_scalar_vars, + size_t num_elem_tensor_vars, + size_t num_node_scalar_vars, + size_t num_node_vector_vars, + size_t num_mat_pt_scalar_vars, + size_t num_mat_pt_tensor_vars, + std::vector elem_scalar_var_names, + std::vector elem_tensor_var_names, + std::vector mat_elem_scalar_var_names, + std::vector mat_elem_tensor_var_names, + std::vector node_scalar_var_names, + std::vector node_vector_var_names, + const int den_id, + const int pres_id, + const int sie_id, + const int sspd_id, + const int mass_id, + const int stress_id, + const int vol_id, + const int div_id, + const int level_set_id, + const int vel_grad_id, + const int conductivity_id, + const int specific_heat_id, + const int node_mass_id, + const int node_vel_id, + const int node_accel_id, + const int node_coord_id, + const int node_grad_level_set_id, + const int node_temp_id, + const int mat_den_id, + const int mat_pres_id, + const int mat_sie_id, + const int mat_sspd_id, + const int mat_mass_id, + const int mat_volfrac_id, + const int mat_geo_volfrac_id, + const int mat_eroded_id, + const int mat_stress_id, + const int mat_conductivity_id, + const int mat_specific_heat_id) + { + // // ************************************** + // // build and save element average fields + // // ************************************** + + // // short hand + // const size_t num_nodes = mesh.num_nodes; + // const size_t num_elems = mesh.num_elems; + // const size_t num_dims = mesh.num_dims; + // const size_t num_nodes_in_elem = mesh.num_nodes_in_elem; + // const size_t num_local_elems = mesh.num_local_elems; + // const size_t num_local_nodes = mesh.num_local_nodes; + // DistributedMap local_element_map = mesh.local_element_map; + // DistributedMap node_map = mesh.node_map; + // const int Pn_order = mesh.Pn; + + // const size_t num_mats = State.MaterialPoints.num_material_points.size(); + + // int myrank, nranks; + // MPI_Comm_rank(MPI_COMM_WORLD,&myrank); + // MPI_Comm_size(MPI_COMM_WORLD,&nranks); + + // /* save the elem state to an array for exporting to graphics files*/ + + // //host version of local element map for argument compatibility + // HostDistributedMap host_local_element_map; + // DCArrayKokkos global_indices_of_local_elements(num_local_elems); + // for(int ielem = 0; ielem < num_local_elems; ielem++){ + // global_indices_of_local_elements(ielem) = local_element_map.getGlobalIndex(ielem); + // } + // host_local_element_map = HostDistributedMap(global_indices_of_local_elements); + // DistributedDFArray elem_scalar_fields(local_element_map, num_elem_scalar_vars, "elem_scalars"); + // DistributedDFArray elem_tensor_fields(local_element_map, num_elem_tensor_vars, 3, 3, "elem_tensors"); + // elem_scalar_fields.set_values(0.0); + // elem_tensor_fields.set_values(0.0); + // //duplicate for now to allow compatibility with comm plan object when using Tpetra (src and dst device type must be equal) + // //We dont want to make a dual view of the rank 0 collector since that will blow device memory constraints sooner than this duplicate + // //one other option is to just do the concatenation ops on the host + // DistributedFArray host_elem_scalar_fields(host_local_element_map, num_elem_scalar_vars, "elem_scalars"); + // DistributedFArray host_elem_tensor_fields(host_local_element_map, num_elem_tensor_vars, 3, 3, "elem_tensors"); + + + // // ----------------------------------------------------------------------- + // // save the output fields to a single element average array for all state + // // ----------------------------------------------------------------------- + // for (int mat_id = 0; mat_id < num_mats; mat_id++) { + + // // material point and guass point state are concatenated together + // concatenate_elem_fields(State.MaterialPoints, + // State.GaussPoints, + // elem_scalar_fields, + // elem_tensor_fields, + // State.MaterialToMeshMaps.elem_in_mat_elem, + // SimulationParameters.output_options.output_elem_state, + // SimulationParameters.output_options.output_gauss_pt_state, + // State.MaterialToMeshMaps.num_mat_local_elems.host(mat_id), + // mat_id, + // num_local_elems, + // den_id, + // pres_id, + // sie_id, + // sspd_id, + // mass_id, + // stress_id, + // vol_id, + // div_id, + // level_set_id, + // vel_grad_id, + // conductivity_id, + // specific_heat_id); + // } // end for mats + + // // make specific fields for the element average + // if (sie_id>=0){ + // FOR_ALL(elem_gid, 0, num_local_elems, { + // // get sie by dividing by the mass + // elem_scalar_fields(elem_gid, sie_id) /= (elem_scalar_fields(elem_gid, mass_id)+1.e-20); + // }); + // } // end if + + // Kokkos::fence(); + // elem_scalar_fields.update_host(); + // elem_tensor_fields.update_host(); + + // // ----------------------------------------------------------------------- + // // copy the output fields to host side array compatible with Tpetra Comms + // // ----------------------------------------------------------------------- + + // for (int mat_id = 0; mat_id < num_mats; mat_id++) { + + // // material point and guass point state are concatenated together + // copy_elem_fields(elem_scalar_fields, + // elem_tensor_fields, + // host_elem_scalar_fields, + // host_elem_tensor_fields, + // State.MaterialToMeshMaps.elem_in_mat_elem, + // SimulationParameters.output_options.output_elem_state, + // SimulationParameters.output_options.output_gauss_pt_state, + // State.MaterialToMeshMaps.num_mat_local_elems.host(mat_id), + // mat_id, + // num_local_elems, + // den_id, + // pres_id, + // sie_id, + // sspd_id, + // mass_id, + // stress_id, + // vol_id, + // div_id, + // level_set_id, + // vel_grad_id, + // conductivity_id, + // specific_heat_id); + // } // end for mats + - } // end if state is to be written + // // ************************ + // // Build the nodal fields + // // ************************ + + // //host version of local node map for argument compatibility + // HostDistributedMap host_node_map; + // DCArrayKokkos global_indices_of_local_nodes(mesh.num_local_nodes); + // for(int inode = 0; inode < mesh.num_local_nodes; inode++){ + // global_indices_of_local_nodes(inode) = mesh.node_map.getGlobalIndex(inode); + // } + // host_node_map = HostDistributedMap(global_indices_of_local_nodes); + + // // save the nodal fields to an array for exporting to graphics files + // DistributedDFArray node_scalar_fields(node_map, num_node_scalar_vars, "node_scalars"); + // DistributedDFArray node_vector_fields(node_map, num_node_vector_vars, 3, "node_tenors"); + // DistributedFArray host_node_scalar_fields(host_node_map, num_node_scalar_vars, "node_scalars"); + // DistributedFArray host_node_vector_fields(host_node_map, num_node_vector_vars, 3, "node_tenors"); + + // concatenate_nodal_fields(State.node, + // node_scalar_fields, + // node_vector_fields, + // SimulationParameters.output_options.output_node_state, + // dt, + // num_local_nodes, + // num_dims, + // node_mass_id, + // node_vel_id, + // node_accel_id, + // node_coord_id, + // node_grad_level_set_id, + // node_temp_id); + + // Kokkos::fence(); + // node_scalar_fields.update_host(); + // node_vector_fields.update_host(); + + // copy_nodal_fields(host_node_scalar_fields, + // host_node_vector_fields, + // node_scalar_fields, + // node_vector_fields, + // SimulationParameters.output_options.output_node_state, + // dt, + // num_local_nodes, + // num_dims, + // node_mass_id, + // node_vel_id, + // node_accel_id, + // node_coord_id, + // node_grad_level_set_id, + // node_temp_id); + + // // ************************************************** + // // Collective communications for node and elem data + // // ************************************************** + + // //elem data collective comms + // //collective map has all indices on rank 0 and non on other ranks + // HostDistributedMap collective_elem_map; + // long long int num_collective_elem_indices = 0; + // if(myrank==0) num_collective_elem_indices = mesh.global_num_elems; + // collective_elem_map = HostDistributedMap(mesh.global_num_elems, num_collective_elem_indices); + + // //collective vector and comms to the collective vector for elem fields + // DistributedFArray collective_elem_scalar_fields(collective_elem_map, num_elem_scalar_vars, "elem_scalars_collective"); + // DistributedFArray collective_elem_tensor_fields(collective_elem_map, num_elem_tensor_vars, 3, 3, "elem_tensors_collective"); + // HostCommPlan collective_elem_scalars_comms(collective_elem_scalar_fields, host_elem_scalar_fields); + // HostCommPlan collective_elem_tensors_comms(collective_elem_tensor_fields, host_elem_tensor_fields, collective_elem_scalars_comms); + // collective_elem_scalars_comms.execute_comms(); + // collective_elem_tensors_comms.execute_comms(); + + // //host of node in elem for Trilinos template argument compatibility + // DistributedFArray host_local_nodes_in_elem(host_local_element_map, mesh.num_nodes_in_elem); + + // //convert nodes in elem back to global (convert back to local after we've collected global ids in collective vector) + // for (size_t elem_id = 0; elem_id < num_local_elems; elem_id++) { + // for (int node_lid = 0; node_lid < mesh.num_nodes_in_elem; node_lid++) { + // host_local_nodes_in_elem(elem_id, node_lid) = mesh.all_node_map.getGlobalIndex(mesh.local_nodes_in_elem(elem_id, node_lid)); + // } + // } // end for elem_gid + + // //collect nodes in elem with a conversion back to global node ids + // DistributedFArray collective_nodes_in_elem(collective_elem_map, mesh.num_nodes_in_elem); + // HostCommPlan nodes_in_elem_comms(collective_nodes_in_elem, host_local_nodes_in_elem); + + // nodes_in_elem_comms.execute_comms(); + + // //node data collective comms + // //collective map has all indices on rank 0 and non on other ranks + // HostDistributedMap collective_node_map; + // long long int num_collective_node_indices = 0; + // if(myrank==0) num_collective_node_indices = mesh.global_num_nodes; + // collective_node_map = HostDistributedMap(mesh.global_num_nodes, num_collective_node_indices); + + // //collective vector and comms to the collective vector for node fields + // DistributedFArray collective_node_scalar_fields(collective_node_map, num_node_scalar_vars); + // DistributedFArray collective_node_vector_fields(collective_node_map, num_node_vector_vars, 3); + // HostCommPlan collective_node_scalars_comms(collective_node_scalar_fields, host_node_scalar_fields); + // HostCommPlan collective_node_vectors_comms(collective_node_vector_fields, host_node_vector_fields, collective_node_scalars_comms); + // collective_node_scalars_comms.execute_comms(); + // collective_node_vectors_comms.execute_comms(); + + // //collect nodal coordinates + // //convert nodes in elem back to global (convert back to local after we've collected global ids in collective vector) + // DistributedFArray host_node_coords(host_node_map, mesh.num_dims); + // for (size_t node_id = 0; node_id < num_local_nodes; node_id++) { + // for (int idim = 0; idim < mesh.num_dims; idim++) { + // host_node_coords(node_id, idim) = State.node.coords.host(node_id, idim); + // } + // } // end for elem_gid + // DistributedFArray collective_node_coords(collective_node_map, mesh.num_dims); + // HostCommPlan collective_node_coords_comms(collective_node_coords, host_node_coords, collective_node_scalars_comms); + // collective_node_coords_comms.execute_comms(); + + // if(myrank==0){ + // // create the folder structure if it does not exist + // struct stat st; + + // if (stat("vtk", &st) != 0) { + // int returnCode = system("mkdir vtk"); + + // if (returnCode == 1) { + // std::cout << "Unable to make vtk directory" << std::endl; + // } + // } + // else{ + // if(solver_id==0 && graphics_id==0){ + // // delete the existing files inside + // int returnCode = system("rm vtk/Fierro*"); + // if (returnCode == 1) { + // std::cout << "Unable to clear vtk/Fierro directory" << std::endl; + // } + // } + // } + + // if (stat("vtk/data", &st) != 0) { + // int returnCode = system("mkdir vtk/data"); + // if (returnCode == 1) { + // std::cout << "Unable to make vtk/data directory" << std::endl; + // } + // } + // else{ + // if(solver_id==0 && graphics_id==0){ + // // delete the existing files inside the folder + // int returnCode = system("rm vtk/data/Fierro*"); + // if (returnCode == 1) { + // std::cout << "Unable to clear vtk/data directory" << std::endl; + // } + // } + // } + // } + // // call the .vtu writer for element fields + // std::string elem_fields_name = "fields"; + + // if(myrank==0){ + // write_vtu(collective_node_coords, + // collective_nodes_in_elem, + // collective_elem_scalar_fields, + // collective_elem_tensor_fields, + // collective_node_scalar_fields, + // collective_node_vector_fields, + // elem_scalar_var_names, + // elem_tensor_var_names, + // node_scalar_var_names, + // node_vector_var_names, + // elem_fields_name, + // graphics_id, + // mesh.global_num_nodes, + // mesh.global_num_elems, + // num_nodes_in_elem, + // Pn_order, + // num_dims, + // solver_id); + // } + + // // ******************************** + // // Build and write the mat fields + // // ******************************** + + + // // note: the file path and folder was created in the elem and node outputs + // size_t num_mat_files_written = 0; + // if(num_mat_pt_scalar_vars > 0 || num_mat_pt_tensor_vars >0){ + + // for (int mat_id = 0; mat_id < num_mats; mat_id++) { + + // const size_t num_mat_local_elems = State.MaterialToMeshMaps.num_mat_local_elems.host(mat_id); + // //array storing number of local elems for this material on each process + // CArray processes_num_local_mat_elems, gatherv_displacements; + // if(myrank==0){ + // processes_num_local_mat_elems = CArray(nranks); + // gatherv_displacements = CArray(nranks); + // } + // MPI_Gather(&num_mat_local_elems,1,MPI_INT,processes_num_local_mat_elems.pointer(),1, + // MPI_INT, 0, MPI_COMM_WORLD); + + // //set global element indices on this rank + // HostDistributedMap host_mat_elem_map; + // DistributedMap element_map = mesh.element_map; + // DCArrayKokkos global_indices_of_local_mat_elems(num_mat_local_elems); + // for(int ielem = 0; ielem < num_mat_local_elems; ielem++){ + // global_indices_of_local_mat_elems(ielem) = element_map.getGlobalIndex(State.MaterialToMeshMaps.elem_in_mat_elem.host(mat_id, ielem)); + // } + // host_mat_elem_map = HostDistributedMap(global_indices_of_local_mat_elems); + + // //allocate arrays for distributed mat elem data + // DistributedFArray host_mat_elem_scalar_fields(host_mat_elem_map, num_mat_pt_scalar_vars, "mat_elem_scalars"); + // DistributedFArray host_mat_elem_tensor_fields(host_mat_elem_map, num_mat_pt_tensor_vars, 3, 3, "mat_elem_tensors"); + + // //collect global element indices on rank 0 for this mat + // //tally total number of mat elems for rank 0 + // DCArrayKokkos global_indices_of_collective_mat_elems; + // long long int num_mat_collective_elems = 0; + // if(myrank==0){ + // for(int irank=0; irank < nranks; irank++){ + // gatherv_displacements(irank) = num_mat_collective_elems; + // num_mat_collective_elems += processes_num_local_mat_elems(irank); + // } + // global_indices_of_collective_mat_elems = DCArrayKokkos(num_mat_collective_elems); + // } + // // if(myrank==0){ + // // for(int irank=0; irank < nranks; irank++){ + // // std::cout << "NUM local mat elem on rank " << irank << " is " << processes_num_local_mat_elems(irank) << std::endl; + // // std::cout << "gatherv displacement on rank " << irank << " is " << gatherv_displacements(irank) << std::endl; + // // } + // // } + // MPI_Gatherv(global_indices_of_local_mat_elems.device_pointer(), num_mat_local_elems, MPI_LONG_LONG_INT, + // global_indices_of_collective_mat_elems.device_pointer(), processes_num_local_mat_elems.pointer(), + // gatherv_displacements.pointer(), MPI_LONG_LONG_INT, 0, MPI_COMM_WORLD); + + // //use indices on rank 0 to construct rank 0 collective map for this mat + // HostDistributedMap collective_mat_elem_map; + // collective_mat_elem_map = HostDistributedMap(global_indices_of_collective_mat_elems); + // //collective_mat_elem_map.print(); + + // //collective storage for scalars and tensors using collective elem mat map + // DistributedFArray collective_mat_elem_scalar_fields(collective_mat_elem_map, num_mat_pt_scalar_vars, "mat_elem_scalars_collective"); + // DistributedFArray collective_mat_elem_tensor_fields(collective_mat_elem_map, num_mat_pt_tensor_vars, 3, 3, "mat_elem_tensors_collective"); + + // // set the nodal vars to zero size, we don't write these fields again + // node_scalar_var_names.clear(); + // node_vector_var_names.clear(); + + // // concatenate material fields into a single array + // concatenate_mat_fields(State.MaterialPoints, + // host_mat_elem_scalar_fields, + // host_mat_elem_tensor_fields, + // State.MaterialToMeshMaps.elem_in_mat_elem, + // SimulationParameters.output_options.output_mat_pt_state, + // num_mat_local_elems, + // mat_id, + // mat_den_id, + // mat_pres_id, + // mat_sie_id, + // mat_sspd_id, + // mat_mass_id, + // mat_volfrac_id, + // mat_geo_volfrac_id, + // mat_eroded_id, + // mat_stress_id, + // mat_conductivity_id, + // mat_specific_heat_id); + + + // std::string str_mat_val = std::to_string(mat_id); + // std::string mat_fields_name = "mat"; + // mat_fields_name += str_mat_val; // add the mat number + + // // the number of actual nodes belonging to the part (i.e., the material) + // size_t num_mat_nodes = 0; + + // //communicate scalars, tensors, and nodes in elem to collective mat arrays on rank 0 + + // //collect nodes in elem for this material on rank 0 + // DistributedFArray collective_mat_nodes_in_mat_elem(collective_mat_elem_map, num_nodes_in_elem, "collective_mat_nodes_in_mat_elem"); + // HostCommPlan mat_nodes_in_elem_comms(collective_mat_nodes_in_mat_elem,collective_nodes_in_elem); //doesnt really do comms since all on rank 0 + // mat_nodes_in_elem_comms.execute_comms(); - // will drop ensight outputs in the near future - if (SimulationParamaters.output_options.format == output_options::ensight){ - write_ensight(mesh, - State, - SimulationParamaters, - dt, - time_value, - graphics_times, - node_states, - gauss_pt_states, - material_pt_states); - } + // HostCommPlan mat_elem_scalars_comms(collective_mat_elem_scalar_fields,host_mat_elem_scalar_fields); + // mat_elem_scalars_comms.execute_comms(); - return; + // HostCommPlan mat_elem_tensors_comms(collective_mat_elem_tensor_fields,host_mat_elem_tensor_fields); + // mat_elem_tensors_comms.execute_comms(); - } // end write_mesh + // //define set of nodes for this mat, collect on rank 0, comms on coords, scalars, and vectors for nodes for this mat + + // // build a unique mesh (element and nodes) for the material (i.e., the part) + // DCArrayKokkos collective_mat_node_indices; + // if(myrank==0){ + // build_material_node_list(mesh, + // collective_mat_node_indices, + // collective_mat_nodes_in_mat_elem, + // State.MaterialToMeshMaps.elem_in_mat_elem, + // mat_id, + // num_mat_nodes, + // num_mat_collective_elems, + // num_nodes_in_elem, + // num_dims); + // } + + // //map object for mat node indices + // HostDistributedMap collective_mat_node_map = HostDistributedMap(collective_mat_node_indices); + + // DistributedFArray collective_mat_node_coords(collective_mat_node_map, num_dims, "collective_mat_node_coords"); + // HostCommPlan mat_node_coords_comms(collective_mat_node_coords,collective_node_coords); //doesnt really do comms since all on rank 0 + // mat_node_coords_comms.execute_comms(); + + // DistributedFArray collective_mat_node_scalar_fields(collective_mat_node_map, num_node_scalar_vars, "collective_mat_node_scalars"); + // HostCommPlan mat_node_scalars_comms(collective_mat_node_scalar_fields,collective_node_scalar_fields); //doesnt really do comms since all on rank 0 + // mat_node_scalars_comms.execute_comms(); + + // DistributedFArray collective_mat_node_vector_fields(collective_mat_node_map, num_node_vector_vars, 3, "collective_mat_node_vectors"); + // HostCommPlan mat_node_vectors_comms(collective_mat_node_vector_fields,collective_node_vector_fields); //doesnt really do comms since all on rank 0 + // mat_node_vectors_comms.execute_comms(); + + // //convert collective mat_nodes_in_mat_elem so it uses contiguous node ids for this mat portion of the mesh + // for (size_t elem_id = 0; elem_id < num_mat_collective_elems; elem_id++) { + // for (int node_lid = 0; node_lid < mesh.num_nodes_in_elem; node_lid++) { + // collective_mat_nodes_in_mat_elem(elem_id, node_lid) = collective_mat_node_map.getLocalIndex(collective_mat_nodes_in_mat_elem(elem_id, node_lid)); + // } + // } // end for elem_gid + + // // only write material data if the mat lives on the mesh, ie. has state allocated + // if (num_mat_collective_elems>0&&myrank==0){ + // // write out a vtu file this + // write_vtu(collective_mat_node_coords, + // collective_mat_nodes_in_mat_elem, + // collective_mat_elem_scalar_fields, + // collective_mat_elem_tensor_fields, + // collective_mat_node_scalar_fields, + // collective_mat_node_vector_fields, + // mat_elem_scalar_var_names, + // mat_elem_tensor_var_names, + // node_scalar_var_names, + // node_vector_var_names, + // mat_fields_name, + // graphics_id, + // num_mat_nodes, + // num_mat_collective_elems, + // num_nodes_in_elem, + // Pn_order, + // num_dims, + // solver_id); + + + // num_mat_files_written++; + + // } // end for mat_id + + // } // end if material is on the mesh + + // } // end if mat variables are to be written + + + // // ************************************************* + // // write Paraview files to open the graphics files + // // ************************************************* + + // // save the graphics time + // graphics_times(graphics_id) = time_value; + + // // check to see if an mesh state was written + // bool write_mesh_state = false; + // if( num_elem_scalar_vars > 0 || + // num_elem_tensor_vars > 0 || + // num_node_scalar_vars > 0 || + // num_node_vector_vars > 0) + // { + // write_mesh_state = true; + // } + // // check to see if a mat state was written + // bool write_mat_pt_state = false; + // if( num_mat_pt_scalar_vars > 0 || + // num_mat_pt_tensor_vars > 0) + // { + // write_mat_pt_state = true; + // } + + // // call the vtm file writer + // std::string mat_fields_name = "mat"; + // if(myrank==0){ + // write_vtm(graphics_times, + // elem_fields_name, + // mat_fields_name, + // time_value, + // graphics_id, + // num_mat_files_written, + // write_mesh_state, + // write_mat_pt_state, + // solver_id); + + // // call the pvd file writer + // write_pvd(graphics_times, + // time_value, + // graphics_id, + // solver_id); + // } + + // // increment graphics id counter + // graphics_id++; // this is private variable in the class + } ///////////////////////////////////////////////////////////////////////////// /// @@ -2900,7 +5092,7 @@ class MeshWriter ///////////////////////////////////////////////////////////////////////////// void write_ensight(Mesh_t& mesh, State_t& State, - SimulationParameters_t& SimulationParamaters, + SimulationParameters_t& SimulationParameters, double dt, double time_value, CArray graphics_times, @@ -2909,7 +5101,10 @@ class MeshWriter std::vector material_pt_states) { size_t num_mats = State.MaterialPoints.num_material_points.size(); - + + int myrank, nranks; + MPI_Comm_rank(MPI_COMM_WORLD,&myrank); + MPI_Comm_size(MPI_COMM_WORLD,&nranks); // ---- Update host data ---- // material point values @@ -2957,12 +5152,19 @@ class MeshWriter const size_t num_dims = mesh.num_dims; // save the cell state to an array for exporting to graphics files - auto elem_fields = CArray(num_elems, num_scalar_vars); + + //host version of local element map for argument compatibility + HostDistributedMap host_local_element_map; + DCArrayKokkos global_indices_of_local_elements(mesh.num_local_elems, "global_indices_of_local_elements"); + for(int ielem = 0; ielem < mesh.num_local_elems; ielem++){ + global_indices_of_local_elements(ielem) = mesh.local_element_map.getGlobalIndex(ielem); + } + host_local_element_map = HostDistributedMap(global_indices_of_local_elements); + DistributedCArray elem_fields = DistributedCArray(host_local_element_map, num_scalar_vars, "elem_fields"); int elem_switch = 1; - DCArrayKokkos speed(num_elems, "speed"); - FOR_ALL(elem_gid, 0, num_elems, { + FOR_ALL(elem_gid, 0, mesh.num_local_elems, { double elem_vel[3]; // note:initialization with a list won't work elem_vel[0] = 0.0; elem_vel[1] = 0.0; @@ -2995,7 +5197,7 @@ class MeshWriter // export material centeric data to the elements for (int mat_id = 0; mat_id < num_mats; mat_id++) { - size_t num_mat_elems = State.MaterialToMeshMaps.num_mat_elems.host(mat_id); + size_t num_mat_elems = State.MaterialToMeshMaps.num_mat_local_elems.host(mat_id); for (size_t mat_elem_sid = 0; mat_elem_sid < num_mat_elems; mat_elem_sid++) { // 1 material per element @@ -3019,17 +5221,53 @@ class MeshWriter // export element centric data double e_switch = 1; - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { + for (size_t elem_gid = 0; elem_gid < mesh.num_local_elems; elem_gid++) { elem_fields(elem_gid, 3) = State.GaussPoints.vol.host(elem_gid); elem_fields(elem_gid, 6) = speed.host(elem_gid); elem_fields(elem_gid, 8) = e_switch; elem_switch *= -1; } // end for elem_gid + //collective map has all indices on rank 0 and non on other ranks + HostDistributedMap collective_elem_map; + long long int num_collective_elem_indices = 0; + if(myrank==0) num_collective_elem_indices = mesh.global_num_elems; + collective_elem_map = HostDistributedMap(mesh.global_num_elems, num_collective_elem_indices); + + //collective vector and comms to the collective vector for elem fields + DistributedCArray collective_elem_fields(collective_elem_map, num_scalar_vars, "collective_elem_fields"); + HostCommPlanLR collective_elem_comms(collective_elem_fields, elem_fields); + collective_elem_comms.execute_comms(); + + //host of node in elem for Trilinos template argument compatibility + DistributedCArray host_local_nodes_in_elem(host_local_element_map, mesh.num_nodes_in_elem, "host_local_nodes_in_elem"); + + //convert nodes in elem back to global (convert back to local after we've collected global ids in collective vector) + for (size_t elem_id = 0; elem_id < mesh.num_local_elems; elem_id++) { + for (int node_lid = 0; node_lid < mesh.num_nodes_in_elem; node_lid++) { + host_local_nodes_in_elem(elem_id, node_lid) = mesh.all_node_map.getGlobalIndex(mesh.local_nodes_in_elem.host(elem_id, node_lid)); + } + } // end for elem_gid + + //collect nodes in elem with a conversion back to global node ids + DistributedCArray collective_nodes_in_elem(collective_elem_map, mesh.num_nodes_in_elem); + HostCommPlanLR nodes_in_elem_comms(collective_nodes_in_elem, host_local_nodes_in_elem); + + nodes_in_elem_comms.execute_comms(); + + //NODE DATA COLLECTION + //host version of local node map for argument compatibility + HostDistributedMap host_node_map; + DCArrayKokkos global_indices_of_local_nodes(mesh.num_local_nodes, "global_indices_of_local_nodes"); + for(int inode = 0; inode < mesh.num_local_nodes; inode++){ + global_indices_of_local_nodes(inode) = mesh.node_map.getGlobalIndex(inode); + } + host_node_map = HostDistributedMap(global_indices_of_local_nodes); + // save the vertex vector fields to an array for exporting to graphics files - CArray vec_fields(num_nodes, num_vec_vars, 3); + DistributedCArray vec_fields(host_node_map, num_vec_vars, 3, "vec_fields"); - for (size_t node_gid = 0; node_gid < num_nodes; node_gid++) { + for (size_t node_gid = 0; node_gid < mesh.num_local_nodes; node_gid++) { // position, var 0 vec_fields(node_gid, 0, 0) = State.node.coords.host(node_gid, 0); vec_fields(node_gid, 0, 1) = State.node.coords.host(node_gid, 1); @@ -3063,227 +5301,238 @@ class MeshWriter } // end for loop over vertices + //collective map has all indices on rank 0 and non on other ranks + HostDistributedMap collective_node_map; + long long int num_collective_node_indices = 0; + if(myrank==0) num_collective_node_indices = mesh.global_num_nodes; + collective_node_map = HostDistributedMap(mesh.global_num_nodes, num_collective_node_indices); + + //collective vector and comms to the collective vector for node fields + DistributedCArray collective_vec_fields(collective_node_map, num_vec_vars, 3, "collective_vec_fields"); + HostCommPlanLR collective_node_comms(collective_vec_fields, vec_fields); + collective_node_comms.execute_comms(); + + if(myrank==0){ + // --------------------------------------------------------------------------- + // Setup of file and directoring for exporting + // --------------------------------------------------------------------------- + FILE* out[20]; // the output files that are written to + char filename[128]; + int max_len = sizeof filename; + int str_output_len; - // --------------------------------------------------------------------------- - // Setup of file and directoring for exporting - // --------------------------------------------------------------------------- - FILE* out[20]; // the output files that are written to - char filename[128]; - int max_len = sizeof filename; - int str_output_len; - - struct stat st; - - if (stat("ensight", &st) != 0) { - system("mkdir ensight"); - } - - if (stat("ensight/data", &st) != 0) { - system("mkdir ensight/data"); - } - - // --------------------------------------------------------------------------- - // Write the Geometry file - // --------------------------------------------------------------------------- - // sprintf(filename, "ensight/data/%s.%05d.geo", name, graphics_id); - str_output_len = snprintf(filename, max_len, "ensight/data/%s.%05d.geo", name, graphics_id); - // filename has the full string - if (str_output_len >= max_len) { fputs("Filename length exceeded; string truncated", stderr); } - - out[0] = fopen(filename, "w"); - - fprintf(out[0], "A graphics dump by Fierro \n"); - - fprintf(out[0], "%s", "EnSight Gold geometry\n"); - fprintf(out[0], "%s", "node id assign\n"); - fprintf(out[0], "%s", "element id assign\n"); - - fprintf(out[0], "part\n"); - fprintf(out[0], "%10d\n", 1); - fprintf(out[0], "Mesh\n"); - - // --- vertices --- - fprintf(out[0], "coordinates\n"); - fprintf(out[0], "%10lu\n", num_nodes); - - // write all components of the point coordinates - for (int node_gid = 0; node_gid < num_nodes; node_gid++) { - fprintf(out[0], "%12.5e\n", State.node.coords.host(node_gid, 0)); - } - - for (int node_gid = 0; node_gid < num_nodes; node_gid++) { - fprintf(out[0], "%12.5e\n", State.node.coords.host(node_gid, 1)); - } - - for (int node_gid = 0; node_gid < num_nodes; node_gid++) { - if (num_dims == 3) { - fprintf(out[0], "%12.5e\n", State.node.coords.host(node_gid, 2)); - } - else{ - fprintf(out[0], "%12.5e\n", 0.0); - } - } - - // --- elements --- - if (num_dims == 3) { - fprintf(out[0], "hexa8\n"); - } - else{ - fprintf(out[0], "quad4\n"); - } - fprintf(out[0], "%10lu\n", num_elems); - - - int convert_ijk_to_ensight[8]; - if(mesh.num_dims==3){ - convert_ijk_to_ensight[0] = 0; - convert_ijk_to_ensight[1] = 1; - convert_ijk_to_ensight[2] = 3; - convert_ijk_to_ensight[3] = 2; - convert_ijk_to_ensight[4] = 4; - convert_ijk_to_ensight[5] = 5; - convert_ijk_to_ensight[6] = 7; - convert_ijk_to_ensight[7] = 6; - } - else{ - - convert_ijk_to_ensight[0] = 0; - convert_ijk_to_ensight[1] = 1; - convert_ijk_to_ensight[2] = 2; - convert_ijk_to_ensight[3] = 3; - convert_ijk_to_ensight[4] = 4; - convert_ijk_to_ensight[5] = 5; - convert_ijk_to_ensight[6] = 6; - convert_ijk_to_ensight[7] = 7; - } // end if - + struct stat st; - // write all global point numbers for this cell - for (int elem_gid = 0; elem_gid < num_elems; elem_gid++) { - for (int node_lid = 0; node_lid < mesh.num_nodes_in_elem; node_lid++) { - fprintf(out[0], "%10lu\t", mesh.nodes_in_elem.host(elem_gid, convert_ijk_to_ensight[node_lid]) + 1); // note: node_gid starts at 1 + if (stat("ensight", &st) != 0) { + system("mkdir ensight"); } - fprintf(out[0], "\n"); - } - fclose(out[0]); - - // --------------------------------------------------------------------------- - // Write the Scalar variable files - // --------------------------------------------------------------------------- + if (stat("ensight/data", &st) != 0) { + system("mkdir ensight/data"); + } - // ensight_vars = (den, pres,...) - for (int var = 0; var < num_scalar_vars; var++) { - // write a scalar value - // sprintf(filename, "ensight/data/%s.%05d.%s", name, graphics_id, scalar_var_names[var]); - str_output_len = snprintf(filename, max_len, "ensight/data/%s.%05d.%s", name, graphics_id, scalar_var_names[var]); + // --------------------------------------------------------------------------- + // Write the Geometry file + // --------------------------------------------------------------------------- + // sprintf(filename, "ensight/data/%s.%05d.geo", name, graphics_id); + str_output_len = snprintf(filename, max_len, "ensight/data/%s.%05d.geo", name, graphics_id); + // filename has the full string if (str_output_len >= max_len) { fputs("Filename length exceeded; string truncated", stderr); } out[0] = fopen(filename, "w"); - fprintf(out[0], "Per_elem scalar values\n"); + fprintf(out[0], "A graphics dump by Fierro \n"); + + fprintf(out[0], "%s", "EnSight Gold geometry\n"); + fprintf(out[0], "%s", "node id assign\n"); + fprintf(out[0], "%s", "element id assign\n"); + fprintf(out[0], "part\n"); fprintf(out[0], "%10d\n", 1); + fprintf(out[0], "Mesh\n"); + + // --- vertices --- + fprintf(out[0], "coordinates\n"); + fprintf(out[0], "%10lu\n", mesh.global_num_nodes); + + // write all components of the point coordinates + for (int node_gid = 0; node_gid < mesh.global_num_nodes; node_gid++) { + fprintf(out[0], "%12.5e\n", collective_vec_fields(node_gid, 0, 0)); + } + + for (int node_gid = 0; node_gid < mesh.global_num_nodes; node_gid++) { + fprintf(out[0], "%12.5e\n", collective_vec_fields(node_gid, 0, 1)); + } + + for (int node_gid = 0; node_gid < mesh.global_num_nodes; node_gid++) { + if (num_dims == 3) { + fprintf(out[0], "%12.5e\n", collective_vec_fields(node_gid, 0, 2)); + } + else{ + fprintf(out[0], "%12.5e\n", 0.0); + } + } + + // --- elements --- if (num_dims == 3) { fprintf(out[0], "hexa8\n"); } else{ fprintf(out[0], "quad4\n"); } + fprintf(out[0], "%10lu\n", mesh.global_num_elems); + + + int convert_ijk_to_ensight[8]; + if(mesh.num_dims==3){ + convert_ijk_to_ensight[0] = 0; + convert_ijk_to_ensight[1] = 1; + convert_ijk_to_ensight[2] = 3; + convert_ijk_to_ensight[3] = 2; + convert_ijk_to_ensight[4] = 4; + convert_ijk_to_ensight[5] = 5; + convert_ijk_to_ensight[6] = 7; + convert_ijk_to_ensight[7] = 6; + } + else{ + + convert_ijk_to_ensight[0] = 0; + convert_ijk_to_ensight[1] = 1; + convert_ijk_to_ensight[2] = 2; + convert_ijk_to_ensight[3] = 3; + convert_ijk_to_ensight[4] = 4; + convert_ijk_to_ensight[5] = 5; + convert_ijk_to_ensight[6] = 6; + convert_ijk_to_ensight[7] = 7; + } // end if - for (int elem_id = 0; elem_id < num_elems; elem_id++) { - fprintf(out[0], "%12.5e\n", elem_fields(elem_id, var)); + + // write all global point numbers for this cell + for (int elem_gid = 0; elem_gid < mesh.global_num_elems; elem_gid++) { + for (int node_lid = 0; node_lid < mesh.num_nodes_in_elem; node_lid++) { + fprintf(out[0], "%10lu\t", collective_nodes_in_elem(elem_gid, convert_ijk_to_ensight[node_lid]) + 1); // note: node_gid starts at 1 + } + fprintf(out[0], "\n"); } fclose(out[0]); - } // end for var - // --------------------------------------------------------------------------- - // Write the Vector variable files - // --------------------------------------------------------------------------- + // --------------------------------------------------------------------------- + // Write the Scalar variable files + // --------------------------------------------------------------------------- - // ensight vector vars = (position, velocity, force) - for (int var = 0; var < num_vec_vars; var++) { - // sprintf(filename, "ensight/data/%s.%05d.%s", name, graphics_id, vec_var_names[var]); - str_output_len = snprintf(filename, max_len, "ensight/data/%s.%05d.%s", name, graphics_id, vec_var_names[var]); - if (str_output_len >= max_len) { fputs("Filename length exceeded; string truncated", stderr); } + // ensight_vars = (den, pres,...) + for (int var = 0; var < num_scalar_vars; var++) { + // write a scalar value + // sprintf(filename, "ensight/data/%s.%05d.%s", name, graphics_id, scalar_var_names[var]); + str_output_len = snprintf(filename, max_len, "ensight/data/%s.%05d.%s", name, graphics_id, scalar_var_names[var]); + if (str_output_len >= max_len) { fputs("Filename length exceeded; string truncated", stderr); } - out[0] = fopen(filename, "w"); - // fprintf(out[0],"Per_node vector values\n"); - // fprintf(out[0],"part\n"); - // fprintf(out[0],"%10d \n",1); - // fprintf(out[0],"hexa8\n"); // WARNING, maybe bug here? + out[0] = fopen(filename, "w"); - fprintf(out[0], "Per_node vector values\n"); - fprintf(out[0], "part\n"); - fprintf(out[0], "%10d\n", 1); - fprintf(out[0], "block\n"); + fprintf(out[0], "Per_elem scalar values\n"); + fprintf(out[0], "part\n"); + fprintf(out[0], "%10d\n", 1); + if (num_dims == 3) { + fprintf(out[0], "hexa8\n"); + } + else{ + fprintf(out[0], "quad4\n"); + } - for (int node_gid = 0; node_gid < num_nodes; node_gid++) { - fprintf(out[0], "%12.5e\n", vec_fields(node_gid, var, 0)); - } + for (int elem_id = 0; elem_id < mesh.global_num_elems; elem_id++) { + fprintf(out[0], "%12.5e\n", collective_elem_fields(elem_id, var)); + } - for (int node_gid = 0; node_gid < num_nodes; node_gid++) { - fprintf(out[0], "%12.5e\n", vec_fields(node_gid, var, 1)); - } + fclose(out[0]); + } // end for var - for (int node_gid = 0; node_gid < num_nodes; node_gid++) { - fprintf(out[0], "%12.5e\n", vec_fields(node_gid, var, 2)); - } + // --------------------------------------------------------------------------- + // Write the Vector variable files + // --------------------------------------------------------------------------- - fclose(out[0]); - } // end for var + // ensight vector vars = (position, velocity, force) + for (int var = 0; var < num_vec_vars; var++) { + // sprintf(filename, "ensight/data/%s.%05d.%s", name, graphics_id, vec_var_names[var]); + str_output_len = snprintf(filename, max_len, "ensight/data/%s.%05d.%s", name, graphics_id, vec_var_names[var]); + if (str_output_len >= max_len) { fputs("Filename length exceeded; string truncated", stderr); } - // --------------------------------------------------------------------------- - // Write the case file - // --------------------------------------------------------------------------- + out[0] = fopen(filename, "w"); + // fprintf(out[0],"Per_node vector values\n"); + // fprintf(out[0],"part\n"); + // fprintf(out[0],"%10d \n",1); + // fprintf(out[0],"hexa8\n"); // WARNING, maybe bug here? - // sprintf(filename, "ensight/%s.case", name); - str_output_len = snprintf(filename, max_len, "ensight/%s.case", name); - if (str_output_len >= max_len) { fputs("Filename length exceeded; string truncated", stderr); } + fprintf(out[0], "Per_node vector values\n"); + fprintf(out[0], "part\n"); + fprintf(out[0], "%10d\n", 1); + fprintf(out[0], "block\n"); - out[0] = fopen(filename, "w"); + for (int node_gid = 0; node_gid < mesh.global_num_nodes; node_gid++) { + fprintf(out[0], "%12.5e\n", collective_vec_fields(node_gid, var, 0)); + } - fprintf(out[0], "FORMAT\n"); - fprintf(out[0], "type: ensight gold\n"); - fprintf(out[0], "GEOMETRY\n"); + for (int node_gid = 0; node_gid < mesh.global_num_nodes; node_gid++) { + fprintf(out[0], "%12.5e\n", collective_vec_fields(node_gid, var, 1)); + } - // sprintf(filename, "model: data/%s.*****.geo\n", name); - str_output_len = snprintf(filename, max_len, "model: data/%s.*****.geo\n", name); - if (str_output_len >= max_len) { fputs("Filename length exceeded; string truncated", stderr); } + for (int node_gid = 0; node_gid < mesh.global_num_nodes; node_gid++) { + fprintf(out[0], "%12.5e\n", collective_vec_fields(node_gid, var, 2)); + } + + fclose(out[0]); + } // end for var - fprintf(out[0], "%s", filename); - fprintf(out[0], "VARIABLE\n"); + // --------------------------------------------------------------------------- + // Write the case file + // --------------------------------------------------------------------------- - for (int var = 0; var < num_scalar_vars; var++) { - // sprintf(filename, "scalar per element: %s data/%s.*****.%s\n", scalar_var_names[var], name, scalar_var_names[var]); - str_output_len = snprintf(filename, max_len, "scalar per element: %s data/%s.*****.%s\n", scalar_var_names[var], name, scalar_var_names[var]); + // sprintf(filename, "ensight/%s.case", name); + str_output_len = snprintf(filename, max_len, "ensight/%s.case", name); if (str_output_len >= max_len) { fputs("Filename length exceeded; string truncated", stderr); } - fprintf(out[0], "%s", filename); - } + out[0] = fopen(filename, "w"); + + fprintf(out[0], "FORMAT\n"); + fprintf(out[0], "type: ensight gold\n"); + fprintf(out[0], "GEOMETRY\n"); - for (int var = 0; var < num_vec_vars; var++) { - // sprintf(filename, "vector per node: %s data/%s.*****.%s\n", vec_var_names[var], name, vec_var_names[var]); - str_output_len = snprintf(filename, max_len, "vector per node: %s data/%s.*****.%s\n", vec_var_names[var], name, vec_var_names[var]); + // sprintf(filename, "model: data/%s.*****.geo\n", name); + str_output_len = snprintf(filename, max_len, "model: data/%s.*****.geo\n", name); if (str_output_len >= max_len) { fputs("Filename length exceeded; string truncated", stderr); } + fprintf(out[0], "%s", filename); - } + fprintf(out[0], "VARIABLE\n"); - fprintf(out[0], "TIME\n"); - fprintf(out[0], "time set: 1\n"); - fprintf(out[0], "number of steps: %4d\n", graphics_id + 1); - fprintf(out[0], "filename start number: 0\n"); - fprintf(out[0], "filename increment: 1\n"); - fprintf(out[0], "time values: \n"); + for (int var = 0; var < num_scalar_vars; var++) { + // sprintf(filename, "scalar per element: %s data/%s.*****.%s\n", scalar_var_names[var], name, scalar_var_names[var]); + str_output_len = snprintf(filename, max_len, "scalar per element: %s data/%s.*****.%s\n", scalar_var_names[var], name, scalar_var_names[var]); + if (str_output_len >= max_len) { fputs("Filename length exceeded; string truncated", stderr); } - graphics_times(graphics_id) = time_value; + fprintf(out[0], "%s", filename); + } - for (int i = 0; i <= graphics_id; i++) { - fprintf(out[0], "%12.5e\n", graphics_times(i)); - } - fclose(out[0]); + for (int var = 0; var < num_vec_vars; var++) { + // sprintf(filename, "vector per node: %s data/%s.*****.%s\n", vec_var_names[var], name, vec_var_names[var]); + str_output_len = snprintf(filename, max_len, "vector per node: %s data/%s.*****.%s\n", vec_var_names[var], name, vec_var_names[var]); + if (str_output_len >= max_len) { fputs("Filename length exceeded; string truncated", stderr); } + fprintf(out[0], "%s", filename); + } + fprintf(out[0], "TIME\n"); + fprintf(out[0], "time set: 1\n"); + fprintf(out[0], "number of steps: %4d\n", graphics_id + 1); + fprintf(out[0], "filename start number: 0\n"); + fprintf(out[0], "filename increment: 1\n"); + fprintf(out[0], "time values: \n"); + + graphics_times(graphics_id) = time_value; + + for (int i = 0; i <= graphics_id; i++) { + fprintf(out[0], "%12.5e\n", graphics_times(i)); + } + fclose(out[0]); + } // --------------------------------------------------------------------------- // Done writing the graphics dump // --------------------------------------------------------------------------- @@ -3292,7 +5541,7 @@ class MeshWriter graphics_id++; delete[] name; - + return; } @@ -3312,7 +5561,7 @@ class MeshWriter ///////////////////////////////////////////////////////////////////////////// void write_vtk_old(Mesh_t& mesh, State_t& State, - SimulationParameters_t& SimulationParamaters, + SimulationParameters_t& SimulationParameters, double dt, double time_value, CArray graphics_times, @@ -3320,7 +5569,10 @@ class MeshWriter std::vector gauss_pt_states, std::vector material_pt_states) { - + + int myrank, nranks; + MPI_Comm_rank(MPI_COMM_WORLD,&myrank); + MPI_Comm_size(MPI_COMM_WORLD,&nranks); size_t num_mats = State.MaterialPoints.num_material_points.size(); // ---- Update host data ---- @@ -3379,8 +5631,16 @@ class MeshWriter const size_t num_elems = mesh.num_elems; const size_t num_dims = mesh.num_dims; + //host version of local element map for argument compatibility + HostDistributedMap host_local_element_map; + DCArrayKokkos global_indices_of_local_elements; + for(int ielem = 0; ielem < mesh.num_local_elems; ielem++){ + global_indices_of_local_elements(ielem) = mesh.local_element_map.getGlobalIndex(ielem); + } + host_local_element_map = HostDistributedMap(global_indices_of_local_elements); + // save the cell state to an array for exporting to graphics files - auto elem_fields = CArray(num_elems, num_cell_scalar_vars); + auto elem_fields = DistributedCArray(host_local_element_map, num_cell_scalar_vars); int elem_switch = 1; DCArrayKokkos speed(num_elems, "speed"); @@ -3418,7 +5678,7 @@ class MeshWriter // export material centeric data to the elements for (int mat_id = 0; mat_id < num_mats; mat_id++) { - size_t num_mat_elems = State.MaterialToMeshMaps.num_mat_elems.host(mat_id); + size_t num_mat_elems = State.MaterialToMeshMaps.num_mat_local_elems.host(mat_id); for (size_t mat_elem_sid = 0; mat_elem_sid < num_mat_elems; mat_elem_sid++) { // 1 material per element @@ -3445,16 +5705,30 @@ class MeshWriter // export element centric data double e_switch = 1; - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { + for (size_t elem_gid = 0; elem_gid < mesh.num_local_elems; elem_gid++) { elem_fields(elem_gid, 3) = State.GaussPoints.vol.host(elem_gid); elem_fields(elem_gid, 6) = speed.host(elem_gid); elem_fields(elem_gid, 8) = State.GaussPoints.div.host(elem_gid); elem_switch *= -1; } // end for elem_gid + //collective map has all indices on rank 0 and non on other ranks + HostDistributedMap collective_elem_map; + long long int num_collective_elem_indices = 0; + if(myrank==0) num_collective_elem_indices = mesh.global_num_elems; + collective_elem_map = HostDistributedMap(mesh.global_num_elems, num_collective_elem_indices); + + //host version of local element map for argument compatibility + HostDistributedMap host_node_map; + DCArrayKokkos global_indices_of_local_nodes; + for(int inode = 0; inode < mesh.num_local_nodes; inode++){ + global_indices_of_local_nodes(inode) = mesh.node_map.getGlobalIndex(inode); + } + host_node_map = HostDistributedMap(global_indices_of_local_nodes); + // save the vertex vector fields to an array for exporting to graphics files - CArray vec_fields(num_nodes, num_point_vec_vars, 3); - CArray point_scalar_fields(num_nodes, num_point_scalar_vars); + DistributedCArray vec_fields(host_node_map, num_point_vec_vars, 3); + DistributedCArray point_scalar_fields(host_node_map, num_point_scalar_vars); for (size_t node_gid = 0; node_gid < num_nodes; node_gid++) { // position, var 0 @@ -3617,34 +5891,262 @@ class MeshWriter if (str_output_len >= max_len) { fputs("Filename length exceeded; string truncated", stderr); } // mesh file - out[0] = fopen(filename, "w"); + out[0] = fopen(filename, "w"); + + fprintf(out[0], "{\n"); + fprintf(out[0], " \"file-series-version\" : \"1.0\",\n"); + fprintf(out[0], " \"files\" : [\n"); + + for (int i = 0; i <= graphics_id; i++) { + fprintf(out[0], " { \"name\" : \"Fierro.%05d.vtk\", \"time\" : %12.5e },\n", i, graphics_times(i) ); + } + + // fprintf(out[0], "%12.5e\n", graphics_times(i)); + fprintf(out[0], " ]\n"); // part 4 + fprintf(out[0], "}"); // part 4 + + fclose(out[0]); + + // increment graphics id counter + graphics_id++; + + + } // end write vtk old + + + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn concatenate_elem_fields + /// + /// \brief A function to calculate the average of elem fields and concatentate into 1 array + /// + /// + /// \param MaterialPoints a struct containing the material point state arrays + /// \param elem_scalar_fields the scalar fields + /// \param elem_tensor_fields the tensor fields + /// \param elem_in_mat_elem a listing of the element ids the material resides in + /// \param output_elem_state a std::vector of enums specifying the elem avg outputs + /// \param num_mat_elems the number of elements the material resides in + /// \param mat_id the index for the material + /// + ///////////////////////////////////////////////////////////////////////////// + void concatenate_elem_fields(const MaterialPoint_t& MaterialPoints, + const GaussPoint_t& GaussPoints, + DistributedDFArray& elem_scalar_fields, + DistributedDFArray& elem_tensor_fields, + const DRaggedRightArrayKokkos& elem_in_mat_elem, + const std::vector& output_elem_state, + const std::vector& output_gauss_pt_states, + const size_t num_mat_elems, + const size_t mat_id, + const size_t num_elems, + const int den_id, + const int pres_id, + const int sie_id, + const int sspd_id, + const int mass_id, + const int stress_id, + const int vol_id, + const int div_id, + const int level_set_id, + const int vel_grad_id, + const int conductivity_id, + const int specific_heat_id) + { + + // --- loop over the material point states + + for (auto field : output_elem_state){ + switch(field){ + // scalar vars + case material_pt_state::density: + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { + + // get elem gid + size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_lid); + + // field + elem_scalar_fields(elem_gid, den_id) += MaterialPoints.den(mat_id, mat_elem_lid)* + MaterialPoints.volfrac(mat_id, mat_elem_lid)* + MaterialPoints.geo_volfrac(mat_id, mat_elem_lid); + }); + break; + case material_pt_state::pressure: + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { + + // get elem gid + size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_lid); + + // field + elem_scalar_fields(elem_gid, pres_id) += MaterialPoints.pres(mat_id, mat_elem_lid)* + MaterialPoints.volfrac(mat_id, mat_elem_lid)* + MaterialPoints.geo_volfrac(mat_id, mat_elem_lid); + }); + break; + case material_pt_state::specific_internal_energy: + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { + + // get elem gid + size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_lid); + + // field + // extensive ie here, but after this function, it will become specific ie + elem_scalar_fields(elem_gid, sie_id) += MaterialPoints.mass(mat_id, mat_elem_lid)* + MaterialPoints.sie(mat_id, mat_elem_lid); + }); + break; + case material_pt_state::sound_speed: + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { + + // get elem gid + size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_lid); + + // field + elem_scalar_fields(elem_gid, sspd_id) += MaterialPoints.sspd(mat_id, mat_elem_lid)* + MaterialPoints.volfrac(mat_id, mat_elem_lid)* + MaterialPoints.geo_volfrac(mat_id, mat_elem_lid); + }); + break; + case material_pt_state::mass: + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { + + // get elem gid + size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_lid); + + // field + elem_scalar_fields(elem_gid, mass_id) += MaterialPoints.mass(mat_id, mat_elem_lid); + }); + break; + // --------------- + // tensor vars + // --------------- + case material_pt_state::stress: + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { + + // get elem gid + size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_lid); + + // field + // average tensor fields, it is always 3D + // note: paraview is row-major, CArray convention + for (size_t i=0; i<3; i++){ + for(size_t j=0; j<3; j++){ + + // stress tensor + elem_tensor_fields(elem_gid, stress_id, i, j) += + MaterialPoints.stress(mat_id, mat_elem_lid,i,j) * + MaterialPoints.volfrac(mat_id, mat_elem_lid)* + MaterialPoints.geo_volfrac(mat_id, mat_elem_lid); + } // end for + } // end for + }); + break; + + // thermal solver vars + case material_pt_state::thermal_conductivity: + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { + + // get elem gid + size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_lid); + + // field + elem_scalar_fields(elem_gid, conductivity_id) += MaterialPoints.conductivity(mat_id, mat_elem_lid)* + MaterialPoints.volfrac(mat_id, mat_elem_lid)* + MaterialPoints.geo_volfrac(mat_id, mat_elem_lid); + }); + break; + + case material_pt_state::specific_heat: + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { + + // get elem gid + size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_lid); + + // field + elem_scalar_fields(elem_gid, specific_heat_id) += MaterialPoints.specific_heat(mat_id, mat_elem_lid)* + MaterialPoints.volfrac(mat_id, mat_elem_lid)* + MaterialPoints.geo_volfrac(mat_id, mat_elem_lid); + }); + break; + + + // add other variables here + + // not used variables + case material_pt_state::volume_fraction: + break; + case material_pt_state::eroded_flag: + break; + case material_pt_state::elastic_modulii: + break; + case material_pt_state::shear_modulii: + break; + case material_pt_state::poisson_ratios: + break; + case material_pt_state::heat_flux: + break; + } // end switch + }// end for over mat point state + + + // --- add loop over gauss points --- + + // export element centric data + for (auto field : output_gauss_pt_states){ + switch(field){ + // scalars + case gauss_pt_state::volume: + + FOR_ALL(elem_gid, 0, num_elems, { + elem_scalar_fields(elem_gid, vol_id) = GaussPoints.vol(elem_gid); + }); + + break; + case gauss_pt_state::divergence_velocity: + + FOR_ALL(elem_gid, 0, num_elems, { + elem_scalar_fields(elem_gid, div_id) = GaussPoints.div(elem_gid); + }); + + break; + + case gauss_pt_state::level_set: - fprintf(out[0], "{\n"); - fprintf(out[0], " \"file-series-version\" : \"1.0\",\n"); - fprintf(out[0], " \"files\" : [\n"); + FOR_ALL(elem_gid, 0, num_elems, { + elem_scalar_fields(elem_gid, level_set_id) = GaussPoints.level_set(elem_gid); + }); - for (int i = 0; i <= graphics_id; i++) { - fprintf(out[0], " { \"name\" : \"Fierro.%05d.vtk\", \"time\" : %12.5e },\n", i, graphics_times(i) ); - } + break; - // fprintf(out[0], "%12.5e\n", graphics_times(i)); - fprintf(out[0], " ]\n"); // part 4 - fprintf(out[0], "}"); // part 4 + // tensors + case gauss_pt_state::gradient_velocity: + // note: paraview is row-major, CArray convention + FOR_ALL(elem_gid, 0, num_elems, { + for (size_t i=0; i<3; i++){ + for(size_t j=0; j<3; j++){ + elem_tensor_fields(elem_gid, vel_grad_id, i, j) = + GaussPoints.vel_grad(elem_gid, i, j); + } + } // end for + }); - fclose(out[0]); + break; - // increment graphics id counter - graphics_id++; + // add other gauss variables here + + } // end switch + } // end loop over gauss_pt_states - } // end write vtk old + // --- add end gauss point loop -- + } // end of function ///////////////////////////////////////////////////////////////////////////// /// - /// \fn concatenate_elem_fields + /// \fn copy_elem_fields /// - /// \brief A function to calculate the average of elem fields and concatentate into 1 array + /// \brief A function to assign dual values to host only array for tpetra compatibility /// /// /// \param MaterialPoints a struct containing the material point state arrays @@ -3656,28 +6158,28 @@ class MeshWriter /// \param mat_id the index for the material /// ///////////////////////////////////////////////////////////////////////////// - void concatenate_elem_fields(const MaterialPoint_t& MaterialPoints, - const GaussPoint_t& GaussPoints, - DCArrayKokkos& elem_scalar_fields, - DCArrayKokkos& elem_tensor_fields, - const DRaggedRightArrayKokkos& elem_in_mat_elem, - const std::vector& output_elem_state, - const std::vector& output_gauss_pt_states, - const size_t num_mat_elems, - const size_t mat_id, - const size_t num_elems, - const int den_id, - const int pres_id, - const int sie_id, - const int sspd_id, - const int mass_id, - const int stress_id, - const int vol_id, - const int div_id, - const int level_set_id, - const int vel_grad_id, - const int conductivity_id, - const int specific_heat_id) + void copy_elem_fields(DistributedDFArray& elem_scalar_fields, + DistributedDFArray& elem_tensor_fields, + DistributedFArray& host_elem_scalar_fields, + DistributedFArray& host_elem_tensor_fields, + const DRaggedRightArrayKokkos& elem_in_mat_elem, + const std::vector& output_elem_state, + const std::vector& output_gauss_pt_states, + const size_t num_mat_elems, + const size_t mat_id, + const size_t num_elems, + const int den_id, + const int pres_id, + const int sie_id, + const int sspd_id, + const int mass_id, + const int stress_id, + const int vol_id, + const int div_id, + const int level_set_id, + const int vel_grad_id, + const int conductivity_id, + const int specific_heat_id) { // --- loop over the material point states @@ -3686,71 +6188,46 @@ class MeshWriter switch(field){ // scalar vars case material_pt_state::density: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { - - // get elem gid - size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_sid); + for(int elem_gid = 0; elem_gid < num_elems; elem_gid++){ // field - elem_scalar_fields(den_id, elem_gid) += MaterialPoints.den(mat_id, mat_elem_sid)* - MaterialPoints.volfrac(mat_id, mat_elem_sid)* - MaterialPoints.geo_volfrac(mat_id, mat_elem_sid); - }); + host_elem_scalar_fields(elem_gid, den_id) = elem_scalar_fields.host(elem_gid, den_id); + } break; case material_pt_state::pressure: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { - - // get elem gid - size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_sid); + for(int elem_gid = 0; elem_gid < num_elems; elem_gid++){ // field - elem_scalar_fields(pres_id, elem_gid) += MaterialPoints.pres(mat_id, mat_elem_sid)* - MaterialPoints.volfrac(mat_id, mat_elem_sid)* - MaterialPoints.geo_volfrac(mat_id, mat_elem_sid); - }); + host_elem_scalar_fields(elem_gid, pres_id) = elem_scalar_fields.host(elem_gid, pres_id); + } break; case material_pt_state::specific_internal_energy: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { - - // get elem gid - size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_sid); + for(int elem_gid = 0; elem_gid < num_elems; elem_gid++){ // field // extensive ie here, but after this function, it will become specific ie - elem_scalar_fields(sie_id, elem_gid) += MaterialPoints.mass(mat_id, mat_elem_sid)* - MaterialPoints.sie(mat_id, mat_elem_sid); - }); + host_elem_scalar_fields(elem_gid, sie_id) = elem_scalar_fields.host(elem_gid, sie_id); + } break; case material_pt_state::sound_speed: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { - - // get elem gid - size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_sid); + for(int elem_gid = 0; elem_gid < num_elems; elem_gid++){ // field - elem_scalar_fields(sspd_id, elem_gid) += MaterialPoints.sspd(mat_id, mat_elem_sid)* - MaterialPoints.volfrac(mat_id, mat_elem_sid)* - MaterialPoints.geo_volfrac(mat_id, mat_elem_sid); - }); + host_elem_scalar_fields(elem_gid, sspd_id) = elem_scalar_fields.host(elem_gid, sspd_id); + } break; case material_pt_state::mass: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { - - // get elem gid - size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_sid); + for(int elem_gid = 0; elem_gid < num_elems; elem_gid++){ // field - elem_scalar_fields(mass_id, elem_gid) += MaterialPoints.mass(mat_id, mat_elem_sid); - }); + host_elem_scalar_fields(elem_gid, mass_id) = elem_scalar_fields.host(elem_gid, mass_id); + } break; // --------------- // tensor vars // --------------- case material_pt_state::stress: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { - - // get elem gid - size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_sid); + for(int elem_gid = 0; elem_gid < num_elems; elem_gid++){ // field // average tensor fields, it is always 3D @@ -3759,40 +6236,27 @@ class MeshWriter for(size_t j=0; j<3; j++){ // stress tensor - elem_tensor_fields(stress_id, elem_gid, i, j) += - MaterialPoints.stress(mat_id, mat_elem_sid,i,j) * - MaterialPoints.volfrac(mat_id, mat_elem_sid)* - MaterialPoints.geo_volfrac(mat_id, mat_elem_sid); + host_elem_tensor_fields(elem_gid, stress_id, i, j) = elem_tensor_fields.host(elem_gid, stress_id, i, j); } // end for } // end for - }); + } break; // thermal solver vars case material_pt_state::thermal_conductivity: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { - - // get elem gid - size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_sid); + for(int elem_gid = 0; elem_gid < num_elems; elem_gid++){ // field - elem_scalar_fields(conductivity_id, elem_gid) += MaterialPoints.conductivity(mat_id, mat_elem_sid)* - MaterialPoints.volfrac(mat_id, mat_elem_sid)* - MaterialPoints.geo_volfrac(mat_id, mat_elem_sid); - }); + host_elem_scalar_fields(elem_gid, conductivity_id) = elem_scalar_fields.host(elem_gid, conductivity_id); + } break; case material_pt_state::specific_heat: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { - - // get elem gid - size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_sid); + for(int elem_gid = 0; elem_gid < num_elems; elem_gid++){ // field - elem_scalar_fields(specific_heat_id, elem_gid) += MaterialPoints.specific_heat(mat_id, mat_elem_sid)* - MaterialPoints.volfrac(mat_id, mat_elem_sid)* - MaterialPoints.geo_volfrac(mat_id, mat_elem_sid); - }); + host_elem_scalar_fields(elem_gid, specific_heat_id) = elem_scalar_fields.host(elem_gid, specific_heat_id); + } break; @@ -3823,38 +6287,38 @@ class MeshWriter // scalars case gauss_pt_state::volume: - FOR_ALL(elem_gid, 0, num_elems, { - elem_scalar_fields(vol_id, elem_gid) = GaussPoints.vol(elem_gid); - }); + for(int elem_gid = 0; elem_gid < num_elems; elem_gid++){ + host_elem_scalar_fields(elem_gid, vol_id) = elem_scalar_fields.host(elem_gid, vol_id); + } break; case gauss_pt_state::divergence_velocity: - FOR_ALL(elem_gid, 0, num_elems, { - elem_scalar_fields(div_id, elem_gid) = GaussPoints.div(elem_gid); - }); + for(int elem_gid = 0; elem_gid < num_elems; elem_gid++){ + host_elem_scalar_fields(elem_gid, div_id) = elem_scalar_fields.host(elem_gid, div_id); + } break; case gauss_pt_state::level_set: - FOR_ALL(elem_gid, 0, num_elems, { - elem_scalar_fields(level_set_id, elem_gid) = GaussPoints.level_set(elem_gid); - }); + for(int elem_gid = 0; elem_gid < num_elems; elem_gid++){ + host_elem_scalar_fields(elem_gid, level_set_id) = elem_scalar_fields.host(elem_gid, level_set_id); + } break; // tensors case gauss_pt_state::gradient_velocity: // note: paraview is row-major, CArray convention - FOR_ALL(elem_gid, 0, num_elems, { + for(int elem_gid = 0; elem_gid < num_elems; elem_gid++){ for (size_t i=0; i<3; i++){ for(size_t j=0; j<3; j++){ - elem_tensor_fields(vel_grad_id, elem_gid, i, j) = - GaussPoints.vel_grad(elem_gid, i, j); + host_elem_tensor_fields(elem_gid, vel_grad_id, i, j) = + elem_tensor_fields.host(elem_gid, vel_grad_id, i, j); } } // end for - }); + } break; @@ -3885,8 +6349,8 @@ class MeshWriter /// ///////////////////////////////////////////////////////////////////////////// void concatenate_mat_fields(const MaterialPoint_t& MaterialPoints, - DCArrayKokkos& mat_elem_scalar_fields, - DCArrayKokkos& mat_elem_tensor_fields, + DistributedDFArray& mat_elem_scalar_fields, + DistributedDFArray& mat_elem_tensor_fields, const DRaggedRightArrayKokkos& elem_in_mat_elem, const std::vector& output_material_pt_states, const size_t num_mat_elems, @@ -3904,76 +6368,74 @@ class MeshWriter const int mat_specific_heat_id) { - // --- loop over the material point states - for (auto field : output_material_pt_states){ switch(field){ // scalar vars case material_pt_state::density: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { // field - mat_elem_scalar_fields(mat_den_id, mat_elem_sid) = MaterialPoints.den(mat_id, mat_elem_sid); + mat_elem_scalar_fields(mat_elem_lid, mat_den_id) = MaterialPoints.den(mat_id, mat_elem_lid); }); break; case material_pt_state::pressure: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { // field - mat_elem_scalar_fields(mat_pres_id, mat_elem_sid) = MaterialPoints.pres(mat_id, mat_elem_sid); + mat_elem_scalar_fields(mat_elem_lid, mat_pres_id) = MaterialPoints.pres(mat_id, mat_elem_lid); }); break; case material_pt_state::specific_internal_energy: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { // field // extensive ie here, but after this function, it will become specific ie - mat_elem_scalar_fields(mat_sie_id, mat_elem_sid) = MaterialPoints.sie(mat_id, mat_elem_sid); + mat_elem_scalar_fields(mat_elem_lid, mat_sie_id) = MaterialPoints.sie(mat_id, mat_elem_lid); }); break; case material_pt_state::sound_speed: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { // field - mat_elem_scalar_fields(mat_sspd_id, mat_elem_sid) = MaterialPoints.sspd(mat_id, mat_elem_sid); + mat_elem_scalar_fields(mat_elem_lid, mat_sspd_id) = MaterialPoints.sspd(mat_id, mat_elem_lid); }); break; case material_pt_state::mass: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { // field - mat_elem_scalar_fields(mat_mass_id, mat_elem_sid) = MaterialPoints.mass(mat_id, mat_elem_sid); + mat_elem_scalar_fields(mat_elem_lid, mat_mass_id) = MaterialPoints.mass(mat_id, mat_elem_lid); }); break; case material_pt_state::volume_fraction: // material volume fraction - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { // field // this is the volume fraction of a material within a part - mat_elem_scalar_fields(mat_volfrac_id, mat_elem_sid) = MaterialPoints.volfrac(mat_id, mat_elem_sid); + mat_elem_scalar_fields(mat_elem_lid, mat_volfrac_id) = MaterialPoints.volfrac(mat_id, mat_elem_lid); }); // geometric volume fraction - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { // field // this is the geometric volume fraction (interface reconstruction) - mat_elem_scalar_fields(mat_geo_volfrac_id, mat_elem_sid) = MaterialPoints.geo_volfrac(mat_id, mat_elem_sid); + mat_elem_scalar_fields(mat_elem_lid, mat_geo_volfrac_id) = MaterialPoints.geo_volfrac(mat_id, mat_elem_lid); }); break; case material_pt_state::eroded_flag: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { // field - mat_elem_scalar_fields(mat_eroded_id, mat_elem_sid) = (double)MaterialPoints.eroded(mat_id, mat_elem_sid); + mat_elem_scalar_fields(mat_elem_lid, mat_eroded_id) = (double)MaterialPoints.eroded(mat_id, mat_elem_lid); }); break; // --------------- // tensor vars // --------------- case material_pt_state::stress: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { // field // average tensor fields, it is always 3D @@ -3982,8 +6444,8 @@ class MeshWriter for(size_t j=0; j<3; j++){ // stress tensor - mat_elem_tensor_fields(mat_stress_id, mat_elem_sid, i, j) = - MaterialPoints.stress(mat_id, mat_elem_sid,i,j); + mat_elem_tensor_fields(mat_elem_lid, mat_stress_id, i, j) = + MaterialPoints.stress(mat_id, mat_elem_lid,i,j); } // end for } // end for }); @@ -3991,24 +6453,24 @@ class MeshWriter // thermal solver vars case material_pt_state::thermal_conductivity: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { // get elem gid - size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_sid); + size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_lid); // field - mat_elem_scalar_fields(mat_conductivity_id, elem_gid) += MaterialPoints.conductivity(mat_id, mat_elem_sid); + mat_elem_scalar_fields(elem_gid, mat_conductivity_id) += MaterialPoints.conductivity(mat_id, mat_elem_lid); }); break; case material_pt_state::specific_heat: - FOR_ALL(mat_elem_sid, 0, num_mat_elems, { + FOR_ALL(mat_elem_lid, 0, num_mat_elems, { // get elem gid - size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_sid); + size_t elem_gid = elem_in_mat_elem(mat_id, mat_elem_lid); // field - mat_elem_scalar_fields(mat_specific_heat_id, elem_gid) += MaterialPoints.specific_heat(mat_id, mat_elem_sid); + mat_elem_scalar_fields(elem_gid, mat_specific_heat_id) += MaterialPoints.specific_heat(mat_id, mat_elem_lid); }); break; @@ -4025,6 +6487,7 @@ class MeshWriter break; } // end switch }// end for over mat point state + } // end of function @@ -4033,7 +6496,7 @@ class MeshWriter /// /// \fn concatenate_nodal_fields /// - /// \brief A function to calculate the average of elem fields + /// \brief A function to calculate the average of nodal fields /// /// /// \param Node a struct containing the material point state arrays @@ -4046,8 +6509,8 @@ class MeshWriter /// ///////////////////////////////////////////////////////////////////////////// void concatenate_nodal_fields(const node_t& Node, - DCArrayKokkos& node_scalar_fields, - DCArrayKokkos& node_vector_fields, + DistributedDFArray& node_scalar_fields, + DistributedDFArray& node_vector_fields, std::vector& output_node_states, double dt, const size_t num_nodes, @@ -4065,13 +6528,13 @@ class MeshWriter case node_state::mass: FOR_ALL(node_gid, 0, num_nodes, { - node_scalar_fields(node_mass_id, node_gid) = Node.mass(node_gid); + node_scalar_fields(node_gid, node_mass_id) = Node.mass(node_gid); }); break; case node_state::temp: FOR_ALL(node_gid, 0, num_nodes, { - node_scalar_fields(node_temp_id, node_gid) = Node.temp(node_gid); + node_scalar_fields(node_gid, node_temp_id) = Node.temp(node_gid); }); break; @@ -4082,13 +6545,13 @@ class MeshWriter FOR_ALL(node_gid, 0, num_nodes, { - node_vector_fields(node_coord_id, node_gid, 0) = Node.coords(node_gid, 0); - node_vector_fields(node_coord_id, node_gid, 1) = Node.coords(node_gid, 1); + node_vector_fields(node_gid, node_coord_id, 0) = Node.coords(node_gid, 0); + node_vector_fields(node_gid, node_coord_id, 1) = Node.coords(node_gid, 1); if (num_dims == 2) { - node_vector_fields(node_coord_id, node_gid, 2) = 0.0; + node_vector_fields(node_gid, node_coord_id, 2) = 0.0; } else{ - node_vector_fields(node_coord_id, node_coord_id, 2) = Node.coords(node_gid, 2); + node_vector_fields(node_gid, node_coord_id, 2) = Node.coords(node_gid, 2); } // end if }); // end parallel for @@ -4099,23 +6562,23 @@ class MeshWriter FOR_ALL(node_gid, 0, num_nodes, { // velocity, var is node_vel_id - node_vector_fields(node_vel_id, node_gid, 0) = Node.vel(node_gid, 0); - node_vector_fields(node_vel_id, node_gid, 1) = Node.vel(node_gid, 1); + node_vector_fields(node_gid, node_vel_id, 0) = Node.vel(node_gid, 0); + node_vector_fields(node_gid, node_vel_id, 1) = Node.vel(node_gid, 1); if (num_dims == 2) { - node_vector_fields(node_vel_id, node_gid, 2) = 0.0; + node_vector_fields(node_gid, node_vel_id, 2) = 0.0; } else{ - node_vector_fields(node_vel_id, node_gid, 2) = Node.vel(node_gid, 2); + node_vector_fields(node_gid, node_vel_id, 2) = Node.vel(node_gid, 2); } // end if // accellerate, var is node_accel_id - node_vector_fields(node_accel_id, node_gid, 0) = (Node.vel(node_gid, 0) - Node.vel_n0(node_gid, 0))/dt; - node_vector_fields(node_accel_id, node_gid, 1) = (Node.vel(node_gid, 1) - Node.vel_n0(node_gid, 1))/dt; + node_vector_fields(node_gid, node_accel_id, 0) = (Node.vel(node_gid, 0) - Node.vel_n0(node_gid, 0))/dt; + node_vector_fields(node_gid, node_accel_id, 1) = (Node.vel(node_gid, 1) - Node.vel_n0(node_gid, 1))/dt; if (num_dims == 2) { - node_vector_fields(node_accel_id, node_gid, 2) = 0.0; + node_vector_fields(node_gid, node_accel_id, 2) = 0.0; } else{ - node_vector_fields(node_accel_id, node_gid, 2) = (Node.vel(node_gid, 2) - Node.vel_n0(node_gid, 2))/dt; + node_vector_fields(node_gid, node_accel_id, 2) = (Node.vel(node_gid, 2) - Node.vel_n0(node_gid, 2))/dt; } // end if }); // end parallel for @@ -4128,13 +6591,13 @@ class MeshWriter FOR_ALL(node_gid, 0, num_nodes, { // velocity, var is node_vel_id - node_vector_fields(node_grad_level_set_id, node_gid, 0) = Node.gradient_level_set(node_gid, 0); - node_vector_fields(node_grad_level_set_id, node_gid, 1) = Node.gradient_level_set(node_gid, 1); + node_vector_fields(node_gid, node_grad_level_set_id, 0) = Node.gradient_level_set(node_gid, 0); + node_vector_fields(node_gid, node_grad_level_set_id, 1) = Node.gradient_level_set(node_gid, 1); if (num_dims == 2) { - node_vector_fields(node_grad_level_set_id, node_gid, 2) = 0.0; + node_vector_fields(node_gid, node_grad_level_set_id, 2) = 0.0; } else{ - node_vector_fields(node_grad_level_set_id, node_gid, 2) = Node.gradient_level_set(node_gid, 2); + node_vector_fields(node_gid, node_grad_level_set_id, 2) = Node.gradient_level_set(node_gid, 2); } // end if }); // end parallel for @@ -4154,6 +6617,134 @@ class MeshWriter + } // end function + + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn copy_nodal_fields + /// + /// \brief A function to calculate the average of nodal fields + /// + /// + /// \param Node a struct containing the material point state arrays + /// \param elem_scalar_fields the scalar fields + /// \param elem_tensor_fields the tensor fields + /// \param elem_in_mat_elem a listing of the element ids the material resides in + /// \param output_node_states a std::vector of enums specifying the model + /// \param num_mat_elems the number of elements the material resides in + /// \param mat_id the index for the material + /// + ///////////////////////////////////////////////////////////////////////////// + void copy_nodal_fields(DistributedFArray& host_node_scalar_fields, + DistributedFArray& host_node_vector_fields, + DistributedDFArray& node_scalar_fields, + DistributedDFArray& node_vector_fields, + std::vector& output_node_states, + double dt, + const size_t num_nodes, + const size_t num_dims, + const int node_mass_id, + const int node_vel_id, + const int node_accel_id, + const int node_coord_id, + const int node_grad_level_set_id, + const int node_temp_id) + { + for (auto field : output_node_states){ + switch(field){ + // scalars + case node_state::mass: + + for(long long int node_gid = 0; node_gid < num_nodes; node_gid++) { + host_node_scalar_fields(node_gid, node_mass_id) = node_scalar_fields.host(node_gid, node_mass_id); + } + + break; + case node_state::temp: + for(long long int node_gid = 0; node_gid < num_nodes; node_gid++) { + host_node_scalar_fields(node_gid, node_temp_id) = node_scalar_fields.host(node_gid, node_temp_id); + } + + break; + + // vector fields + + case node_state::coords: + + for(long long int node_gid = 0; node_gid < num_nodes; node_gid++) { + + host_node_vector_fields(node_gid, node_coord_id, 0) = node_vector_fields.host(node_gid, node_coord_id, 0); + host_node_vector_fields(node_gid, node_coord_id, 1) = node_vector_fields.host(node_gid, node_coord_id, 1); + if (num_dims == 2) { + host_node_vector_fields(node_gid, node_coord_id, 2) = 0.0; + } + else{ + host_node_vector_fields(node_gid, node_coord_id, 2) = node_vector_fields.host(node_gid, node_coord_id, 2); + } // end if + + } // end parallel for + + break; + case node_state::velocity: + + for(long long int node_gid = 0; node_gid < num_nodes; node_gid++) { + + // velocity, var is node_vel_id + host_node_vector_fields(node_gid, node_vel_id, 0) = node_vector_fields.host(node_gid, node_vel_id, 0); + host_node_vector_fields(node_gid, node_vel_id, 1) = node_vector_fields.host(node_gid, node_vel_id, 1); + if (num_dims == 2) { + host_node_vector_fields(node_gid, node_vel_id, 2) = 0.0; + } + else{ + host_node_vector_fields(node_gid, node_vel_id, 2) = node_vector_fields.host(node_gid, node_vel_id, 2); + } // end if + + // accellerate, var is node_accel_id + host_node_vector_fields(node_gid, node_accel_id, 0) = node_vector_fields.host(node_gid, node_accel_id, 0); + host_node_vector_fields(node_gid, node_accel_id, 1) = node_vector_fields.host(node_gid, node_accel_id, 1); + if (num_dims == 2) { + host_node_vector_fields(node_gid, node_accel_id, 2) = 0.0; + } + else{ + host_node_vector_fields(node_gid, node_accel_id, 2) = node_vector_fields.host(node_gid, node_accel_id, 2); + } // end if + + } // end parallel for + + break; + + + case node_state::gradient_level_set: + + for(long long int node_gid = 0; node_gid < num_nodes; node_gid++) { + + // velocity, var is node_vel_id + host_node_vector_fields(node_gid, node_grad_level_set_id, 0) = node_vector_fields.host(node_gid, node_grad_level_set_id, 0); + host_node_vector_fields(node_gid, node_grad_level_set_id, 1) = node_vector_fields.host(node_gid, node_grad_level_set_id, 1); + if (num_dims == 2) { + host_node_vector_fields(node_gid, node_grad_level_set_id, 2) = 0.0; + } + else{ + host_node_vector_fields(node_gid, node_grad_level_set_id, 2) = node_vector_fields.host(node_gid, node_grad_level_set_id, 2); + } // end if + + } // end parallel for + + break; + + // -- not used vars + case node_state::force: + break; + + // heat transer vars + case node_state::heat_transfer: + break; + // tensors + } // end switch + } // end for over + + + } // end function ///////////////////////////////////////////////////////////////////////////// @@ -4170,12 +6761,12 @@ class MeshWriter /// ///////////////////////////////////////////////////////////////////////////// void write_vtu( - const ViewCArray& node_coords_host, - const ViewCArray& nodes_in_elem_host, - const DCArrayKokkos& elem_scalar_fields, - const DCArrayKokkos& elem_tensor_fields, - const DCArrayKokkos& node_scalar_fields, - const DCArrayKokkos& node_vector_fields, + const DistributedDCArray& node_coords, + const DistributedDCArray& nodes_in_elem, + const DistributedDFArray& elem_scalar_fields, + const DistributedDFArray& elem_tensor_fields, + const DistributedDFArray& node_scalar_fields, + const DistributedDFArray& node_vector_fields, const std::vector& elem_scalar_var_names, const std::vector& elem_tensor_var_names, const std::vector& node_scalar_var_names, @@ -4189,11 +6780,17 @@ class MeshWriter const size_t num_dims, const size_t solver_id ) - { + { + FILE* out[20]; // the output files that are written to char filename[100]; // char string int max_len = sizeof filename; int str_output_len; + int myrank; + MPI_Comm_rank(MPI_COMM_WORLD,&myrank); + std::string str_rank_val = std::to_string(myrank); + std::string rank_fields_name = "rank"; + rank_fields_name += str_rank_val; // add the mat number const size_t num_elem_scalar_vars = elem_scalar_var_names.size(); const size_t num_elem_tensor_vars = elem_tensor_var_names.size(); @@ -4203,8 +6800,8 @@ class MeshWriter // create filename - str_output_len = snprintf(filename, max_len, "vtk/data/Fierro.solver%zu.%s.%05d.vtu", - solver_id, partname.c_str(), graphics_id); + str_output_len = snprintf(filename, max_len, "vtk/data/Fierro.solver%zu.%s_%s.%05d.vtu", + solver_id, partname.c_str(), rank_fields_name.c_str(), graphics_id); if (str_output_len >= max_len) { fputs("Filename length exceeded; string truncated", stderr); } // mesh file @@ -4230,12 +6827,12 @@ class MeshWriter for (size_t node_gid = 0; node_gid < num_nodes; node_gid++) { double coord_z = 0.0; if(num_dims==3){ - coord_z = node_coords_host(node_gid, 2); + coord_z = node_coords.host(node_gid, 2); } fprintf(out[0], " %f %f %f\n", - node_coords_host(node_gid, 0), - node_coords_host(node_gid, 1), + node_coords.host(node_gid, 0), + node_coords.host(node_gid, 1), coord_z); } // end for fprintf(out[0], " \n"); @@ -4268,7 +6865,7 @@ class MeshWriter for (int j = 0; j <= Pn_order; j++) { for (int i = 0; i <= Pn_order; i++) { size_t node_lid = PointIndexFromIJK(i, j, k, order); - fprintf(out[0], "%lu ", nodes_in_elem_host(elem_gid, node_lid)); + fprintf(out[0], "%lu ", nodes_in_elem.host(elem_gid, node_lid)); } } } // end for @@ -4276,13 +6873,13 @@ class MeshWriter else if (num_dims == 3 && Pn_order == 1){ // 3D linear hexahedral elements for (int node_lid = 0; node_lid < 8; node_lid++) { - fprintf(out[0], "%lu ", nodes_in_elem_host(elem_gid, node_lid)); + fprintf(out[0], "%lu ", nodes_in_elem.host(elem_gid, node_lid)); } // end for } else if (num_dims == 2){ // 2D linear is the only supported option for (int node_lid = 0; node_lid < 4; node_lid++) { - fprintf(out[0], "%lu ", nodes_in_elem_host(elem_gid, node_lid)); + fprintf(out[0], "%lu ", nodes_in_elem.host(elem_gid, node_lid)); } // end for } else { @@ -4352,9 +6949,9 @@ class MeshWriter for (size_t node_gid = 0; node_gid < num_nodes; node_gid++) { fprintf(out[0], " %f %f %f\n", - node_vector_fields.host(a_var, node_gid, 0), - node_vector_fields.host(a_var, node_gid, 1), - node_vector_fields.host(a_var, node_gid, 2)); + node_vector_fields.host(node_gid, a_var, 0), + node_vector_fields.host(node_gid, a_var, 1), + node_vector_fields.host(node_gid, a_var, 2)); } // end for nodes fprintf(out[0], " \n"); @@ -4365,7 +6962,7 @@ class MeshWriter for (int a_var = 0; a_var < num_node_scalar_vars; a_var++) { fprintf(out[0], " \n", node_scalar_var_names[a_var].c_str()); for (size_t node_gid = 0; node_gid < num_nodes; node_gid++) { - fprintf(out[0], " %f\n", node_scalar_fields.host(a_var, node_gid)); + fprintf(out[0], " %f\n", node_scalar_fields.host(node_gid, a_var)); } // end for nodes fprintf(out[0], " \n"); } // end for vec_vars @@ -4390,7 +6987,7 @@ class MeshWriter fprintf(out[0], " \n", elem_scalar_var_names[a_var].c_str()); // the 1 is number of scalar components [1:4] for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { - fprintf(out[0], " %f\n", elem_scalar_fields.host(a_var, elem_gid)); + fprintf(out[0], " %f\n", elem_scalar_fields.host(elem_gid, a_var)); } // end for elem fprintf(out[0], " \n"); } // end for elem scalar_vars @@ -4405,7 +7002,7 @@ class MeshWriter // Txx Txy Txz Tyx Tyy Tyz Tzx Tzy Tzz for (size_t i=0; i<3; i++){ for(size_t j=0; j<3; j++){ - fprintf(out[0], " %f ", elem_tensor_fields.host(a_var, elem_gid, i, j)); + fprintf(out[0], " %f ", elem_tensor_fields.host(elem_gid, a_var, i, j)); } // end j } // end i } // end for elem @@ -4496,72 +7093,139 @@ class MeshWriter double time_value, int graphics_id, int num_mats, + int local_num_mats, + CArray local_mats_in_rank, + int nranks, + int myrank, bool write_mesh_state, bool write_mat_pt_state, const size_t solver_id) - { - // loop over all the files that were written - for(int file_id=0; file_id<=graphics_id; file_id++){ + { + //gather number of mats on each rank + //array storing number of local elems for this material on each process + CArray num_mats_in_rank, gatherv_displacements; + CArray interface_mats_in_rank; + if(myrank==0){ + num_mats_in_rank = CArray(nranks); + interface_mats_in_rank = CArray(nranks); + gatherv_displacements = CArray(nranks); + } + MPI_Gather(&local_num_mats,1,MPI_INT,num_mats_in_rank.pointer(),1, + MPI_INT, 0, MPI_COMM_WORLD); + + //gather which mats are present on each rank + long long int length_mats_in_rank = 0; + RaggedRightArray mats_in_rank; + if(myrank==0){ + for(int irank=0; irank < nranks; irank++){ + gatherv_displacements(irank) = length_mats_in_rank; + interface_mats_in_rank(irank) = num_mats_in_rank(irank); + length_mats_in_rank += num_mats_in_rank(irank); + } + mats_in_rank = RaggedRightArray(interface_mats_in_rank); + } + // if(myrank==0){ + // for(int irank=0; irank < nranks; irank++){ + // std::cout << "NUM local mat elem on rank " << irank << " is " << num_mats_in_rank(irank) << std::endl; + // std::cout << "gatherv displacement on rank " << irank << " is " << gatherv_displacements(irank) << std::endl; + // } + // } + MPI_Gatherv(local_mats_in_rank.pointer(), local_num_mats, MPI_INT, + mats_in_rank.pointer(), num_mats_in_rank.pointer(), + gatherv_displacements.pointer(), MPI_INT, 0, MPI_COMM_WORLD); + + + if(myrank==0){ + + //invert map of rank to mat so its mat to rank + CArray num_ranks_in_mat(num_mats); + num_ranks_in_mat.set_values(0); + + //count how many ranks each material is in + for(int irank = 0; irank < nranks; irank++){ + for(int imat = 0; imat < num_mats_in_rank(irank); imat++){ + num_ranks_in_mat(mats_in_rank(irank,imat))++; + } + } - FILE* out[20]; // the output files that are written to - char filename[100]; // char string - int max_len = sizeof filename; - int str_output_len; + //allocate ragged storage and assign ranks to each mat + RaggedRightArray ranks_in_mat(num_ranks_in_mat); + num_ranks_in_mat.set_values(0); + for(int irank = 0; irank < nranks; irank++){ + for(int imat = 0; imat < num_mats_in_rank(irank); imat++){ + ranks_in_mat(mats_in_rank(irank,imat),num_ranks_in_mat(mats_in_rank(irank,imat))) = irank; + num_ranks_in_mat(mats_in_rank(irank,imat))++; + } + } + + // loop over all the files that need to be written + for(int file_id=0; file_id<=graphics_id; file_id++){ - // Write time series metadata to the data file - str_output_len = snprintf(filename, max_len, "vtk/data/Fierro.solver%zu.%05d.vtm", solver_id, file_id); + FILE* out[20]; // the output files that are written to + char filename[100]; // char string + int max_len = sizeof filename; + int str_output_len; - if (str_output_len >= max_len) { fputs("Filename length exceeded; string truncated", stderr); } - // mesh file - out[0] = fopen(filename, "w"); - - fprintf(out[0], "\n"); - fprintf(out[0], "\n"); - fprintf(out[0], " \n"); + // Write time series metadata to the data file + str_output_len = snprintf(filename, max_len, "vtk/data/Fierro.solver%zu.%05d.vtm", solver_id, file_id); - - // Average mesh fields -- node and elem state written - size_t block_id = 0; // this will need to be incremented based on the number of mesh fields written - if (write_mesh_state){ - fprintf(out[0], " \n", block_id); - { - block_id++; // increment block id for material outputs that follow the element avg block + if (str_output_len >= max_len) { fputs("Filename length exceeded; string truncated", stderr); } + // mesh file - // elem and nodal fields are in this file - fprintf(out[0], " \n"); - fprintf(out[0], " \n", - file_id, solver_id, elem_part_name.c_str(), file_id, graphics_times(file_id) ); - fprintf(out[0], " \n"); + out[0] = fopen(filename, "w"); + + fprintf(out[0], "\n"); + fprintf(out[0], "\n"); + fprintf(out[0], " \n"); - // add other Mesh average output Pieces here - } - fprintf(out[0], " \n"); - } // end if write elem and node state is true + + // Average mesh fields -- node and elem state written + size_t block_id = 0; // this will need to be incremented based on the number of mesh fields written + if (write_mesh_state){ + fprintf(out[0], " \n", block_id); + { + block_id++; // increment block id for material outputs that follow the element avg block + + // elem and nodal fields are in this file + fprintf(out[0], " \n"); + for(int irank = 0; irank < nranks; irank++){ + fprintf(out[0], " \n", + irank, solver_id, elem_part_name.c_str(), irank, file_id ); + } + fprintf(out[0], " \n"); - // note: the block_id was incremented if an element average field output was made - if (write_mat_pt_state){ - fprintf(out[0], " \n", block_id); - for (size_t mat_id=0; mat_id\n", mat_id, mat_id); - fprintf(out[0], " \n", - file_id, solver_id, mat_part_name.c_str(), mat_id, file_id, graphics_times(file_id) ); - fprintf(out[0], " \n"); + // add other Mesh average output Pieces here + } + fprintf(out[0], " \n"); + } // end if write elem and node state is true - } // end for loop mat_id - fprintf(out[0], " \n"); - } // end if write mat satte is true + // note: the block_id was incremented if an element average field output was made + if (write_mat_pt_state){ + fprintf(out[0], " \n", block_id); + for (size_t mat_id=0; mat_id\n", mat_id, mat_id); + for(int irank = 0; irank < num_ranks_in_mat(mat_id); irank++){ + fprintf(out[0], " \n", + irank, solver_id, mat_part_name.c_str(), mat_id, ranks_in_mat(mat_id, irank), file_id ); + } + fprintf(out[0], " \n"); - // done writing the files to be read by the vtm file - fprintf(out[0], " \n"); - fprintf(out[0], ""); + } // end for loop mat_id + fprintf(out[0], " \n"); + } // end if write mat satte is true - fclose(out[0]); + // done writing the files to be read by the vtm file + fprintf(out[0], " \n"); + fprintf(out[0], ""); - } // end for file_id + fclose(out[0]); + + } // end for file_id + } } // end vtm @@ -4585,7 +7249,7 @@ class MeshWriter ///////////////////////////////////////////////////////////////////////////// void build_material_elem_node_lists( const Mesh_t& mesh, - const DCArrayKokkos& state_node_coords, + const DistributedDCArray& state_node_coords, DCArrayKokkos& mat_node_coords, DCArrayKokkos & mat_nodes_in_mat_elem, const DRaggedRightArrayKokkos& elem_in_mat_elem, @@ -4666,7 +7330,65 @@ class MeshWriter } // end build part (i.e., material elem and point lists) function + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn build_material_elem_node_lists + /// + /// \brief Creates elems and nodes for a unique mesh of a material (i.e, a part) + /// + /// \param Simulation mesh + /// \param State node data + /// \param Material node coordinates + /// \param Material nodes in the material element + /// \param Material to mesh map for elements + /// \param number of material nodes + /// \param number of material elements + /// \param number of nodes in the element + /// \param number of dimensions + /// + ///////////////////////////////////////////////////////////////////////////// + void build_material_node_list( + const Mesh_t& mesh, + DCArrayKokkos& mat_node_indices, + DistributedDCArray& mat_nodes_in_mat_elem, + const DRaggedRightArrayKokkos& elem_in_mat_elem, + const size_t mat_id, + size_t& num_mat_nodes, + const size_t num_mat_elems, + const size_t num_nodes_in_elem, + const size_t num_dims) + { + + + std::set mat_node_set; + long long int node_gid; + for (int elem_mat_id = 0; elem_mat_id < num_mat_elems; elem_mat_id++) + { + // set nodes per element + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + node_gid = mat_nodes_in_mat_elem.host(elem_mat_id, node_lid); //nodes in elem still stores global indices + mat_node_set.insert(node_gid); + } + } + // save the number of nodes defining the material region, i.e., the part + num_mat_nodes = mat_node_set.size(); + + //copy set to matar view + int inode = 0; + int ighost = 0; + auto it = mat_node_set.begin(); + + // create a Map for ghost node indices + mat_node_indices = DCArrayKokkos(num_mat_nodes, "mat_nodes"); + while (it != mat_node_set.end()) { + mat_node_indices.host(ighost++) = *it; + it++; + } + mat_node_indices.update_device(); + + } // end build part (i.e., material elem and point lists) function @@ -4685,7 +7407,7 @@ class MeshWriter ///////////////////////////////////////////////////////////////////////////// void write_material_point_state(Mesh_t& mesh, State_t& State, - SimulationParameters_t& SimulationParamaters, + SimulationParameters_t& SimulationParameters, double time_value, CArray graphics_times, std::vector node_states, diff --git a/single-node-refactor/src/common/include/region_fill.h b/single-node-refactor/src/common/include/region_fill.h index 964389344..06a4b11e9 100644 --- a/single-node-refactor/src/common/include/region_fill.h +++ b/single-node-refactor/src/common/include/region_fill.h @@ -44,7 +44,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "geometry_new.h" -struct SimulationParamaters_t; +struct SimulationParameters_t; struct Material_t; struct Mesh_t; struct BoundaryCondition_t; @@ -58,7 +58,7 @@ using namespace mtr; // ----------------------------------------------------------------------------- // The functions to setup fields on a mesh // ------------------------------------------------------------------------------ -void simulation_setup(SimulationParameters_t& SimulationParamaters, +void simulation_setup(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, @@ -69,9 +69,9 @@ void simulation_setup(SimulationParameters_t& SimulationParamaters, void fill_regions( const Material_t& Materials, const Mesh_t& mesh, - const DCArrayKokkos & node_coords, - DCArrayKokkos & node_vel, - DCArrayKokkos & node_temp, + const DistributedDCArray & node_coords, + DistributedDCArray & node_vel, + DistributedDCArray & node_temp, DCArrayKokkos & gauss_den, DCArrayKokkos & gauss_sie, DCArrayKokkos & gauss_use_sie, @@ -99,7 +99,7 @@ void fill_regions( // ----------------------------------------------------------------------------- // A function to populate the material point and material zone state // ------------------------------------------------------------------------------ -void material_state_setup(SimulationParameters_t& SimulationParamaters, +void material_state_setup(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, @@ -250,7 +250,7 @@ void paint_multi_scalar(const DCArrayKokkos& field_scalar, /// ///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION -void paint_scalar(const DCArrayKokkos& field_scalar, +void paint_scalar(const DistributedDCArray& field_scalar, const ViewCArrayKokkos mesh_coords, const double scalar, const double slope, @@ -275,7 +275,7 @@ void paint_scalar(const DCArrayKokkos& field_scalar, /// ///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION -void paint_vector(const DCArrayKokkos& vector_field, +void paint_vector(const DistributedDCArray& vector_field, const ViewCArrayKokkos & mesh_coords, const double u, const double v, @@ -386,8 +386,8 @@ void init_press_sspd_stress(const Material_t& Materials, ///////////////////////////////////////////////////////////////////////////// void calc_corner_mass(const Material_t& Materials, const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_mass, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_mass, const DCArrayKokkos& corner_mass, const DRaggedRightArrayKokkos& MaterialPoints_mass, const DRaggedRightArrayKokkos& elem_in_mat_elem, @@ -410,8 +410,8 @@ void calc_corner_mass(const Material_t& Materials, /// ///////////////////////////////////////////////////////////////////////////// void calc_node_mass(const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_mass, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_mass, const DCArrayKokkos& corner_mass); @@ -419,7 +419,7 @@ void calc_node_mass(const Mesh_t& mesh, void init_corner_node_masses_zero( const Mesh_t& mesh, - const DCArrayKokkos& node_mass, + const DistributedDCArray& node_mass, const DCArrayKokkos& corner_mass); #endif \ No newline at end of file diff --git a/single-node-refactor/src/common/include/state.h b/single-node-refactor/src/common/include/state.h index e2d8e3367..cbc204cc1 100644 --- a/single-node-refactor/src/common/include/state.h +++ b/single-node-refactor/src/common/include/state.h @@ -59,6 +59,32 @@ enum class fill_gauss_state level_set }; +//distributed vector type in use +using DistributedMap = TpetraPartitionMap<>; +using HostDistributedMap = TpetraPartitionMap; +template +using DistributedDFArray = TpetraDFArray; +template +using DistributedDCArray = TpetraDCArray; +template +using DistributedCArray = TpetraDCArray; +template +using DistributedFArray = TpetraDFArray; +template +using CommPlan = TpetraLRCommunicationPlan; +template +using OutputCommPlan = TpetraCommunicationPlan; +template +using HostCommPlanLR = TpetraLRCommunicationPlan; +template +using HostCommPlan = TpetraCommunicationPlan; + + +template +void super_vector_initialization(DistributedDCArray super_vector, DistributedDCArray sub_vector, size_t extent); + +template +void super_vector_initialization(DistributedDFArray super_vector, DistributedDFArray sub_vector, size_t extent); ///////////////////////////////////////////////////////////////////////////// /// @@ -269,16 +295,16 @@ enum class node_state ///////////////////////////////////////////////////////////////////////////// struct node_t { - DCArrayKokkos coords; ///< Nodal coordinates - DCArrayKokkos coords_n0; ///< Nodal coordinates at tn=0 of time integration - DCArrayKokkos vel; ///< Nodal velocity - DCArrayKokkos vel_n0; ///< Nodal velocity at tn=0 of time integration - DCArrayKokkos mass; ///< Nodal mass - DCArrayKokkos force; ///< Nodal force - DCArrayKokkos temp; ///< Nodal temperature - DCArrayKokkos temp_n0; ///< Nodal temperature at tn=0 of time integration - DCArrayKokkos q_transfer; ///< Nodal heat flux - DCArrayKokkos gradient_level_set; ///< Nodal gradient of the level set function + DistributedDCArray coords, local_coords, ghost_coords; ///< Nodal coordinates + DistributedDCArray coords_n0, local_coords_n0, ghost_coords_n0; ///< Nodal coordinates at tn=0 of time integration + DistributedDCArray vel, local_vel, ghost_vel; ///< Nodal velocity + DistributedDCArray vel_n0, local_vel_n0, ghost_vel_n0; ///< Nodal velocity at tn=0 of time integration + DistributedDCArray mass, local_mass, ghost_mass; ///< Nodal mass + DistributedDCArray force, local_force, ghost_force; ///< Nodal force + DistributedDCArray temp, local_temp, ghost_temp; ///< Nodal temperature + DistributedDCArray temp_n0, local_temp_n0, ghost_temp_n0; ///< Nodal temperature at tn=0 of time integration + DistributedDCArray q_transfer, local_q_transfer, ghost_q_transfer; ///< Nodal heat flux + DistributedDCArray gradient_level_set, local_gradient_level_set, ghost_gradient_level_set; ///< Nodal gradient of the level set function // initialization method (num_nodes, num_dims, state to allocate) void initialize(size_t num_nodes, size_t num_dims, std::vector node_states) @@ -286,28 +312,28 @@ struct node_t for (auto field : node_states){ switch(field){ case node_state::coords: - if (coords.size() == 0) this->coords = DCArrayKokkos(num_nodes, num_dims, "node_coordinates"); - if (coords_n0.size() == 0) this->coords_n0 = DCArrayKokkos(num_nodes, num_dims, "node_coordinates_n0"); + if (coords.size() == 0) this->coords = DistributedDCArray(num_nodes, num_dims, "node_coordinates"); + if (coords_n0.size() == 0) this->coords_n0 = DistributedDCArray(num_nodes, num_dims, "node_coordinates_n0"); break; case node_state::velocity: - if (vel.size() == 0) this->vel = DCArrayKokkos(num_nodes, num_dims, "node_velocity"); - if (vel_n0.size() == 0) this->vel_n0 = DCArrayKokkos(num_nodes, num_dims, "node_velocity_n0"); + if (vel.size() == 0) this->vel = DistributedDCArray(num_nodes, num_dims, "node_velocity"); + if (vel_n0.size() == 0) this->vel_n0 = DistributedDCArray(num_nodes, num_dims, "node_velocity_n0"); break; case node_state::force: - if (force.size() == 0) this->force = DCArrayKokkos(num_nodes, num_dims, "node_force"); + if (force.size() == 0) this->force = DistributedDCArray(num_nodes, num_dims, "node_force"); break; case node_state::mass: - if (mass.size() == 0) this->mass = DCArrayKokkos(num_nodes, "node_mass"); + if (mass.size() == 0) this->mass = DistributedDCArray(num_nodes, "node_mass"); break; case node_state::temp: - if (temp.size() == 0) this->temp = DCArrayKokkos(num_nodes, "node_temp"); - if (temp_n0.size() == 0) this->temp_n0 = DCArrayKokkos(num_nodes, "node_temp_n0"); + if (temp.size() == 0) this->temp = DistributedDCArray(num_nodes, "node_temp"); + if (temp_n0.size() == 0) this->temp_n0 = DistributedDCArray(num_nodes, "node_temp_n0"); break; case node_state::heat_transfer: - if (q_transfer.size() == 0) this->q_transfer = DCArrayKokkos(num_nodes, "node_q_transfer"); + if (q_transfer.size() == 0) this->q_transfer = DistributedDCArray(num_nodes, "node_q_transfer"); break; case node_state::gradient_level_set: - if (gradient_level_set.size() == 0) this->gradient_level_set = DCArrayKokkos(num_nodes, num_dims, "node_grad_levelset"); + if (gradient_level_set.size() == 0) this->gradient_level_set = DistributedDCArray(num_nodes, num_dims, "node_grad_levelset"); break; default: std::cout<<"Desired node state not understood in node_t initialize"< node_states, DistributedMap subview_map = DistributedMap(), size_t offset=0) + { + if(subview_map.size() == 0){ + for (auto field : node_states){ + switch(field){ + case node_state::coords: + if (coords.size() == 0) this->coords = DistributedDCArray(partitioned_map, num_dims, "node_coordinates"); + if (coords_n0.size() == 0) this->coords_n0 = DistributedDCArray(partitioned_map, num_dims, "node_coordinates_n0"); + break; + case node_state::velocity: + if (vel.size() == 0) this->vel = DistributedDCArray(partitioned_map, num_dims, "node_velocity"); + if (vel_n0.size() == 0) this->vel_n0 = DistributedDCArray(partitioned_map, num_dims, "node_velocity_n0"); + break; + case node_state::force: + if (force.size() == 0) this->force = DistributedDCArray(partitioned_map, num_dims, "node_force"); + break; + case node_state::mass: + if (mass.size() == 0) this->mass = DistributedDCArray(partitioned_map, "node_mass"); + break; + case node_state::temp: + if (temp.size() == 0) this->temp = DistributedDCArray(partitioned_map, "node_temp"); + if (temp_n0.size() == 0) this->temp_n0 = DistributedDCArray(partitioned_map, "node_temp_n0"); + break; + case node_state::heat_transfer: + if (q_transfer.size() == 0) this->q_transfer = DistributedDCArray(partitioned_map, "node_q_transfer"); + break; + case node_state::gradient_level_set: + if (gradient_level_set.size() == 0) this->gradient_level_set = DistributedDCArray(partitioned_map, num_dims, "node_grad_levelset"); + break; + default: + std::cout<<"Desired node state not understood in node_t initialize"<coords.size()==0||this->local_coords.size()==0){ + //store local data with existing managed view made by mesh read for now + if(this->local_coords.size()==0&&this->coords.size()!=0){ + this->local_coords = this->coords; + } + if(this->local_coords_n0.size()==0&&this->coords_n0.size()!=0){ + this->local_coords_n0 = this->coords_n0; + } + //storage for nlocal+nghost + this->coords = DistributedDCArray(partitioned_map, num_dims, "node_coordinates"); + this->coords_n0 = DistributedDCArray(partitioned_map, num_dims, "node_coordinates_n0"); + //assign local data to new storage if local data was allocated + if(this->local_coords.size()!=0){ + super_vector_initialization(this->coords, this->local_coords, subview_map.size()); + } + if(this->local_coords_n0.size()!=0){ + super_vector_initialization(this->coords_n0, this->local_coords_n0, subview_map.size()); + } + //replace local data storage with subview of nlocal+nghost; previous managed view should self-destruct here + this->local_coords = DistributedDCArray(this->coords,subview_map); + this->local_coords_n0 = DistributedDCArray(this->coords_n0,subview_map); + } + break; + case node_state::velocity: + //if both local and all vector were already allocated skip + if(this->vel.size()==0||this->local_vel.size()==0){ + //store local data with existing managed view made by mesh read for now + if(this->local_vel.size()==0&&this->vel.size()!=0){ + this->local_vel = this->vel; + } + if(this->local_vel_n0.size()==0&&this->vel_n0.size()!=0){ + this->local_vel_n0 = this->vel_n0; + } + //storage for nlocal+nghost + this->vel = DistributedDCArray(partitioned_map, num_dims, "node_velocity"); + this->vel_n0 = DistributedDCArray(partitioned_map, num_dims, "node_velocity_n0"); + //assign local data to new storage + if(this->local_vel.size()!=0){ + super_vector_initialization(this->vel, this->local_vel, subview_map.size()); + } + if(this->local_vel_n0.size()!=0){ + super_vector_initialization(this->vel_n0, this->local_vel_n0, subview_map.size()); + } + //replace local data storage with subview of nlocal+nghost; previous managed view should self-destruct here + this->local_vel = DistributedDCArray(this->vel,subview_map); + this->local_vel_n0 = DistributedDCArray(this->vel_n0,subview_map); + } + break; + case node_state::force: + //if both local and all vector were already allocated skip + if(this->force.size()==0||this->local_force.size()==0){ + //store local data with existing managed view made by mesh read for now + if(this->local_force.size()==0&&this->force.size()!=0){ + this->local_force = this->force; + } + //storage for nlocal+nghost + this->force = DistributedDCArray(partitioned_map, num_dims, "node_force"); + //assign local data to new storage + if(this->local_force.size()!=0){ + super_vector_initialization(this->force, this->local_force, subview_map.size()); + } + //replace local data storage with subview of nlocal+nghost; previous managed view should self-destruct here + this->local_force = DistributedDCArray(this->force,subview_map); + } + break; + case node_state::mass: + //if both local and all vector were already allocated skip + if(this->mass.size()==0||this->local_mass.size()==0){ + //store local data with existing managed view made by mesh read for now + if(this->local_mass.size()==0&&this->mass.size()!=0){ + this->local_mass = this->mass; + } + //storage for nlocal+nghost + this->mass = DistributedDCArray(partitioned_map, "node_mass"); + //assign local data to new storage + if(this->local_mass.size()!=0){ + super_vector_initialization(this->mass, this->local_mass, subview_map.size()); + } + //replace local data storage with subview of nlocal+nghost; previous managed view should self-destruct here + this->local_mass = DistributedDCArray(this->mass,subview_map); + } + break; + case node_state::temp: + //if both local and all vector were already allocated skip + if(this->temp.size()==0||this->local_temp.size()==0){ + //store local data with existing managed view made by mesh read for now + if(this->local_temp.size()==0&&this->temp.size()!=0){ + this->local_temp = this->temp; + } + if(this->local_temp_n0.size()==0&&this->temp_n0.size()!=0){ + this->local_temp_n0 = this->temp_n0; + } + //storage for nlocal+nghost + this->temp = DistributedDCArray(partitioned_map, "node_temp"); + this->temp_n0 = DistributedDCArray(partitioned_map, "node_temp_n0"); + //assign local data to new storage + if(this->local_temp.size()!=0){ + super_vector_initialization(this->temp, this->local_temp, subview_map.size()); + } + if(this->local_temp_n0.size()!=0){ + super_vector_initialization(this->temp_n0, this->local_temp_n0, subview_map.size()); + } + //replace local data storage with subview of nlocal+nghost; previous managed view should self-destruct here + this->local_temp = DistributedDCArray(this->temp,subview_map); + this->local_temp_n0 = DistributedDCArray(this->temp_n0,subview_map); + } + break; + case node_state::heat_transfer: + //if both local and all vector were already allocated skip + if(this->q_transfer.size()==0||this->local_q_transfer.size()==0){ + //store local data with existing managed view made by mesh read for now + if(this->local_q_transfer.size()==0&&this->q_transfer.size()!=0){ + this->local_q_transfer = this->q_transfer; + } + //storage for nlocal+nghost + this->q_transfer = DistributedDCArray(partitioned_map, "node_q_transfer"); + + //assign local data to new storage + if(this->local_q_transfer.size()!=0){ + super_vector_initialization(this->q_transfer, this->local_q_transfer, subview_map.size()); + } + //replace local data storage with subview of nlocal+nghost; previous managed view should self-destruct here + this->local_q_transfer = DistributedDCArray(this->q_transfer,subview_map); + } + break; + case node_state::gradient_level_set: + //if both local and all vector were already allocated skip + if(this->gradient_level_set.size()==0||this->local_gradient_level_set.size()==0){ + //store local data with existing managed view made by mesh read for now + if(this->local_gradient_level_set.size()==0&&this->gradient_level_set.size()!=0){ + this->local_gradient_level_set = this->gradient_level_set; + } + //storage for nlocal+nghost + this->gradient_level_set = DistributedDCArray(partitioned_map, num_dims, "node_grad_levelset"); + //assign local data to new storage + if(this->gradient_level_set.size()!=0){ + super_vector_initialization(this->gradient_level_set, this->local_gradient_level_set, subview_map.size()); + } + //replace local data storage with subview of nlocal+nghost; previous managed view should self-destruct here + this->local_gradient_level_set = DistributedDCArray(this->gradient_level_set,subview_map); + } + break; + default: + std::cout<<"Desired node state not understood in node_t initialize"< num_mat_elems; ///< returns the exact number of matpts DCArrayKokkos num_mat_elems_buffer; ///< returns the number of matpts plus buffer + DCArrayKokkos num_mat_local_elems; ///< returns the exact number of matpts DRaggedRightArrayKokkos elem_in_mat_elem; ///< returns the elem for this material @@ -439,6 +660,12 @@ struct MaterialToMeshMap_t this->num_mat_elems_buffer = DCArrayKokkos (num_mats, "num_mat_elems_with_buffer"); } + // Note: num_mat_elems is allocated in problem setup + if (num_mat_local_elems.size() == 0){ + this->num_mat_local_elems = DCArrayKokkos (num_mats, "num_mat_local_elems"); + } + + }; // end method }; // end MaterialtoMeshMaps_t @@ -473,6 +700,7 @@ enum class material_pt_state struct MaterialPoint_t { DCArrayKokkos num_material_points; ///< the actual number of material points, omitting the buffer + DCArrayKokkos num_material_local_points; ///< the actual number of material points, omitting the buffer DCArrayKokkos num_material_points_buffer; ///< number of material points plus a buffer DRaggedRightArrayKokkos den; ///< MaterialPoint density @@ -520,6 +748,10 @@ struct MaterialPoint_t this->num_material_points_buffer = DCArrayKokkos (num_mats, "num_material_points_with_buffer"); } + if (num_material_local_points.size() == 0){ + this->num_material_local_points = DCArrayKokkos (num_mats, "num_material_local_points"); + } + }; // end method // initialization method (num_dims) @@ -949,7 +1181,23 @@ struct State_t MaterialZone_t MaterialZones; ///< access as MaterialZones.var(mat_id, mat_zone), only used with arbitrary-order FE }; // end state_t - +template +void super_vector_initialization(DistributedDCArray super_vector, DistributedDCArray sub_vector, size_t extent){ + FOR_ALL(i, 0, extent, { + for (size_t dim = 0; dim < super_vector.component_length(); dim++) { + super_vector(i, dim) = sub_vector(i, dim); + } + }); // end parallel for corners +} + +template +void super_vector_initialization(DistributedDFArray super_vector, DistributedDFArray sub_vector, size_t extent){ + FOR_ALL(i, 0, extent, { + for (size_t dim = 0; dim < super_vector.component_length(); dim++) { + super_vector(i, dim) = sub_vector(i, dim); + } + }); // end parallel for corners +} diff --git a/single-node-refactor/src/common/src/geometry_new.cpp b/single-node-refactor/src/common/src/geometry_new.cpp index ea89773b1..d4c61243e 100644 --- a/single-node-refactor/src/common/src/geometry_new.cpp +++ b/single-node-refactor/src/common/src/geometry_new.cpp @@ -61,7 +61,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KOKKOS_FUNCTION void geometry::get_bmatrix(const ViewCArrayKokkos& B_matrix, const size_t elem_gid, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const ViewCArrayKokkos& elem_node_gids) { const size_t num_nodes = 8; @@ -269,7 +269,7 @@ void geometry::get_bmatrix(const ViewCArrayKokkos& B_matrix, KOKKOS_FUNCTION void geometry::get_vol_quad(const DCArrayKokkos& elem_vol, const size_t elem_gid, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const ViewCArrayKokkos& elem_node_gids) { elem_vol(elem_gid) = 0.0; @@ -315,7 +315,7 @@ void geometry::get_vol_quad(const DCArrayKokkos& elem_vol, KOKKOS_FUNCTION void geometry::get_vol_hex(const DCArrayKokkos& elem_vol, const size_t elem_gid, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const ViewCArrayKokkos& elem_node_gids) { const size_t num_nodes = 8; @@ -365,7 +365,7 @@ void geometry::get_vol_hex(const DCArrayKokkos& elem_vol, /// ///////////////////////////////////////////////////////////////////////////// void geometry::get_vol(const DCArrayKokkos& elem_vol, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const Mesh_t& mesh) { const size_t num_dims = mesh.num_dims; @@ -406,7 +406,7 @@ void geometry::get_vol(const DCArrayKokkos& elem_vol, KOKKOS_FUNCTION void geometry::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, const size_t elem_gid, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const ViewCArrayKokkos& elem_node_gids) { const size_t num_nodes = 4; @@ -481,7 +481,7 @@ void geometry::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, ///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION double geometry::get_area_quad(const size_t elem_gid, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const ViewCArrayKokkos& elem_node_gids) { double elem_area = 0.0; @@ -567,7 +567,7 @@ double geometry::heron(const double x1, KOKKOS_FUNCTION void geometry::get_area_weights2D(const ViewCArrayKokkos& corner_areas, const size_t elem_gid, - const DCArrayKokkos& node_coords, + const DistributedDCArray& node_coords, const ViewCArrayKokkos& elem_node_gids) { const size_t num_nodes = 4; @@ -631,7 +631,7 @@ size_t check_bdy(const size_t patch_gid, const double orig_y, const double orig_z, const Mesh_t& mesh, - const DCArrayKokkos& node_coords) + const DistributedDCArray& node_coords) { size_t num_dims = mesh.num_dims; @@ -714,7 +714,7 @@ size_t check_bdy(const size_t patch_gid, ///////////////////////////////////////////////////////////////////////////// void tag_bdys(const BoundaryCondition_t& boundary, Mesh_t& mesh, - const DCArrayKokkos& node_coords) + const DistributedDCArray& node_coords) { // create a temporary storage for the bdy patches in a set diff --git a/single-node-refactor/src/common/src/region_fill.cpp b/single-node-refactor/src/common/src/region_fill.cpp index 34c9a2d7f..332741596 100644 --- a/single-node-refactor/src/common/src/region_fill.cpp +++ b/single-node-refactor/src/common/src/region_fill.cpp @@ -47,7 +47,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -void simulation_setup(SimulationParameters_t& SimulationParamaters, +void simulation_setup(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, @@ -59,11 +59,12 @@ void simulation_setup(SimulationParameters_t& SimulationParamaters, // the number of elems and nodes in the mesh const size_t num_dims = mesh.num_dims; const size_t num_elems = mesh.num_elems; + const size_t num_local_elems = mesh.num_local_elems; const size_t num_nodes = mesh.num_nodes; const size_t num_gauss_points = mesh.num_gauss_in_elem*mesh.num_elems; const size_t num_mats = Materials.num_mats; // the number of materials on the mesh - + // Calculate element volume geometry::get_vol(State.GaussPoints.vol, State.node.coords, mesh); @@ -78,7 +79,7 @@ void simulation_setup(SimulationParameters_t& SimulationParamaters, fillGaussState.initialize(num_gauss_points, max_num_mats_per_elem, 3, - SimulationParamaters.region_setups.fill_gauss_states); + SimulationParameters.region_setups.fill_gauss_states); // the elem state is always used, thus always initialized fillElemState.initialize(num_elems, @@ -118,11 +119,11 @@ void simulation_setup(SimulationParameters_t& SimulationParamaters, State.MeshtoMaterialMaps.mats_in_elem, State.MeshtoMaterialMaps.num_mats_in_elem, voxel_elem_mat_id, - SimulationParamaters.mesh_input.object_ids, - SimulationParamaters.region_setups.region_fills, - SimulationParamaters.region_setups.region_fills_host, - SimulationParamaters.region_setups.fill_gauss_states, - SimulationParamaters.region_setups.fill_node_states, + SimulationParameters.mesh_input.object_ids, + SimulationParameters.region_setups.region_fills, + SimulationParameters.region_setups.region_fills_host, + SimulationParameters.region_setups.fill_gauss_states, + SimulationParameters.region_setups.fill_node_states, max_num_mats_per_elem); @@ -136,6 +137,7 @@ void simulation_setup(SimulationParameters_t& SimulationParamaters, // a counter for the Material index spaces DCArrayKokkos num_elems_saved_for_mat(num_mats, "num_elems_saved_for_mat"); + DCArrayKokkos num_local_elems_saved_for_mat(num_mats, "num_local_elems_saved_for_mat"); for (int mat_id = 0; mat_id < num_mats; mat_id++) { size_t sum_local; @@ -159,7 +161,30 @@ void simulation_setup(SimulationParameters_t& SimulationParamaters, num_elems_saved_for_mat.host(mat_id) = sum_total; } // end for + for (int mat_id = 0; mat_id < num_mats; mat_id++) { + size_t sum_local; + size_t sum_total; + + FOR_REDUCE_SUM(elem_gid, 0, num_local_elems, sum_local, { + + // loop over the materials in the element + for (size_t a_mat_in_elem=0; a_mat_in_elem < State.MeshtoMaterialMaps.num_mats_in_elem(elem_gid); a_mat_in_elem++){ + + // check to see if it is mat_id + if (State.MeshtoMaterialMaps.mats_in_elem(elem_gid, a_mat_in_elem) == mat_id) { + // increment the number of elements the materials live in + sum_local++; + } // end if a_mat is equal to mat_id + + } // end loop over materials in elem + }, sum_total); + + // material index space size + num_local_elems_saved_for_mat.host(mat_id) = sum_total; + } // end for + num_elems_saved_for_mat.update_device(); + num_local_elems_saved_for_mat.update_device(); Kokkos::fence(); @@ -186,6 +211,8 @@ void simulation_setup(SimulationParameters_t& SimulationParamaters, // The exact size plus a buffer is for e.g., remap. The buffers are shortly below here. State.MaterialToMeshMaps.num_mat_elems.host(mat_id) = num_elems_saved_for_mat.host(mat_id); State.MaterialPoints.num_material_points.host(mat_id) = num_elems_saved_for_mat.host(mat_id) * num_mat_pts_in_elem; + State.MaterialToMeshMaps.num_mat_local_elems.host(mat_id) = num_local_elems_saved_for_mat.host(mat_id); + State.MaterialPoints.num_material_local_points.host(mat_id) = num_local_elems_saved_for_mat.host(mat_id) * num_mat_pts_in_elem; State.MaterialCorners.num_material_corners.host(mat_id) = num_elems_saved_for_mat.host(mat_id) * mesh.num_nodes_in_elem; State.MaterialZones.num_material_zones.host(mat_id) = num_elems_saved_for_mat.host(mat_id) * mesh.num_zones_in_elem; @@ -202,6 +229,8 @@ void simulation_setup(SimulationParameters_t& SimulationParamaters, // copy to device the actual sizes State.MaterialToMeshMaps.num_mat_elems.update_device(); State.MaterialPoints.num_material_points.update_device(); + State.MaterialToMeshMaps.num_mat_local_elems.update_device(); + State.MaterialPoints.num_material_local_points.update_device(); State.MaterialCorners.num_material_corners.update_device(); State.MaterialZones.num_material_zones.update_device(); @@ -249,9 +278,9 @@ void simulation_setup(SimulationParameters_t& SimulationParamaters, void fill_regions( const Material_t& Materials, const Mesh_t& mesh, - const DCArrayKokkos & node_coords, - DCArrayKokkos & node_vel, - DCArrayKokkos & node_temp, + const DistributedDCArray & node_coords, + DistributedDCArray & node_vel, + DistributedDCArray & node_temp, DCArrayKokkos & gauss_den, DCArrayKokkos & gauss_sie, DCArrayKokkos & gauss_use_sie, @@ -670,7 +699,7 @@ void fill_regions( /// /// \brief a function to setup the material point and zone state /// -/// \param SimulationParamaters holds the simulation parameters +/// \param SimulationParameters holds the simulation parameters /// \param Materials is the material object /// \param mesh is the mesh object /// \param Boundary is the boundary condition object @@ -679,7 +708,7 @@ void fill_regions( /// \param fillElemState is a vector of enums telling what elem state to set /// ///////////////////////////////////////////////////////////////////////////// -void material_state_setup(SimulationParameters_t& SimulationParamaters, +void material_state_setup(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, @@ -772,6 +801,7 @@ void material_state_setup(SimulationParameters_t& SimulationParamaters, State.MaterialPoints.mass.host(mat_id,mat_point_sid) = fillGaussState.den.host(gauss_gid,a_mat_in_elem) * mat_vol; + } // --- set eroded flag to false --- @@ -1706,7 +1736,7 @@ void paint_multi_scalar(const DCArrayKokkos& field_scalar, /// ///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION -void paint_scalar(const DCArrayKokkos& field_scalar, +void paint_scalar(const DistributedDCArray& field_scalar, const ViewCArrayKokkos mesh_coords, const double scalar, const double slope, @@ -1836,7 +1866,7 @@ void paint_scalar(const DCArrayKokkos& field_scalar, /// ///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION -void paint_vector(const DCArrayKokkos& vector_field, +void paint_vector(const DistributedDCArray& vector_field, const ViewCArrayKokkos & mesh_coords, const double u, const double v, @@ -2262,8 +2292,8 @@ void init_press_sspd_stress(const Material_t& Materials, ///////////////////////////////////////////////////////////////////////////// void calc_corner_mass(const Material_t& Materials, const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_mass, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_mass, const DCArrayKokkos& corner_mass, const DRaggedRightArrayKokkos& MaterialPoints_mass, const DRaggedRightArrayKokkos& elem_in_mat_elem, @@ -2308,8 +2338,8 @@ void calc_corner_mass(const Material_t& Materials, /// ///////////////////////////////////////////////////////////////////////////// void calc_node_mass(const Mesh_t& mesh, - const DCArrayKokkos& node_coords, - const DCArrayKokkos& node_mass, + const DistributedDCArray& node_coords, + const DistributedDCArray& node_mass, const DCArrayKokkos& corner_mass) { @@ -2338,7 +2368,7 @@ void calc_node_mass(const Mesh_t& mesh, /// ///////////////////////////////////////////////////////////////////////////// void init_corner_node_masses_zero(const Mesh_t& mesh, - const DCArrayKokkos& node_mass, + const DistributedDCArray& node_mass, const DCArrayKokkos& corner_mass) { // calculate the nodal mass diff --git a/single-node-refactor/src/driver.cpp b/single-node-refactor/src/driver.cpp index aecaca973..d39ab723f 100644 --- a/single-node-refactor/src/driver.cpp +++ b/single-node-refactor/src/driver.cpp @@ -42,6 +42,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "level_set_solver.h" #include "region_fill.h" +#include @@ -50,7 +51,13 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Will be parsed from YAML input void Driver::initialize() { - std::cout << "Initializing Driver" << std::endl; + + MPI_Comm_rank(MPI_COMM_WORLD,&myrank); + MPI_Comm_size(MPI_COMM_WORLD,&nranks); + + if(myrank == 0){ + std::cout << "Initializing Driver" << std::endl; + } Yaml::Node root; try { @@ -58,53 +65,76 @@ void Driver::initialize() } catch (const Yaml::Exception e) { - std::cout << "Exception " << e.Type() << ": " << e.what() << std::endl; + if(myrank == 0){ + std::cout << "Exception " << e.Type() << ": " << e.what() << std::endl; + } + + MPI_Barrier(MPI_COMM_WORLD); + MPI_Finalize(); exit(0); } - parse_yaml(root, SimulationParamaters, Materials, BoundaryConditions); - std::cout << "Finished parsing YAML file" << std::endl; + parse_yaml(root, SimulationParameters, Materials, BoundaryConditions); + + if(myrank == 0){ + std::cout << "Finished parsing YAML file" << std::endl; + } - if (SimulationParamaters.mesh_input.source == mesh_input::file) { + if (SimulationParameters.mesh_input.source == mesh_input::file) { // Create and/or read mesh - std::cout << "Mesh file path: " << SimulationParamaters.mesh_input.file_path << std::endl; - mesh_reader.set_mesh_file(SimulationParamaters.mesh_input.file_path.data()); + if(myrank == 0){ + std::cout << "Mesh file path: " << SimulationParameters.mesh_input.file_path << std::endl; + } + mesh_reader.set_mesh_file(SimulationParameters.mesh_input.file_path.data()); + mesh.num_dims = num_dims; mesh_reader.read_mesh(mesh, State, - SimulationParamaters.mesh_input, + SimulationParameters.mesh_input, num_dims); } - else if (SimulationParamaters.mesh_input.source == mesh_input::generate) { + else if (SimulationParameters.mesh_input.source == mesh_input::generate) { mesh_builder.build_mesh(mesh, State.GaussPoints, State.node, State.corner, - SimulationParamaters); + SimulationParameters); } else{ throw std::runtime_error("**** NO MESH INPUT OPTIONS PROVIDED IN YAML ****"); - return; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Finalize(); + exit(0); } + //build relevant partition maps for ghost nodes, elements, etc. + mesh.init_maps(State.node, SimulationParameters.mesh_input); + + // Build connectivity + mesh.build_connectivity(); + // Build boundary conditions const int num_bcs = BoundaryConditions.num_bcs; + //make bcs MPI parallel? // --- calculate bdy sets ---// mesh.init_bdy_sets(num_bcs); tag_bdys(BoundaryConditions, mesh, State.node.coords); + build_boundry_node_sets(mesh); // Setup the Solvers - double time_final = SimulationParamaters.dynamic_options.time_final; - for (size_t solver_id = 0; solver_id < SimulationParamaters.solver_inputs.size(); solver_id++) { + double time_final = SimulationParameters.dynamic_options.time_final; + for (size_t solver_id = 0; solver_id < SimulationParameters.solver_inputs.size(); solver_id++) { - if (SimulationParamaters.solver_inputs[solver_id].method == solver_input::SGH3D) { - - std::cout << "Initializing dynx_FE solver" << std::endl; + if (SimulationParameters.solver_inputs[solver_id].method == solver_input::SGH3D) { + + if(myrank == 0){ + std::cout << "Initializing dynx_FE solver" << std::endl; + } SGH3D* sgh_solver = new SGH3D(); - sgh_solver->initialize(SimulationParamaters, + sgh_solver->initialize(SimulationParameters, Materials, mesh, BoundaryConditions, @@ -118,12 +148,14 @@ void Driver::initialize() } // end if SGH solver - else if (SimulationParamaters.solver_inputs[solver_id].method == solver_input::SGHRZ) { - - std::cout << "Initializing dynx_FE_RZ solver" << std::endl; + else if (SimulationParameters.solver_inputs[solver_id].method == solver_input::SGHRZ) { + + if(myrank == 0){ + std::cout << "Initializing dynx_FE_RZ solver" << std::endl; + } SGHRZ* sgh_solver_rz = new SGHRZ(); - sgh_solver_rz->initialize(SimulationParamaters, + sgh_solver_rz->initialize(SimulationParameters, Materials, mesh, BoundaryConditions, @@ -135,12 +167,14 @@ void Driver::initialize() solvers.push_back(sgh_solver_rz); } // end if SGHRZ solver - else if (SimulationParamaters.solver_inputs[solver_id].method == solver_input::SGTM3D) { - - std::cout << "Initializing thrmex_FE solver" << std::endl; + else if (SimulationParameters.solver_inputs[solver_id].method == solver_input::SGTM3D) { + + if(myrank == 0){ + std::cout << "Initializing thrmex_FE solver" << std::endl; + } SGTM3D* sgtm_solver_3d = new SGTM3D(); - sgtm_solver_3d->initialize(SimulationParamaters, + sgtm_solver_3d->initialize(SimulationParameters, Materials, mesh, BoundaryConditions, @@ -153,12 +187,12 @@ void Driver::initialize() solvers.push_back(sgtm_solver_3d); } // end if SGTM solver - else if (SimulationParamaters.solver_inputs[solver_id].method == solver_input::levelSet) { + else if (SimulationParameters.solver_inputs[solver_id].method == solver_input::levelSet) { std::cout << "Initializing level set solver" << std::endl; LevelSet* level_set_solver = new LevelSet(); - level_set_solver->initialize(SimulationParamaters, + level_set_solver->initialize(SimulationParameters, Materials, mesh, BoundaryConditions, @@ -173,7 +207,9 @@ void Driver::initialize() } // end if level set solver else { throw std::runtime_error("**** NO SOLVER INPUT OPTIONS PROVIDED IN YAML, OR OPTION NOT UNDERSTOOD ****"); - return; + MPI_Barrier(MPI_COMM_WORLD); + MPI_Finalize(); + exit(0); } } // end for loop over solvers @@ -185,7 +221,7 @@ void Driver::initialize() fillGaussState_t fillGaussState; fillElemState_t fillElemState; - simulation_setup(SimulationParamaters, + simulation_setup(SimulationParameters, Materials, mesh, BoundaryConditions, @@ -196,7 +232,7 @@ void Driver::initialize() // Allocate material state for (auto& solver : solvers) { - solver->initialize_material_state(SimulationParamaters, + solver->initialize_material_state(SimulationParameters, Materials, mesh, BoundaryConditions, @@ -205,7 +241,7 @@ void Driver::initialize() // populate the material point state - material_state_setup(SimulationParamaters, + material_state_setup(SimulationParameters, Materials, mesh, BoundaryConditions, @@ -222,12 +258,14 @@ void Driver::initialize() /// ///////////////////////////////////////////////////////////////////////////// void Driver::setup() -{ - std::cout << "Inside driver setup" << std::endl; +{ + if(myrank == 0){ + std::cout << "Inside driver setup" << std::endl; + } // allocate state, setup models, and apply fill instructions for (auto& solver : solvers) { - solver->setup(SimulationParamaters, + solver->setup(SimulationParameters, Materials, mesh, BoundaryConditions, @@ -245,10 +283,12 @@ void Driver::setup() /// ///////////////////////////////////////////////////////////////////////////// void Driver::execute() -{ - std::cout << "Inside driver execute" << std::endl; +{ + if(myrank == 0){ + std::cout << "Inside driver execute" << std::endl; + } for (auto& solver : solvers) { - solver->execute(SimulationParamaters, + solver->execute(SimulationParameters, Materials, BoundaryConditions, mesh, @@ -267,18 +307,22 @@ void Driver::execute() /// ///////////////////////////////////////////////////////////////////////////// void Driver::finalize() -{ - std::cout << "Inside driver finalize" << std::endl; +{ + if(myrank == 0){ + std::cout << "Inside driver finalize" << std::endl; + } for (auto& solver : solvers) { if (solver->finalize_flag) { - solver->finalize(SimulationParamaters, + solver->finalize(SimulationParameters, Materials, BoundaryConditions); } } // destroy FEA modules for (auto& solver : solvers) { - std::cout << "Deleting solver" << std::endl; + if(myrank == 0){ + std::cout << "Deleting solver" << std::endl; + } delete solver; } } @@ -297,7 +341,7 @@ void Driver::setup_solver_vars(T& a_solver, // the final time of the simulation - double time_final = this->SimulationParamaters.dynamic_options.time_final; + double time_final = this->SimulationParameters.dynamic_options.time_final; // save the solver_id a_solver->solver_id = solver_id; @@ -309,7 +353,7 @@ void Driver::setup_solver_vars(T& a_solver, // setting the ending times are tricky, requiring logic // set the start and ending times - double t_end = this->SimulationParamaters.solver_inputs[solver_id].time_end; // default is t=0 + double t_end = this->SimulationParameters.solver_inputs[solver_id].time_end; // default is t=0 if(solver_id==0){ a_solver->time_start = 0.0; diff --git a/single-node-refactor/src/driver.h b/single-node-refactor/src/driver.h index 7f99ec3ae..5a52c5f58 100644 --- a/single-node-refactor/src/driver.h +++ b/single-node-refactor/src/driver.h @@ -65,7 +65,7 @@ class Driver MeshReader mesh_reader; MeshBuilder mesh_builder; - SimulationParameters_t SimulationParamaters; ///< the input simulation parameters + SimulationParameters_t SimulationParameters; ///< the input simulation parameters // --------------------------------------------------------------------- // Material and Boundary declarations @@ -91,6 +91,9 @@ class Driver // set of enabled solvers std::vector solvers; + //MPI data + int myrank, nranks; + Driver(char* YAML) { yaml_file = YAML; diff --git a/single-node-refactor/src/input/parse_material_inputs.hpp b/single-node-refactor/src/input/parse_material_inputs.hpp index 694f9a988..f17bd01d6 100644 --- a/single-node-refactor/src/input/parse_material_inputs.hpp +++ b/single-node-refactor/src/input/parse_material_inputs.hpp @@ -53,7 +53,8 @@ struct MaterialFunctions_t; struct MaterialEnums_t; using namespace mtr; - +template +using DistributedDCArray = TpetraDCArray; // parse the material text void parse_materials(Yaml::Node& root, Material_t& Materials, const size_t num_dims); diff --git a/single-node-refactor/src/input/parse_yaml.cpp b/single-node-refactor/src/input/parse_yaml.cpp index 5f2875ae8..5cd49737e 100644 --- a/single-node-refactor/src/input/parse_yaml.cpp +++ b/single-node-refactor/src/input/parse_yaml.cpp @@ -79,34 +79,34 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // ================================================================================= // Parse YAML file // ================================================================================= -void parse_yaml(Yaml::Node& root, SimulationParameters_t& SimulationParamaters, Material_t& Materials, BoundaryCondition_t& Boundary) +void parse_yaml(Yaml::Node& root, SimulationParameters_t& SimulationParameters, Material_t& Materials, BoundaryCondition_t& Boundary) { - parse_mesh_inputs(root, SimulationParamaters.mesh_input); + parse_mesh_inputs(root, SimulationParameters.mesh_input); - parse_dynamic_options(root, SimulationParamaters.dynamic_options); + parse_dynamic_options(root, SimulationParameters.dynamic_options); - parse_output_options(root, SimulationParamaters.output_options); + parse_output_options(root, SimulationParameters.output_options); - parse_solver_input(root, SimulationParamaters.solver_inputs); + parse_solver_input(root, SimulationParameters.solver_inputs); // parse the region yaml text into a vector of boundary conditions - size_t num_solvers = SimulationParamaters.solver_inputs.size(); + size_t num_solvers = SimulationParameters.solver_inputs.size(); parse_bcs(root, Boundary, num_solvers); // parse the region yaml text into a vector of region_fills parse_regions(root, - SimulationParamaters.region_setups.reg_fills_in_solver, - SimulationParamaters.region_setups.num_reg_fills_in_solver, - SimulationParamaters.region_setups.region_fills, - SimulationParamaters.region_setups.region_fills_host, - SimulationParamaters.region_setups.fill_gauss_states, - SimulationParamaters.region_setups.fill_node_states, + SimulationParameters.region_setups.reg_fills_in_solver, + SimulationParameters.region_setups.num_reg_fills_in_solver, + SimulationParameters.region_setups.region_fills, + SimulationParameters.region_setups.region_fills_host, + SimulationParameters.region_setups.fill_gauss_states, + SimulationParameters.region_setups.fill_node_states, num_solvers); // parse the material yaml text into a vector of materials - parse_materials(root, Materials, SimulationParamaters.mesh_input.num_dims); + parse_materials(root, Materials, SimulationParameters.mesh_input.num_dims); parse_multimaterial_options(root, Materials); } diff --git a/single-node-refactor/src/input/parse_yaml.hpp b/single-node-refactor/src/input/parse_yaml.hpp index 19cc1cf85..ed5ef4496 100644 --- a/single-node-refactor/src/input/parse_yaml.hpp +++ b/single-node-refactor/src/input/parse_yaml.hpp @@ -55,7 +55,7 @@ using namespace mtr; // utility function for parsing YAML file -void parse_yaml(Yaml::Node& root, SimulationParameters_t& SimulationParamaters, Material_t& Materials, BoundaryCondition_t& Boundary); +void parse_yaml(Yaml::Node& root, SimulationParameters_t& SimulationParameters, Material_t& Materials, BoundaryCondition_t& Boundary); #endif // end Header Guard \ No newline at end of file diff --git a/single-node-refactor/src/main.cpp b/single-node-refactor/src/main.cpp index f3b1db91d..543c0c90a 100644 --- a/single-node-refactor/src/main.cpp +++ b/single-node-refactor/src/main.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include "matar.h" #include "driver.h" @@ -51,21 +52,33 @@ /// ///////////////////////////////////////////////////////////////////////////// int main(int argc, char* argv[]) -{ +{ + // initialize MPI + MPI_Init(&argc, &argv); + int myrank, nranks; + MPI_Comm_rank(MPI_COMM_WORLD,&myrank); + MPI_Comm_size(MPI_COMM_WORLD,&nranks); + bool will_run = false; + // check to see of an input file was supplied when running the code if (argc == 1) { - std::cout << "\n\n**********************************\n\n"; - std::cout << " ERROR:\n"; - std::cout << " Please supply a YAML input, \n"; - std::cout << " ./Fierro input.yaml \n\n"; - std::cout << "**********************************\n\n" << std::endl; + if(myrank == 0){ + std::cout << "\n\n**********************************\n\n"; + std::cout << " ERROR:\n"; + std::cout << " Please supply a YAML input, \n"; + std::cout << " mpirun -np n Fierro input.yaml \n\n"; + std::cout << "**********************************\n\n" << std::endl; + } + + MPI_Finalize(); return 0; } // end if if (std::string(argv[1]) == "--help"){ - - print_inputs(); - + if(myrank == 0){ + print_inputs(); + } + MPI_Finalize(); return 0; } @@ -75,7 +88,6 @@ int main(int argc, char* argv[]) // Create driver Driver* driver = new Driver(argv[1]); - // Timing data for each step auto time_start = std::chrono::high_resolution_clock::now(); auto time_init = std::chrono::high_resolution_clock::now(); @@ -84,28 +96,35 @@ int main(int argc, char* argv[]) auto time_now = std::chrono::high_resolution_clock::now(); auto calc_time = std::chrono::duration_cast(time_now - time_init).count(); - printf("\n**** Total time to initialize driver in seconds %f ****\n\n", calc_time * 1e-9); + if(myrank == 0){ + printf("\n**** Total time to initialize driver in seconds %f ****\n\n", calc_time * 1e-9); + } auto time_setup = std::chrono::high_resolution_clock::now(); driver->setup(); time_now = std::chrono::high_resolution_clock::now(); calc_time = std::chrono::duration_cast(time_now - time_setup).count(); - printf("\n**** Total time to setup driver in seconds %f ****\n\n", calc_time * 1e-9); + if(myrank == 0){ + printf("\n**** Total time to setup driver in seconds %f ****\n\n", calc_time * 1e-9); + } auto time_run = std::chrono::high_resolution_clock::now(); driver->execute(); time_now = std::chrono::high_resolution_clock::now(); calc_time = std::chrono::duration_cast(time_now - time_setup).count(); - printf("\n**** Total time to execute driver in seconds %f ****\n\n", calc_time * 1e-9); + if(myrank == 0){ + printf("\n**** Total time to execute driver in seconds %f ****\n\n", calc_time * 1e-9); + } driver->finalize(); time_now = std::chrono::high_resolution_clock::now(); calc_time = std::chrono::duration_cast(time_now - time_start).count(); - - printf("\n**** Total time to run simulation in seconds %f ****\n\n", calc_time * 1e-9); + if(myrank == 0){ + printf("\n**** Total time to run simulation in seconds %f ****\n\n", calc_time * 1e-9); + } // Delete driver delete driver; @@ -113,6 +132,11 @@ int main(int argc, char* argv[]) Kokkos::finalize(); - std::cout << "**** End of main **** " << std::endl; + if(myrank == 0){ + std::cout << "**** End of main **** " << std::endl; + } + + MPI_Barrier(MPI_COMM_WORLD); + MPI_Finalize(); return 0; } diff --git a/single-node-refactor/src/material_models/artificial_viscosity/mars.h b/single-node-refactor/src/material_models/artificial_viscosity/mars.h index 00f136f59..e09602dbd 100644 --- a/single-node-refactor/src/material_models/artificial_viscosity/mars.h +++ b/single-node-refactor/src/material_models/artificial_viscosity/mars.h @@ -48,7 +48,7 @@ namespace MARSDissipationModel { const RaggedRightArrayKokkos & dissipation_global_vars, const DCArrayKokkos& GaussPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_eroded, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_vel, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_sspd, const ViewCArrayKokkos& disp_corner_forces, @@ -364,7 +364,7 @@ namespace DirMARSDissipationModel { const RaggedRightArrayKokkos & dissipation_global_vars, const DCArrayKokkos& GaussPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_eroded, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_vel, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_sspd, const ViewCArrayKokkos& disp_corner_forces, @@ -659,7 +659,7 @@ namespace MARSRZDissipationModel { const RaggedRightArrayKokkos & dissipation_global_vars, const DCArrayKokkos& GaussPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_eroded, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_vel, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_sspd, const ViewCArrayKokkos& disp_corner_forces, @@ -932,7 +932,7 @@ namespace DirMARSRZDissipationModel { const RaggedRightArrayKokkos & dissipation_global_vars, const DCArrayKokkos& GaussPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_eroded, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_vel, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_sspd, const ViewCArrayKokkos& disp_corner_forces, diff --git a/single-node-refactor/src/material_models/artificial_viscosity/no_dissipation.h b/single-node-refactor/src/material_models/artificial_viscosity/no_dissipation.h index 4c356304e..d45d72e4d 100644 --- a/single-node-refactor/src/material_models/artificial_viscosity/no_dissipation.h +++ b/single-node-refactor/src/material_models/artificial_viscosity/no_dissipation.h @@ -45,7 +45,7 @@ namespace NoDissipationModel { const RaggedRightArrayKokkos & dissipation_global_vars, const DCArrayKokkos& GaussPoints_vel_grad, const DRaggedRightArrayKokkos& MaterialPoints_eroded, - const DCArrayKokkos& node_vel, + const DistributedDCArray& node_vel, const DRaggedRightArrayKokkos& MaterialPoints_den, const DRaggedRightArrayKokkos& MaterialPoints_sspd, const ViewCArrayKokkos& disp_corner_forces, diff --git a/single-node-refactor/src/material_models/strength/decoupled_plasticity.h b/single-node-refactor/src/material_models/strength/decoupled_plasticity.h index 64c0735f2..ef10fd991 100644 --- a/single-node-refactor/src/material_models/strength/decoupled_plasticity.h +++ b/single-node-refactor/src/material_models/strength/decoupled_plasticity.h @@ -123,9 +123,9 @@ namespace HypoPlasticityModel { KOKKOS_FUNCTION static void calc_stress( const DCArrayKokkos &GaussPoints_vel_grad, - const DCArrayKokkos &node_coords, - const DCArrayKokkos &node_vel, - const DCArrayKokkos &nodes_in_elem, + const DistributedDCArray &node_coords, + const DistributedDCArray &node_vel, + const DistributedDCArray &nodes_in_elem, const DRaggedRightArrayKokkos &MaterialPoints_pres, const DRaggedRightArrayKokkos &MaterialPoints_stress, const DRaggedRightArrayKokkos &MaterialPoints_stress_n0, @@ -416,9 +416,9 @@ namespace HypoPlasticityRZModel { KOKKOS_FUNCTION static void calc_stress( const DCArrayKokkos &GaussPoints_vel_grad, - const DCArrayKokkos &node_coords, - const DCArrayKokkos &node_vel, - const DCArrayKokkos &nodes_in_elem, + const DistributedDCArray &node_coords, + const DistributedDCArray &node_vel, + const DistributedDCArray &nodes_in_elem, const DRaggedRightArrayKokkos &MaterialPoints_pres, const DRaggedRightArrayKokkos &MaterialPoints_stress, const DRaggedRightArrayKokkos &MaterialPoints_stress_n0, diff --git a/single-node-refactor/src/material_models/strength/host_ann_strength.h b/single-node-refactor/src/material_models/strength/host_ann_strength.h index 688d06c17..1622a7eb2 100644 --- a/single-node-refactor/src/material_models/strength/host_ann_strength.h +++ b/single-node-refactor/src/material_models/strength/host_ann_strength.h @@ -219,9 +219,9 @@ namespace HostANNStrengthModel { // this model is launched from the CPU, coding inside is run on GPUS static void calc_stress( const DCArrayKokkos &GaussPoints_vel_grad, - const DCArrayKokkos &node_coords, - const DCArrayKokkos &node_vel, - const DCArrayKokkos &nodes_in_elem, + const DistributedDCArray &node_coords, + const DistributedDCArray &node_vel, + const DistributedDCArray &nodes_in_elem, const DRaggedRightArrayKokkos &MaterialPoints_pres, const DRaggedRightArrayKokkos &MaterialPoints_stress, const DRaggedRightArrayKokkos &MaterialPoints_stress_n0, diff --git a/single-node-refactor/src/material_models/strength/host_user_defined_strength.h b/single-node-refactor/src/material_models/strength/host_user_defined_strength.h index 32c2269a1..5d3c4ffc8 100644 --- a/single-node-refactor/src/material_models/strength/host_user_defined_strength.h +++ b/single-node-refactor/src/material_models/strength/host_user_defined_strength.h @@ -94,8 +94,8 @@ namespace HostUserDefinedStrengthModel { // function is accessed on the Host static void calc_stress( const DCArrayKokkos& GaussPoints_vel_grad, - const DCArrayKokkos &node_coords, - const DCArrayKokkos &node_vel, + const DistributedDCArray &node_coords, + const DistributedDCArray &node_vel, const ViewCArrayKokkos& elem_node_gids, const DRaggedRightArrayKokkos& MaterialPoints_pres, const DRaggedRightArrayKokkos& MaterialPoints_stress, @@ -204,7 +204,7 @@ namespace HostNotionalStrengthModel { const DCArrayKokkos &vel_grad, const DCArrayKokkos &node_coords, const DCArrayKokkos &node_vel, - const DCArrayKokkos &nodes_in_elem, + const DistributedDCArray &nodes_in_elem, const DRaggedRightArrayKokkos &MaterialPoints_pres, const DRaggedRightArrayKokkos &MaterialPoints_stress, const DRaggedRightArrayKokkos &MaterialPoints_sspd, diff --git a/single-node-refactor/src/material_models/strength/no_strength.h b/single-node-refactor/src/material_models/strength/no_strength.h index faf14440e..4528670ea 100644 --- a/single-node-refactor/src/material_models/strength/no_strength.h +++ b/single-node-refactor/src/material_models/strength/no_strength.h @@ -73,9 +73,9 @@ namespace NoStrengthModel { KOKKOS_FUNCTION static void calc_stress( const DCArrayKokkos &vel_grad, - const DCArrayKokkos &node_coords, - const DCArrayKokkos &node_vel, - const DCArrayKokkos &nodes_in_elem, + const DistributedDCArray &node_coords, + const DistributedDCArray &node_vel, + const DistributedDCArray &nodes_in_elem, const DRaggedRightArrayKokkos &MaterialPoints_pres, const DRaggedRightArrayKokkos &MaterialPoints_stress, const DRaggedRightArrayKokkos &MaterialPoints_stress_n0, diff --git a/single-node-refactor/src/material_models/strength/user_defined_strength.h b/single-node-refactor/src/material_models/strength/user_defined_strength.h index b293e7670..b62c3663a 100644 --- a/single-node-refactor/src/material_models/strength/user_defined_strength.h +++ b/single-node-refactor/src/material_models/strength/user_defined_strength.h @@ -94,9 +94,9 @@ namespace UserDefinedStrengthModel { KOKKOS_FUNCTION static void calc_stress( const DCArrayKokkos &GaussPoints_vel_grad, - const DCArrayKokkos &node_coords, - const DCArrayKokkos &node_vel, - const DCArrayKokkos &nodes_in_elem, + const DistributedDCArray &node_coords, + const DistributedDCArray &node_vel, + const DistributedDCArray &nodes_in_elem, const DRaggedRightArrayKokkos &MaterialPoints_pres, const DRaggedRightArrayKokkos &MaterialPoints_stress, const DRaggedRightArrayKokkos &MaterialPoints_stress_n0, @@ -201,7 +201,7 @@ namespace NotionalStrengthModel { const DCArrayKokkos &GaussPoints_vel_grad, const DCArrayKokkos &node_coords, const DCArrayKokkos &node_vel, - const DCArrayKokkos &nodes_in_elem, + const DistributedDCArray &nodes_in_elem, const DRaggedRightArrayKokkos &MaterialPoints_pres, const DRaggedRightArrayKokkos &MaterialPoints_stress, const DRaggedRightArrayKokkos &MaterialPoints_sspd, diff --git a/single-node-refactor/src/solver.h b/single-node-refactor/src/solver.h index 3ec2a23d2..760ffb424 100644 --- a/single-node-refactor/src/solver.h +++ b/single-node-refactor/src/solver.h @@ -59,31 +59,31 @@ class Solver Solver(); virtual ~Solver(); - virtual void initialize(SimulationParameters_t& SimulationParamaters, + virtual void initialize(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, - State_t& State) const = 0; + State_t& State) {} - virtual void initialize_material_state(SimulationParameters_t& SimulationParamaters, + virtual void initialize_material_state(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, State_t& State) const = 0; - virtual void setup(SimulationParameters_t& SimulationParamaters, + virtual void setup(SimulationParameters_t& SimulationParameters, Material_t& Materials, Mesh_t& mesh, BoundaryCondition_t& Boundary, State_t& State) = 0; - virtual void execute(SimulationParameters_t& SimulationParamaters, + virtual void execute(SimulationParameters_t& SimulationParameters, Material_t& Materials, BoundaryCondition_t& BoundaryConditions, Mesh_t& mesh, State_t& State) = 0; - virtual void finalize(SimulationParameters_t& SimulationParamaters, + virtual void finalize(SimulationParameters_t& SimulationParameters, Material_t& Materials, BoundaryCondition_t& Boundary) const = 0;