From 372ad8c0aa8b285e49ecbd32793ef667e9b70571 Mon Sep 17 00:00:00 2001 From: Jeffrey Garretson Date: Mon, 16 Aug 2021 15:29:43 -0600 Subject: [PATCH] Adding option to perform blocking neighbor calls in halo updates, and improving cmake MPI code --- cmake/compilers.cmake | 28 ++++++++++-------- src/gamera/mpi/gamapp_mpi.F90 | 53 ++++++++++++++++++++++++++--------- 2 files changed, 56 insertions(+), 25 deletions(-) diff --git a/cmake/compilers.cmake b/cmake/compilers.cmake index d0d9234d..1cf00b2e 100644 --- a/cmake/compilers.cmake +++ b/cmake/compilers.cmake @@ -14,18 +14,22 @@ set(CMAKE_REQUIRED_LIBRARIES ${OpenMP_Fortran_LIBRARIES}) if (ENABLE_MPI) #mpi is a nightmare - #try to find a fortran 2008 specific wrapper first - set(MPI_Fortran_COMPILER mpif08) - find_package(MPI COMPONENTS Fortran QUIET) + #try to explicitly find intel mpi first + set(MPI_Fortran_COMPILER mpiifort) + find_package(MPI COMPONENTS Fortran QUIET) if(NOT MPI_FOUND OR NOT MPI_Fortran_HAVE_F08_MODULE) - #just look for whatever - unset(MPI_Fortran_COMPILER) - find_package(MPI REQUIRED COMPONENTS Fortran) - else() - message("-- Found MPI") + #try to find a fortran 2008 specific wrapper + set(MPI_Fortran_COMPILER mpif08) + find_package(MPI COMPONENTS Fortran QUIET) + if(NOT MPI_FOUND OR NOT MPI_Fortran_HAVE_F08_MODULE) + #just look for whatever + unset(MPI_Fortran_COMPILER) + find_package(MPI REQUIRED COMPONENTS Fortran) + endif() endif() - - if(NOT MPI_Fortran_HAVE_F08_MODULE) + if(MPI_Fortran_HAVE_F08_MODULE) + message("-- Found MPI") + else() message(FATAL_ERROR "MPI Library does not support F08 interface") endif() endif() @@ -91,8 +95,8 @@ if(CMAKE_Fortran_COMPILER_ID MATCHES Intel) endif() #Check Intel Fortran version - if(NOT ALLOW_INVALID_COMPILERS AND CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER "19") - message(FATAL_ERROR "Intel Fortran compilers 19 or newer are not supported. Set the ALLOW_INVALID_COMPILERS variable to ON to force compilation at your own risk.") + if(NOT ALLOW_INVALID_COMPILERS AND CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER "22") + message(FATAL_ERROR "Intel Fortran compilers newer than 21 are not supported. Set the ALLOW_INVALID_COMPILERS variable to ON to force compilation at your own risk.") endif() elseif(CMAKE_Fortran_COMPILER_ID MATCHES GNU) diff --git a/src/gamera/mpi/gamapp_mpi.F90 b/src/gamera/mpi/gamapp_mpi.F90 index 4c7e97ff..1bae146f 100644 --- a/src/gamera/mpi/gamapp_mpi.F90 +++ b/src/gamera/mpi/gamapp_mpi.F90 @@ -15,6 +15,7 @@ module gamapp_mpi type, extends(GamApp_T) :: gamAppMpi_T type(MPI_Comm) :: gamMpiComm integer, dimension(:), allocatable :: sendRanks, recvRanks + logical :: blockHalo = .false. ! Gas Data Transfer Variables integer, dimension(:), allocatable :: sendCountsGas @@ -103,6 +104,8 @@ module gamapp_mpi stop endif + call xmlInp%Set_Val(gamAppMpi%blockHalo,"coupling/blockHalo",.false.) + ! read debug flags call xmlInp%Set_Val(writeGhosts,"debug/writeGhosts",.false.) call xmlInp%Set_Val(writeMagFlux,"debug/writeMagFlux",.false.) @@ -680,11 +683,22 @@ module gamapp_mpi ! just tell MPI to use the arrays we defined during initialization to send and receive data! ! Gas Cell Data - call mpi_ineighbor_alltoallw(gamAppMpi%State%Gas, gamAppMpi%sendCountsGas, & - gamAppMpi%sendDisplsGas, gamAppMpi%sendTypesGas, & - gamAppMpi%State%Gas, gamAppMpi%recvCountsGas, & - gamAppMpi%recvDisplsGas, gamAppMpi%recvTypesGas, & - gamAppMpi%gamMpiComm, gasReq, ierr) + if(gamAppMpi%blockHalo) then + ! synchronous + call mpi_neighbor_alltoallw(gamAppMpi%State%Gas, gamAppMpi%sendCountsGas, & + gamAppMpi%sendDisplsGas, gamAppMpi%sendTypesGas, & + gamAppMpi%State%Gas, gamAppMpi%recvCountsGas, & + gamAppMpi%recvDisplsGas, gamAppMpi%recvTypesGas, & + gamAppMpi%gamMpiComm, ierr) + else + !asynchronous + call mpi_ineighbor_alltoallw(gamAppMpi%State%Gas, gamAppMpi%sendCountsGas, & + gamAppMpi%sendDisplsGas, gamAppMpi%sendTypesGas, & + gamAppMpi%State%Gas, gamAppMpi%recvCountsGas, & + gamAppMpi%recvDisplsGas, gamAppMpi%recvTypesGas, & + gamAppMpi%gamMpiComm, gasReq, ierr) + endif + if(ierr /= MPI_Success) then call MPI_Error_string( ierr, message, length, ierr) print *,message(1:length) @@ -702,11 +716,22 @@ module gamapp_mpi endif ! Magnetic Face Flux Data - call mpi_ineighbor_alltoallw(gamAppMpi%State%magFlux, gamAppMpi%sendCountsMagFlux, & - gamAppMpi%sendDisplsMagFlux, gamAppMpi%sendTypesMagFlux, & - gamAppMpi%State%magFlux, gamAppMpi%recvCountsMagFlux, & - gamAppMpi%recvDisplsMagFlux, gamAppMpi%recvTypesMagFlux, & - gamAppMpi%gamMpiComm, mfReq, ierr) + if(gamAppMpi%blockHalo) then + ! synchronous + call mpi_neighbor_alltoallw(gamAppMpi%State%magFlux, gamAppMpi%sendCountsMagFlux, & + gamAppMpi%sendDisplsMagFlux, gamAppMpi%sendTypesMagFlux, & + gamAppMpi%State%magFlux, gamAppMpi%recvCountsMagFlux, & + gamAppMpi%recvDisplsMagFlux, gamAppMpi%recvTypesMagFlux, & + gamAppMpi%gamMpiComm, ierr) + else + ! asynchronous + call mpi_ineighbor_alltoallw(gamAppMpi%State%magFlux, gamAppMpi%sendCountsMagFlux, & + gamAppMpi%sendDisplsMagFlux, gamAppMpi%sendTypesMagFlux, & + gamAppMpi%State%magFlux, gamAppMpi%recvCountsMagFlux, & + gamAppMpi%recvDisplsMagFlux, gamAppMpi%recvTypesMagFlux, & + gamAppMpi%gamMpiComm, mfReq, ierr) + endif + if(ierr /= MPI_Success) then call MPI_Error_string( ierr, message, length, ierr) print *,message(1:length) @@ -721,9 +746,11 @@ module gamapp_mpi endif - call mpi_wait(gasReq, MPI_STATUS_IGNORE, ierr) - if(gamAppMpi%Model%doMHD) then - call mpi_wait(mfReq, MPI_STATUS_IGNORE, ierr) + if(.not. gamAppMpi%blockHalo) then + call mpi_wait(gasReq, MPI_STATUS_IGNORE, ierr) + if(gamAppMpi%Model%doMHD) then + call mpi_wait(mfReq, MPI_STATUS_IGNORE, ierr) + endif endif endif