module testMpiFields
  use testHelperMpi
  use gamapp_mpi
  use usergamic

  implicit none

  type(gamAppMpi_T), allocatable :: gamAppMpi

contains

  @before
  subroutine setup(this)
      class (MpiTestMethod), intent(inout) :: this

      call setMpiReal()

      allocate(gamAppMpi)

      call initGamera_mpi(gamAppMpi,initUser,getMpiF08Communicator(this),'lfmbw_241.xml',.false.)

      do while ((gamAppMpi%Model%tFin - gamAppMpi%Model%t) > 1e-15)
           call stepGamera_mpi(gamAppMpi)
      enddo

  end subroutine setup

  @after
  subroutine teardown(this)
      class (MpiTestMethod), intent(inout) :: this

      deallocate(gamAppMpi)
  end subroutine teardown

  subroutine copyFaces4(var,is,ie,js,je,ks,ke)
      integer, intent(in) :: is,ie,js,je,ks,ke
      real(rp), intent(inout) :: var(is:ie,js:je,ks:ke,NDIM)

      integer :: i,j,k,rank,ierr,maxDim,buffCount
      real(rp), dimension(:), allocatable :: transmitBuffer

      maxDim = max(gamAppMpi%Grid%Nip,gamAppMpi%Grid%Njp,gamAppMpi%Grid%nkp)
      allocate(transmitBuffer(maxDim*maxDim))

      call mpi_comm_rank(gamAppMpi%gamMpiComm,rank,ierr)

      if(.not. gamAppMpi%Grid%hasUpperBC(IDIR)) then
          ! send data at upper I faces
          buffCount = gamAppMpi%grid%Njp*gamAppMpi%Grid%Nkp
          transmitBuffer(1:buffCount) = reshape(var(gamAppMpi%Grid%ie+1,&
                            gamAppMpi%Grid%js:gamAppMpi%Grid%je,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke,&
                            IDIR), (/buffCount/))
          call mpi_send(transmitBuffer,buffCount,&
              MPI_MYFLOAT,rank+gamAppMpi%Grid%NumRj,1234,gamAppMpi%gamMpiComm,ierr)
      endif
      if(.not. gamAppMpi%Grid%hasLowerBC(IDIR)) then
          ! receive data at lower I faces
          buffCount = gamAppMpi%grid%Njp*gamAppMpi%Grid%Nkp
          call mpi_recv(transmitBuffer,buffCount,&
              MPI_MYFLOAT,rank-gamAppMpi%Grid%NumRj,1234,gamAppMpi%gamMpiComm,MPI_STATUS_IGNORE,ierr)
          var(gamAppMpi%Grid%ie+1,&
              gamAppMpi%Grid%js:gamAppMpi%Grid%je,&
              gamAppMpi%Grid%ks:gamAppMpi%Grid%ke,&
              IDIR) = reshape(transmitBuffer(1:buffCount),(/gamAppMpi%Grid%Njp,gamAppMpi%Grid%Nkp/))
      endif

      if(.not. gamAppMpi%Grid%hasUpperBC(JDIR)) then
          ! send data at upper J faces
          buffCount = gamAppMpi%grid%Nip*gamAppMpi%Grid%Nkp
          transmitBuffer(1:buffCount) = reshape(var(gamAppMpi%Grid%is:gamAppMpi%Grid%ie,&
                            gamAppMpi%Grid%je+1,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke,&
                            JDIR),(/buffCount/))
          call mpi_send(transmitBuffer,buffCount,&
              MPI_MYFLOAT,rank+1,1235,gamAppMpi%gamMpiComm,ierr)
      endif
      if(.not. gamAppMpi%Grid%hasLowerBC(JDIR)) then
          ! receive data at lower J faces
          buffCount = gamAppMpi%grid%Nip*gamAppMpi%Grid%Nkp
          call mpi_recv(transmitBuffer,buffCount,&
              MPI_MYFLOAT,rank-1,1235,gamAppMpi%gamMpiComm,MPI_STATUS_IGNORE,ierr)
          var(gamAppMpi%Grid%is:gamAppMpi%Grid%ie,&
              gamAppMpi%Grid%je+1,&
              gamAppMpi%Grid%ks:gamAppMpi%Grid%ke,&
              JDIR) = reshape(transmitBuffer(1:buffCount),(/gamAppMpi%Grid%Nip,gamAppMpi%Grid%Nkp/))
      endif

      deallocate(transmitBuffer)

  end subroutine copyFaces4

  subroutine verifyFaces4(var,varName,is,ie,js,je,ks,ke)
      integer, intent(in) :: is,ie,js,je,ks,ke
      real(rp), intent(in) :: var(is:ie,js:je,ks:ke,NDIM)
      character(len=*), intent(in) :: varName

      integer :: i,j,k
      character(len=strLen) :: checkMessage
      real(rp) :: checkValue

      ! min/max K faces
      do i=gamAppMpi%Grid%is,gamAppMpi%Grid%ie
          do j=gamAppMpi%Grid%js,gamAppMpi%Grid%je
              write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad K face value in ',trim(varName),' at (',i,',',j,',',gamAppMpi%Grid%ks,')'
              checkValue = abs(var(i,j,gamAppMpi%Grid%ks,KDIR) - var(i,j,gamAppMpi%Grid%ke+1,KDIR))
              @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
          enddo
      enddo

      if(.not. gamAppMpi%Grid%hasLowerBC(IDIR)) then
          ! check lower I faces
          do j=gamAppMpi%Grid%js,gamAppMpi%Grid%je
              do k=gamAppMpi%Grid%ks,gamAppMpi%Grid%ke
                  write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad I face value in ',trim(varName),' at (',gamAppMpi%Grid%is,',',j,',',k,')'
                  checkValue = abs(var(gamAppMpi%Grid%ie+1,j,k,IDIR)-var(gamAppMpi%Grid%is,j,k,IDIR))
                  @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
              enddo
          enddo
      endif

      if(.not. gamAppMpi%Grid%hasLowerBC(JDIR)) then
          ! check lower J faces
          do i=gamAppMpi%Grid%is,gamAppMpi%Grid%ie
              do k=gamAppMpi%Grid%ks,gamAppMpi%Grid%ke
                  write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad J face value in ',trim(varName),' at (',i,',',gamAppMpi%Grid%js,',',k,')'
                  checkValue = abs(var(i,gamAppMpi%Grid%je+1,k,JDIR)-var(i,gamAppMpi%grid%js,k,JDIR))
                  @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
              enddo
          enddo
      endif

  end subroutine verifyFaces4

  subroutine copyFaces5(var,is,ie,js,je,ks,ke,dn)
      integer, intent(in) :: is,ie,js,je,ks,ke,dn
      real(rp), intent(inout) :: var(is:ie,js:je,ks:ke,dn,NDIM)

      integer :: i,j,k,rank,ierr,maxDim,buffCount
      real(rp), dimension(:), allocatable :: transmitBuffer

      maxDim = max(gamAppMpi%Grid%Nip,gamAppMpi%Grid%Njp,gamAppMpi%Grid%nkp)
      allocate(transmitBuffer(dn*maxDim*maxDim))
      if(.not. allocated(transmitBuffer)) write (*,*) 'Allocated failed!'

      call mpi_comm_rank(gamAppMpi%gamMpiComm,rank,ierr)

      if(.not. gamAppMpi%Grid%hasUpperBC(IDIR)) then
          ! send data at upper I faces
          buffCount = dn*gamAppMpi%grid%Njp*gamAppMpi%Grid%Nkp
          transmitBuffer(1:buffCount) = reshape(var(gamAppMpi%Grid%ie+1,&
                            gamAppMpi%Grid%js:gamAppMpi%Grid%je,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke,&
                            :,IDIR), (/buffCount/))
          call mpi_send(transmitBuffer,buffCount,&
              MPI_MYFLOAT,rank+gamAppMpi%Grid%NumRj,1236,gamAppMpi%gamMpiComm,ierr)
      endif
      if(.not. gamAppMpi%Grid%hasLowerBC(IDIR)) then
          ! receive data at lower I faces
          buffCount = dn*gamAppMpi%grid%Njp*gamAppMpi%Grid%Nkp
          call mpi_recv(transmitBuffer,buffCount,&
              MPI_MYFLOAT,rank-gamAppMpi%Grid%NumRj,1236,gamAppMpi%gamMpiComm,MPI_STATUS_IGNORE,ierr)
          var(gamAppMpi%Grid%ie+1,&
              gamAppMpi%Grid%js:gamAppMpi%Grid%je,&
              gamAppMpi%Grid%ks:gamAppMpi%Grid%ke,&
              :,IDIR) = reshape(transmitBuffer(1:buffCount),(/gamAppMpi%Grid%Njp,gamAppMpi%Grid%Nkp,dn/))
      endif

      if(.not. gamAppMpi%Grid%hasUpperBC(JDIR)) then
          ! send data at upper J faces
          buffCount = dn*gamAppMpi%grid%Nip*gamAppMpi%Grid%Nkp
          transmitBuffer(1:buffCount) = reshape(var(gamAppMpi%Grid%is:gamAppMpi%Grid%ie,&
                            gamAppMpi%Grid%je+1,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke,&
                            :,JDIR),(/buffCount/))
          call mpi_send(transmitBuffer,buffCount,&
              MPI_MYFLOAT,rank+1,1237,gamAppMpi%gamMpiComm,ierr)
      endif
      if(.not. gamAppMpi%Grid%hasLowerBC(JDIR)) then
          ! receive data at lower J faces
          buffCount = dn*gamAppMpi%grid%Nip*gamAppMpi%Grid%Nkp
          call mpi_recv(transmitBuffer,buffCount,&
              MPI_MYFLOAT,rank-1,1237,gamAppMpi%gamMpiComm,MPI_STATUS_IGNORE,ierr)
          var(gamAppMpi%Grid%is:gamAppMpi%Grid%ie,&
              gamAppMpi%Grid%je+1,&
              gamAppMpi%Grid%ks:gamAppMpi%Grid%ke,&
              :,JDIR) = reshape(transmitBuffer(1:buffCount),(/gamAppMpi%Grid%Nip,gamAppMpi%Grid%Nkp,dn/))
      endif

      deallocate(transmitBuffer)

  end subroutine copyFaces5

  subroutine verifyFaces5(var,varName,is,ie,js,je,ks,ke,dn)
      integer, intent(in) :: is,ie,js,je,ks,ke,dn
      real(rp), intent(in) :: var(is:ie,js:je,ks:ke,dn,NDIM)
      character(len=*), intent(in) :: varName

      integer :: i,j,k
      character(len=strLen) :: checkMessage
      real(rp) :: checkValue

      ! min/max K faces
      do i=gamAppMpi%Grid%is,gamAppMpi%Grid%ie
          do j=gamAppMpi%Grid%js,gamAppMpi%Grid%je
              write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad K face value in ',trim(varName),' at (',i,',',j,',',gamAppMpi%Grid%ks,')'
              checkValue = sum(abs(var(i,j,gamAppMpi%Grid%ks,:,KDIR) - var(i,j,gamAppMpi%Grid%ke+1,:,KDIR)))
              @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
          enddo
      enddo

      if(.not. gamAppMpi%Grid%hasLowerBC(IDIR)) then
          ! check lower I faces
          do j=gamAppMpi%Grid%js,gamAppMpi%Grid%je
              do k=gamAppMpi%Grid%ks,gamAppMpi%Grid%ke
                  write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad I face value in ',trim(varName),' at (',gamAppMpi%Grid%is,',',j,',',k,')'
                  checkValue = sum(abs(var(gamAppMpi%Grid%ie+1,j,k,:,IDIR)-var(gamAppMpi%Grid%is,j,k,:,IDIR)))
                  @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
              enddo
          enddo
      endif

      if(.not. gamAppMpi%Grid%hasLowerBC(JDIR)) then
          ! check lower J faces
          do i=gamAppMpi%Grid%is,gamAppMpi%Grid%ie
              do k=gamAppMpi%Grid%ks,gamAppMpi%Grid%ke
                  write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad J face value in ',trim(varName),' at (',i,',',gamAppMpi%Grid%js,',',k,')'
                  checkValue = sum(abs(var(i,gamAppMpi%Grid%je+1,k,:,JDIR)-var(i,gamAppMpi%grid%js,k,:,JDIR)))
                  @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
              enddo
          enddo
      endif

  end subroutine verifyFaces5

  subroutine copyEdges4(var,is,ie,js,je,ks,ke)
      integer, intent(in) :: is,ie,js,je,ks,ke
      real(rp), intent(inout) :: var(is:ie,js:je,ks:ke,NDIM)

      integer :: i,j,k,rank,ierr

      call mpi_comm_rank(gamAppMpi%gamMpiComm,rank,ierr)

      if(.not. gamAppMpi%Grid%hasUpperBC(IDIR)) then
          ! send upper I data

          ! J edges
          call mpi_send(var(gamAppMpi%Grid%ie+1,&
                            gamAppMpi%Grid%js:gamAppMpi%Grid%je,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke+1,&
                            JDIR),&
              gamAppMpi%Grid%Njp*(gamAppMpi%Grid%Nkp+1),&
              MPI_MYFLOAT,rank+gamAppMpi%Grid%NumRj,1236,gamAppMpi%gamMpiComm,ierr)

          ! K edges
          call mpi_send(var(gamAppMpi%Grid%ie+1,&
                            gamAppMpi%Grid%js:gamAppMpi%Grid%je+1,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke,&
                            KDIR),&
              (gamAppMpi%Grid%Njp+1)*gamAppMpi%Grid%Nkp,&
              MPI_MYFLOAT,rank+gamAppMpi%Grid%NumRj,1237,gamAppMpi%gamMpiComm,ierr)
      endif
      if(.not. gamAppMpi%Grid%hasLowerBC(IDIR)) then
          ! receive lower I data

          ! J edges
          call mpi_recv(var(gamAppMpi%Grid%ie+1,&
                            gamAppMpi%Grid%js:gamAppMpi%Grid%je,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke+1,&
                            JDIR),&
              gamAppMpi%Grid%Njp*(gamAppMpi%Grid%Nkp+1),&
              MPI_MYFLOAT,rank-gamAppMpi%Grid%NumRj,1236,gamAppMpi%gamMpiComm,MPI_STATUS_IGNORE,ierr)

          ! K edges
          call mpi_recv(var(gamAppMpi%Grid%ie+1,&
                            gamAppMpi%Grid%js:gamAppMpi%Grid%je+1,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke,&
                            KDIR),&
              (gamAppMpi%Grid%Njp+1)*gamAppMpi%Grid%Nkp,&
              MPI_MYFLOAT,rank-gamAppMpi%Grid%NumRj,1237,gamAppMpi%gamMpiComm,MPI_STATUS_IGNORE,ierr)
      endif

      if(.not. gamAppMpi%Grid%hasUpperBC(JDIR)) then
          ! send upper J data

          ! I edges
          call mpi_send(var(gamAppMpi%Grid%is:gamAppMpi%Grid%ie,&
                            gamAppMpi%Grid%je+1,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke+1,&
                            IDIR),&
              gamAppMpi%Grid%Nip*(gamAppMpi%Grid%Nkp+1),&
              MPI_MYFLOAT,rank+1,1238,gamAppMpi%gamMpiComm,ierr)

          ! K edges
          call mpi_send(var(gamAppMpi%Grid%is:gamAppMpi%Grid%ie+1,&
                            gamAppMpi%Grid%je+1,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke,&
                            KDIR),&
              (gamAppMpi%Grid%Nip+1)*gamAppMpi%Grid%Nkp,&
              MPI_MYFLOAT,rank+1,1239,gamAppMpi%gamMpiComm,ierr)
      endif
      if(.not. gamAppMpi%Grid%hasLowerBC(JDIR)) then
          ! receive lower J data

          ! I edges
          call mpi_recv(var(gamAppMpi%Grid%is:gamAppMpi%Grid%ie,&
                            gamAppMpi%Grid%je+1,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke+1,&
                            IDIR),&
              gamAppMpi%Grid%Nip*(gamAppMpi%Grid%Nkp+1),&
              MPI_MYFLOAT,rank-1,1238,gamAppMpi%gamMpiComm,MPI_STATUS_IGNORE,ierr)

          ! K edges
          call mpi_recv(var(gamAppMpi%Grid%is:gamAppMpi%Grid%ie+1,&
                            gamAppMpi%Grid%je+1,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke,&
                            KDIR),&
              (gamAppMpi%Grid%Nip+1)*gamAppMpi%Grid%Nkp,&
              MPI_MYFLOAT,rank-1,1239,gamAppMpi%gamMpiComm,MPI_STATUS_IGNORE,ierr)
      endif

  end subroutine copyEdges4

  subroutine verifyEdges4(var,varName,is,ie,js,je,ks,ke)
      integer, intent(in) :: is,ie,js,je,ks,ke
      real(rp), intent(in) :: var(is:ie,js:je,ks:ke,NDIM)
      character(len=*), intent(in) :: varName

      integer :: i,j,k
      character(len=strLen) :: checkMessage
      real(rp) :: checkValue

      ! I edges
      do i=gamAppMpi%Grid%is,gamAppMpi%Grid%ie
          do j=gamAppMpi%Grid%js,gamAppMpi%Grid%je+1
              write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad I edge value in ',trim(varName),' at (',i,',',j,',',gamAppMpi%Grid%ks,')'
              checkValue = abs(var(i,j,gamAppMpi%Grid%ks,IDIR) - var(i,j,gamAppMpi%Grid%ke+1,IDIR))
              @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
          enddo
      enddo

      ! J edges
      do i=gamAppMpi%Grid%is,gamAppMpi%Grid%ie+1
          do j=gamAppMpi%Grid%js,gamAppMpi%Grid%je
              write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad J edge value in ',trim(varName),' at (',i,',',j,',',gamAppMpi%Grid%ks,')'
              checkValue = abs(var(i,j,gamAppMpi%Grid%ks,JDIR) - var(i,j,gamAppMpi%Grid%ke+1,JDIR))
              @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
          enddo
      enddo

      if(.not. gamAppMpi%Grid%hasLowerBC(IDIR)) then
          ! check lower I data
          ! J edges
          do k=gamAppMpi%Grid%ks,gamAppMpi%Grid%ke+1
              do j=gamAppMpi%Grid%js,gamAppMpi%Grid%je
                  write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad J edge value in ',trim(varName),' at (',gamAppMpi%Grid%is,',',j,',',k,')'
                  checkValue = abs(var(gamAppMpi%Grid%is,j,k,JDIR) - var(gamAppMpi%Grid%ie+1,j,k,JDIR))
                  @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
              enddo
          enddo
          ! K edges
          do k=gamAppMpi%Grid%ks,gamAppMpi%Grid%ke
              do j=gamAppMpi%Grid%js,gamAppMpi%Grid%je+1
                  write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad K edge value in ',trim(varName),' at (',gamAppMpi%Grid%is,',',j,',',k,')'
                  checkValue = abs(var(gamAppMpi%Grid%is,j,k,KDIR) - var(gamAppMpi%Grid%ie+1,j,k,KDIR))
                  @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
              enddo
          enddo
      endif

      if(.not. gamAppMpi%Grid%hasLowerBC(JDIR)) then
          ! check lower J data
          ! I edges
          do k=gamAppMpi%Grid%ks,gamAppMpi%Grid%ke+1
              do i=gamAppMpi%Grid%is,gamAppMpi%Grid%ie
                  write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad I edge value in ',trim(varName),' at (',i,',',gamAppMpi%Grid%js,',',k,')'
                  checkValue = abs(var(i,gamAppMpi%Grid%js,k,IDIR) - var(i,gamAppMpi%Grid%je+1,k,IDIR))
                  @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
              enddo
          enddo
          ! K edges
          do k=gamAppMpi%Grid%ks,gamAppMpi%Grid%ke
              do i=gamAppMpi%Grid%js,gamAppMpi%Grid%ie+1
                  write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad K edge value in ',trim(varName),' at (',i,',',gamAppMpi%Grid%js,',',k,')'
                  checkValue = abs(var(i,gamAppMpi%Grid%js,k,KDIR) - var(i,gamAppMpi%Grid%je+1,k,KDIR))
                  @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
              enddo
          enddo
      endif

  end subroutine verifyEdges4

  subroutine copyEdges5(var,is,ie,js,je,ks,ke,dn)
      integer, intent(in) :: is,ie,js,je,ks,ke,dn
      real(rp), intent(inout) :: var(is:ie,js:je,ks:ke,dn,NDIM)

      integer :: i,j,k,rank,ierr

      call mpi_comm_rank(gamAppMpi%gamMpiComm,rank,ierr)

      if(.not. gamAppMpi%Grid%hasUpperBC(IDIR)) then
          ! send upper I data

          ! J edges
          call mpi_send(var(gamAppMpi%Grid%ie+1,&
                            gamAppMpi%Grid%js:gamAppMpi%Grid%je,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke+1,&
                            :,JDIR),&
              dn*gamAppMpi%Grid%Njp*(gamAppMpi%Grid%Nkp+1),&
              MPI_MYFLOAT,rank+gamAppMpi%Grid%NumRj,1240,gamAppMpi%gamMpiComm,ierr)

          ! K edges
          call mpi_send(var(gamAppMpi%Grid%ie+1,&
                            gamAppMpi%Grid%js:gamAppMpi%Grid%je+1,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke,&
                            :,KDIR),&
              dn*(gamAppMpi%Grid%Njp+1)*gamAppMpi%Grid%Nkp,&
              MPI_MYFLOAT,rank+gamAppMpi%Grid%NumRj,1241,gamAppMpi%gamMpiComm,ierr)
      endif
      if(.not. gamAppMpi%Grid%hasLowerBC(IDIR)) then
          ! receive lower I data

          ! J edges
          call mpi_recv(var(gamAppMpi%Grid%ie+1,&
                            gamAppMpi%Grid%js:gamAppMpi%Grid%je,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke+1,&
                            :,JDIR),&
              dn*gamAppMpi%Grid%Njp*(gamAppMpi%Grid%Nkp+1),&
              MPI_MYFLOAT,rank-gamAppMpi%Grid%NumRj,1240,gamAppMpi%gamMpiComm,MPI_STATUS_IGNORE,ierr)

          ! K edges
          call mpi_recv(var(gamAppMpi%Grid%ie+1,&
                            gamAppMpi%Grid%js:gamAppMpi%Grid%je+1,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke,&
                            :,KDIR),&
              dn*(gamAppMpi%Grid%Njp+1)*gamAppMpi%Grid%Nkp,&
              MPI_MYFLOAT,rank-gamAppMpi%Grid%NumRj,1241,gamAppMpi%gamMpiComm,MPI_STATUS_IGNORE,ierr)
      endif

      if(.not. gamAppMpi%Grid%hasUpperBC(JDIR)) then
          ! send upper J data

          ! I edges
          call mpi_send(var(gamAppMpi%Grid%is:gamAppMpi%Grid%ie,&
                            gamAppMpi%Grid%je+1,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke+1,&
                            :,IDIR),&
              dn*gamAppMpi%Grid%Nip*(gamAppMpi%Grid%Nkp+1),&
              MPI_MYFLOAT,rank+1,1242,gamAppMpi%gamMpiComm,ierr)

          ! K edges
          call mpi_send(var(gamAppMpi%Grid%is:gamAppMpi%Grid%ie+1,&
                            gamAppMpi%Grid%je+1,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke,&
                            :,KDIR),&
              dn*(gamAppMpi%Grid%Nip+1)*gamAppMpi%Grid%Nkp,&
              MPI_MYFLOAT,rank+1,1243,gamAppMpi%gamMpiComm,ierr)
      endif
      if(.not. gamAppMpi%Grid%hasLowerBC(JDIR)) then
          ! receive lower J data

          ! I edges
          call mpi_recv(var(gamAppMpi%Grid%is:gamAppMpi%Grid%ie,&
                            gamAppMpi%Grid%je+1,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke+1,&
                            :,IDIR),&
              dn*gamAppMpi%Grid%Nip*(gamAppMpi%Grid%Nkp+1),&
              MPI_MYFLOAT,rank-1,1242,gamAppMpi%gamMpiComm,MPI_STATUS_IGNORE,ierr)

          ! K edges
          call mpi_recv(var(gamAppMpi%Grid%is:gamAppMpi%Grid%ie+1,&
                            gamAppMpi%Grid%je+1,&
                            gamAppMpi%Grid%ks:gamAppMpi%Grid%ke,&
                            :,KDIR),&
              dn*(gamAppMpi%Grid%Nip+1)*gamAppMpi%Grid%Nkp,&
              MPI_MYFLOAT,rank-1,1243,gamAppMpi%gamMpiComm,MPI_STATUS_IGNORE,ierr)
      endif

  end subroutine copyEdges5

  subroutine verifyEdges5(var,varName,is,ie,js,je,ks,ke,dn)
      integer, intent(in) :: is,ie,js,je,ks,ke,dn
      real(rp), intent(in) :: var(is:ie,js:je,ks:ke,dn,NDIM)
      character(len=*), intent(in) :: varName

      integer :: i,j,k
      character(len=strLen) :: checkMessage
      real(rp) :: checkValue

      ! I edges
      do i=gamAppMpi%Grid%is,gamAppMpi%Grid%ie
          do j=gamAppMpi%Grid%js,gamAppMpi%Grid%je+1
              write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad I edge value in ',trim(varName),' at (',i,',',j,',',gamAppMpi%Grid%ks,')'
              checkValue = sum(abs(var(i,j,gamAppMpi%Grid%ks,:,IDIR) - var(i,j,gamAppMpi%Grid%ke+1,:,IDIR)))
              @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
          enddo
      enddo

      ! J edges
      do i=gamAppMpi%Grid%is,gamAppMpi%Grid%ie+1
          do j=gamAppMpi%Grid%js,gamAppMpi%Grid%je
              write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad J edge value in ',trim(varName),' at (',i,',',j,',',gamAppMpi%Grid%ks,')'
              checkValue = sum(abs(var(i,j,gamAppMpi%Grid%ks,:,JDIR) - var(i,j,gamAppMpi%Grid%ke+1,:,JDIR)))
              @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
          enddo
      enddo

      if(.not. gamAppMpi%Grid%hasLowerBC(IDIR)) then
          ! check lower I data
          ! J edges
          do k=gamAppMpi%Grid%ks,gamAppMpi%Grid%ke+1
              do j=gamAppMpi%Grid%js,gamAppMpi%Grid%je
                  write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad J edge value in ',trim(varName),' at (',gamAppMpi%Grid%is,',',j,',',k,')'
                  checkValue = sum(abs(var(gamAppMpi%Grid%is,j,k,:,JDIR) - var(gamAppMpi%Grid%ie+1,j,k,:,JDIR)))
                  @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
              enddo
          enddo
          ! K edges
          do k=gamAppMpi%Grid%ks,gamAppMpi%Grid%ke
              do j=gamAppMpi%Grid%js,gamAppMpi%Grid%je+1
                  write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad K edge value in ',trim(varName),' at (',gamAppMpi%Grid%is,',',j,',',k,')'
                  checkValue = sum(abs(var(gamAppMpi%Grid%is,j,k,:,KDIR) - var(gamAppMpi%Grid%ie+1,j,k,:,KDIR)))
                  @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
              enddo
          enddo
      endif

      if(.not. gamAppMpi%Grid%hasLowerBC(JDIR)) then
          ! check lower J data
          ! I edges
          do k=gamAppMpi%Grid%ks,gamAppMpi%Grid%ke+1
              do i=gamAppMpi%Grid%is,gamAppMpi%Grid%ie
                  write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad I edge value in ',trim(varName),' at (',i,',',gamAppMpi%Grid%js,',',k,')'
                  checkValue = sum(abs(var(i,gamAppMpi%Grid%js,k,:,IDIR) - var(i,gamAppMpi%Grid%je+1,k,:,IDIR)))
                  @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
              enddo
          enddo
          ! K edges
          do k=gamAppMpi%Grid%ks,gamAppMpi%Grid%ke
              do i=gamAppMpi%Grid%js,gamAppMpi%Grid%ie+1
                  write (checkMessage,'(A,A,A,I0,A,I0,A,I0,A)') 'Bad K edge value in ',trim(varName),' at (',i,',',gamAppMpi%Grid%js,',',k,')'
                  checkValue = sum(abs(var(i,gamAppMpi%Grid%js,k,:,KDIR) - var(i,gamAppMpi%Grid%je+1,k,:,KDIR)))
                  @assertLessThanOrEqual(checkValue, 1e-8_rp,trim(checkMessage))
              enddo
          enddo
      endif

  end subroutine verifyEdges5

  @test(npes=[8])
  subroutine testFaceMetricFields(this)
      class (MpiTestMethod), intent(inout) :: this

      call copyFaces5(gamAppMpi%Grid%Tf,&
          gamAppMpi%Grid%isg,gamAppMpi%Grid%ieg+1,&
          gamAppMpi%Grid%jsg,gamAppMpi%Grid%jeg+1,&
          gamAppMpi%Grid%ksg,gamAPpMpi%Grid%keg+1, NDIM*NDIM)
      call copyFaces4(gamAppMpi%Grid%Face,&
          gamAppMpi%Grid%isg,gamAppMpi%Grid%ieg+1,&
          gamAppMpi%Grid%jsg,gamAppMpi%Grid%jeg+1,&
          gamAppMpi%Grid%ksg,gamAPpMpi%Grid%keg+1)

      call verifyFaces5(gamAppMpi%Grid%Tf,'Tf',&
          gamAppMpi%Grid%isg,gamAppMpi%Grid%ieg+1,&
          gamAppMpi%Grid%jsg,gamAppMpi%Grid%jeg+1,&
          gamAppMpi%Grid%ksg,gamAPpMpi%Grid%keg+1, NDIM*NDIM) ! g+1,g+1,g+1,DIM*DIM,DIM
      call verifyFaces4(gamAppMpi%Grid%Face,'Face',&
          gamAppMpi%Grid%isg,gamAppMpi%Grid%ieg+1,&
          gamAppMpi%Grid%jsg,gamAppMpi%Grid%jeg+1,&
          gamAppMpi%Grid%ksg,gamAPpMpi%Grid%keg+1) ! g+1,g+1,g+1,DIM
  end subroutine testFaceMetricFields

  @test(npes=[8])
  subroutine testEdgeMetricFields(this)
      class (MpiTestMethod), intent(inout) :: this

      call copyEdges5(gamAppMpi%Grid%Te,&
          gamAppMpi%Grid%isg,gamAppMpi%Grid%ieg+1,&
          gamAppMpi%Grid%jsg,gamAppMpi%Grid%jeg+1,&
          gamAppMpi%Grid%ksg,gamAPpMpi%Grid%keg+1, NDIM*NDIM)
      call copyEdges5(gamAppMpi%Grid%Teb,&
          gamAppMpi%Grid%isg,gamAppMpi%Grid%ieg+1,&
          gamAppMpi%Grid%jsg,gamAppMpi%Grid%jeg+1,&
          gamAppMpi%Grid%ksg,gamAPpMpi%Grid%keg+1, 4)
      call copyEdges4(gamAppMpi%Grid%Edge,&
          gamAppMpi%Grid%isg,gamAppMpi%Grid%ieg+1,&
          gamAppMpi%Grid%jsg,gamAppMpi%Grid%jeg+1,&
          gamAppMpi%Grid%ksg,gamAPpMpi%Grid%keg+1)

      call verifyEdges5(gamAppMpi%Grid%Te,'Te',&
          gamAppMpi%Grid%isg,gamAppMpi%Grid%ieg+1,&
          gamAppMpi%Grid%jsg,gamAppMpi%Grid%jeg+1,&
          gamAppMpi%Grid%ksg,gamAPpMpi%Grid%keg+1, NDIM*NDIM) ! g,g,g,DIM*DIM,DIM
      call verifyEdges5(gamAppMpi%Grid%Teb,'Teb',&
          gamAppMpi%Grid%isg,gamAppMpi%Grid%ieg+1,&
          gamAppMpi%Grid%jsg,gamAppMpi%Grid%jeg+1,&
          gamAppMpi%Grid%ksg,gamAPpMpi%Grid%keg+1, 4) ! g,g,g,4,DIM
      call verifyEdges4(gamAppMpi%Grid%Edge,'Edge',&
          gamAppMpi%Grid%isg,gamAppMpi%Grid%ieg+1,&
          gamAppMpi%Grid%jsg,gamAppMpi%Grid%jeg+1,&
          gamAppMpi%Grid%ksg,gamAPpMpi%Grid%keg+1) ! g,g,g,DIM
  end subroutine testEdgeMetricFields

  @test(npes=[8])
  subroutine testFaceDataFields(this)
      class (MpiTestMethod), intent(inout) :: this

      call copyFaces4(gamAPpMpi%State%magFlux,&
          gamAppMpi%Grid%isg,gamAppMpi%Grid%ieg+1,&
          gamAppMpi%Grid%jsg,gamAppMpi%Grid%jeg+1,&
          gamAppMpi%Grid%ksg,gamAPpMpi%Grid%keg+1)

      call verifyFaces4(gamAPpMpi%State%magFlux,'magFlux',&
          gamAppMpi%Grid%isg,gamAppMpi%Grid%ieg+1,&
          gamAppMpi%Grid%jsg,gamAppMpi%Grid%jeg+1,&
          gamAppMpi%Grid%ksg,gamAPpMpi%Grid%keg+1) ! g+1,g+1,g+1,NDIM
  end subroutine testFaceDataFields

  @test(npes=[8])
  subroutine testEdgeDataFields(this)
      class (MpiTestMethod), intent(inout) :: this

      call copyEdges4(gamAppMpi%State%Efld,&
          gamAppMpi%Grid%isg,gamAppMpi%Grid%ieg,&
          gamAppMpi%Grid%jsg,gamAppMpi%Grid%jeg,&
          gamAppMpi%Grid%ksg,gamAPpMpi%Grid%keg)

      call verifyEdges4(gamAppMpi%State%Efld,'Efld',&
          gamAppMpi%Grid%isg,gamAppMpi%Grid%ieg,&
          gamAppMpi%Grid%jsg,gamAppMpi%Grid%jeg,&
          gamAppMpi%Grid%ksg,gamAPpMpi%Grid%keg) ! g,g,g,NDIM
  end subroutine testEdgeDataFields

end module testMpiFields

