Trcadv async + serial kernel fix

src/General/memory.f90

@@ -516,7 +516,7 @@ subroutine alloc_tot()
       gphiv    = huge(gphiv(1,1))
       allocate(gphif(jpj,jpi))          
       gphif    = huge(gphif(1,1))
-      allocate(e1t(jpj,jpi))            
+      allocate(e1t(jpj,jpi))
       e1t      = huge(e1t(1,1))
       allocate(e1u(jpj,jpi))            
       e1u      = huge(e1u(1,1))
@@ -790,10 +790,14 @@ subroutine alloc_tot()
         allocate(DAY_LENGTH(jpj,jpi))   
        DAY_LENGTH = huge(DAY_LENGTH(1,1))
        forcing_phys_initialized = .false.
+
+       !$acc enter data create(e1t,e2t,e3t,e3w,e3t_back,tra,trb,tmask,avt,&
+       !$acc& e1u,e2u,e3u,e1v,e2v,e3v,un,vn,wn,trn)
+
 #ifdef Mem_Monitor
       mem_all=get_mem(err) - aux_mem
 #endif
-  
+
         END subroutine alloc_tot
 
 
@@ -802,6 +806,8 @@ subroutine clean_memory()
 
             ! myalloc (memory.f90)
 
+            !$acc exit data delete(e1t,e2t,e3t,e3w,e3t_back,tra,trb,tmask,avt)
+
 #ifdef key_mpp
 
             !$acc exit data delete(te_send, tw_send, tn_send, ts_send) finalize
@@ -953,7 +959,7 @@ subroutine clean_memory()
             deallocate(tra_DIA_2d_IO_HIGH)
             deallocate(tra_PHYS_2d_IO)
             deallocate(tra_PHYS_2d_IO_HIGH)
-            
+
 
             if(lwp) then
                 deallocate(tottrn)
@@ -995,6 +1001,8 @@ subroutine clean_memory()
             deallocate(highfreq_table_dia)
             deallocate(highfreq_table_dia2d)
 
+            !$acc exit data delete(trn, e1u, e2u, e3u, e1v, e2v, e3v, un, vn, wn)
+
         end subroutine clean_memory
 
     INTEGER FUNCTION find_index_var(string)

src/General/step.f90

@@ -265,6 +265,10 @@ SUBROUTINE trcstp
 !         with surface boundary condition
 !         with IMPLICIT vertical diffusion
 
+      ! XXX: to be removed
+      use DIA_mem, only: diaflx
+      use myalloc, only: tra,trb,e1t,e3t_back,e2t,e3t,e3w,tmask,avt
+
        IMPLICIT NONE
       integer jn,jk,ji,jj
       trcstpparttime = MPI_WTIME() ! cronometer-start
@@ -296,7 +300,9 @@ SUBROUTINE trcstp
 
       IF (lbfm) CALL trcsms
 
+      !$acc update device(e1t,diaflx,e3t_back,e2t,trb,tmask,e3t,tra,avt,e3w) if (lzdf)
       IF (lzdf) CALL trczdf ! tracers: vertical diffusion
+      !$acc update host(diaflx,tra) if (lzdf)
 
       IF (lsnu) CALL snutel
 

src/IO/DIA_mem.f90

@@ -50,6 +50,7 @@ SUBROUTINE alloc_DIA_local_flx()
        INDflxDUMP = huge(INDflxDUMP(1))
        allocate(diaflx        (7, Fsize, jptra   ))
        diaflx = 0
+       !$acc enter data create(flx_ridxt,diaflx)
       END SUBROUTINE alloc_DIA_local_flx
 
 
@@ -95,6 +96,7 @@ subroutine clean_memory_dia()
 
           if (allocated(diaflx)) then
               deallocate(diaflx)
+              !$acc exit data delete(diaflx)
           endif
 
           if (allocated(INDflxDUMPZERO)) then

src/MPI/ogstm_mpi.f90

@@ -413,6 +413,7 @@ SUBROUTINE mpplnk_my_openacc(ptab,gpu)
       !$acc kernels default(present) if(use_gpu)
       IF(nbondj.eq.0.or.nbondj.eq.1) THEN ! All but south boundary, we received from south
 
+         !$acc loop independent
          DO jw=1,SOUTH_count_recv
               ji = SOUTHpoints_recv(1,jw)
               jk = SOUTHpoints_recv(2,jw)
@@ -423,6 +424,7 @@ SUBROUTINE mpplnk_my_openacc(ptab,gpu)
 
       IF(nbondj.eq.-1.or.nbondj.eq.0) THEN ! All but north boundary, we received from north
 
+        !$acc loop independent
         DO jw=1,NORTH_count_recv
               ji = NORTHpoints_recv(1,jw)
               jk = NORTHpoints_recv(2,jw)

src/PHYS/ADV_mem.f90

@@ -114,6 +114,9 @@ subroutine myalloc_ADV()
       mem_all=get_mem(err) - aux_mem
 #endif
 
+      !$acc enter data create(advmask,zaa,zbb,zcc,inv_eu,inv_ev,inv_et,&
+      !$acc& big_fact_zaa,big_fact_zbb,big_fact_zcc,zbtr_arr)
+
       END subroutine myalloc_ADV
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1
 
@@ -133,6 +136,9 @@ subroutine clean_memory_adv()
           deallocate(big_fact_zcc)
           deallocate(zbtr_arr)
 
+          !$acc exit data delete(advmask,zaa,zbb,zcc,inv_eu,inv_ev,inv_et,&
+          !$acc& big_fact_zaa,big_fact_zbb,big_fact_zcc,zbtr_arr)
+
       end subroutine clean_memory_adv
 
 

src/PHYS/ZDF_mem.f90

@@ -46,6 +46,9 @@ subroutine myalloc_ZDF()
         jarr_zdf     = huge(jarr_zdf(1,1))
        allocate(jarr_zdf_flx(jpi*jpj,jpk)) 
         jarr_zdf_flx = huge(jarr_zdf_flx(1,1))
+        !$acc enter data create(jarr_zdf,jarr_zdf_flx)
+        !$acc update device(jarr_zdf,jarr_zdf_flx)
+#ifndef _OPENACC
        allocate(zwd(jpk, ntids))           
         zwd          = huge(zwd(1,1))
        allocate(zws(jpk, ntids))           
@@ -60,14 +63,36 @@ subroutine myalloc_ZDF()
         zwz          = huge(zwz(1,1))
        allocate(zwt(jpk, ntids))           
         zwt          = huge(zwt(1,1))
+#endif
 
 #ifdef Mem_Monitor
       mem_all=get_mem(err) - aux_mem
 #endif
 
 
       END subroutine myalloc_ZDF
-
+
+#ifdef _OPENACC
+      subroutine myalloc_ZDF_gpu()
+        allocate(zwd(jpk, dimen_jvzdf))
+        zwd          = huge(zwd(1,1))
+        allocate(zws(jpk, dimen_jvzdf))
+        zws          = huge(zws(1,1))
+        allocate(zwi(jpk, dimen_jvzdf))
+        zwi          = huge(zwi(1,1))
+        allocate(zwx(jpk, dimen_jvzdf))
+        zwx          = huge(zwx(1,1))
+        allocate(zwy(jpk, dimen_jvzdf))
+        zwy          = huge(zwy(1,1))
+        allocate(zwz(jpk, dimen_jvzdf))
+        zwz          = huge(zwz(1,1))
+        allocate(zwt(jpk, dimen_jvzdf))
+        zwt          = huge(zwt(1,1))
+
+        !$acc enter data create(zwd,zwi,zwx,zws,zwz,zwy,zwt)
+        !$acc update device(zwd,zwi,zwx,zws,zwz,zwy,zwt)
+      END subroutine myalloc_ZDF_gpu
+#endif
 
 
       subroutine clean_memory_zdf()
@@ -82,6 +107,8 @@ subroutine clean_memory_zdf()
           deallocate(zwz)
           deallocate(zwt)
 
+          !$acc exit data delete(jarr_zdf,jarr_zdf_flx,zwd,zwi,zwx,zws,zwz,zwy,zwt)
+
       end subroutine clean_memory_zdf