Changeset d005a67 in flexpart.git

Timestamp:

May 21, 2019, 1:09:18 PM (5 years ago)

Author:

Sabine <sabine.eckhardt@…>

Branches:

master, 10.4.1_pesei, GFS_025, bugfixes+enhancements, dev, release-10, release-10.4.1, scaling-bug

Children:

f963113, 0a98afe

Parents:

5d74ed9 (diff), 0c8c7f2 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge remote-tracking branch 'refs/remotes/origin/dev' into dev

Files:

• options/COMMAND (modified) (1 diff)
• src/FLEXPART.f90 (modified) (2 diffs)
• src/FLEXPART_MPI.f90 (modified) (3 diffs)
• src/com_mod.f90 (modified) (7 diffs)
• src/init_domainfill.f90 (modified) (2 diffs)
• src/init_domainfill_mpi.f90 (modified) (4 diffs)
• src/makefile (modified) (4 diffs)
• src/mpi_mod.f90 (modified) (17 diffs)
• src/netcdf_output_mod.f90 (modified) (1 diff)
• src/par_mod.f90 (modified) (1 diff)
• src/partoutput.f90 (modified) (1 diff)
• src/partoutput_average.f90 (added)
• src/partoutput_average_mpi.f90 (added)
• src/partoutput_mpi.f90 (modified) (1 diff)
• src/partpos_average.f90 (added)
• src/readcommand.f90 (modified) (4 diffs)
• src/timemanager.f90 (modified) (2 diffs)
• src/timemanager_mpi.f90 (modified) (6 diffs)
• src/verttransform_ecmwf.f90 (modified) (1 diff)
• src/readwind_gfs.f90 (modified) (3 diffs)
• src/releaseparticles.f90 (modified) (1 diff)
• src/verttransform_gfs.f90 (modified) (1 diff)

options/COMMAND

@@ -19 +19 @@
  IFINE=                 4, ! Reduction for time step in vertical transport, used only if CTL>1 
  IOUT=                  1, ! Output type: [1]mass 2]pptv 3]1&2 4]plume 5]1&4, +8 for NetCDF output     
- IPOUT=                 0, ! Particle position output: 0]no 1]every output 2]only at end   
+ IPOUT=                 0, ! Particle position output: 0]no 1]every output 2]only at end 3]time averaged
  LSUBGRID=              0, ! Increase of ABL heights due to sub-grid scale orographic variations;[0]off 1]on 
  LCONVECTION=           1, ! Switch for convection parameterization;0]off [1]on    

src/FLEXPART.f90

@@ -68 +68 @@
   integer :: detectformat
 
-
-
-  ! Initialize arrays in com_mod
-  !*****************************
-  call com_mod_allocate_part(maxpart)
   
-
   ! Generate a large number of random numbers
   !******************************************
@@ -172 +166 @@
   endif
 
+  ! Initialize arrays in com_mod
+  !*****************************
+  call com_mod_allocate_part(maxpart)
+
+
   ! Read the age classes to be used
   !********************************

src/FLEXPART_MPI.f90

@@ -77 +77 @@
   if (mp_measure_time) call mpif_mtime('flexpart',0)
 
-  ! Initialize arrays in com_mod 
-  !*****************************
-
-  if(.not.(lmpreader.and.lmp_use_reader)) call com_mod_allocate_part(maxpart_mpi)
-
-
+  
   ! Generate a large number of random numbers
   !******************************************
@@ -180 +175 @@
   endif
 
+  ! Initialize arrays in com_mod 
+  !*****************************
+
+  if(.not.(lmpreader.and.lmp_use_reader)) call com_mod_allocate_part(maxpart_mpi)
+
 
 ! Read the age classes to be used
@@ -413 +413 @@
   end if ! (mpif_pid == 0) 
 
-  if (mp_measure_time) call mpif_mtime('iotime',0)
+  if (mp_measure_time) call mpif_mtime('iotime',1)
 
   if (verbosity.gt.0 .and. lroot) then

src/com_mod.f90

@@ -18 +18 @@
 
   implicit none
+
+
 
   !****************************************************************
@@ -69 +71 @@
 
   real :: ctl,fine
-  integer :: ifine,iout,ipout,ipin,iflux,mdomainfill
+  integer :: ifine,iout,ipout,ipin,iflux,mdomainfill,ipoutfac
   integer :: mquasilag,nested_output,ind_source,ind_receptor
   integer :: ind_rel,ind_samp,ioutputforeachrelease,linit_cond,surf_only
@@ -82 +84 @@
   ! iout     output options: 1 conc. output (ng/m3), 2 mixing ratio (pptv), 3 both
   ! ipout    particle dump options: 0 no, 1 every output interval, 2 only at end
+  ! ipoutfac increase particle dump interval by factor (default 1)
   ! ipin     read in particle positions from dumped file from a previous run
   ! fine     real(ifine)
@@ -128 +131 @@
 
   logical :: gdomainfill
-
   ! gdomainfill             .T., if domain-filling is global, .F. if not
 
@@ -651 +653 @@
   real :: receptorarea(maxreceptor)
   real :: creceptor(maxreceptor,maxspec)
+  real, allocatable, dimension(:,:) :: creceptor0
   character(len=16) :: receptorname(maxreceptor)
   integer :: numreceptor
@@ -673 +676 @@
   real, allocatable, dimension(:,:) :: xmass1
   real, allocatable, dimension(:,:) :: xscav_frac1
+
+! Variables used for writing out interval averages for partoutput
+!****************************************************************
+
+  integer, allocatable, dimension(:) :: npart_av
+  real, allocatable, dimension(:) :: part_av_cartx,part_av_carty,part_av_cartz,part_av_z,part_av_topo
+  real, allocatable, dimension(:) :: part_av_pv,part_av_qv,part_av_tt,part_av_rho,part_av_tro,part_av_hmix
+  real, allocatable, dimension(:) :: part_av_uu,part_av_vv,part_av_energy
 
   ! eso: Moved from timemanager
@@ -780 +791 @@
          & idt(nmpart),itramem(nmpart),itrasplit(nmpart),&
          & xtra1(nmpart),ytra1(nmpart),ztra1(nmpart),&
-         & xmass1(nmpart, maxspec),&
-         & checklifetime(nmpart,maxspec), species_lifetime(maxspec,2))!CGZ-lifetime
+         & xmass1(nmpart, maxspec))  ! ,&
+!         & checklifetime(nmpart,maxspec), species_lifetime(maxspec,2))!CGZ-lifetime
+
+    if (ipout.eq.3) then
+      allocate(npart_av(nmpart),part_av_cartx(nmpart),part_av_carty(nmpart),&
+           & part_av_cartz(nmpart),part_av_z(nmpart),part_av_topo(nmpart))
+      allocate(part_av_pv(nmpart),part_av_qv(nmpart),part_av_tt(nmpart),&
+           & part_av_rho(nmpart),part_av_tro(nmpart),part_av_hmix(nmpart))
+      allocate(part_av_uu(nmpart),part_av_vv(nmpart),part_av_energy(nmpart))
+    end if
 
 
     allocate(uap(nmpart),ucp(nmpart),uzp(nmpart),us(nmpart),&
          & vs(nmpart),ws(nmpart),cbt(nmpart))
-    
+
   end subroutine com_mod_allocate_part
 

src/init_domainfill.f90

@@ -86 +86 @@
     endif
   endif
+
+! Exit here if resuming a run from particle dump
+!***********************************************
+  if (gdomainfill.and.ipin.ne.0) return
 
 ! Do not release particles twice (i.e., not at both in the leftmost and rightmost
@@ -414 +418 @@
 !***************************************************************************
 
-  if (ipin.eq.1) then
+  if ((ipin.eq.1).and.(.not.gdomainfill)) then
     open(unitboundcond,file=path(2)(1:length(2))//'boundcond.bin', &
          form='unformatted')

src/init_domainfill_mpi.f90

@@ -110 +110 @@
   endif
 
+! Exit here if resuming a run from particle dump
+!***********************************************
+  if (gdomainfill.and.ipin.ne.0) return
+
 ! Do not release particles twice (i.e., not at both in the leftmost and rightmost
 ! grid cell) for a global domain
@@ -213 +217 @@
         colmass(ix,jy)=(pp(1)-pp(nz))/ga*gridarea(jy)
         colmasstotal=colmasstotal+colmass(ix,jy)
-
       end do
     end do
@@ -466 +469 @@
 
 ! eso TODO: only needed for root process
-    if (ipin.eq.1) then
+    if ((ipin.eq.1).and.(.not.gdomainfill)) then
       open(unitboundcond,file=path(2)(1:length(2))//'boundcond.bin', &
            form='unformatted')
@@ -474 +477 @@
     endif
 
-    numpart = numpart/mp_partgroup_np
-    if (mod(numpart,mp_partgroup_np).ne.0) numpart=numpart+1
-
-  else ! Allocate dummy arrays for receiving processes 
-    allocate(itra1_tmp(nullsize),npoint_tmp(nullsize),nclass_tmp(nullsize),&
-         & idt_tmp(nullsize),itramem_tmp(nullsize),itrasplit_tmp(nullsize),&
-         & xtra1_tmp(nullsize),ytra1_tmp(nullsize),ztra1_tmp(nullsize),&
-         & xmass1_tmp(nullsize, nullsize))
+    if (ipin.eq.0) then    
+      numpart = numpart/mp_partgroup_np
+      if (mod(numpart,mp_partgroup_np).ne.0) numpart=numpart+1
+    end if
+
+  else ! Allocate dummy arrays for receiving processes
+    if (ipin.eq.0) then    
+      allocate(itra1_tmp(nullsize),npoint_tmp(nullsize),nclass_tmp(nullsize),&
+           & idt_tmp(nullsize),itramem_tmp(nullsize),itrasplit_tmp(nullsize),&
+           & xtra1_tmp(nullsize),ytra1_tmp(nullsize),ztra1_tmp(nullsize),&
+           & xmass1_tmp(nullsize, nullsize))
+    end if
     
-  end if ! end if(lroot)  
+  end if ! end if(lroot)
+
 
 
 ! Distribute particles to other processes (numpart is 'per-process', not total)
-  call MPI_Bcast(numpart, 1, MPI_INTEGER, id_root, mp_comm_used, mp_ierr)
-! eso TODO: xmassperparticle: not necessary to send
-  call MPI_Bcast(xmassperparticle, 1, mp_sp, id_root, mp_comm_used, mp_ierr)
-  call mpif_send_part_properties(numpart)
+! Only if not restarting from previous run
+  if (ipin.eq.0) then
+    call MPI_Bcast(numpart, 1, MPI_INTEGER, id_root, mp_comm_used, mp_ierr)
+    call mpif_send_part_properties(npart(1)/mp_partgroup_np)
 
 ! Deallocate the temporary arrays used for all particles
-  deallocate(itra1_tmp,npoint_tmp,nclass_tmp,idt_tmp,itramem_tmp,&
+    deallocate(itra1_tmp,npoint_tmp,nclass_tmp,idt_tmp,itramem_tmp,&
          & itrasplit_tmp,xtra1_tmp,ytra1_tmp,ztra1_tmp,xmass1_tmp)
+  end if
 
 

src/makefile

@@ -118 +118 @@
 OBJECTS_SERIAL = \
         releaseparticles.o      partoutput.o \
+        partoutput_average.o \
         conccalc.o \
         init_domainfill.o       concoutput.o  \
@@ -132 +133 @@
 ## For MPI version 
 OBJECTS_MPI = releaseparticles_mpi.o partoutput_mpi.o \
-        conccalc_mpi.o \
+        partoutput_average_mpi.o conccalc_mpi.o \
         init_domainfill_mpi.o concoutput_mpi.o  \
         timemanager_mpi.o FLEXPART_MPI.o        \
@@ -149 +150 @@
 advance.o               initialize.o            \
 writeheader.o           writeheader_txt.o       \
-writeprecip.o \
+partpos_average.o       writeprecip.o \
 writeheader_surf.o      assignland.o\
 part0.o                 gethourlyOH.o\
@@ -348 +349 @@
 part0.o: par_mod.o
 partdep.o: par_mod.o
+partpos_average.o: com_mod.o par_mod.o
 partoutput.o: com_mod.o par_mod.o
+partoutput_average.o: com_mod.o par_mod.o
+partoutput_average_mpi.o: com_mod.o par_mod.o mpi_mod.o
 partoutput_mpi.o: com_mod.o mpi_mod.o par_mod.o
 partoutput_short.o: com_mod.o par_mod.o

src/mpi_mod.f90

@@ -88 +88 @@
 ! Variables for MPI processes in the 'particle' group
   integer, allocatable, dimension(:) :: mp_partgroup_rank
+  integer, allocatable, dimension(:) :: npart_per_process
   integer :: mp_partgroup_comm, mp_partgroup_pid, mp_partgroup_np
 
@@ -125 +126 @@
 ! mp_time_barrier   Measure MPI barrier time
 ! mp_exact_numpart  Use an extra MPI communication to give the exact number of particles
-!                   to standard output (this does *not* otherwise affect the simulation) 
+!                   to standard output (this does not otherwise affect the simulation)
   logical, parameter :: mp_dbg_mode = .false.
   logical, parameter :: mp_dev_mode = .false.
@@ -190 +191 @@
 !   mp_np       number of running processes, decided at run-time
 !***********************************************************************
-    use par_mod, only: maxpart, numwfmem, dep_prec
-    use com_mod, only: mpi_mode, verbosity
+    use par_mod, only: maxpart, numwfmem, dep_prec, maxreceptor, maxspec
+    use com_mod, only: mpi_mode, verbosity, creceptor0
 
     implicit none
@@ -337 +338 @@
 
 ! Set maxpart per process
-! eso 08/2016: Increase maxpart per process, in case of unbalanced distribution
+! ESO 08/2016: Increase maxpart per process, in case of unbalanced distribution
     maxpart_mpi=int(mp_maxpart_factor*real(maxpart)/real(mp_partgroup_np))
     if (mp_np == 1) maxpart_mpi = maxpart
@@ -365 +366 @@
     end if
 
+! Allocate array for number of particles per process    
+    allocate(npart_per_process(0:mp_partgroup_np-1))
+
+! Write whether MPI_IN_PLACE is used or not
+#ifdef USE_MPIINPLACE
+    if (lroot) write(*,*) 'Using MPI_IN_PLACE operations'
+#else
+    if (lroot) allocate(creceptor0(maxreceptor,maxspec))
+    if (lroot) write(*,*) 'Not using MPI_IN_PLACE operations'
+#endif
     goto 101
 
@@ -559 +570 @@
 ! invalid particles at the end of the arrays
 
-601 do i=num_part, 1, -1
+601 do i=numpart, 1, -1
       if (itra1(i).eq.-999999999) then
         numpart=numpart-1
@@ -598 +609 @@
     integer :: i,jj,nn, num_part=1,m,imin, num_trans
     logical :: first_iter
-    integer,allocatable,dimension(:) :: numparticles_mpi, idx_arr
+    integer,allocatable,dimension(:) :: idx_arr
     real,allocatable,dimension(:) :: sorted ! TODO: we don't really need this
 
@@ -607 +618 @@
 ! All processes exchange information on number of particles
 !****************************************************************************
-    allocate(numparticles_mpi(0:mp_partgroup_np-1), &
-         &idx_arr(0:mp_partgroup_np-1), sorted(0:mp_partgroup_np-1))
-
-    call MPI_Allgather(numpart, 1, MPI_INTEGER, numparticles_mpi, &
+    allocate( idx_arr(0:mp_partgroup_np-1), sorted(0:mp_partgroup_np-1))
+
+    call MPI_Allgather(numpart, 1, MPI_INTEGER, npart_per_process, &
          & 1, MPI_INTEGER, mp_comm_used, mp_ierr)
 
@@ -616 +626 @@
 ! Sort from lowest to highest
 ! Initial guess: correct order
-    sorted(:) = numparticles_mpi(:)
+    sorted(:) = npart_per_process(:)
     do i=0, mp_partgroup_np-1
       idx_arr(i) = i
     end do
+
+! Do not rebalance particles for ipout=3    
+    if (ipout.eq.3) return
 
 ! For each successive element in index array, see if a lower value exists
@@ -645 +658 @@
     m=mp_partgroup_np-1 ! index for last sorted process (most particles)
     do i=0,mp_partgroup_np/2-1
-      num_trans = numparticles_mpi(idx_arr(m)) - numparticles_mpi(idx_arr(i))
+      num_trans = npart_per_process(idx_arr(m)) - npart_per_process(idx_arr(i))
       if (mp_partid.eq.idx_arr(m).or.mp_partid.eq.idx_arr(i)) then
-        if ( numparticles_mpi(idx_arr(m)).gt.mp_min_redist.and.&
-             & real(num_trans)/real(numparticles_mpi(idx_arr(m))).gt.mp_redist_fract) then
+        if ( npart_per_process(idx_arr(m)).gt.mp_min_redist.and.&
+             & real(num_trans)/real(npart_per_process(idx_arr(m))).gt.mp_redist_fract) then
 ! DBG
-          ! write(*,*) 'mp_partid, idx_arr(m), idx_arr(i), mp_min_redist, num_trans, numparticles_mpi', &
-          !      &mp_partid, idx_arr(m), idx_arr(i), mp_min_redist, num_trans, numparticles_mpi
+          ! write(*,*) 'mp_partid, idx_arr(m), idx_arr(i), mp_min_redist, num_trans, npart_per_process', &
+          !      &mp_partid, idx_arr(m), idx_arr(i), mp_min_redist, num_trans, npart_per_process
 ! DBG
           call mpif_redist_part(itime, idx_arr(m), idx_arr(i), num_trans/2)
@@ -659 +672 @@
     end do
 
-    deallocate(numparticles_mpi, idx_arr, sorted)
+    deallocate(idx_arr, sorted)
 
   end subroutine mpif_calculate_part_redist
@@ -1961 +1974 @@
     if (readclouds) then
       j=j+1
-      call MPI_Irecv(ctwc(:,:,mind),d2s1,mp_sp,id_read,MPI_ANY_TAG,&
+      call MPI_Irecv(ctwc(:,:,mind),d2s1*5,mp_sp,id_read,MPI_ANY_TAG,&
            &MPI_COMM_WORLD,reqs(j),mp_ierr)
       if (mp_ierr /= 0) goto 600
@@ -2326 +2339 @@
       if (readclouds) then
         j=j+1
-        call MPI_Irecv(ctwcn(:,:,mind,k),d2s1,mp_sp,id_read,MPI_ANY_TAG,&
+        call MPI_Irecv(ctwcn(:,:,mind,k),d2s1*5,mp_sp,id_read,MPI_ANY_TAG,&
              &MPI_COMM_WORLD,reqs(j),mp_ierr)
         if (mp_ierr /= 0) goto 600
@@ -2462 +2475 @@
     end if
 
+  ! Receptor concentrations    
+    if (lroot) then
+      call MPI_Reduce(MPI_IN_PLACE,creceptor,rcpt_size,mp_sp,MPI_SUM,id_root, &
+           & mp_comm_used,mp_ierr)
+      if (mp_ierr /= 0) goto 600
+    else
+      call MPI_Reduce(creceptor,0,rcpt_size,mp_sp,MPI_SUM,id_root, &
+           & mp_comm_used,mp_ierr)
+    end if
+
 #else
 
       call MPI_Reduce(gridunc, gridunc0, grid_size3d, mp_sp, MPI_SUM, id_root, &
            & mp_comm_used, mp_ierr)
+      if (mp_ierr /= 0) goto 600
       if (lroot) gridunc = gridunc0
+
+      call MPI_Reduce(creceptor, creceptor0,rcpt_size,mp_sp,MPI_SUM,id_root, &
+           & mp_comm_used,mp_ierr)
+      if (mp_ierr /= 0) goto 600
+      if (lroot) creceptor = creceptor0
 
 #endif
@@ -2482 +2511 @@
     end if
 
-! Receptor concentrations    
-    if (lroot) then
-      call MPI_Reduce(MPI_IN_PLACE,creceptor,rcpt_size,mp_sp,MPI_SUM,id_root, &
-           & mp_comm_used,mp_ierr)
-      if (mp_ierr /= 0) goto 600
-    else
-      call MPI_Reduce(creceptor,0,rcpt_size,mp_sp,MPI_SUM,id_root, &
-           & mp_comm_used,mp_ierr)
-    end if
 
     if (mp_measure_time) call mpif_mtime('commtime',1)
@@ -2700 +2720 @@
       end if
 
-    case ('readwind')
-      if (imode.eq.0) then
-        call cpu_time(mp_readwind_time_beg)
-        mp_readwind_wtime_beg = mpi_wtime()
-      else
-        call cpu_time(mp_readwind_time_end)
-        mp_readwind_wtime_end = mpi_wtime()
-
-        mp_readwind_time_total = mp_readwind_time_total + &
-             &(mp_readwind_time_end - mp_readwind_time_beg)
-        mp_readwind_wtime_total = mp_readwind_wtime_total + &
-             &(mp_readwind_wtime_end - mp_readwind_wtime_beg)
-      end if
+   case ('readwind')
+     if (imode.eq.0) then
+       call cpu_time(mp_readwind_time_beg)
+       mp_readwind_wtime_beg = mpi_wtime()
+     else
+       call cpu_time(mp_readwind_time_end)
+       mp_readwind_wtime_end = mpi_wtime()
+
+       mp_readwind_time_total = mp_readwind_time_total + &
+            &(mp_readwind_time_end - mp_readwind_time_beg)
+       mp_readwind_wtime_total = mp_readwind_wtime_total + &
+            &(mp_readwind_wtime_end - mp_readwind_wtime_beg)
+     end if
 
     case ('commtime')
@@ -2788 +2808 @@
           write(*,FMT='(A60,TR1,F9.2)') 'TOTAL CPU TIME FOR GETFIELDS:',&
                & mp_getfields_time_total
-          write(*,FMT='(A60,TR1,F9.2)') 'TOTAL WALL TIME FOR READWIND:',&
-               & mp_readwind_wtime_total
-          write(*,FMT='(A60,TR1,F9.2)') 'TOTAL CPU TIME FOR READWIND:',&
-               & mp_readwind_time_total
+!          write(*,FMT='(A60,TR1,F9.2)') 'TOTAL WALL TIME FOR READWIND:',&
+!               & mp_readwind_wtime_total
+!          write(*,FMT='(A60,TR1,F9.2)') 'TOTAL CPU TIME FOR READWIND:',&
+!               & mp_readwind_time_total
           write(*,FMT='(A60,TR1,F9.2)') 'TOTAL WALL TIME FOR FILE IO:',&
                & mp_io_wtime_total

src/netcdf_output_mod.f90

@@ -93 +93 @@
   character(len=255), parameter :: institution = 'NILU'
 
-  integer            :: tpointer
+  integer            :: tpointer=0
   character(len=255) :: ncfname, ncfnamen
 

src/par_mod.f90

@@ -280 +280 @@
 
   integer,parameter :: unitpath=1, unitcommand=1, unitageclasses=1, unitgrid=1
-  integer,parameter :: unitavailab=1, unitreleases=88, unitpartout=93
+  integer,parameter :: unitavailab=1, unitreleases=88, unitpartout=93, unitpartout_average=105
   integer,parameter :: unitpartin=93, unitflux=98, unitouttraj=96
   integer,parameter :: unitvert=1, unitoro=1, unitpoin=1, unitreceptor=1

src/partoutput.f90

@@ -71 +71 @@
   !**************************************
 
-  if (ipout.eq.1) then
+  if (ipout.eq.1.or.ipout.eq.3) then
     open(unitpartout,file=path(2)(1:length(2))//'partposit_'//adate// &
          atime,form='unformatted')

src/partoutput_mpi.f90

@@ -78 +78 @@
   !**************************************
 
-  if (ipout.eq.1) then
+  if (ipout.eq.1.or.ipout.eq.3) then
     open(unitpartout,file=path(2)(1:length(2))//'partposit_'//adate// &
          atime,form='unformatted',status=file_stat,position='append')

src/readcommand.f90

@@ -50 +50 @@
   ! ipin                 1 continue simulation with dumped particle data, 0 no *
   ! ipout                0 no particle dump, 1 every output time, 3 only at end*
+  ! ipoutfac             increase particle dump interval by factor (default 1) *
   ! itsplit [s]          time constant for particle splitting                  *
   ! loutaver [s]         concentration output is an average over loutaver      *
@@ -97 +98 @@
   iout, &
   ipout, &
+  ipoutfac, &
   lsubgrid, &
   lconvection, &
@@ -129 +131 @@
   iout=3
   ipout=0
+  ipoutfac=1
   lsubgrid=1
   lconvection=1
@@ -507 +510 @@
   !****************************************************************
 
-  if ((ipout.ne.0).and.(ipout.ne.1).and.(ipout.ne.2)) then
+  if ((ipout.ne.0).and.(ipout.ne.1).and.(ipout.ne.2).and.(ipout.ne.3)) then
     write(*,*) ' #### FLEXPART MODEL ERROR! FILE COMMAND:     #### '
-    write(*,*) ' #### IPOUT MUST BE 1, 2 OR 3!                #### '
+    write(*,*) ' #### IPOUT MUST BE 0, 1, 2 OR 3!             #### '
     stop
   endif

src/timemanager.f90

@@ -451 +451 @@
 45      format(i13,' Seconds simulated: ',i13, ' Particles:    Uncertainty: ',3f7.3)
 46      format(' Simulated ',f7.1,' hours (',i13,' s), ',i13, ' particles')
-        if (ipout.ge.1) call partoutput(itime)    ! dump particle positions
+        if (ipout.ge.1) then
+          if (mod(itime,ipoutfac*loutstep).eq.0) call partoutput(itime) ! dump particle positions
+          if (ipout.eq.3) call partoutput_average(itime) ! dump particle positions
+        endif
         loutnext=loutnext+loutstep
         loutstart=loutnext-loutaver/2
@@ -609 +612 @@
 !        write (*,*) 'advance: ',prob(1),xmass1(j,1),ztra1(j)
 
+  ! Calculate average position for particle dump output
+  !****************************************************
+
+        if (ipout.eq.3) call partpos_average(itime,j)
+
+
   ! Calculate the gross fluxes across layer interfaces
   !***************************************************

src/timemanager_mpi.f90

@@ -113 +113 @@
   integer :: j,ks,kp,l,n,itime=0,nstop,nstop1,memstat=0
 ! integer :: ksp
-  integer :: ip
+  integer :: ip,irec
   integer :: loutnext,loutstart,loutend
   integer :: ix,jy,ldeltat,itage,nage,idummy
@@ -129 +129 @@
 ! Measure time spent in timemanager
   if (mp_measure_time) call mpif_mtime('timemanager',0)
+
 
 ! First output for time 0
@@ -532 +533 @@
                 end if
 
-              else  ! :TODO: check for zeroing in the netcdf module
+              else
                 call concoutput_surf_nest(itime,outnum)
               end if
@@ -593 +594 @@
 46      format(' Simulated ',f7.1,' hours (',i13,' s), ',i13, ' particles')
         if (ipout.ge.1) then
+          if (mp_measure_time) call mpif_mtime('iotime',0)
+          irec=0
           do ip=0, mp_partgroup_np-1
-            if (ip.eq.mp_partid) call partoutput(itime) ! dump particle positions
+            if (ip.eq.mp_partid) then
+              if (mod(itime,ipoutfac*loutstep).eq.0) call partoutput(itime) ! dump particle positions
+              if (ipout.eq.3) call partoutput_average(itime,irec) ! dump particle positions
+            endif
+            if (ipout.eq.3) irec=irec+npart_per_process(ip)
             call mpif_mpi_barrier
           end do
+          if (mp_measure_time) call mpif_mtime('iotime',1)
         end if
 
@@ -757 +765 @@
         if (mp_measure_time) call mpif_mtime('advance',1)
 
+  ! Calculate average position for particle dump output
+  !****************************************************
+
+        if (ipout.eq.3) call partpos_average(itime,j)
+
 
 ! Calculate the gross fluxes across layer interfaces
@@ -895 +908 @@
     do ip=0, mp_partgroup_np-1
       if (ip.eq.mp_partid) then 
-        !if (mp_dbg_mode) write(*,*) 'call partoutput(itime), proc, mp_partid',ip,mp_partid
+        if (mp_dbg_mode) write(*,*) 'call partoutput(itime), proc, mp_partid',ip,mp_partid
         call partoutput(itime)    ! dump particle positions
       end if

src/verttransform_ecmwf.f90

@@ -73 +73 @@
   use com_mod
   use cmapf_mod, only: cc2gll
-!  use mpi_mod
 
   implicit none

src/readwind_gfs.f90

@@ -83 +83 @@
 
   ! NCEP
-  integer :: numpt,numpu,numpv,numpw,numprh
+  integer :: numpt,numpu,numpv,numpw,numprh,numpclwch
   real :: help, temp, ew
   real :: elev
@@ -134 +134 @@
   numpw=0
   numprh=0
+  numpclwch=0
   ifield=0
 10   ifield=ifield+1
@@ -557 +558 @@
       endif
 ! SEC & IP 12/2018 read GFS clouds
-      if(isec1(6).eq.153) then  !! CLWCR  Cloud liquid water content [kg/kg]
-        clwch(i,j,nlev_ec-k+2,n)=zsec4(nxfield*(ny-j-1)+i+1)
+      if((isec1(6).eq.153).and.(isec1(7).eq.100)) then  !! CLWCR  Cloud liquid water content [kg/kg] 
+         if((i.eq.0).and.(j.eq.0)) then
+            do ii=1,nuvz
+              if ((isec1(8)*100.0).eq.akz(ii)) numpclwch=ii
+            end do
+        endif
+        help=zsec4(nxfield*(ny-j-1)+i+1)
+        if(i.le.i180) then
+          clwch(i179+i,j,numpclwch,n)=help
+        else
+          clwch(i-i181,j,numpclwch,n)=help
+        endif
         readclouds=.true.
         sumclouds=.true.
+!        readclouds=.false.
+!       sumclouds=.false.
       endif
 

src/releaseparticles.f90

@@ -114 +114 @@
       average_timecorrect=0.
       do k=1,nspec
-        if (zpoint1(i).gt.0.5) then      ! point source
+        if(abs(xpoint2(i)-xpoint1(i)).lt.1.E-4.and.abs(ypoint2(i)-ypoint1(i)).lt.1.E-4) then
+!        if (zpoint1(i).gt.0.5) then      ! point source
           timecorrect(k)=point_hour(k,nhour)*point_dow(k,ndayofweek)
         else                             ! area source

src/verttransform_gfs.f90

@@ -548 +548 @@
         if ((lsp.gt.0.01).or.(convp.gt.0.01)) then ! cloud and precipitation
 
-          do kz=nz,1,-1 !go Bottom up!
+          do kz=nz,2,-1 !go Bottom up!
             if (clw(ix,jy,kz,n).gt. 0) then ! is in cloud
               cloudsh(ix,jy,n)=cloudsh(ix,jy,n)+height(kz)-height(kz-1)
               clouds(ix,jy,kz,n)=1                               ! is a cloud
               if (lsp.ge.convp) then
-                clouds(ix,jy,kz,n)=3                            ! lsp in-cloud
+                clouds(ix,jy,kz,n)=3                             ! lsp in-cloud
               else
                 clouds(ix,jy,kz,n)=2                             ! convp in-cloud