Context Navigation

-                      r0f7835d
+                      r16b61a5
 subroutine boundcond_domainfill(itime,loutend)
+  !                                  i      i
+  !*****************************************************************************
+  !                                                                            *
+  ! Particles are created by this subroutine continuously throughout the       *
+  ! simulation at the boundaries of the domain-filling box.                    *
+  ! All particles carry the same amount of mass which alltogether comprises the*
+  ! mass of air within the box, which remains (more or less) constant.         *
+  !                                                                            *
+  !     Author: A. Stohl                                                       *
+  !                                                                            *
+  !     16 October 2002                                                        *
+  !                                                                            *
+  !*****************************************************************************
+  !                                                                            *
+  ! Variables:                                                                 *
+  !                                                                            *
+  ! nx_we(2)       grid indices for western and eastern boundary of domain-    *
+  !                filling trajectory calculations                             *
+  ! ny_sn(2)       grid indices for southern and northern boundary of domain-  *
+  !                filling trajectory calculations                             *
+  !                                                                            *
+  !*****************************************************************************
+!                                  i      i
+!*****************************************************************************
+!                                                                            *
+! Particles are created by this subroutine continuously throughout the       *
+! simulation at the boundaries of the domain-filling box.                    *
+! All particles carry the same amount of mass which alltogether comprises the*
+! mass of air within the box, which remains (more or less) constant.         *
+!                                                                            *
+!     Author: A. Stohl                                                       *
+!                                                                            *
+!     16 October 2002                                                        *
+!                                                                            *
+!*****************************************************************************
+!                                                                            *
+! Variables:                                                                 *
+!                                                                            *
+! nx_we(2)       grid indices for western and eastern boundary of domain-    *
+!                filling trajectory calculations                             *
+! ny_sn(2)       grid indices for southern and northern boundary of domain-  *
+!                filling trajectory calculations                             *
+!                                                                            *
+!*****************************************************************************
+!  CHANGES
+!    08/2016 eso: MPI version:
+!
+!   -Root process release particles and distributes to other processes.
+!    Temporary arrays are used, also for the non-root (receiving) processes.
+!   -The scheme can be improved by having all processes report numpart
+!    (keeping track of how many particles have left the domain), so that
+!    a proportional amount of new particles can be distributed (however
+!    we have a separate function called from timemanager that will
+!    redistribute particles among processes if there are imbalances)
+!*****************************************************************************
   use point_mod
 …
   integer :: itime,in,indz,indzp,i,loutend
   integer :: j,k,ix,jy,m,indzh,indexh,minpart,ipart,mmass
+  integer :: numactiveparticles
+  integer :: numactiveparticles, numpart_total, rel_counter
+  integer,allocatable,dimension(:) ::  numrel_mpi !, numactiveparticles_mpi
   real :: windl(2),rhol(2)
 …
   integer :: idummy = -11
+  integer :: mtag
   logical :: first_call=.true.
+  ! Use different seed for each process
+! Sizes of temporary arrays are maxpartfract*maxpart. Increase maxpartfract if
+! needed.
+  real,parameter :: maxpartfract=0.1
+  integer :: tmp_size = int(maxpartfract*maxpart)
+! Use different seed for each process
   if (first_call) then
     idummy=idummy+mp_seed
 …
   ! If domain-filling is global, no boundary conditions are needed
   !***************************************************************
+! If domain-filling is global, no boundary conditions are needed
+!***************************************************************
   if (gdomainfill) return
 …
   accmasst=0.
   numactiveparticles=0
+  ! Terminate trajectories that have left the domain, if domain-filling
+  ! trajectory calculation domain is not global
+  !********************************************************************
+! Keep track of active particles on each process
+  allocate(numrel_mpi(0:mp_partgroup_np-1))
+! numactiveparticles_mpi(0:mp_partgroup_np-1)
+! New particles to be released on each process
+  numrel_mpi(:)=0
+! Terminate trajectories that have left the domain, if domain-filling
+! trajectory calculation domain is not global. Done for all processes
+!********************************************************************
   do i=1,numpart
 …
          numactiveparticles+1
   end do
+  ! Determine auxiliary variables for time interpolation
+  !*****************************************************
+  dt1=real(itime-memtime(1))
+  dt2=real(memtime(2)-itime)
+  dtt=1./(dt1+dt2)
+  ! Initialize auxiliary variable used to search for vacant storage space
+  !**********************************************************************
+  minpart=1
+  !***************************************
+  ! Western and eastern boundary condition
+  !***************************************
+  ! Loop from south to north
+  !*************************
+  do jy=ny_sn(1),ny_sn(2)
+  ! Loop over western (index 1) and eastern (index 2) boundary
+  !***********************************************************
+    do k=1,2
+  ! Loop over all release locations in a column
+  !********************************************
+      do j=1,numcolumn_we(k,jy)
+  ! Determine, for each release location, the area of the corresponding boundary
+  !*****************************************************************************
+        if (j.eq.1) then
+          deltaz=(zcolumn_we(k,jy,2)+zcolumn_we(k,jy,1))/2.
+        else if (j.eq.numcolumn_we(k,jy)) then
+  !        deltaz=height(nz)-(zcolumn_we(k,jy,j-1)+
+  !    +        zcolumn_we(k,jy,j))/2.
+  ! In order to avoid taking a very high column for very many particles,
+  ! use the deltaz from one particle below instead
+          deltaz=(zcolumn_we(k,jy,j)-zcolumn_we(k,jy,j-2))/2.
+        else
+          deltaz=(zcolumn_we(k,jy,j+1)-zcolumn_we(k,jy,j-1))/2.
+        endif
+        if ((jy.eq.ny_sn(1)).or.(jy.eq.ny_sn(2))) then
+          boundarea=deltaz*111198.5/2.*dy
+        else
+          boundarea=deltaz*111198.5*dy
+        endif
+  ! Interpolate the wind velocity and density to the release location
+  !******************************************************************
+  ! Determine the model level below the release position
+  !*****************************************************
+        do i=2,nz
+          if (height(i).gt.zcolumn_we(k,jy,j)) then
+            indz=i-1
+            indzp=i
+            goto 6
+!  numactiveparticles_mpi(mp_partid) = numactiveparticles
+! Collect number of active particles from all processes
+  ! call MPI_Allgather(numactiveparticles, 1, MPI_INTEGER, &
+  !      &numactiveparticles_mpi, 1, MPI_INTEGER, mp_comm_used, mp_ierr)
+! Total number of new releases
+  numpart_total = 0
+! This section only done by root process
+!***************************************
+  if (lroot) then
+! Use separate arrays for newly released particles
+!*************************************************
+    allocate(itra1_tmp(tmp_size),npoint_tmp(tmp_size),nclass_tmp(tmp_size),&
+         & idt_tmp(tmp_size),itramem_tmp(tmp_size),itrasplit_tmp(tmp_size),&
+         & xtra1_tmp(tmp_size),ytra1_tmp(tmp_size),ztra1_tmp(tmp_size),&
+         & xmass1_tmp(tmp_size, maxspec))
+! Initialize all particles as non-existent
+    itra1_tmp(:)=-999999999
+! Determine auxiliary variables for time interpolation
+!*****************************************************
+    dt1=real(itime-memtime(1))
+    dt2=real(memtime(2)-itime)
+    dtt=1./(dt1+dt2)
+! Initialize auxiliary variable used to search for vacant storage space
+!**********************************************************************
+    minpart=1
+!***************************************
+! Western and eastern boundary condition
+!***************************************
+! Loop from south to north
+!*************************
+    do jy=ny_sn(1),ny_sn(2)
+! Loop over western (index 1) and eastern (index 2) boundary
+!***********************************************************
+      do k=1,2
+! Loop over all release locations in a column
+!********************************************
+        do j=1,numcolumn_we(k,jy)
+! Determine, for each release location, the area of the corresponding boundary
+!*****************************************************************************
+          if (j.eq.1) then
+            deltaz=(zcolumn_we(k,jy,2)+zcolumn_we(k,jy,1))/2.
+          else if (j.eq.numcolumn_we(k,jy)) then
+!        deltaz=height(nz)-(zcolumn_we(k,jy,j-1)+
+!    +        zcolumn_we(k,jy,j))/2.
+! In order to avoid taking a very high column for very many particles,
+! use the deltaz from one particle below instead
+            deltaz=(zcolumn_we(k,jy,j)-zcolumn_we(k,jy,j-2))/2.
+          else
+            deltaz=(zcolumn_we(k,jy,j+1)-zcolumn_we(k,jy,j-1))/2.
           endif
+        end do
+       continue
+  ! Vertical distance to the level below and above current position
+  !****************************************************************
+        dz1=zcolumn_we(k,jy,j)-height(indz)
+        dz2=height(indzp)-zcolumn_we(k,jy,j)
+        dz=1./(dz1+dz2)
+  ! Vertical and temporal interpolation
+  !************************************
+        do m=1,2
+          indexh=memind(m)
+          do in=1,2
+            indzh=indz+in-1
+            windl(in)=uu(nx_we(k),jy,indzh,indexh)
+            rhol(in)=rho(nx_we(k),jy,indzh,indexh)
+          end do
+          windhl(m)=(dz2*windl(1)+dz1*windl(2))*dz
+          rhohl(m)=(dz2*rhol(1)+dz1*rhol(2))*dz
+        end do
+        windx=(windhl(1)*dt2+windhl(2)*dt1)*dtt
+        rhox=(rhohl(1)*dt2+rhohl(2)*dt1)*dtt
+  ! Calculate mass flux, divided by number of processes
+  !****************************************************
+        fluxofmass=windx*rhox*boundarea*real(lsynctime)/mp_partgroup_np
+  ! If the mass flux is directed into the domain, add it to previous mass fluxes;
+  ! if it is out of the domain, set accumulated mass flux to zero
+  !******************************************************************************
+        if (k.eq.1) then
+          if (fluxofmass.ge.0.) then
+            acc_mass_we(k,jy,j)=acc_mass_we(k,jy,j)+fluxofmass
+          if ((jy.eq.ny_sn(1)).or.(jy.eq.ny_sn(2))) then
+            boundarea=deltaz*111198.5/2.*dy
           else
             acc_mass_we(k,jy,j)=0.
+            boundarea=deltaz*111198.5*dy
           endif
+        else
+          if (fluxofmass.le.0.) then
+            acc_mass_we(k,jy,j)=acc_mass_we(k,jy,j)+abs(fluxofmass)
+          else
+            acc_mass_we(k,jy,j)=0.
+          endif
+        endif
+        accmasst=accmasst+acc_mass_we(k,jy,j)
+  ! If the accumulated mass exceeds half the mass that each particle shall carry,
+  ! one (or more) particle(s) is (are) released and the accumulated mass is
+  ! reduced by the mass of this (these) particle(s)
+  !******************************************************************************
+        if (acc_mass_we(k,jy,j).ge.xmassperparticle/2.) then
+          mmass=int((acc_mass_we(k,jy,j)+xmassperparticle/2.)/ &
+               xmassperparticle)
+          acc_mass_we(k,jy,j)=acc_mass_we(k,jy,j)- &
+               real(mmass)*xmassperparticle
+        else
+          mmass=0
+        endif
+        do m=1,mmass
+          do ipart=minpart,maxpart_mpi
+  ! If a vacant storage space is found, attribute everything to this array element
+  !*****************************************************************************
+            if (itra1(ipart).ne.itime) then
+  ! Assign particle positions
+  !**************************
+              xtra1(ipart)=real(nx_we(k))
+              if (jy.eq.ny_sn(1)) then
+                ytra1(ipart)=real(jy)+0.5*ran1(idummy)
+              else if (jy.eq.ny_sn(2)) then
+                ytra1(ipart)=real(jy)-0.5*ran1(idummy)
+              else
+                ytra1(ipart)=real(jy)+(ran1(idummy)-.5)
+              endif
+              if (j.eq.1) then
+                ztra1(ipart)=zcolumn_we(k,jy,1)+(zcolumn_we(k,jy,2)- &
+                     zcolumn_we(k,jy,1))/4.
+              else if (j.eq.numcolumn_we(k,jy)) then
+                ztra1(ipart)=(2.*zcolumn_we(k,jy,j)+ &
+                     zcolumn_we(k,jy,j-1)+height(nz))/4.
+              else
+                ztra1(ipart)=zcolumn_we(k,jy,j-1)+ran1(idummy)* &
+                     (zcolumn_we(k,jy,j+1)-zcolumn_we(k,jy,j-1))
+              endif
+  ! Interpolate PV to the particle position
+  !****************************************
+              ixm=int(xtra1(ipart))
+              jym=int(ytra1(ipart))
+              ixp=ixm+1
+              jyp=jym+1
+              ddx=xtra1(ipart)-real(ixm)
+              ddy=ytra1(ipart)-real(jym)
+              rddx=1.-ddx
+              rddy=1.-ddy
+              p1=rddx*rddy
+              p2=ddx*rddy
+              p3=rddx*ddy
+              p4=ddx*ddy
+              do i=2,nz
+                if (height(i).gt.ztra1(ipart)) then
+                  indzm=i-1
+                  indzp=i
+                  goto 26
+                endif
+              end do
+            continue
+              dz1=ztra1(ipart)-height(indzm)
+              dz2=height(indzp)-ztra1(ipart)
+              dz=1./(dz1+dz2)
+              do mm=1,2
+                indexh=memind(mm)
+                do in=1,2
+                  indzh=indzm+in-1
+                  y1(in)=p1*pv(ixm,jym,indzh,indexh) &
+                       +p2*pv(ixp,jym,indzh,indexh) &
+                       +p3*pv(ixm,jyp,indzh,indexh) &
+                       +p4*pv(ixp,jyp,indzh,indexh)
+                end do
+                yh1(mm)=(dz2*y1(1)+dz1*y1(2))*dz
+              end do
+              pvpart=(yh1(1)*dt2+yh1(2)*dt1)*dtt
+              ylat=ylat0+ytra1(ipart)*dy
+              if (ylat.lt.0.) pvpart=-1.*pvpart
+  ! For domain-filling option 2 (stratospheric O3), do the rest only in the stratosphere
+  !*****************************************************************************
+              if (((ztra1(ipart).gt.3000.).and. &
+                   (pvpart.gt.pvcrit)).or.(mdomainfill.eq.1)) then
+                nclass(ipart)=min(int(ran1(idummy)* &
+                     real(nclassunc))+1,nclassunc)
+                numactiveparticles=numactiveparticles+1
+                numparticlecount=numparticlecount+1
+                npoint(ipart)=numparticlecount
+                idt(ipart)=mintime
+                itra1(ipart)=itime
+                itramem(ipart)=itra1(ipart)
+                itrasplit(ipart)=itra1(ipart)+ldirect*itsplit
+                xmass1(ipart,1)=xmassperparticle
+                if (mdomainfill.eq.2) xmass1(ipart,1)= &
+                     xmass1(ipart,1)*pvpart*48./29.*ozonescale/10.**9
+              else
+                goto 71
+              endif
+  ! Increase numpart, if necessary
+  !*******************************
+              numpart=max(numpart,ipart)
+              goto 73      ! Storage space has been found, stop searching
+! Interpolate the wind velocity and density to the release location
+!******************************************************************
+! Determine the model level below the release position
+!*****************************************************
+          do i=2,nz
+            if (height(i).gt.zcolumn_we(k,jy,j)) then
+              indz=i-1
+              indzp=i
+              goto 6
             endif
           end do
+          if (ipart.gt.maxpart_mpi) &
+               stop 'boundcond_domainfill.f: too many particles required'
+        minpart=ipart+1
+        continue
+         continue
+! Vertical distance to the level below and above current position
+!****************************************************************
+          dz1=zcolumn_we(k,jy,j)-height(indz)
+          dz2=height(indzp)-zcolumn_we(k,jy,j)
+          dz=1./(dz1+dz2)
+! Vertical and temporal interpolation
+!************************************
+          do m=1,2
+            indexh=memind(m)
+            do in=1,2
+              indzh=indz+in-1
+              windl(in)=uu(nx_we(k),jy,indzh,indexh)
+              rhol(in)=rho(nx_we(k),jy,indzh,indexh)
+            end do
+            windhl(m)=(dz2*windl(1)+dz1*windl(2))*dz
+            rhohl(m)=(dz2*rhol(1)+dz1*rhol(2))*dz
+          end do
+          windx=(windhl(1)*dt2+windhl(2)*dt1)*dtt
+          rhox=(rhohl(1)*dt2+rhohl(2)*dt1)*dtt
+! Calculate mass flux
+!********************
+          fluxofmass=windx*rhox*boundarea*real(lsynctime)
+! If the mass flux is directed into the domain, add it to previous mass fluxes;
+! if it is out of the domain, set accumulated mass flux to zero
+!******************************************************************************
+          if (k.eq.1) then
+            if (fluxofmass.ge.0.) then
+              acc_mass_we(k,jy,j)=acc_mass_we(k,jy,j)+fluxofmass
+            else
+              acc_mass_we(k,jy,j)=0.
+            endif
+          else
+            if (fluxofmass.le.0.) then
+              acc_mass_we(k,jy,j)=acc_mass_we(k,jy,j)+abs(fluxofmass)
+            else
+              acc_mass_we(k,jy,j)=0.
+            endif
+          endif
+          accmasst=accmasst+acc_mass_we(k,jy,j)
+! If the accumulated mass exceeds half the mass that each particle shall carry,
+! one (or more) particle(s) is (are) released and the accumulated mass is
+! reduced by the mass of this (these) particle(s)
+!******************************************************************************
+          if (acc_mass_we(k,jy,j).ge.xmassperparticle/2.) then
+            mmass=int((acc_mass_we(k,jy,j)+xmassperparticle/2.)/ &
+                 xmassperparticle)
+            acc_mass_we(k,jy,j)=acc_mass_we(k,jy,j)- &
+                 real(mmass)*xmassperparticle
+          else
+            mmass=0
+          endif
+          do m=1,mmass
+            do ipart=minpart,maxpart
+! If a vacant storage space is found, attribute everything to this array element
+! TODO: for the MPI version this test can be removed, as all
+!       elements in _tmp arrays are initialized to zero
+!*****************************************************************************
+              if (itra1_tmp(ipart).ne.itime) then
+! Assign particle positions
+!**************************
+                xtra1_tmp(ipart)=real(nx_we(k))
+                if (jy.eq.ny_sn(1)) then
+                  ytra1_tmp(ipart)=real(jy)+0.5*ran1(idummy)
+                else if (jy.eq.ny_sn(2)) then
+                  ytra1_tmp(ipart)=real(jy)-0.5*ran1(idummy)
+                else
+                  ytra1_tmp(ipart)=real(jy)+(ran1(idummy)-.5)
+                endif
+                if (j.eq.1) then
+                  ztra1_tmp(ipart)=zcolumn_we(k,jy,1)+(zcolumn_we(k,jy,2)- &
+                       zcolumn_we(k,jy,1))/4.
+                else if (j.eq.numcolumn_we(k,jy)) then
+                  ztra1_tmp(ipart)=(2.*zcolumn_we(k,jy,j)+ &
+                       zcolumn_we(k,jy,j-1)+height(nz))/4.
+                else
+                  ztra1_tmp(ipart)=zcolumn_we(k,jy,j-1)+ran1(idummy)* &
+                       (zcolumn_we(k,jy,j+1)-zcolumn_we(k,jy,j-1))
+                endif
+! Interpolate PV to the particle position
+!****************************************
+                ixm=int(xtra1_tmp(ipart))
+                jym=int(ytra1_tmp(ipart))
+                ixp=ixm+1
+                jyp=jym+1
+                ddx=xtra1_tmp(ipart)-real(ixm)
+                ddy=ytra1_tmp(ipart)-real(jym)
+                rddx=1.-ddx
+                rddy=1.-ddy
+                p1=rddx*rddy
+                p2=ddx*rddy
+                p3=rddx*ddy
+                p4=ddx*ddy
+                do i=2,nz
+                  if (height(i).gt.ztra1_tmp(ipart)) then
+                    indzm=i-1
+                    indzp=i
+                    goto 26
+                  endif
+                end do
+              continue
+                dz1=ztra1_tmp(ipart)-height(indzm)
+                dz2=height(indzp)-ztra1_tmp(ipart)
+                dz=1./(dz1+dz2)
+                do mm=1,2
+                  indexh=memind(mm)
+                  do in=1,2
+                    indzh=indzm+in-1
+                    y1(in)=p1*pv(ixm,jym,indzh,indexh) &
+                         +p2*pv(ixp,jym,indzh,indexh) &
+                         +p3*pv(ixm,jyp,indzh,indexh) &
+                         +p4*pv(ixp,jyp,indzh,indexh)
+                  end do
+                  yh1(mm)=(dz2*y1(1)+dz1*y1(2))*dz
+                end do
+                pvpart=(yh1(1)*dt2+yh1(2)*dt1)*dtt
+                ylat=ylat0+ytra1_tmp(ipart)*dy
+                if (ylat.lt.0.) pvpart=-1.*pvpart
+! For domain-filling option 2 (stratospheric O3), do the rest only in the stratosphere
+!*****************************************************************************
+                if (((ztra1_tmp(ipart).gt.3000.).and. &
+                     (pvpart.gt.pvcrit)).or.(mdomainfill.eq.1)) then
+                  nclass_tmp(ipart)=min(int(ran1(idummy)* &
+                       real(nclassunc))+1,nclassunc)
+                  numactiveparticles=numactiveparticles+1
+                  numparticlecount=numparticlecount+1
+                  npoint_tmp(ipart)=numparticlecount
+                  idt_tmp(ipart)=mintime
+                  itra1_tmp(ipart)=itime
+                  itramem_tmp(ipart)=itra1_tmp(ipart)
+                  itrasplit_tmp(ipart)=itra1_tmp(ipart)+ldirect*itsplit
+                  xmass1_tmp(ipart,1)=xmassperparticle
+                  if (mdomainfill.eq.2) xmass1_tmp(ipart,1)= &
+                       xmass1_tmp(ipart,1)*pvpart*48./29.*ozonescale/10.**9
+                else
+                  goto 71
+                endif
+! Increase numpart, if necessary
+!*******************************
+                numpart_total=max(numpart_total,ipart)
+                goto 73      ! Storage space has been found, stop searching
+              endif
+            end do
+            if (ipart.gt.tmp_size) &
+                 stop 'boundcond_domainfill_mpi.f90: too many particles required'
+          minpart=ipart+1
+          continue
+          end do
         end do
       end do
     end do
+  end do
+  !*****************************************
+  ! Southern and northern boundary condition
+  !*****************************************
+  ! Loop from west to east
+  !***********************
+  do ix=nx_we(1),nx_we(2)
+  ! Loop over southern (index 1) and northern (index 2) boundary
+  !*************************************************************
+    do k=1,2
+      ylat=ylat0+real(ny_sn(k))*dy
+      cosfact=cos(ylat*pi180)
+  ! Loop over all release locations in a column
+  !********************************************
+      do j=1,numcolumn_sn(k,ix)
+  ! Determine, for each release location, the area of the corresponding boundary
+  !*****************************************************************************
+        if (j.eq.1) then
+          deltaz=(zcolumn_sn(k,ix,2)+zcolumn_sn(k,ix,1))/2.
+        else if (j.eq.numcolumn_sn(k,ix)) then
+  !        deltaz=height(nz)-(zcolumn_sn(k,ix,j-1)+
+  !    +        zcolumn_sn(k,ix,j))/2.
+  ! In order to avoid taking a very high column for very many particles,
+  ! use the deltaz from one particle below instead
+          deltaz=(zcolumn_sn(k,ix,j)-zcolumn_sn(k,ix,j-2))/2.
+        else
+          deltaz=(zcolumn_sn(k,ix,j+1)-zcolumn_sn(k,ix,j-1))/2.
+        endif
+        if ((ix.eq.nx_we(1)).or.(ix.eq.nx_we(2))) then
+          boundarea=deltaz*111198.5/2.*cosfact*dx
+        else
+          boundarea=deltaz*111198.5*cosfact*dx
+        endif
+  ! Interpolate the wind velocity and density to the release location
+  !******************************************************************
+  ! Determine the model level below the release position
+  !*****************************************************
+        do i=2,nz
+          if (height(i).gt.zcolumn_sn(k,ix,j)) then
+            indz=i-1
+            indzp=i
+            goto 16
+!*****************************************
+! Southern and northern boundary condition
+!*****************************************
+! Loop from west to east
+!***********************
+    do ix=nx_we(1),nx_we(2)
+! Loop over southern (index 1) and northern (index 2) boundary
+!*************************************************************
+      do k=1,2
+        ylat=ylat0+real(ny_sn(k))*dy
+        cosfact=cos(ylat*pi180)
+! Loop over all release locations in a column
+!********************************************
+        do j=1,numcolumn_sn(k,ix)
+! Determine, for each release location, the area of the corresponding boundary
+!*****************************************************************************
+          if (j.eq.1) then
+            deltaz=(zcolumn_sn(k,ix,2)+zcolumn_sn(k,ix,1))/2.
+          else if (j.eq.numcolumn_sn(k,ix)) then
+!        deltaz=height(nz)-(zcolumn_sn(k,ix,j-1)+
+!    +        zcolumn_sn(k,ix,j))/2.
+! In order to avoid taking a very high column for very many particles,
+! use the deltaz from one particle below instead
+            deltaz=(zcolumn_sn(k,ix,j)-zcolumn_sn(k,ix,j-2))/2.
+          else
+            deltaz=(zcolumn_sn(k,ix,j+1)-zcolumn_sn(k,ix,j-1))/2.
           endif
+        end do
+      continue
+  ! Vertical distance to the level below and above current position
+  !****************************************************************
+        dz1=zcolumn_sn(k,ix,j)-height(indz)
+        dz2=height(indzp)-zcolumn_sn(k,ix,j)
+        dz=1./(dz1+dz2)
+  ! Vertical and temporal interpolation
+  !************************************
+        do m=1,2
+          indexh=memind(m)
+          do in=1,2
+            indzh=indz+in-1
+            windl(in)=vv(ix,ny_sn(k),indzh,indexh)
+            rhol(in)=rho(ix,ny_sn(k),indzh,indexh)
+          end do
+          windhl(m)=(dz2*windl(1)+dz1*windl(2))*dz
+          rhohl(m)=(dz2*rhol(1)+dz1*rhol(2))*dz
+        end do
+        windx=(windhl(1)*dt2+windhl(2)*dt1)*dtt
+        rhox=(rhohl(1)*dt2+rhohl(2)*dt1)*dtt
+  ! Calculate mass flux, divided by number of processes
+  !****************************************************
+        fluxofmass=windx*rhox*boundarea*real(lsynctime)/mp_partgroup_np
+  ! If the mass flux is directed into the domain, add it to previous mass fluxes;
+  ! if it is out of the domain, set accumulated mass flux to zero
+  !******************************************************************************
+        if (k.eq.1) then
+          if (fluxofmass.ge.0.) then
+            acc_mass_sn(k,ix,j)=acc_mass_sn(k,ix,j)+fluxofmass
+          if ((ix.eq.nx_we(1)).or.(ix.eq.nx_we(2))) then
+            boundarea=deltaz*111198.5/2.*cosfact*dx
           else
             acc_mass_sn(k,ix,j)=0.
+            boundarea=deltaz*111198.5*cosfact*dx
           endif
+        else
+          if (fluxofmass.le.0.) then
+            acc_mass_sn(k,ix,j)=acc_mass_sn(k,ix,j)+abs(fluxofmass)
+          else
+            acc_mass_sn(k,ix,j)=0.
+          endif
+        endif
+        accmasst=accmasst+acc_mass_sn(k,ix,j)
+  ! If the accumulated mass exceeds half the mass that each particle shall carry,
+  ! one (or more) particle(s) is (are) released and the accumulated mass is
+  ! reduced by the mass of this (these) particle(s)
+  !******************************************************************************
+        if (acc_mass_sn(k,ix,j).ge.xmassperparticle/2.) then
+          mmass=int((acc_mass_sn(k,ix,j)+xmassperparticle/2.)/ &
+               xmassperparticle)
+          acc_mass_sn(k,ix,j)=acc_mass_sn(k,ix,j)- &
+               real(mmass)*xmassperparticle
+        else
+          mmass=0
+        endif
+        do m=1,mmass
+          do ipart=minpart,maxpart_mpi
+  ! If a vacant storage space is found, attribute everything to this array element
+  !*****************************************************************************
+            if (itra1(ipart).ne.itime) then
+  ! Assign particle positions
+  !**************************
+              ytra1(ipart)=real(ny_sn(k))
+              if (ix.eq.nx_we(1)) then
+                xtra1(ipart)=real(ix)+0.5*ran1(idummy)
+              else if (ix.eq.nx_we(2)) then
+                xtra1(ipart)=real(ix)-0.5*ran1(idummy)
+              else
+                xtra1(ipart)=real(ix)+(ran1(idummy)-.5)
+              endif
+              if (j.eq.1) then
+                ztra1(ipart)=zcolumn_sn(k,ix,1)+(zcolumn_sn(k,ix,2)- &
+                     zcolumn_sn(k,ix,1))/4.
+              else if (j.eq.numcolumn_sn(k,ix)) then
+                ztra1(ipart)=(2.*zcolumn_sn(k,ix,j)+ &
+                     zcolumn_sn(k,ix,j-1)+height(nz))/4.
+              else
+                ztra1(ipart)=zcolumn_sn(k,ix,j-1)+ran1(idummy)* &
+                     (zcolumn_sn(k,ix,j+1)-zcolumn_sn(k,ix,j-1))
+              endif
+  ! Interpolate PV to the particle position
+  !****************************************
+              ixm=int(xtra1(ipart))
+              jym=int(ytra1(ipart))
+              ixp=ixm+1
+              jyp=jym+1
+              ddx=xtra1(ipart)-real(ixm)
+              ddy=ytra1(ipart)-real(jym)
+              rddx=1.-ddx
+              rddy=1.-ddy
+              p1=rddx*rddy
+              p2=ddx*rddy
+              p3=rddx*ddy
+              p4=ddx*ddy
+              do i=2,nz
+                if (height(i).gt.ztra1(ipart)) then
+                  indzm=i-1
+                  indzp=i
+                  goto 126
+                endif
+              end do
+           continue
+              dz1=ztra1(ipart)-height(indzm)
+              dz2=height(indzp)-ztra1(ipart)
+              dz=1./(dz1+dz2)
+              do mm=1,2
+                indexh=memind(mm)
+                do in=1,2
+                  indzh=indzm+in-1
+                  y1(in)=p1*pv(ixm,jym,indzh,indexh) &
+                       +p2*pv(ixp,jym,indzh,indexh) &
+                       +p3*pv(ixm,jyp,indzh,indexh) &
+                       +p4*pv(ixp,jyp,indzh,indexh)
+                end do
+                yh1(mm)=(dz2*y1(1)+dz1*y1(2))*dz
+              end do
+              pvpart=(yh1(1)*dt2+yh1(2)*dt1)*dtt
+              if (ylat.lt.0.) pvpart=-1.*pvpart
+  ! For domain-filling option 2 (stratospheric O3), do the rest only in the stratosphere
+  !*****************************************************************************
+              if (((ztra1(ipart).gt.3000.).and. &
+                   (pvpart.gt.pvcrit)).or.(mdomainfill.eq.1)) then
+                nclass(ipart)=min(int(ran1(idummy)* &
+                     real(nclassunc))+1,nclassunc)
+                numactiveparticles=numactiveparticles+1
+                numparticlecount=numparticlecount+1
+                npoint(ipart)=numparticlecount
+                idt(ipart)=mintime
+                itra1(ipart)=itime
+                itramem(ipart)=itra1(ipart)
+                itrasplit(ipart)=itra1(ipart)+ldirect*itsplit
+                xmass1(ipart,1)=xmassperparticle
+                if (mdomainfill.eq.2) xmass1(ipart,1)= &
+                     xmass1(ipart,1)*pvpart*48./29.*ozonescale/10.**9
+              else
+                goto 171
+              endif
+  ! Increase numpart, if necessary
+  !*******************************
+              numpart=max(numpart,ipart)
+              goto 173      ! Storage space has been found, stop searching
+! Interpolate the wind velocity and density to the release location
+!******************************************************************
+! Determine the model level below the release position
+!*****************************************************
+          do i=2,nz
+            if (height(i).gt.zcolumn_sn(k,ix,j)) then
+              indz=i-1
+              indzp=i
+              goto 16
             endif
           end do
+          if (ipart.gt.maxpart_mpi) &
+               stop 'boundcond_domainfill.f: too many particles required'
+       minpart=ipart+1
+       continue
+        continue
+! Vertical distance to the level below and above current position
+!****************************************************************
+          dz1=zcolumn_sn(k,ix,j)-height(indz)
+          dz2=height(indzp)-zcolumn_sn(k,ix,j)
+          dz=1./(dz1+dz2)
+! Vertical and temporal interpolation
+!************************************
+          do m=1,2
+            indexh=memind(m)
+            do in=1,2
+              indzh=indz+in-1
+              windl(in)=vv(ix,ny_sn(k),indzh,indexh)
+              rhol(in)=rho(ix,ny_sn(k),indzh,indexh)
+            end do
+            windhl(m)=(dz2*windl(1)+dz1*windl(2))*dz
+            rhohl(m)=(dz2*rhol(1)+dz1*rhol(2))*dz
+          end do
+          windx=(windhl(1)*dt2+windhl(2)*dt1)*dtt
+          rhox=(rhohl(1)*dt2+rhohl(2)*dt1)*dtt
+! Calculate mass flux
+!********************
+          fluxofmass=windx*rhox*boundarea*real(lsynctime)
+! If the mass flux is directed into the domain, add it to previous mass fluxes;
+! if it is out of the domain, set accumulated mass flux to zero
+!******************************************************************************
+          if (k.eq.1) then
+            if (fluxofmass.ge.0.) then
+              acc_mass_sn(k,ix,j)=acc_mass_sn(k,ix,j)+fluxofmass
+            else
+              acc_mass_sn(k,ix,j)=0.
+            endif
+          else
+            if (fluxofmass.le.0.) then
+              acc_mass_sn(k,ix,j)=acc_mass_sn(k,ix,j)+abs(fluxofmass)
+            else
+              acc_mass_sn(k,ix,j)=0.
+            endif
+          endif
+          accmasst=accmasst+acc_mass_sn(k,ix,j)
+! If the accumulated mass exceeds half the mass that each particle shall carry,
+! one (or more) particle(s) is (are) released and the accumulated mass is
+! reduced by the mass of this (these) particle(s)
+!******************************************************************************
+          if (acc_mass_sn(k,ix,j).ge.xmassperparticle/2.) then
+            mmass=int((acc_mass_sn(k,ix,j)+xmassperparticle/2.)/ &
+                 xmassperparticle)
+            acc_mass_sn(k,ix,j)=acc_mass_sn(k,ix,j)- &
+                 real(mmass)*xmassperparticle
+          else
+            mmass=0
+          endif
+          do m=1,mmass
+            do ipart=minpart,maxpart
+! If a vacant storage space is found, attribute everything to this array element
+!*****************************************************************************
+              if (itra1_tmp(ipart).ne.itime) then
+! Assign particle positions
+!**************************
+                ytra1_tmp(ipart)=real(ny_sn(k))
+                if (ix.eq.nx_we(1)) then
+                  xtra1_tmp(ipart)=real(ix)+0.5*ran1(idummy)
+                else if (ix.eq.nx_we(2)) then
+                  xtra1_tmp(ipart)=real(ix)-0.5*ran1(idummy)
+                else
+                  xtra1_tmp(ipart)=real(ix)+(ran1(idummy)-.5)
+                endif
+                if (j.eq.1) then
+                  ztra1_tmp(ipart)=zcolumn_sn(k,ix,1)+(zcolumn_sn(k,ix,2)- &
+                       zcolumn_sn(k,ix,1))/4.
+                else if (j.eq.numcolumn_sn(k,ix)) then
+                  ztra1_tmp(ipart)=(2.*zcolumn_sn(k,ix,j)+ &
+                       zcolumn_sn(k,ix,j-1)+height(nz))/4.
+                else
+                  ztra1_tmp(ipart)=zcolumn_sn(k,ix,j-1)+ran1(idummy)* &
+                       (zcolumn_sn(k,ix,j+1)-zcolumn_sn(k,ix,j-1))
+                endif
+! Interpolate PV to the particle position
+!****************************************
+                ixm=int(xtra1_tmp(ipart))
+                jym=int(ytra1_tmp(ipart))
+                ixp=ixm+1
+                jyp=jym+1
+                ddx=xtra1_tmp(ipart)-real(ixm)
+                ddy=ytra1_tmp(ipart)-real(jym)
+                rddx=1.-ddx
+                rddy=1.-ddy
+                p1=rddx*rddy
+                p2=ddx*rddy
+                p3=rddx*ddy
+                p4=ddx*ddy
+                do i=2,nz
+                  if (height(i).gt.ztra1_tmp(ipart)) then
+                    indzm=i-1
+                    indzp=i
+                    goto 126
+                  endif
+                end do
+             continue
+                dz1=ztra1_tmp(ipart)-height(indzm)
+                dz2=height(indzp)-ztra1_tmp(ipart)
+                dz=1./(dz1+dz2)
+                do mm=1,2
+                  indexh=memind(mm)
+                  do in=1,2
+                    indzh=indzm+in-1
+                    y1(in)=p1*pv(ixm,jym,indzh,indexh) &
+                         +p2*pv(ixp,jym,indzh,indexh) &
+                         +p3*pv(ixm,jyp,indzh,indexh) &
+                         +p4*pv(ixp,jyp,indzh,indexh)
+                  end do
+                  yh1(mm)=(dz2*y1(1)+dz1*y1(2))*dz
+                end do
+                pvpart=(yh1(1)*dt2+yh1(2)*dt1)*dtt
+                if (ylat.lt.0.) pvpart=-1.*pvpart
+! For domain-filling option 2 (stratospheric O3), do the rest only in the stratosphere
+!*****************************************************************************
+                if (((ztra1_tmp(ipart).gt.3000.).and. &
+                     (pvpart.gt.pvcrit)).or.(mdomainfill.eq.1)) then
+                  nclass_tmp(ipart)=min(int(ran1(idummy)* &
+                       real(nclassunc))+1,nclassunc)
+                  numactiveparticles=numactiveparticles+1
+                  numparticlecount=numparticlecount+1
+                  npoint_tmp(ipart)=numparticlecount
+                  idt_tmp(ipart)=mintime
+                  itra1_tmp(ipart)=itime
+                  itramem_tmp(ipart)=itra1_tmp(ipart)
+                  itrasplit_tmp(ipart)=itra1_tmp(ipart)+ldirect*itsplit
+                  xmass1_tmp(ipart,1)=xmassperparticle
+                  if (mdomainfill.eq.2) xmass1_tmp(ipart,1)= &
+                       xmass1_tmp(ipart,1)*pvpart*48./29.*ozonescale/10.**9
+                else
+                  goto 171
+                endif
+! Increase numpart, if necessary
+!*******************************
+                numpart_total=max(numpart_total,ipart)
+                goto 173      ! Storage space has been found, stop searching
+              endif
+            end do
+            if (ipart.gt.tmp_size) &
+                 stop 'boundcond_domainfill.f: too many particles required'
+         minpart=ipart+1
+         continue
+          end do
         end do
       end do
     end do
+! xm=0.
+! do i=1,numpart_total
+!   if (itra1_tmp(i).eq.itime) xm=xm+xmass1(i,1)
+! end do
+!write(*,*) itime,numactiveparticles,numparticlecount,numpart,
+!    +xm,accmasst,xm+accmasst
+  end if ! if lroot
+! Distribute the number of particles to be released
+! *************************************************
+  call MPI_Bcast(numpart_total, 1, MPI_INTEGER, id_root, mp_comm_used, mp_ierr)
+  do i=0, mp_partgroup_np-1
+    numrel_mpi(i) = numpart_total/mp_partgroup_np
+    if (i.lt.mod(numpart_total,mp_partgroup_np)) numrel_mpi(i) = numrel_mpi(i) + 1
   end do
+  xm=0.
+  do i=1,numpart
+    if (itra1(i).eq.itime) xm=xm+xmass1(i,1)
+! Allocate temporary arrays for receiving processes
+  if (.not.lroot) then
+    allocate(itra1_tmp(numrel_mpi(mp_partid)),&
+         & npoint_tmp(numrel_mpi(mp_partid)),&
+         & nclass_tmp(numrel_mpi(mp_partid)),&
+         & idt_tmp(numrel_mpi(mp_partid)),&
+         & itramem_tmp(numrel_mpi(mp_partid)),&
+         & itrasplit_tmp(numrel_mpi(mp_partid)),&
+         & xtra1_tmp(numrel_mpi(mp_partid)),&
+         & ytra1_tmp(numrel_mpi(mp_partid)),&
+         & ztra1_tmp(numrel_mpi(mp_partid)),&
+         & xmass1_tmp(numrel_mpi(mp_partid),maxspec))
+! Initialize all particles as non-existent
+    itra1_tmp(:)=-999999999
+  end if
+! Distribute particles
+! Keep track of released particles so far
+  rel_counter = 0
+  mtag = 1000
+  do i=0, mp_partgroup_np-1
+! For root process, nothing to do except update release count
+    if (i.eq.0) then
+      rel_counter = rel_counter + numrel_mpi(i)
+      cycle
+    end if
+! Send particles from root to non-root processes
+    if (lroot.and.numrel_mpi(i).gt.0) then
+      call MPI_SEND(nclass_tmp(rel_counter+1:rel_counter+numrel_mpi(i)),&
+           &numrel_mpi(i),MPI_INTEGER,i,mtag+1*i,mp_comm_used,mp_ierr)
+      call MPI_SEND(npoint_tmp(rel_counter+1:rel_counter+numrel_mpi(i)),&
+           &numrel_mpi(i),MPI_INTEGER,i,mtag+2*i,mp_comm_used,mp_ierr)
+      call MPI_SEND(itra1_tmp(rel_counter+1:rel_counter+numrel_mpi(i)),&
+           &numrel_mpi(i),MPI_INTEGER,i,mtag+3*i,mp_comm_used,mp_ierr)
+      call MPI_SEND(idt_tmp(rel_counter+1:rel_counter+numrel_mpi(i)),&
+           &numrel_mpi(i),MPI_INTEGER,i,mtag+4*i,mp_comm_used,mp_ierr)
+      call MPI_SEND(itramem_tmp(rel_counter+1:rel_counter+numrel_mpi(i)),&
+           &numrel_mpi(i),MPI_INTEGER,i,mtag+5*i,mp_comm_used,mp_ierr)
+      call MPI_SEND(itrasplit_tmp(rel_counter+1:rel_counter+numrel_mpi(i)),&
+           &numrel_mpi(i),MPI_INTEGER,i,mtag+6*i,mp_comm_used,mp_ierr)
+      call MPI_SEND(xtra1_tmp(rel_counter+1:rel_counter+numrel_mpi(i)),&
+           &numrel_mpi(i),mp_dp,i,mtag+7*i,mp_comm_used,mp_ierr)
+      call MPI_SEND(ytra1_tmp(rel_counter+1:rel_counter+numrel_mpi(i)),&
+           &numrel_mpi(i),mp_dp,i,mtag+8*i,mp_comm_used,mp_ierr)
+      call MPI_SEND(ztra1_tmp(rel_counter+1:rel_counter+numrel_mpi(i)),&
+           &numrel_mpi(i),mp_sp,i,mtag+9*i,mp_comm_used,mp_ierr)
+      do j=1,nspec
+        call MPI_SEND(xmass1_tmp(rel_counter+1:rel_counter+numrel_mpi(i),j),&
+             &numrel_mpi(i),mp_sp,i,mtag+(9+j)*i,mp_comm_used,mp_ierr)
+      end do
+! Non-root processes issue receive requests
+    else if (i.eq.mp_partid.and.numrel_mpi(i).gt.0) then
+      call MPI_RECV(nclass_tmp(1:numrel_mpi(i)),numrel_mpi(i),&
+           &MPI_INTEGER,id_root,mtag+1*i,mp_comm_used,mp_status,mp_ierr)
+      call MPI_RECV(npoint_tmp(1:numrel_mpi(i)),numrel_mpi(i),&
+           &MPI_INTEGER,id_root,mtag+2*i,mp_comm_used,mp_status,mp_ierr)
+      call MPI_RECV(itra1_tmp(1:numrel_mpi(i)),numrel_mpi(i),&
+           &MPI_INTEGER,id_root,mtag+3*i,mp_comm_used,mp_status,mp_ierr)
+      call MPI_RECV(idt_tmp(1:numrel_mpi(i)),numrel_mpi(i),&
+           &MPI_INTEGER,id_root,mtag+4*i,mp_comm_used,mp_status,mp_ierr)
+      call MPI_RECV(itramem_tmp(1:numrel_mpi(i)),numrel_mpi(i),&
+           &MPI_INTEGER,id_root,mtag+5*i,mp_comm_used,mp_status,mp_ierr)
+      call MPI_RECV(itrasplit_tmp(1:numrel_mpi(i)),numrel_mpi(i),&
+           &MPI_INTEGER,id_root,mtag+6*i,mp_comm_used,mp_status,mp_ierr)
+      call MPI_RECV(xtra1_tmp(1:numrel_mpi(i)),numrel_mpi(i),&
+           &mp_dp,id_root,mtag+7*i,mp_comm_used,mp_status,mp_ierr)
+      call MPI_RECV(ytra1_tmp(1:numrel_mpi(i)),numrel_mpi(i),&
+           &mp_dp,id_root,mtag+8*i,mp_comm_used,mp_status,mp_ierr)
+      call MPI_RECV(ztra1_tmp(1:numrel_mpi(i)),numrel_mpi(i),&
+           &mp_sp,id_root,mtag+9*i,mp_comm_used,mp_status,mp_ierr)
+      do j=1,nspec
+        call MPI_RECV(xmass1_tmp(1:numrel_mpi(i),j),numrel_mpi(i),&
+             &mp_sp,id_root,mtag+(9+j)*i,mp_comm_used,mp_status,mp_ierr)
+      end do
+    end if
+    rel_counter = rel_counter + numrel_mpi(i)
   end do
+  !write(*,*) itime,numactiveparticles,numparticlecount,numpart,
+  !    +xm,accmasst,xm+accmasst
+  ! If particles shall be dumped, then accumulated masses at the domain boundaries
+  ! must be dumped, too, to be used for later runs
+  !*****************************************************************************
+! :TODO: eso parallelize
+! Find free storage space for the new particles.
+! This section is independent of the redistribution scheme used
+! ********************************************************************
+! Keep track of released particles so far
+  minpart=1
+! The algorithm should be correct also for root process
+  do i=1, numrel_mpi(mp_partid)
+    do ipart=minpart, maxpart
+      if (itra1(ipart).ne.itime) then
+        itra1(ipart) = itra1_tmp(i)
+        npoint(ipart) = npoint_tmp(i)
+        nclass(ipart) = nclass_tmp(i)
+        idt(ipart) = idt_tmp(i)
+        itramem(ipart) = itramem_tmp(i)
+        itrasplit(ipart) = itrasplit_tmp(i)
+        xtra1(ipart) = xtra1_tmp(i)
+        ytra1(ipart) = ytra1_tmp(i)
+        ztra1(ipart) = ztra1_tmp(i)
+        xmass1(ipart,:) = xmass1_tmp(i,:)
+! Increase numpart, if necessary
+        numpart=max(numpart,ipart)
+        goto 200 ! Storage space has been found, stop searching
+      end if
+    end do
+minpart=ipart+1
+  end do
+! If particles shall be dumped, then accumulated masses at the domain boundaries
+! must be dumped, too, to be used for later runs
+!*****************************************************************************
   if ((ipout.gt.0).and.(itime.eq.loutend)) then
     if (lroot) then
+      call mpif_mtime('iotime',0)
       open(unitboundcond,file=path(2)(1:length(2))//'boundcond.bin', &
            form='unformatted')
 …
            zcolumn_we,zcolumn_sn,acc_mass_we,acc_mass_sn
       close(unitboundcond)
+      call mpif_mtime('iotime',1)
     end if
   endif
+! Deallocate temporary arrays
+  deallocate(itra1_tmp,npoint_tmp,nclass_tmp,idt_tmp,itramem_tmp,itrasplit_tmp,&
+       & xtra1_tmp,ytra1_tmp,ztra1_tmp,xmass1_tmp,numrel_mpi)
+! numactiveparticles_mpi
 end subroutine boundcond_domainfill

Note: See TracChangeset for help on using the changeset viewer.

Context Navigation

Changeset 16b61a5 in flexpart.git for src/boundcond_domainfill_mpi.f90

Legend:

src/boundcond_domainfill_mpi.f90

Download in other formats: