source: flexpart.git/src/init_domainfill_mpi.f90 @ 328fdf9

10.4.1_peseiGFS_025bugfixes+enhancementsdevrelease-10release-10.4.1scaling-bug
Last change on this file since 328fdf9 was 328fdf9, checked in by Espen Sollum ATMOS <eso@…>, 5 years ago

bugfix: particles were lost at start of global domain filling run. Error in restarting domain filling run from particle dump

  • Property mode set to 100644
File size: 19.5 KB
Line 
1!**********************************************************************
2! Copyright 1998,1999,2000,2001,2002,2005,2007,2008,2009,2010         *
3! Andreas Stohl, Petra Seibert, A. Frank, Gerhard Wotawa,             *
4! Caroline Forster, Sabine Eckhardt, John Burkhart, Harald Sodemann   *
5!                                                                     *
6! This file is part of FLEXPART.                                      *
7!                                                                     *
8! FLEXPART is free software: you can redistribute it and/or modify    *
9! it under the terms of the GNU General Public License as published by*
10! the Free Software Foundation, either version 3 of the License, or   *
11! (at your option) any later version.                                 *
12!                                                                     *
13! FLEXPART is distributed in the hope that it will be useful,         *
14! but WITHOUT ANY WARRANTY; without even the implied warranty of      *
15! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the       *
16! GNU General Public License for more details.                        *
17!                                                                     *
18! You should have received a copy of the GNU General Public License   *
19! along with FLEXPART.  If not, see <http://www.gnu.org/licenses/>.   *
20!**********************************************************************
21
22subroutine init_domainfill
23!
24!*****************************************************************************
25!                                                                            *
26! Initializes particles equally distributed over the first release location  *
27! specified in file RELEASES. This box is assumed to be the domain for doing *
28! domain-filling trajectory calculations.                                    *
29! All particles carry the same amount of mass which alltogether comprises the*
30! mass of air within the box.                                                *
31!                                                                            *
32!     Author: A. Stohl                                                       *
33!                                                                            *
34!     15 October 2002                                                        *
35!                                                                            *
36!  CHANGES                                                                   *
37!    12/2014 eso: MPI version                                                *
38!*****************************************************************************
39!                                                                            *
40! Variables:                                                                 *
41!                                                                            *
42! numparticlecount    consecutively counts the number of particles released  *
43! nx_we(2)       grid indices for western and eastern boundary of domain-    *
44!                filling trajectory calculations                             *
45! ny_sn(2)       grid indices for southern and northern boundary of domain-  *
46!                filling trajectory calculations                             *
47!                                                                            *
48!*****************************************************************************
49! MPI version:
50!
51!   -Root process allocates temporary arrays holding properties for
52!    all particles in the simulation
53!   -An index array is used to assign 1st particle to 1st process, 2nd particle
54!    to 2nd process and so on so that they are evenly distibuted geographically
55!   -Inititialization for domain-filling is done as in the serial code
56!   -Root process distributes particles evenly to other processes
57!
58!*****************************************************************************
59
60  use point_mod
61  use par_mod
62  use com_mod
63  use random_mod, only: ran1
64  use mpi_mod
65
66  implicit none
67
68! ncolumn_mpi,numparttot_mpi        ncolumn,numparttot per process
69  integer :: j,ix,jy,kz,ncolumn,numparttot,ncolumn_mpi,numparttot_mpi, arr_size
70  real :: gridarea(0:nymax-1),pp(nzmax),ylat,ylatp,ylatm,hzone
71  real :: cosfactm,cosfactp,deltacol,dz1,dz2,dz,pnew,fractus
72  real,parameter :: pih=pi/180.
73  real :: colmass(0:nxmax-1,0:nymax-1),colmasstotal,zposition
74
75  integer :: ixm,ixp,jym,jyp,indzm,indzp,in,indzh,i,jj
76  real :: pvpart,ddx,ddy,rddx,rddy,p1,p2,p3,p4,y1(2)
77
78  integer :: idummy = -11
79  integer,allocatable,dimension(:) :: idx ! index array
80  integer :: stride
81  integer, parameter :: nullsize=0
82  logical :: first_call=.true.
83
84! Use different seed for each process ! TODO: not needed anymore
85  if (first_call) then
86    idummy=idummy+mp_seed
87    first_call=.false.
88  end if
89
90! Determine the release region (only full grid cells), over which particles
91! shall be initialized
92! Use 2 fields for west/east and south/north boundary
93!**************************************************************************
94
95  nx_we(1)=max(int(xpoint1(1)),0)
96  nx_we(2)=min((int(xpoint2(1))+1),nxmin1)
97  ny_sn(1)=max(int(ypoint1(1)),0)
98  ny_sn(2)=min((int(ypoint2(1))+1),nymin1)
99
100! For global simulations (both global wind data and global domain-filling),
101! set a switch, such that no boundary conditions are used
102!**************************************************************************
103  if (xglobal.and.sglobal.and.nglobal) then
104    if ((nx_we(1).eq.0).and.(nx_we(2).eq.nxmin1).and. &
105         (ny_sn(1).eq.0).and.(ny_sn(2).eq.nymin1)) then
106      gdomainfill=.true.
107    else
108      gdomainfill=.false.
109    endif
110  endif
111
112! Exit here if resuming a run from particle dump
113!***********************************************
114  if (gdomainfill.and.ipin.ne.0) return
115
116! Do not release particles twice (i.e., not at both in the leftmost and rightmost
117! grid cell) for a global domain
118!*****************************************************************************
119  if (xglobal) nx_we(2)=min(nx_we(2),nx-2)
120
121
122! This section only done by the root process
123!*******************************************
124  if (lroot) then
125! Arrays for particles to be released. Add a small number to npart(1) in case of
126! round-off errors
127    arr_size = npart(1) + mp_np
128    allocate(itra1_tmp(arr_size),npoint_tmp(arr_size),nclass_tmp(arr_size),&
129         & idt_tmp(arr_size),itramem_tmp(arr_size),itrasplit_tmp(arr_size),&
130         & xtra1_tmp(arr_size),ytra1_tmp(arr_size),ztra1_tmp(arr_size),&
131         & xmass1_tmp(arr_size, maxspec))
132
133! Index array for particles. This is to avoid having particles
134! near edges of domain all on one process.
135!****************************************************************************
136    allocate(idx(npart(1)))
137    stride = npart(1)/mp_partgroup_np
138
139    jj=0
140    do j=1, stride
141      do i=0, mp_partgroup_np-1
142        jj = jj+1
143        if (jj.gt.npart(1)) exit
144        idx(jj) = i*stride+j
145      end do
146    end do
147
148! Add extra indices if npart(1) not evenly divisible by number of processes
149    do i=1, mod(npart(1),mp_partgroup_np)
150      jj = jj+1
151      if (jj.gt.npart(1)) exit
152      idx(jj) = jj
153    end do
154
155! Initialize all particles as non-existent   
156    itra1_tmp(:)=-999999999
157
158! Calculate area of grid cell with formula M=2*pi*R*h*dx/360,
159! see Netz, Formeln der Mathematik, 5. Auflage (1983), p.90
160!************************************************************
161
162    do jy=ny_sn(1),ny_sn(2)      ! loop about latitudes
163      ylat=ylat0+real(jy)*dy
164      ylatp=ylat+0.5*dy
165      ylatm=ylat-0.5*dy
166      if ((ylatm.lt.0).and.(ylatp.gt.0.)) then
167        hzone=1./dyconst
168      else
169        cosfactp=cos(ylatp*pih)*r_earth
170        cosfactm=cos(ylatm*pih)*r_earth
171        if (cosfactp.lt.cosfactm) then
172          hzone=sqrt(r_earth**2-cosfactp**2)- &
173               sqrt(r_earth**2-cosfactm**2)
174        else
175          hzone=sqrt(r_earth**2-cosfactm**2)- &
176               sqrt(r_earth**2-cosfactp**2)
177        endif
178      endif
179      gridarea(jy)=2.*pi*r_earth*hzone*dx/360.
180    end do
181
182! Do the same for the south pole
183
184    if (sglobal) then
185      ylat=ylat0
186      ylatp=ylat+0.5*dy
187      ylatm=ylat
188      cosfactm=0.
189      cosfactp=cos(ylatp*pih)*r_earth
190      hzone=sqrt(r_earth**2-cosfactm**2)- &
191           sqrt(r_earth**2-cosfactp**2)
192      gridarea(0)=2.*pi*r_earth*hzone*dx/360.
193    endif
194
195! Do the same for the north pole
196
197    if (nglobal) then
198      ylat=ylat0+real(nymin1)*dy
199      ylatp=ylat
200      ylatm=ylat-0.5*dy
201      cosfactp=0.
202      cosfactm=cos(ylatm*pih)*r_earth
203      hzone=sqrt(r_earth**2-cosfactp**2)- &
204           sqrt(r_earth**2-cosfactm**2)
205      gridarea(nymin1)=2.*pi*r_earth*hzone*dx/360.
206    endif
207
208
209! Calculate total mass of each grid column and of the whole atmosphere
210!*********************************************************************
211
212    colmasstotal=0.
213    do jy=ny_sn(1),ny_sn(2)          ! loop about latitudes
214      do ix=nx_we(1),nx_we(2)      ! loop about longitudes
215        pp(1)=rho(ix,jy,1,1)*r_air*tt(ix,jy,1,1)
216        pp(nz)=rho(ix,jy,nz,1)*r_air*tt(ix,jy,nz,1)
217        colmass(ix,jy)=(pp(1)-pp(nz))/ga*gridarea(jy)
218        colmasstotal=colmasstotal+colmass(ix,jy)
219      end do
220    end do
221
222    write(*,*) 'Atm. mass: ',colmasstotal
223
224
225    if (ipin.eq.0) numpart=0
226
227! Determine the particle positions
228!*********************************
229
230    numparttot=0
231    numcolumn=0
232    do jy=ny_sn(1),ny_sn(2)      ! loop about latitudes
233      ylat=ylat0+real(jy)*dy
234      do ix=nx_we(1),nx_we(2)      ! loop about longitudes
235        ncolumn=nint(0.999*real(npart(1))*colmass(ix,jy)/ &
236             colmasstotal)
237        if (ncolumn.eq.0) goto 30
238        if (ncolumn.gt.numcolumn) numcolumn=ncolumn
239
240! Calculate pressure at the altitudes of model surfaces, using the air density
241! information, which is stored as a 3-d field
242!*****************************************************************************
243
244        do kz=1,nz
245          pp(kz)=rho(ix,jy,kz,1)*r_air*tt(ix,jy,kz,1)
246        end do
247
248
249        deltacol=(pp(1)-pp(nz))/real(ncolumn)
250        pnew=pp(1)+deltacol/2.
251        jj=0
252        do j=1,ncolumn
253          jj=jj+1
254
255
256! For columns with many particles (i.e. around the equator), distribute
257! the particles equally, for columns with few particles (i.e. around the
258! poles), distribute the particles randomly
259!***********************************************************************
260
261
262          if (ncolumn.gt.20) then
263            pnew=pnew-deltacol
264          else
265            pnew=pp(1)-ran1(idummy)*(pp(1)-pp(nz))
266          endif
267
268          do kz=1,nz-1
269            if ((pp(kz).ge.pnew).and.(pp(kz+1).lt.pnew)) then
270              dz1=pp(kz)-pnew
271              dz2=pnew-pp(kz+1)
272              dz=1./(dz1+dz2)
273
274! Assign particle position
275!*************************
276! Do the following steps only if particles are not read in from previous model run
277!*****************************************************************************
278              if (ipin.eq.0) then
279                xtra1_tmp(idx(numpart+jj))=real(ix)-0.5+ran1(idummy)
280                if (ix.eq.0) xtra1_tmp(idx(numpart+jj))=ran1(idummy)
281                if (ix.eq.nxmin1) xtra1_tmp(idx(numpart+jj))= &
282                     real(nxmin1)-ran1(idummy)
283                ytra1_tmp(idx(numpart+jj))=real(jy)-0.5+ran1(idummy)
284                ztra1_tmp(idx(numpart+jj))=(height(kz)*dz2+height(kz+1)*dz1)*dz
285                if (ztra1_tmp(idx(numpart+jj)).gt.height(nz)-0.5) &
286                     ztra1_tmp(idx(numpart+jj))=height(nz)-0.5
287
288
289! Interpolate PV to the particle position
290!****************************************
291                ixm=int(xtra1_tmp(idx(numpart+jj)))
292                jym=int(ytra1_tmp(idx(numpart+jj)))
293                ixp=ixm+1
294                jyp=jym+1
295                ddx=xtra1_tmp(idx(numpart+jj))-real(ixm)
296                ddy=ytra1_tmp(idx(numpart+jj))-real(jym)
297                rddx=1.-ddx
298                rddy=1.-ddy
299                p1=rddx*rddy
300                p2=ddx*rddy
301                p3=rddx*ddy
302                p4=ddx*ddy
303                do i=2,nz
304                  if (height(i).gt.ztra1_tmp(idx(numpart+jj))) then
305                    indzm=i-1
306                    indzp=i
307                    goto 6
308                  endif
309                end do
3106               continue
311                dz1=ztra1_tmp(idx(numpart+jj))-height(indzm)
312                dz2=height(indzp)-ztra1_tmp(idx(numpart+jj))
313                dz=1./(dz1+dz2)
314                do in=1,2
315                  indzh=indzm+in-1
316                  y1(in)=p1*pv(ixm,jym,indzh,1) &
317                       +p2*pv(ixp,jym,indzh,1) &
318                       +p3*pv(ixm,jyp,indzh,1) &
319                       +p4*pv(ixp,jyp,indzh,1)
320                end do
321                pvpart=(dz2*y1(1)+dz1*y1(2))*dz
322                if (ylat.lt.0.) pvpart=-1.*pvpart
323
324
325! For domain-filling option 2 (stratospheric O3), do the rest only in the stratosphere
326!*****************************************************************************
327
328                if (((ztra1_tmp(idx(numpart+jj)).gt.3000.).and. &
329                     (pvpart.gt.pvcrit)).or.(mdomainfill.eq.1)) then
330
331! Assign certain properties to the particle
332!******************************************
333                  nclass_tmp(idx(numpart+jj))=min(int(ran1(idummy)* &
334                       real(nclassunc))+1,nclassunc)
335                  numparticlecount=numparticlecount+1
336                  npoint_tmp(idx(numpart+jj))=numparticlecount
337                  idt_tmp(idx(numpart+jj))=mintime
338                  itra1_tmp(idx(numpart+jj))=0
339                  itramem_tmp(idx(numpart+jj))=0
340                  itrasplit_tmp(idx(numpart+jj))=itra1_tmp(idx(numpart+jj))+ldirect* &
341                       itsplit
342                  xmass1_tmp(idx(numpart+jj),1)=colmass(ix,jy)/real(ncolumn)
343                  if (mdomainfill.eq.2) xmass1_tmp(idx(numpart+jj),1)= &
344                       xmass1_tmp(idx(numpart+jj),1)*pvpart*48./29.*ozonescale/10.**9
345                else
346                  jj=jj-1
347                endif
348              endif
349            endif
350          end do
351        end do
352        numparttot=numparttot+ncolumn
353        if (ipin.eq.0) numpart=numpart+jj
35430      continue
355      end do
356    end do
357
358
359! Check whether numpart is really smaller than maxpart
360!*****************************************************
361
362! ESO :TODO: this warning need to be moved further up, else out-of-bounds error earlier
363    if (numpart.gt.maxpart) then
364      write(*,*) 'numpart too large: change source in init_atm_mass.f'
365      write(*,*) 'numpart: ',numpart,' maxpart: ',maxpart
366    endif
367
368
369    xmassperparticle=colmasstotal/real(numparttot)
370
371
372! Make sure that all particles are within domain
373!***********************************************
374
375    do j=1,numpart
376      if ((xtra1_tmp(j).lt.0.).or.(xtra1_tmp(j).ge.real(nxmin1)).or. &
377           (ytra1_tmp(j).lt.0.).or.(ytra1_tmp(j).ge.real(nymin1))) then
378        itra1_tmp(j)=-999999999
379      endif
380    end do
381
382
383
384
385! For boundary conditions, we need fewer particle release heights per column,
386! because otherwise it takes too long until enough mass has accumulated to
387! release a particle at the boundary (would take dx/u seconds), leading to
388! relatively large position errors of the order of one grid distance.
389! It's better to release fewer particles per column, but to do so more often.
390! Thus, use on the order of nz starting heights per column.
391! We thus repeat the above to determine fewer starting heights, that are
392! used furtheron in subroutine boundcond_domainfill.f.
393!****************************************************************************
394
395    fractus=real(numcolumn)/real(nz)
396    write(*,*) 'Total number of particles at model start: ',numpart
397    write(*,*) 'Maximum number of particles per column: ',numcolumn
398    write(*,*) 'If ',fractus,' <1, better use more particles'
399    fractus=sqrt(max(fractus,1.))/2.
400
401    do jy=ny_sn(1),ny_sn(2)      ! loop about latitudes
402      do ix=nx_we(1),nx_we(2)      ! loop about longitudes
403        ncolumn=nint(0.999/fractus*real(npart(1))*colmass(ix,jy) &
404             /colmasstotal)
405        if (ncolumn.gt.maxcolumn) stop 'maxcolumn too small'
406        if (ncolumn.eq.0) goto 80
407
408
409! Memorize how many particles per column shall be used for all boundaries
410! This is further used in subroutine boundcond_domainfill.f
411! Use 2 fields for west/east and south/north boundary
412!************************************************************************
413
414        if (ix.eq.nx_we(1)) numcolumn_we(1,jy)=ncolumn
415        if (ix.eq.nx_we(2)) numcolumn_we(2,jy)=ncolumn
416        if (jy.eq.ny_sn(1)) numcolumn_sn(1,ix)=ncolumn
417        if (jy.eq.ny_sn(2)) numcolumn_sn(2,ix)=ncolumn
418
419! Calculate pressure at the altitudes of model surfaces, using the air density
420! information, which is stored as a 3-d field
421!*****************************************************************************
422
423        do kz=1,nz
424          pp(kz)=rho(ix,jy,kz,1)*r_air*tt(ix,jy,kz,1)
425        end do
426
427! Determine the reference starting altitudes
428!*******************************************
429
430        deltacol=(pp(1)-pp(nz))/real(ncolumn)
431        pnew=pp(1)+deltacol/2.
432        do j=1,ncolumn
433          pnew=pnew-deltacol
434          do kz=1,nz-1
435            if ((pp(kz).ge.pnew).and.(pp(kz+1).lt.pnew)) then
436              dz1=pp(kz)-pnew
437              dz2=pnew-pp(kz+1)
438              dz=1./(dz1+dz2)
439              zposition=(height(kz)*dz2+height(kz+1)*dz1)*dz
440              if (zposition.gt.height(nz)-0.5) zposition=height(nz)-0.5
441
442! Memorize vertical positions where particles are introduced
443! This is further used in subroutine boundcond_domainfill.f
444!***********************************************************
445
446              if (ix.eq.nx_we(1)) zcolumn_we(1,jy,j)=zposition
447              if (ix.eq.nx_we(2)) zcolumn_we(2,jy,j)=zposition
448              if (jy.eq.ny_sn(1)) zcolumn_sn(1,ix,j)=zposition
449              if (jy.eq.ny_sn(2)) zcolumn_sn(2,ix,j)=zposition
450
451! Initialize mass that has accumulated at boundary to zero
452!*********************************************************
453
454              acc_mass_we(1,jy,j)=0.
455              acc_mass_we(2,jy,j)=0.
456              acc_mass_sn(1,jy,j)=0.
457              acc_mass_sn(2,jy,j)=0.
458            endif
459          end do
460        end do
46180      continue
462      end do
463    end do
464
465! If particles shall be read in to continue an existing run,
466! then the accumulated masses at the domain boundaries must be read in, too.
467! This overrides any previous calculations.
468!***************************************************************************
469
470! eso TODO: only needed for root process
471    if ((ipin.eq.1).and.(.not.gdomainfill)) then
472      open(unitboundcond,file=path(2)(1:length(2))//'boundcond.bin', &
473           form='unformatted')
474      read(unitboundcond) numcolumn_we,numcolumn_sn, &
475           zcolumn_we,zcolumn_sn,acc_mass_we,acc_mass_sn
476      close(unitboundcond)
477    endif
478
479    if (ipin.eq.0) then   
480      numpart = numpart/mp_partgroup_np
481      if (mod(numpart,mp_partgroup_np).ne.0) numpart=numpart+1
482    end if
483
484  else ! Allocate dummy arrays for receiving processes
485    if (ipin.eq.0) then   
486      allocate(itra1_tmp(nullsize),npoint_tmp(nullsize),nclass_tmp(nullsize),&
487           & idt_tmp(nullsize),itramem_tmp(nullsize),itrasplit_tmp(nullsize),&
488           & xtra1_tmp(nullsize),ytra1_tmp(nullsize),ztra1_tmp(nullsize),&
489           & xmass1_tmp(nullsize, nullsize))
490    end if
491   
492  end if ! end if(lroot)
493
494
495
496! Distribute particles to other processes (numpart is 'per-process', not total)
497! Only if not restarting from previous run
498  if (ipin.eq.0) then
499    call MPI_Bcast(numpart, 1, MPI_INTEGER, id_root, mp_comm_used, mp_ierr)
500    call mpif_send_part_properties(npart(1)/mp_partgroup_np)
501
502! Deallocate the temporary arrays used for all particles
503    deallocate(itra1_tmp,npoint_tmp,nclass_tmp,idt_tmp,itramem_tmp,&
504         & itrasplit_tmp,xtra1_tmp,ytra1_tmp,ztra1_tmp,xmass1_tmp)
505  end if
506
507
508end subroutine init_domainfill
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG