1 | !*********************************************************************** |
---|
2 | !* Copyright 2012,2013 * |
---|
3 | !* Jerome Brioude, Delia Arnold, Andreas Stohl, Wayne Angevine, * |
---|
4 | !* John Burkhart, Massimo Cassiani, Adam Dingwell, Richard C Easter, Sabine Eckhardt,* |
---|
5 | !* Stephanie Evan, Jerome D Fast, Don Morton, Ignacio Pisso, * |
---|
6 | !* Petra Seibert, Gerard Wotawa, Caroline Forster, Harald Sodemann, * |
---|
7 | !* * |
---|
8 | !* This file is part of FLEXPART WRF * |
---|
9 | !* * |
---|
10 | !* FLEXPART is free software: you can redistribute it and/or modify * |
---|
11 | !* it under the terms of the GNU General Public License as published by* |
---|
12 | !* the Free Software Foundation, either version 3 of the License, or * |
---|
13 | !* (at your option) any later version. * |
---|
14 | !* * |
---|
15 | !* FLEXPART is distributed in the hope that it will be useful, * |
---|
16 | !* but WITHOUT ANY WARRANTY; without even the implied warranty of * |
---|
17 | !* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * |
---|
18 | !* GNU General Public License for more details. * |
---|
19 | !* * |
---|
20 | !* You should have received a copy of the GNU General Public License * |
---|
21 | !* along with FLEXPART. If not, see <http://www.gnu.org/licenses/>. * |
---|
22 | !*********************************************************************** |
---|
23 | |
---|
24 | subroutine timemanager_mpi(mts) |
---|
25 | |
---|
26 | !******************************************************************************* |
---|
27 | ! * |
---|
28 | ! Handles the computation of trajectories, i.e. determines which * |
---|
29 | ! trajectories have to be computed at what time. * |
---|
30 | ! Manages dry+wet deposition routines, radioactive decay and the computation * |
---|
31 | ! of concentrations. * |
---|
32 | ! * |
---|
33 | ! Author: A. Stohl * |
---|
34 | ! * |
---|
35 | ! 20 May 1996 * |
---|
36 | ! * |
---|
37 | ! Dec 2005, J. Fast - Only call conccalc & concoutput when (iout.ge.1) * |
---|
38 | ! Aug 2007, W. Wang - call KFeta convection scheme (lconvection=2or3) |
---|
39 | ! Note, backward is unavailabe for lconvection=2 |
---|
40 | ! Mar 2012, J. Brioude: modifications to handle openmp and mpi * |
---|
41 | !******************************************************************************* |
---|
42 | ! Changes, Bernd C. Krueger, Feb. 2001: * |
---|
43 | ! Call of convmix when new windfield is read * |
---|
44 | !------------------------------------ * |
---|
45 | ! Changes Petra Seibert, Sept 2002 * |
---|
46 | ! fix wet scavenging problem * |
---|
47 | ! Code may not be correct for decay of deposition! * |
---|
48 | ! Changes Petra Seibert, Nov 2002 * |
---|
49 | ! call convection BEFORE new fields are read in BWD mode * |
---|
50 | ! Changes Caroline Forster, Feb 2005 |
---|
51 | ! new interface between flexpart and convection scheme |
---|
52 | ! Emanuel's latest subroutine convect43c.f is used |
---|
53 | !******************************************************************************* |
---|
54 | ! * |
---|
55 | ! Variables: * |
---|
56 | ! DEP .true. if either wet or dry deposition is switched on * |
---|
57 | ! decay(maxspec) [1/s] decay constant for radioactive decay * |
---|
58 | ! DRYDEP .true. if dry deposition is switched on * |
---|
59 | ! ideltas [s] modelling period * |
---|
60 | ! jtime [s] actual temporal position of calculation * |
---|
61 | ! ldeltat [s] time since computation of radioact. decay of depositions * |
---|
62 | ! loutaver [s] averaging period for concentration calculations * |
---|
63 | ! loutend [s] end of averaging for concentration calculations * |
---|
64 | ! loutnext [s] next time at which output fields shall be centered * |
---|
65 | ! loutsample [s] sampling interval for averaging of concentrations * |
---|
66 | ! loutstart [s] start of averaging for concentration calculations * |
---|
67 | ! loutstep [s] time interval for which concentrations shall be calculated* |
---|
68 | ! npoint(maxpart) index, which starting point the trajectory has * |
---|
69 | ! starting positions of trajectories * |
---|
70 | ! nstop serves as indicator for fate of particles * |
---|
71 | ! in the particle loop * |
---|
72 | ! nstop1 serves as indicator for wind fields (see getfields) * |
---|
73 | ! outnum number of samples for each concentration calculation * |
---|
74 | ! outnum number of samples for each concentration calculation * |
---|
75 | ! prob probability of absorption at ground due to dry deposition * |
---|
76 | ! WETDEP .true. if wet deposition is switched on * |
---|
77 | ! weight weight for each concentration sample (1/2 or 1) * |
---|
78 | ! uap(maxpart),ucp(maxpart),uzp(maxpart) = random velocities due to turbulence * |
---|
79 | ! us(maxpart),vs(maxpart),ws(maxpart) = random velocities due to interpolation * |
---|
80 | ! xtra1(maxpart), ytra1(maxpart), ztra1(maxpart) = * |
---|
81 | ! spatial positions of trajectories * |
---|
82 | ! * |
---|
83 | ! Constants: * |
---|
84 | ! maxpart maximum number of trajectories * |
---|
85 | ! * |
---|
86 | !******************************************************************************* |
---|
87 | |
---|
88 | ! include 'includepar' |
---|
89 | ! include 'includecom' |
---|
90 | use unc_mod |
---|
91 | use point_mod |
---|
92 | ! use xmass_mod |
---|
93 | use flux_mod |
---|
94 | use outg_mod |
---|
95 | use oh_mod |
---|
96 | use par_mod |
---|
97 | use com_mod |
---|
98 | use mpi_mod |
---|
99 | use mt_stream |
---|
100 | |
---|
101 | ! use ran_mod |
---|
102 | ! use interpol_mod |
---|
103 | |
---|
104 | implicit none |
---|
105 | |
---|
106 | include 'mpif.h' |
---|
107 | |
---|
108 | integer :: ix,jy,j,ks,kp,l,n,jtime,nstop,nstop1 |
---|
109 | ! integer :: MPI_COMM_WORLD |
---|
110 | ! integer :: ksp |
---|
111 | integer :: loutnext,loutstart,loutend,jj,chunksize |
---|
112 | !,chunksize2 |
---|
113 | integer :: chunksize3,omp_get_num_threads |
---|
114 | integer :: ldeltat,itage,nage,th_itra1,i |
---|
115 | real :: outnum,weight,prob(maxspec),nrand,decfact |
---|
116 | ! real :: uap(maxpart),ucp(maxpart),uzp(maxpart) |
---|
117 | ! real :: us(maxpart),vs(maxpart),ws(maxpart) |
---|
118 | ! integer(kind=2) :: cbt(maxpart) |
---|
119 | ! real,allocatable, dimension (:) :: uap,ucp,uzp |
---|
120 | ! real,allocatable, dimension (:) :: us,vs,ws |
---|
121 | ! integer(kind=2),allocatable, dimension (:) :: cbt |
---|
122 | real :: drydeposit(maxspec),gridtotalunc,wetgridtotalunc |
---|
123 | real :: drygridtotalunc,xold,yold,zold,xmassfract |
---|
124 | ! integer j,k,l,n,jtime,nstop,nstop1 |
---|
125 | ! integer loutnext,loutstart,loutend |
---|
126 | ! integer ix,jy,ldeltat,itage,nage |
---|
127 | ! real outnum,weight,prob(maxspec) |
---|
128 | ! real uap(maxpart),ucp(maxpart),uzp(maxpart),decfact |
---|
129 | ! real us(maxpart),vs(maxpart),ws(maxpart),cbt(maxpart) |
---|
130 | ! real drydeposit(maxspec),gridtotalunc,wetgridtotalunc |
---|
131 | ! real drygridtotalunc,xold,yold,zold |
---|
132 | ! real xm,xm1 |
---|
133 | |
---|
134 | |
---|
135 | integer :: th_npoint,th_idt,th_itramem,jdeb,jfin,stat,th_nclass |
---|
136 | ! integer,save :: cpt(24)=0 |
---|
137 | integer,save :: cpt(maxomp)=0 |
---|
138 | |
---|
139 | real(kind=dp) :: th_xtra1,th_ytra1 |
---|
140 | real :: th_ztra1,th_uap,th_ucp,th_uzp |
---|
141 | real :: th_us,th_vs,th_ws,ran3 |
---|
142 | integer(kind=2) :: th_cbt |
---|
143 | integer :: OMP_GET_THREAD_NUM,from |
---|
144 | |
---|
145 | real :: p1,p2,p3,p4,ddx,ddy,rddx,rddy,dtt,dt1,dt2 |
---|
146 | integer :: ixp,jyp,ngrid,indz,indzp,nbp,jj2,ii,offset |
---|
147 | logical :: depoindicator(maxspec) |
---|
148 | logical,save :: indzindicator(nzmax) |
---|
149 | real :: ust,wst,ol,h,zeta,sigu,sigv,tlu,tlv,tlw |
---|
150 | real :: sigw,dsigwdz,dsigw2dz,th_xmass1(maxspec) |
---|
151 | real :: start, finish |
---|
152 | real :: uprof(nzmax),vprof(nzmax),wprof(nzmax) |
---|
153 | real :: usigprof(nzmax),vsigprof(nzmax),wsigprof(nzmax) |
---|
154 | real :: rhoprof(nzmax),rhogradprof(nzmax) |
---|
155 | real :: tkeprof(nzmax),pttprof(nzmax) |
---|
156 | real :: u,v,w,usig,vsig,wsig,pvi |
---|
157 | integer*4 :: now(3) |
---|
158 | integer :: ttime,cpttra |
---|
159 | integer, dimension(MPI_STATUS_SIZE) :: status |
---|
160 | integer :: myid,ntasks,ierr,islave,tag2,ompid,n_threads,tag3,i_omp |
---|
161 | type (mt_state) :: mts (0: MAX_STREAM) |
---|
162 | !************************ |
---|
163 | |
---|
164 | !JB |
---|
165 | call MPI_COMM_RANK ( MPI_COMM_WORLD, myid, ierr ) |
---|
166 | call MPI_COMM_SIZE ( MPI_COMM_WORLD, ntasks, ierr ) |
---|
167 | ! myid gives the info on the node id |
---|
168 | |
---|
169 | loutnext=loutstep/2 |
---|
170 | outnum=0. |
---|
171 | loutstart=loutnext-loutaver/2 |
---|
172 | loutend=loutnext+loutaver/2 |
---|
173 | |
---|
174 | ! if (myid.eq.0) then |
---|
175 | allocate(uap(maxpart) ,stat=stat) |
---|
176 | allocate(ucp(maxpart) ,stat=stat) |
---|
177 | allocate(uzp(maxpart) ,stat=stat) |
---|
178 | allocate(us(maxpart) ,stat=stat) |
---|
179 | allocate(vs(maxpart) ,stat=stat) |
---|
180 | allocate(ws(maxpart) ,stat=stat) |
---|
181 | allocate(cbt(maxpart) ,stat=stat) |
---|
182 | ! endif |
---|
183 | ! if (chunksize2.eq.0) chunksize2=1 |
---|
184 | chunksize2=int(maxpart/ntasks)+1 !if sent homogeneously |
---|
185 | ! print*,'chunk',myid,chunksize2,numpart |
---|
186 | if (ntasks.gt.1) then |
---|
187 | allocate(mpi_npoint(chunksize2) ,stat=stat) |
---|
188 | if (stat.ne.0) write(*,*)'ERROR: could not 1' |
---|
189 | allocate(mpi_idt(chunksize2) ,stat=stat) |
---|
190 | if (stat.ne.0) write(*,*)'ERROR: could not 2' |
---|
191 | allocate(mpi_itra1(chunksize2) ,stat=stat) |
---|
192 | allocate(mpi_itramem(chunksize2) ,stat=stat) |
---|
193 | if (stat.ne.0) write(*,*)'ERROR: could not 3' |
---|
194 | allocate(mpi_uap(chunksize2) ,stat=stat) |
---|
195 | if (stat.ne.0) write(*,*)'ERROR: could not 4' |
---|
196 | allocate(mpi_ucp(chunksize2) ,stat=stat) |
---|
197 | if (stat.ne.0) write(*,*)'ERROR: could not 5' |
---|
198 | allocate(mpi_uzp(chunksize2) ,stat=stat) |
---|
199 | if (stat.ne.0) write(*,*)'ERROR: could not 6' |
---|
200 | allocate(mpi_us(chunksize2) ,stat=stat) |
---|
201 | if (stat.ne.0) write(*,*)'ERROR: could not 7' |
---|
202 | allocate(mpi_vs(chunksize2) ,stat=stat) |
---|
203 | if (stat.ne.0) write(*,*)'ERROR: could not 8' |
---|
204 | allocate(mpi_ws(chunksize2) ,stat=stat) |
---|
205 | if (stat.ne.0) write(*,*)'ERROR: could not 82' |
---|
206 | allocate(mpi_xtra1(chunksize2) ,stat=stat) |
---|
207 | if (stat.ne.0) write(*,*)'ERROR: could not 9' |
---|
208 | allocate(mpi_ytra1(chunksize2) ,stat=stat) |
---|
209 | if (stat.ne.0) write(*,*)'ERROR: could not10' |
---|
210 | allocate(mpi_ztra1(chunksize2) ,stat=stat) |
---|
211 | if (stat.ne.0) write(*,*)'ERROR: could not11' |
---|
212 | allocate(mpi_cbt(chunksize2) ,stat=stat) |
---|
213 | if (stat.ne.0) write(*,*)'ERROR: could not12' |
---|
214 | allocate(mpi_xmass1(chunksize2,nspec) ,stat=stat) |
---|
215 | ! allocate(mpi_drydep1(chunksize2,nspec) ,stat=stat) |
---|
216 | if (stat.ne.0) write(*,*)'ERROR: could not13' |
---|
217 | allocate(mpi_nclass(chunksize2) ,stat=stat) |
---|
218 | allocate(dummyi2(chunksize2) ,stat=stat) |
---|
219 | if (stat.ne.0) write(*,*)'ERROR: could not14' |
---|
220 | allocate(dummyr2(chunksize2) ,stat=stat) |
---|
221 | if (stat.ne.0) write(*,*)'ERROR: could not15' |
---|
222 | allocate(dummyi22(chunksize2) ,stat=stat) |
---|
223 | if (stat.ne.0) write(*,*)'ERROR: could not16' |
---|
224 | allocate(dummyr22(chunksize2) ,stat=stat) |
---|
225 | if (stat.ne.0) write(*,*)'ERROR: could not17' |
---|
226 | chunksize2=chunksize |
---|
227 | endif |
---|
228 | |
---|
229 | !********************************************************************** |
---|
230 | ! Loop over the whole modelling period in time steps of mintime seconds |
---|
231 | !********************************************************************** |
---|
232 | |
---|
233 | ! print*,'time',myid,ideltas,lsynctime |
---|
234 | do jtime=0,ideltas,lsynctime |
---|
235 | |
---|
236 | |
---|
237 | |
---|
238 | ! print*,'jtime',jtime |
---|
239 | ! Computation of wet deposition every lsynctime seconds |
---|
240 | ! maybe wet depo frequency can be relaxed later but better be on safe side |
---|
241 | ! wetdepo must be called BEFORE new fields are read in but should not |
---|
242 | ! be called in the very beginning before any fields are loaded, or |
---|
243 | ! before particles are in the system |
---|
244 | ! Code may not be correct for decay of deposition |
---|
245 | ! changed by Petra Seibert 9/02 |
---|
246 | !******************************************************************** |
---|
247 | |
---|
248 | if (WETDEP .and. jtime .ne. 0 .and. numpart .gt. 0) & |
---|
249 | call wetdepo(jtime,lsynctime,loutnext) |
---|
250 | |
---|
251 | if (OHREA .and. jtime .ne. 0 .and. numpart .gt. 0) & |
---|
252 | call ohreaction(jtime,lsynctime,loutnext) |
---|
253 | |
---|
254 | ! compute convection for backward runs |
---|
255 | !************************************* |
---|
256 | |
---|
257 | ! if ((ldirect.eq.-1).and.(lconvection.eq.1).and.(jtime.lt.0)) |
---|
258 | ! & call convmix(jtime) |
---|
259 | |
---|
260 | if ((ldirect.eq.-1).and.(jtime.lt.0)) then |
---|
261 | if (lconvection .eq. 1) call convmix(jtime) |
---|
262 | if (lconvection .eq. 2 .or. lconvection .eq. 3) & |
---|
263 | call convmix_kfeta(jtime) |
---|
264 | endif |
---|
265 | |
---|
266 | ! Get necessary wind fields if not available |
---|
267 | !******************************************* |
---|
268 | |
---|
269 | ! call itime(now) |
---|
270 | ! ttime=now(1)*3600+now(2)*60+now(3) |
---|
271 | call cpu_time(start) |
---|
272 | call getfields(jtime,nstop1) |
---|
273 | if (nstop1.gt.1) stop 'NO METEO FIELDS AVAILABLE' |
---|
274 | ! call itime(now) |
---|
275 | ! ttime=now(1)*3600+now(2)*60+now(3)-ttime |
---|
276 | call cpu_time(finish) |
---|
277 | ! print*,'read wind time',ttime |
---|
278 | if (option_verbose.eq.1) then |
---|
279 | print*,'read wind time',finish-start |
---|
280 | endif |
---|
281 | ! Release particles |
---|
282 | !****************** |
---|
283 | |
---|
284 | !JB |
---|
285 | if (myid.eq.0) then ! I let only the master thread releasing the particles and calculate the output |
---|
286 | ! call itime(now) |
---|
287 | call cpu_time(start) |
---|
288 | if (mdomainfill.ge.1) then |
---|
289 | if (jtime.eq.0) then |
---|
290 | call init_domainfill() |
---|
291 | else |
---|
292 | call boundcond_domainfill(jtime,loutend) |
---|
293 | endif |
---|
294 | else |
---|
295 | if (numpoint_option.eq.0) then |
---|
296 | call releaseparticles_irreg(jtime) |
---|
297 | elseif (numpoint_option.eq.1) then |
---|
298 | ! print*,'avant release' |
---|
299 | call releaseparticles_reg(jtime) |
---|
300 | endif |
---|
301 | endif |
---|
302 | ! do i=1,numpart |
---|
303 | ! print*,'ipart 2',myid,i,ztra1(i) |
---|
304 | ! enddo |
---|
305 | ! print*,'test rel',npoint(1),npoint(2),npoint(3) |
---|
306 | |
---|
307 | ! print*,'test rel1',npoint(5139),npoint(6002),npoint(100003) |
---|
308 | ! Compute convective mixing for forward runs |
---|
309 | ! for backward runs it is done before next windfield is read in |
---|
310 | !************************************************************** |
---|
311 | |
---|
312 | ! if ((ldirect.eq.1).and.(lconvection.eq.1)) & |
---|
313 | ! call convmix(jtime) |
---|
314 | |
---|
315 | if (ldirect.eq.1) then |
---|
316 | if (lconvection.eq.1)call convmix(jtime) |
---|
317 | if (lconvection.eq.2 .or. lconvection .eq. 3) & |
---|
318 | call convmix_kfeta(jtime) |
---|
319 | endif |
---|
320 | ! print*,'intermediaire' |
---|
321 | |
---|
322 | ! If middle of averaging period of output fields is reached, accumulated |
---|
323 | ! deposited mass radioactively decays |
---|
324 | !*********************************************************************** |
---|
325 | |
---|
326 | if (DEP.and.(jtime.eq.loutnext).and.(ldirect.gt.0)) then |
---|
327 | do ks=1,nspec |
---|
328 | do kp=1,maxpointspec_act |
---|
329 | if (decay(ks).gt.0.) then |
---|
330 | do nage=1,nageclass |
---|
331 | do l=1,nclassunc |
---|
332 | ! Mother output grid |
---|
333 | do jy=0,numygrid-1 |
---|
334 | do ix=0,numxgrid-1 |
---|
335 | wetgridunc(ix,jy,ks,kp,l,nage)= & |
---|
336 | wetgridunc(ix,jy,ks,kp,l,nage)* & |
---|
337 | exp(-1.*outstep*decay(ks)) |
---|
338 | drygridunc(ix,jy,ks,kp,l,nage)= & |
---|
339 | drygridunc(ix,jy,ks,kp,l,nage)* & |
---|
340 | exp(-1.*outstep*decay(ks)) |
---|
341 | end do |
---|
342 | end do |
---|
343 | ! Nested output grid |
---|
344 | if (nested_output.eq.1) then |
---|
345 | do jy=0,numygridn-1 |
---|
346 | do ix=0,numxgridn-1 |
---|
347 | wetgriduncn(ix,jy,ks,kp,l,nage)= & |
---|
348 | wetgriduncn(ix,jy,ks,kp,l,nage)* & |
---|
349 | exp(-1.*outstep*decay(ks)) |
---|
350 | drygriduncn(ix,jy,ks,kp,l,nage)= & |
---|
351 | drygriduncn(ix,jy,ks,kp,l,nage)* & |
---|
352 | exp(-1.*outstep*decay(ks)) |
---|
353 | end do |
---|
354 | end do |
---|
355 | endif |
---|
356 | end do |
---|
357 | end do |
---|
358 | endif |
---|
359 | end do |
---|
360 | end do |
---|
361 | endif |
---|
362 | |
---|
363 | !!! CHANGE: These lines may be switched on to check the conservation |
---|
364 | !!! of mass within FLEXPART |
---|
365 | |
---|
366 | ! if (mod(jtime,loutsample).eq.0) then |
---|
367 | ! xm=0. |
---|
368 | ! xm1=0. |
---|
369 | ! do 247 j=1,numpart |
---|
370 | !47 if (itra1(j).eq.jtime) xm1=xm1+xmass1(j,1) |
---|
371 | ! xm=xm1 |
---|
372 | ! do 248 nage=1,nageclass |
---|
373 | ! do 248 ix=0,numxgrid-1 |
---|
374 | ! do 248 jy=0,numygrid-1 |
---|
375 | ! do 248 l=1,nclassunc |
---|
376 | !48 xm=xm+wetgridunc(ix,jy,1,l,nage)+drygridunc(ix,jy,1,l,nage) |
---|
377 | ! write(*,'(i6,4f10.3)') jtime,xm,xm1 |
---|
378 | ! endif |
---|
379 | !!! CHANGE |
---|
380 | |
---|
381 | |
---|
382 | ! Check whether concentrations are to be calculated |
---|
383 | !************************************************** |
---|
384 | |
---|
385 | if ((ldirect*jtime.ge.ldirect*loutstart).and. & |
---|
386 | (ldirect*jtime.le.ldirect*loutend)) then ! add to grid |
---|
387 | if (mod(jtime-loutstart,loutsample).eq.0) then |
---|
388 | |
---|
389 | ! If we are exactly at the start or end of the concentration averaging interval, |
---|
390 | ! give only half the weight to this sample |
---|
391 | !******************************************************************************* |
---|
392 | |
---|
393 | if ((jtime.eq.loutstart).or.(jtime.eq.loutend)) then |
---|
394 | weight=0.5 |
---|
395 | else |
---|
396 | weight=1.0 |
---|
397 | endif |
---|
398 | ! print*,'avant conccalc' |
---|
399 | outnum=outnum+weight |
---|
400 | if(iout.ge.1) then |
---|
401 | if (outgrid_option.eq.0) then |
---|
402 | call conccalc_irreg(jtime,weight) |
---|
403 | elseif (outgrid_option.eq.1) then |
---|
404 | call conccalc_reg(jtime,weight) |
---|
405 | endif |
---|
406 | endif |
---|
407 | endif |
---|
408 | |
---|
409 | ! print*,'apres conccalc' |
---|
410 | |
---|
411 | ! if ((mquasilag.eq.1).and.(jtime.eq.(loutstart+loutend)/2)) & |
---|
412 | ! call partoutput_short(jtime) ! dump particle positions in extremely compressed format |
---|
413 | |
---|
414 | |
---|
415 | ! Output and reinitialization of grid |
---|
416 | ! If necessary, first sample of new grid is also taken |
---|
417 | !***************************************************** |
---|
418 | |
---|
419 | if ((jtime.eq.loutend).and.(outnum.gt.0.)) then |
---|
420 | ! print*,'iout',iout,ipout,outgrid_option |
---|
421 | if ((iout.le.3.).or.(iout.eq.5)) then |
---|
422 | if(iout.ge.1) then |
---|
423 | if (outgrid_option.eq.0) then |
---|
424 | call concoutput_irreg(jtime,outnum,gridtotalunc, & |
---|
425 | wetgridtotalunc,drygridtotalunc) |
---|
426 | if (nested_output.eq.1) call concoutput_nest_irreg(jtime,outnum) |
---|
427 | elseif (outgrid_option.eq.1) then |
---|
428 | call concoutput_reg(jtime,outnum,gridtotalunc, & |
---|
429 | wetgridtotalunc,drygridtotalunc) |
---|
430 | if (nested_output.eq.1) call concoutput_nest_reg(jtime,outnum) |
---|
431 | endif |
---|
432 | endif |
---|
433 | |
---|
434 | ! print*,'apres concout' |
---|
435 | ! if (nested_output.eq.1.and.iout.ge.1) |
---|
436 | ! + call concoutput_nest(jtime,outnum) |
---|
437 | outnum=0. |
---|
438 | endif |
---|
439 | if ((iout.eq.4).or.(iout.eq.5)) call plumetraj(jtime) |
---|
440 | if (iflux.eq.1) call fluxoutput(jtime) |
---|
441 | write(*,45) jtime,numpart,gridtotalunc,wetgridtotalunc, & |
---|
442 | drygridtotalunc |
---|
443 | 45 format(i9,' SECONDS SIMULATED: ',i9, & |
---|
444 | ' PARTICLES: Uncertainty: ',3f7.3) |
---|
445 | if (ipout.ge.1) call partoutput(jtime) ! dump particle positions |
---|
446 | loutnext=loutnext+loutstep |
---|
447 | loutstart=loutnext-loutaver/2 |
---|
448 | loutend=loutnext+loutaver/2 |
---|
449 | if (jtime.eq.loutstart) then |
---|
450 | weight=0.5 |
---|
451 | outnum=outnum+weight |
---|
452 | if(iout.ge.1) then |
---|
453 | if (outgrid_option.eq.0) then |
---|
454 | call conccalc_irreg(jtime,weight) |
---|
455 | elseif (outgrid_option.eq.1) then |
---|
456 | call conccalc_reg(jtime,weight) |
---|
457 | endif |
---|
458 | endif |
---|
459 | endif |
---|
460 | |
---|
461 | |
---|
462 | ! Check, whether particles are to be split: |
---|
463 | ! If so, create new particles and attribute all information from the old |
---|
464 | ! particles also to the new ones; old and new particles both get half the |
---|
465 | ! mass of the old ones |
---|
466 | !************************************************************************ |
---|
467 | |
---|
468 | if (ldirect*jtime.ge.ldirect*itsplit) then |
---|
469 | n=numpart |
---|
470 | do j=1,numpart |
---|
471 | if (ldirect*jtime.ge.ldirect*itrasplit(j)) then |
---|
472 | if (n.lt.maxpart) then |
---|
473 | n=n+1 |
---|
474 | itrasplit(j)=2*(itrasplit(j)-itramem(j))+itramem(j) |
---|
475 | itrasplit(n)=itrasplit(j) |
---|
476 | itramem(n)=itramem(j) |
---|
477 | itra1(n)=itra1(j) |
---|
478 | idt(n)=idt(j) |
---|
479 | npoint(n)=npoint(j) |
---|
480 | nclass(n)=nclass(j) |
---|
481 | xtra1(n)=xtra1(j) |
---|
482 | ytra1(n)=ytra1(j) |
---|
483 | ztra1(n)=ztra1(j) |
---|
484 | uap(n)=uap(j) |
---|
485 | ucp(n)=ucp(j) |
---|
486 | uzp(n)=uzp(j) |
---|
487 | us(n)=us(j) |
---|
488 | vs(n)=vs(j) |
---|
489 | ws(n)=ws(j) |
---|
490 | cbt(n)=cbt(j) |
---|
491 | do ks=1,nspec |
---|
492 | xmass1(j,ks)=xmass1(j,ks)/2. |
---|
493 | xmass1(n,ks)=xmass1(j,ks) |
---|
494 | end do |
---|
495 | endif |
---|
496 | endif |
---|
497 | end do |
---|
498 | numpart=n |
---|
499 | endif |
---|
500 | endif |
---|
501 | endif |
---|
502 | |
---|
503 | |
---|
504 | |
---|
505 | |
---|
506 | ! Loop over all particles |
---|
507 | !************************ |
---|
508 | |
---|
509 | |
---|
510 | ! chunksize=int(numpart/ntasks)+1 !if sent homogeneously |
---|
511 | chunksize=int(numpart/ntasks) !if sent homogeneously |
---|
512 | ! call itime(now) |
---|
513 | ! ttime=now(1)*3600+now(2)*60+now(3)-ttime |
---|
514 | call cpu_time(finish) |
---|
515 | |
---|
516 | ! print*,'processing time',ttime |
---|
517 | if (option_verbose.eq.1) then |
---|
518 | print*,'processing time',finish-start |
---|
519 | endif |
---|
520 | endif !over myid |
---|
521 | !JB |
---|
522 | ! at this stage, I assume that each node has the same shared memory because they run getfields. |
---|
523 | ! now we need to split the trajectories into pieces for each node |
---|
524 | ! if (myid.eq.0) then |
---|
525 | |
---|
526 | if (jtime.eq.ideltas) exit |
---|
527 | |
---|
528 | ! Compute interval since radioactive decay of deposited mass was computed |
---|
529 | !************************************************************************ |
---|
530 | |
---|
531 | if (jtime.lt.loutnext) then |
---|
532 | ldeltat=jtime-(loutnext-loutstep) |
---|
533 | else ! first half of next interval |
---|
534 | ldeltat=jtime-loutnext |
---|
535 | endif |
---|
536 | |
---|
537 | |
---|
538 | if (myid.eq.0) then |
---|
539 | ! call itime(now) |
---|
540 | ! ttime=now(1)*3600+now(2)*60+now(3) |
---|
541 | call cpu_time(start) |
---|
542 | do ii=1,ntasks-1 |
---|
543 | call MPI_SEND(chunksize,1, MPI_INTEGER, ii,3001, MPI_COMM_WORLD, ierr) |
---|
544 | call MPI_SEND(numpart,1, MPI_INTEGER, ii,3002, MPI_COMM_WORLD, ierr) |
---|
545 | enddo |
---|
546 | else |
---|
547 | call MPI_RECV(chunksize,1, MPI_INTEGER, 0,3001, MPI_COMM_WORLD,status, ierr) |
---|
548 | call MPI_RECV(numpart,1, MPI_INTEGER, 0,3002, MPI_COMM_WORLD,status, ierr) |
---|
549 | endif |
---|
550 | ! print*,'numpart',numpart |
---|
551 | ! chunksize2=chunksize |
---|
552 | ! print*,'chunksize 0',chunksize2,ntasks |
---|
553 | !JB |
---|
554 | ! here I am going to send the infos to each slave nodes. |
---|
555 | if (numpart.gt.0 .and. ntasks.gt.1 ) then |
---|
556 | call MPI_BARRIER(MPI_COMM_WORLD,ierr) |
---|
557 | |
---|
558 | call sendint_mpi(1,numpart,chunksize,0) |
---|
559 | ! print*,'after npoint',myid,numpart,chunksize |
---|
560 | call sendint_mpi(2,numpart,chunksize,0) |
---|
561 | ! print*,'after idt',myid,numpart,chunksize |
---|
562 | call sendint_mpi(3,numpart,chunksize,0) |
---|
563 | ! print*,'after itra1',myid,numpart,chunksize |
---|
564 | call sendreal_mpi(4,numpart,chunksize,0) |
---|
565 | ! print*,'after uap',myid,numpart,chunksize |
---|
566 | call sendreal_mpi(5,numpart,chunksize,0) |
---|
567 | ! print*,'after ucp',myid,numpart,chunksize |
---|
568 | call sendreal_mpi(6,numpart,chunksize,0) |
---|
569 | ! print*,'after uzp',myid,numpart,chunksize |
---|
570 | call sendreal_mpi(7,numpart,chunksize,0) |
---|
571 | ! print*,'after us',myid,numpart,chunksize |
---|
572 | call sendreal_mpi(8,numpart,chunksize,0) |
---|
573 | ! print*,'after vs',myid,numpart,chunksize |
---|
574 | call sendreal_mpi(9,numpart,chunksize,0) |
---|
575 | ! print*,'after ws',myid,numpart,chunksize |
---|
576 | call sendreal_mpi(10,numpart,chunksize,0) |
---|
577 | ! print*,'after ztra1',myid,numpart,chunksize |
---|
578 | call senddouble_mpi(11,numpart,chunksize,0) |
---|
579 | ! print*,'after xtra1',myid,numpart,chunksize |
---|
580 | call senddouble_mpi(12,numpart,chunksize,0) |
---|
581 | ! print*,'after ytra1',myid,numpart,chunksize |
---|
582 | call sendint2_mpi(13,numpart,chunksize,0) |
---|
583 | ! print*,'after cbt',myid,numpart,chunksize |
---|
584 | call sendint_mpi(14,numpart,chunksize,0) |
---|
585 | ! print*,'after itramem',myid,numpart,chunksize |
---|
586 | call sendint_mpi(15,numpart,chunksize,0) |
---|
587 | ! print*,'after nclass',myid,numpart,chunksize |
---|
588 | call sendreal2d_mpi(20,numpart,nspec,chunksize,0) |
---|
589 | ! print*,'after xmass1',myid,numpart,chunksize |
---|
590 | if (myid.eq.0) then |
---|
591 | ! call itime(now) |
---|
592 | ! ttime=now(1)*3600+now(2)*60+now(3)-ttime |
---|
593 | call cpu_time(finish) |
---|
594 | |
---|
595 | ! print*,'sending time',ttime |
---|
596 | if (option_verbose.eq.1) then |
---|
597 | print*,'sending time',finish-start |
---|
598 | endif |
---|
599 | else |
---|
600 | chunksize2=chunksize |
---|
601 | endif |
---|
602 | endif !if statement on numpart et ntasks |
---|
603 | |
---|
604 | ! print*,'debut chunksize',chunksize,chunksize2,myid |
---|
605 | ! sigw,dsigwdz,dsigw2dz,cpt(nbp),ompid) |
---|
606 | |
---|
607 | ! initialize the temporary drydeposition grid |
---|
608 | |
---|
609 | if (DRYDEP.and.ldirect.gt.0) then |
---|
610 | do ks=1,nspec |
---|
611 | do kp=1,maxpointspec_act |
---|
612 | do nage=1,nageclass |
---|
613 | do jy=0,numygrid-1 |
---|
614 | do ix=0,numxgrid-1 |
---|
615 | do l=1,nclassunc |
---|
616 | drygridunc2(ix,jy,ks,kp,l,nage)=0. |
---|
617 | end do |
---|
618 | end do |
---|
619 | end do |
---|
620 | if (nested_output.eq.1) then |
---|
621 | do jy=0,numygridn-1 |
---|
622 | do ix=0,numxgridn-1 |
---|
623 | do l=1,nclassunc |
---|
624 | drygriduncn2(ix,jy,ks,kp,l,nage)=0. |
---|
625 | end do |
---|
626 | end do |
---|
627 | end do |
---|
628 | endif |
---|
629 | end do |
---|
630 | end do |
---|
631 | end do |
---|
632 | endif |
---|
633 | |
---|
634 | ! |
---|
635 | !JB |
---|
636 | ! now we are entering the openmp zone. |
---|
637 | ! if (myid.eq.0) then |
---|
638 | ! print*,'itra11',numpart,itra1(numpart-2:numpart+2) |
---|
639 | ! print*,'itra12',chunksize2,mpi_itra1(chunksize2-2:chunksize2+1) |
---|
640 | ! else |
---|
641 | ! print*,'itra13',mpi_itra1(chunksize-2:chunksize+1) |
---|
642 | ! endif |
---|
643 | |
---|
644 | ! print*,'continue',myid,chunksize2 |
---|
645 | !!!$OMP PARALLEL NUM_THREADS(10) DEFAULT(SHARED) & |
---|
646 | !$OMP PARALLEL DEFAULT(SHARED) & |
---|
647 | !$OMP PRIVATE(jj, th_npoint, th_idt, th_uap, th_ucp, & |
---|
648 | !$OMP th_uzp, th_us, th_vs, th_ws, th_xtra1, th_ytra1, th_ztra1,decfact, & |
---|
649 | !$OMP th_cbt, xold, yold, zold, kp, itage, prob, nstop, xmassfract, & |
---|
650 | !$OMP th_nclass,chunksize3,start,finish,ngrid,ompid,depoindicator,nbp, & |
---|
651 | !$OMP indzindicator,cpttra,th_xmass1,th_itra1,th_itramem,drydeposit,n_threads) & |
---|
652 | !$OMP SHARED(height,rho,tt,vsetaver,dquer,xtra1,ytra1,ztra1, & |
---|
653 | !$OMP density,cunningham,itra1,ioutputforeachrelease,cbt,iflux, & |
---|
654 | !$OMP uun,vvn,wwn,ustar,wstar,oli,uupol,vvpol,uu,vv,ww,drhodz,ptt,tke, & |
---|
655 | !$OMP rhon,drhodzn,pttn,tken,vdep,vdepn,itramem,nageclass,lage, & |
---|
656 | !$OMP jtime,ldirect,memind,nglobal,switchnorthg,m_x,m_y,m_xn,m_yn, & |
---|
657 | !$OMP switchsouthg,numbnests,xln,xrn,yln,yrn,memtime,xresoln, & |
---|
658 | !$OMP yresoln,hmix,hmixn,tropopause, & |
---|
659 | !$OMP tropopausen,lsynctime,dxconst,dyconst,mdomainfill, & |
---|
660 | !$OMP turb_option,turbswitch,ifine,chunksize,chunksize2, & |
---|
661 | !!!maxrand, & |
---|
662 | !$OMP xmass,xmass1,DRYDEP,DRYDEPSPEC,nspec,rannumb,uniform_rannumb,cpt, & |
---|
663 | !$OMP lwindinterv,npart,npoint,idt,uap,ucp,uzp,us,vs,ws, & |
---|
664 | !$OMP linit_cond,decay,ldeltat,nclass,nested_output,numpart, & |
---|
665 | !$OMP mpi_npoint,mpi_idt, mpi_uap, mpi_ucp, mpi_uzp, mpi_us, mpi_vs, & |
---|
666 | !$OMP mpi_ws, mpi_xtra1, mpi_ytra1, mpi_ztra1, & |
---|
667 | !$OMP mpi_cbt,drygridunc2,drygriduncn2, & |
---|
668 | !$OMP mpi_xmass1, mpi_itra1,myid,mpi_itramem,mpi_nclass, & |
---|
669 | !$OMP mts) |
---|
670 | |
---|
671 | ! call itime(now) |
---|
672 | ! ttime=now(1)*3600+now(2)*60+now(3) |
---|
673 | call cpu_time(start) |
---|
674 | ! chunksize3=int(chunksize2/omp_get_num_threads())+1 |
---|
675 | n_threads=omp_get_num_threads() |
---|
676 | |
---|
677 | ! ompid=omp_get_num_threads() |
---|
678 | ompid=OMP_GET_THREAD_NUM() |
---|
679 | chunksize3=int(real(chunksize2)/real(n_threads)/20.)+1 !more efficient |
---|
680 | |
---|
681 | if (ompid+1.gt.maxomp) then |
---|
682 | print*,'problem with maxomp. modify par_mod.f90',maxomp,ompid+1 |
---|
683 | stop |
---|
684 | endif |
---|
685 | cpttra=0 |
---|
686 | ! print*,'chunksi',chunksize2,chunksize3,myid |
---|
687 | if (chunksize2.gt.0 .and. numpart.gt.0) then |
---|
688 | ! print*,'test rel2',npoint(5139),npoint(6002),npoint(100003) |
---|
689 | !!!$OMP DO SCHEDULE(STATIC,10) |
---|
690 | !$OMP DO SCHEDULE(STATIC,chunksize3) |
---|
691 | !!!$OMP DO SCHEDULE(GUIDED,1) |
---|
692 | !!!$OMP DO |
---|
693 | ! do jj=1,numpart |
---|
694 | ! do jj=numpart,1,-1 |
---|
695 | ! print*,jj |
---|
696 | do jj=1,chunksize2 |
---|
697 | |
---|
698 | ! If integration step is due, do it |
---|
699 | !********************************** |
---|
700 | !$OMP CRITICAL |
---|
701 | if (ntasks.gt.1) then |
---|
702 | th_itra1=mpi_itra1(jj) |
---|
703 | th_itramem=mpi_itramem(jj) |
---|
704 | th_npoint=mpi_npoint(jj) |
---|
705 | else |
---|
706 | th_itra1=itra1(jj) |
---|
707 | th_itramem=itramem(jj) |
---|
708 | th_npoint=npoint(jj) |
---|
709 | endif |
---|
710 | |
---|
711 | !$OMP END CRITICAL |
---|
712 | ! if (th_itra1(jj).eq.jtime) then |
---|
713 | ! if (mod(jj,30000).eq.1) print*,'middle',jj,th_itra1,jtime |
---|
714 | ! if (th_itra1.lt.-9999) print*,'middle',jj,th_itra1,jtime |
---|
715 | ! if (th_itra1.eq.jtime) then |
---|
716 | ! if (th_itra1.eq.jtime .and. th_npoint.gt.0 .and. th_npoint.le.numpoint) then |
---|
717 | if (th_itra1.eq.jtime ) then |
---|
718 | cpttra=cpttra+1 |
---|
719 | |
---|
720 | ! print*,'avant init',j |
---|
721 | ! Initialize newly released particle |
---|
722 | !*********************************** |
---|
723 | !$OMP CRITICAL |
---|
724 | if (ntasks.eq.1) then |
---|
725 | ! th_npoint=npoint(jj) |
---|
726 | th_idt=idt(jj) |
---|
727 | th_uap=uap(jj) |
---|
728 | th_ucp=ucp(jj) |
---|
729 | th_uzp=uzp(jj) |
---|
730 | th_us=us(jj) |
---|
731 | th_vs=vs(jj) |
---|
732 | th_ws=ws(jj) |
---|
733 | th_xtra1=xtra1(jj) |
---|
734 | th_ytra1=ytra1(jj) |
---|
735 | th_ztra1=ztra1(jj) |
---|
736 | th_nclass=nclass(jj) |
---|
737 | th_cbt=cbt(jj) |
---|
738 | do ks=1,nspec |
---|
739 | th_xmass1(ks)=xmass1(jj,ks) |
---|
740 | enddo |
---|
741 | if (ioutputforeachrelease.eq.1) then |
---|
742 | kp=npoint(jj) |
---|
743 | else |
---|
744 | kp=1 |
---|
745 | endif |
---|
746 | else |
---|
747 | ! th_npoint=mpi_npoint(jj) |
---|
748 | th_idt=mpi_idt(jj) |
---|
749 | th_uap=mpi_uap(jj) |
---|
750 | th_ucp=mpi_ucp(jj) |
---|
751 | th_uzp=mpi_uzp(jj) |
---|
752 | th_us=mpi_us(jj) |
---|
753 | th_vs=mpi_vs(jj) |
---|
754 | th_ws=mpi_ws(jj) |
---|
755 | th_xtra1=mpi_xtra1(jj) |
---|
756 | th_ytra1=mpi_ytra1(jj) |
---|
757 | th_ztra1=mpi_ztra1(jj) |
---|
758 | th_nclass=mpi_nclass(jj) |
---|
759 | th_cbt=mpi_cbt(jj) |
---|
760 | do ks=1,nspec |
---|
761 | th_xmass1(ks)=mpi_xmass1(jj,ks) |
---|
762 | enddo |
---|
763 | if (ioutputforeachrelease.eq.1) then |
---|
764 | kp=mpi_npoint(jj) |
---|
765 | else |
---|
766 | kp=1 |
---|
767 | endif |
---|
768 | endif |
---|
769 | !$OMP END CRITICAL |
---|
770 | |
---|
771 | ! Determine age class of the particle |
---|
772 | ! itage=abs(itra1(jj)-itramem(jj)) |
---|
773 | itage=abs(th_itra1-th_itramem) |
---|
774 | do nage=1,nageclass |
---|
775 | if (itage.lt.lage(nage)) exit |
---|
776 | enddo |
---|
777 | ! if (jj.lt.5) print*,'xmass1',th_xmass1(1) |
---|
778 | ! ompid=OMP_GET_THREAD_NUM() |
---|
779 | nbp=ompid+1 |
---|
780 | ! print*,th_npoint,jj,npoint(jj) |
---|
781 | ! print*,'befo',th_xtra1,th_ytra1,th_ztra1 |
---|
782 | ! iff=0 |
---|
783 | ! if ((itramem(jj).eq.jtime).or.(jtime.eq.0)) & |
---|
784 | if ((th_itramem.eq.jtime).or.(jtime.eq.0)) then |
---|
785 | ! call initialize(jtime,idt(j),uap(j),ucp(j),uzp(j), & |
---|
786 | ! us(j),vs(j),ws(j),xtra1(j),ytra1(j),ztra1(j),cbt(j)) |
---|
787 | call initialize(jtime,th_idt,th_uap,th_ucp,th_uzp, & |
---|
788 | th_us,th_vs,th_ws,th_xtra1,th_ytra1,th_ztra1,th_cbt, & |
---|
789 | ngrid,depoindicator,indzindicator,cpt(nbp),ompid,myid,n_threads,mts ) |
---|
790 | endif |
---|
791 | |
---|
792 | ! print*,'after',th_xtra1,th_ytra1,th_ztra1 |
---|
793 | ! Memorize particle positions |
---|
794 | !**************************** |
---|
795 | ! if (mod(jj,100000).eq.1) print*,'middle',jj,myid,ompid |
---|
796 | ! xold=xtra1(j) |
---|
797 | ! yold=ytra1(j) |
---|
798 | ! zold=ztra1(j) |
---|
799 | xold=th_xtra1 |
---|
800 | yold=th_ytra1 |
---|
801 | zold=th_ztra1 |
---|
802 | |
---|
803 | ! Integrate Lagevin equation for lsynctime seconds |
---|
804 | !************************************************* |
---|
805 | ! write(*,*)'numpart,jtime, particle #=',numpart,jtime,j |
---|
806 | |
---|
807 | ! call advance(jtime,npoint(j),idt(j),uap(j),ucp(j),uzp(j),us(j), & |
---|
808 | ! vs(j),ws(j),nstop,xtra1(j),ytra1(j),ztra1(j),prob,cbt(j)) |
---|
809 | ! if ( abs(xold).gt.1000. .or. xold.ne.xold .or. th_xtra1.ne.th_xtra1 ) & |
---|
810 | ! print*,'pb avant 0',xold,yold,th_itramem, & |
---|
811 | ! jtime,jj,chunksize2 |
---|
812 | ! print*,'npoint',mpi_npoint(jj),th_npoint,jj |
---|
813 | call advance(jtime,th_npoint,th_idt,th_uap,th_ucp,th_uzp, & |
---|
814 | th_us,th_vs,th_ws,nstop,th_xtra1,& |
---|
815 | th_ytra1,th_ztra1,prob,th_cbt, & |
---|
816 | ngrid,depoindicator,indzindicator,cpt(nbp),ompid,myid,n_threads,mts ) |
---|
817 | |
---|
818 | ! if ( abs(xold).gt.1000. .or. xold.ne.xold .or. th_xtra1.ne.th_xtra1 ) & |
---|
819 | ! print*,'pb avant 1',xold,yold,th_itramem, & |
---|
820 | ! jtime,jj,chunksize2 |
---|
821 | ! Calculate the gross fluxes across layer interfaces |
---|
822 | !*************************************************** |
---|
823 | |
---|
824 | |
---|
825 | if (iflux.eq.1) call calcfluxes(nage,jj,xold,yold,zold) |
---|
826 | |
---|
827 | ! if (jj.lt.5) print*,'coord after',myid,th_itra1,th_xmass1(1),DRYDEPSPEC(ks) |
---|
828 | |
---|
829 | ! Determine, when next time step is due |
---|
830 | ! If trajectory is terminated, mark it |
---|
831 | !************************************** |
---|
832 | !!!$OMP CRITICAL |
---|
833 | do ks=1,nspec |
---|
834 | drydeposit(ks)=0. |
---|
835 | enddo |
---|
836 | if (nstop.gt.1) then |
---|
837 | if (linit_cond.ge.1) call initial_cond_calc(jtime,jj) |
---|
838 | ! itra1(jj)=-999999999 |
---|
839 | th_itra1=-999999999 |
---|
840 | if (option_verbose.gt.1) print*,'out of domain',th_xtra1,th_ytra1,th_ztra1 |
---|
841 | |
---|
842 | else |
---|
843 | ! itra1(jj)=jtime+lsynctime |
---|
844 | th_itra1=jtime+lsynctime |
---|
845 | |
---|
846 | |
---|
847 | ! if (jj.lt.5) print*,'coord after2',myid,th_itra1,th_xmass1(1),DRYDEPSPEC(ks) |
---|
848 | ! Dry deposition and radioactive decay for each species |
---|
849 | !****************************************************** |
---|
850 | xmassfract=0. |
---|
851 | |
---|
852 | do ks=1,nspec |
---|
853 | if (decay(ks).gt.0.) then ! radioactive decay |
---|
854 | decfact=exp(-real(abs(lsynctime))*decay(ks)) |
---|
855 | else |
---|
856 | decfact=1. |
---|
857 | endif |
---|
858 | |
---|
859 | if (DRYDEPSPEC(ks)) then ! dry deposition |
---|
860 | ! drydeposit(ks)=xmass1(j,ks)*prob(ks)*decfact |
---|
861 | drydeposit(ks)=th_xmass1(ks)*prob(ks)*decfact |
---|
862 | ! xmass1(j,ks)=xmass1(j,ks)*(1.-prob(ks))*decfact |
---|
863 | th_xmass1(ks)=th_xmass1(ks)*(1.-prob(ks))*decfact |
---|
864 | if (decay(ks).gt.0.) then ! correct for decay (see wetdepo) |
---|
865 | drydeposit(ks)=drydeposit(ks)* & |
---|
866 | exp(real(abs(ldeltat))*decay(ks)) |
---|
867 | endif |
---|
868 | else ! no dry deposition |
---|
869 | ! xmass1(j,ks)=xmass1(j,ks)*decfact |
---|
870 | th_xmass1(ks)=th_xmass1(ks)*decfact |
---|
871 | endif |
---|
872 | ! if (jj.lt.5) print*,'coord after3',myid,th_itra1,th_xmass1(1),DRYDEPSPEC(ks),xmass(th_npoint,1) |
---|
873 | |
---|
874 | if (mdomainfill.eq.0) then |
---|
875 | if (xmass(th_npoint,ks).gt.0.) & |
---|
876 | ! xmassfract=max(xmassfract,real(npart(npoint(jj)))* & |
---|
877 | xmassfract=max(xmassfract,real(npart(th_npoint))* & |
---|
878 | ! xmass1(j,ks)/xmass(npoint(j),ks)) |
---|
879 | th_xmass1(ks)/xmass(th_npoint,ks)) |
---|
880 | else |
---|
881 | xmassfract=1. |
---|
882 | endif |
---|
883 | |
---|
884 | end do |
---|
885 | |
---|
886 | if (ipin.eq.0 .and. xmassfract.lt.0.000001) then ! terminate all particles carrying less mass |
---|
887 | ! itra1(jj)=-999999999 |
---|
888 | th_itra1=-999999999 |
---|
889 | endif |
---|
890 | |
---|
891 | ! Sabine Eckhardt, June 2008 |
---|
892 | ! don't create depofield for backward runs |
---|
893 | if (DRYDEP.AND.(ldirect.eq.1)) then |
---|
894 | ! call drydepokernel(nclass(jj),drydeposit,real(xtra1(jj)), & |
---|
895 | call drydepokernel(th_nclass,drydeposit,real(th_xtra1), & |
---|
896 | ! real(ytra1(jj)),nage,kp) |
---|
897 | real(th_ytra1),itage,nage,kp) |
---|
898 | if (nested_output.eq.1) call drydepokernel_nest( & |
---|
899 | ! nclass(jj),drydeposit,real(xtra1(jj)),real(ytra1(jj)), & |
---|
900 | th_nclass,drydeposit,real(th_xtra1),real(th_ytra1), & |
---|
901 | itage,nage,kp) |
---|
902 | endif |
---|
903 | |
---|
904 | ! Terminate trajectories that are older than maximum allowed age |
---|
905 | !*************************************************************** |
---|
906 | |
---|
907 | ! if (abs(itra1(jj)-itramem(jj)).ge.lage(nageclass)) then |
---|
908 | if (abs(th_itra1-th_itramem).ge.lage(nageclass)) then |
---|
909 | if (linit_cond.ge.1) & |
---|
910 | call initial_cond_calc(jtime+lsynctime,jj) |
---|
911 | ! itra1(jj)=-999999999 |
---|
912 | th_itra1=-999999999 |
---|
913 | endif |
---|
914 | endif |
---|
915 | !! print*,xtra1(j),th_xtra1,OMP_GET_THREAD_NUM() |
---|
916 | !$OMP CRITICAL |
---|
917 | if (ntasks.eq.1) then |
---|
918 | npoint(jj)=th_npoint |
---|
919 | idt(jj)=th_idt |
---|
920 | uap(jj)=th_uap |
---|
921 | ucp(jj)=th_ucp |
---|
922 | uzp(jj)=th_uzp |
---|
923 | us(jj)=th_us |
---|
924 | vs(jj)=th_vs |
---|
925 | ws(jj)=th_ws |
---|
926 | xtra1(jj)=th_xtra1 |
---|
927 | ytra1(jj)=th_ytra1 |
---|
928 | ztra1(jj)=th_ztra1 |
---|
929 | cbt(jj)=th_cbt |
---|
930 | do ks=1,nspec |
---|
931 | xmass1(jj,ks)=th_xmass1(ks) |
---|
932 | ! drydep1(jj,ks)=drydeposit(ks) |
---|
933 | enddo |
---|
934 | ! itramem(jj)=th_itramem |
---|
935 | itra1(jj)=th_itra1 |
---|
936 | else |
---|
937 | mpi_npoint(jj)=th_npoint |
---|
938 | mpi_idt(jj)=th_idt |
---|
939 | mpi_uap(jj)=th_uap |
---|
940 | mpi_ucp(jj)=th_ucp |
---|
941 | mpi_uzp(jj)=th_uzp |
---|
942 | mpi_us(jj)=th_us |
---|
943 | mpi_vs(jj)=th_vs |
---|
944 | mpi_ws(jj)=th_ws |
---|
945 | mpi_xtra1(jj)=th_xtra1 |
---|
946 | mpi_ytra1(jj)=th_ytra1 |
---|
947 | mpi_ztra1(jj)=th_ztra1 |
---|
948 | ! mpi_nclass(jj)=th_nclass |
---|
949 | mpi_cbt(jj)=th_cbt |
---|
950 | do ks=1,nspec |
---|
951 | mpi_xmass1(jj,ks)=th_xmass1(ks) |
---|
952 | ! mpi_drydep1(jj,ks)=drydeposit(ks) |
---|
953 | enddo |
---|
954 | ! mpi_itramem(jj)=th_itramem |
---|
955 | mpi_itra1(jj)=th_itra1 |
---|
956 | endif |
---|
957 | ! if (jj.lt.5) print*,'coord cont',th_itra1 |
---|
958 | |
---|
959 | !$OMP END CRITICAL |
---|
960 | |
---|
961 | endif |
---|
962 | |
---|
963 | end do !loop over particles |
---|
964 | !$OMP END DO |
---|
965 | endif |
---|
966 | !!!$OMP |
---|
967 | !!!$OMP FLUSH(npoint,idt,uap,ucp,uzp,us,vs,ws,xtra1,ytra1,ztra1,cbt,xmass1,itra1) |
---|
968 | !!!$OMP FLUSH |
---|
969 | ! call itime(now) |
---|
970 | ! ttime=now(1)*3600+now(2)*60+now(3)-ttime |
---|
971 | call cpu_time(finish) |
---|
972 | ! print*,'time',ttime,cpttra,myid,OMP_GET_THREAD_NUM() |
---|
973 | if (option_verbose.eq.1) then |
---|
974 | print*,'time',finish-start,cpttra,myid,ompid |
---|
975 | endif |
---|
976 | !$OMP END PARALLEL |
---|
977 | !JB |
---|
978 | ! the openmp is done. the output above gives how long it takes to finish the loop over the particles. good benchmark |
---|
979 | |
---|
980 | ! here we use mpi to use the mpi_* arrays to update the big ones in the master |
---|
981 | ! thread |
---|
982 | |
---|
983 | ! call MPI_REDUCE (mypi ,pi ,1, MPI_DOUBLE_PRECISION , MPI_SUM ,0, & |
---|
984 | ! MPI_COMM_WORLD , ierr ) |
---|
985 | ! print*,'after loop',myid,chunksize |
---|
986 | if (chunksize.gt.0 .and. ntasks.gt.1) then |
---|
987 | |
---|
988 | !JB |
---|
989 | ! I need a barrier so each node is a the same place |
---|
990 | ! I am going to send the new results to the master thread now. |
---|
991 | call MPI_BARRIER(MPI_COMM_WORLD,ierr) |
---|
992 | if (myid.eq.0) then |
---|
993 | ! call itime(now) |
---|
994 | ! ttime=now(1)*3600+now(2)*60+now(3) |
---|
995 | call cpu_time(start) |
---|
996 | endif |
---|
997 | call sendint_mpi(1,numpart,chunksize,1) |
---|
998 | call sendint_mpi(2,numpart,chunksize,1) |
---|
999 | call sendint_mpi(3,numpart,chunksize,1) |
---|
1000 | call sendreal_mpi(4,numpart,chunksize,1) |
---|
1001 | call sendreal_mpi(5,numpart,chunksize,1) |
---|
1002 | call sendreal_mpi(6,numpart,chunksize,1) |
---|
1003 | call sendreal_mpi(7,numpart,chunksize,1) |
---|
1004 | call sendreal_mpi(8,numpart,chunksize,1) |
---|
1005 | call sendreal_mpi(9,numpart,chunksize,1) |
---|
1006 | call sendreal_mpi(10,numpart,chunksize,1) |
---|
1007 | call senddouble_mpi(11,numpart,chunksize,1) |
---|
1008 | call senddouble_mpi(12,numpart,chunksize,1) |
---|
1009 | call sendint2_mpi(13,numpart,chunksize,1) |
---|
1010 | call sendint_mpi(14,numpart,chunksize,1) |
---|
1011 | ! call sendint_mpi(15,nclass(1:numpart),numpart,chunksize2,1) |
---|
1012 | call sendreal2d_mpi(99,numpart,nspec,chunksize,1) |
---|
1013 | ! if (DRYDEP) call sendreal2d_mpi(21,numpart,nspec,chunksize,1) |
---|
1014 | if (DRYDEP.and.(ldirect.eq.1)) call senddrydep_mpi(numxgrid*numygrid) |
---|
1015 | if (DRYDEP.and.(ldirect.eq.1).and.nested_output.eq.1) & |
---|
1016 | call senddrydep_nest_mpi(numxgridn*numygridn) |
---|
1017 | ! print*,'size of vector',chunksize,chunksize2,myid,numpart |
---|
1018 | ! |
---|
1019 | ! if (myid.eq.0) then |
---|
1020 | ! print*,'itra1',numpart,itra1(numpart-2:numpart+2) |
---|
1021 | ! print*,'itra2',chunksize2,mpi_itra1(chunksize2-2:chunksize2+1) |
---|
1022 | ! else |
---|
1023 | ! print*,'itra3',mpi_itra1(chunksize-2:chunksize+1) |
---|
1024 | ! endif |
---|
1025 | if (myid.eq.0) then |
---|
1026 | ! call itime(now) |
---|
1027 | ! ttime=now(1)*3600+now(2)*60+now(3)-ttime |
---|
1028 | call cpu_time(finish) |
---|
1029 | ! print*,'receiving time',ttime |
---|
1030 | if (option_verbose.eq.1) then |
---|
1031 | print*,'receiving time',finish-start |
---|
1032 | endif |
---|
1033 | endif |
---|
1034 | endif !finish transfering between nodes |
---|
1035 | |
---|
1036 | ! update the drydepo |
---|
1037 | if (DRYDEP.and.ldirect.gt.0) then |
---|
1038 | do ks=1,nspec |
---|
1039 | do kp=1,maxpointspec_act |
---|
1040 | do nage=1,nageclass |
---|
1041 | |
---|
1042 | do jy=0,numygrid-1 |
---|
1043 | do ix=0,numxgrid-1 |
---|
1044 | do l=1,nclassunc |
---|
1045 | drygridunc(ix,jy,ks,kp,l,nage)=drygridunc(ix,jy,ks,kp,l,nage) & |
---|
1046 | +drygridunc2(ix,jy,ks,kp,l,nage) |
---|
1047 | end do |
---|
1048 | end do |
---|
1049 | end do |
---|
1050 | if (nested_output.eq.1) then |
---|
1051 | do jy=0,numygridn-1 |
---|
1052 | do ix=0,numxgridn-1 |
---|
1053 | do l=1,nclassunc |
---|
1054 | drygriduncn(ix,jy,ks,kp,l,nage)=drygriduncn(ix,jy,ks,kp,l,nage) & |
---|
1055 | +drygriduncn2(ix,jy,ks,kp,l,nage) |
---|
1056 | end do |
---|
1057 | end do |
---|
1058 | end do |
---|
1059 | endif |
---|
1060 | end do |
---|
1061 | end do |
---|
1062 | end do |
---|
1063 | |
---|
1064 | endif |
---|
1065 | |
---|
1066 | ! if (DRYDEP.AND.(ldirect.eq.1)) then |
---|
1067 | !! call drydepokernel(nclass(jj),drydeposit,real(xtra1(jj)), & |
---|
1068 | ! call drydepokernel(th_nclass,drydeposit,real(th_xtra1), & |
---|
1069 | !! real(ytra1(jj)),nage,kp) |
---|
1070 | ! real(th_ytra1),itage,nage,kp) |
---|
1071 | ! if (nested_output.eq.1) call drydepokernel_nest( & |
---|
1072 | !! nclass(jj),drydeposit,real(xtra1(jj)),real(ytra1(jj)), & |
---|
1073 | ! th_nclass,drydeposit,real(th_xtra1),real(th_ytra1), & |
---|
1074 | ! itage,nage,kp) |
---|
1075 | ! endif |
---|
1076 | |
---|
1077 | |
---|
1078 | end do !loop over time |
---|
1079 | |
---|
1080 | |
---|
1081 | ! Complete the calculation of initial conditions for particles not yet terminated |
---|
1082 | !***************************************************************************** |
---|
1083 | call MPI_BARRIER(MPI_COMM_WORLD,ierr) |
---|
1084 | |
---|
1085 | do j=1,numpart |
---|
1086 | if (linit_cond.ge.1) call initial_cond_calc(jtime,j) |
---|
1087 | end do |
---|
1088 | |
---|
1089 | if (ipout.eq.2) call partoutput(jtime) ! dump particle positions |
---|
1090 | |
---|
1091 | if (linit_cond.ge.1) call initial_cond_output(jtime) ! dump initial cond. field |
---|
1092 | |
---|
1093 | close(104) |
---|
1094 | |
---|
1095 | ! De-allocate memory and end |
---|
1096 | !*************************** |
---|
1097 | |
---|
1098 | if (iflux.eq.1) then |
---|
1099 | deallocate(flux) |
---|
1100 | endif |
---|
1101 | if (ntasks.gt.1) then |
---|
1102 | ! print*,'before dealloc',myid |
---|
1103 | deallocate(mpi_npoint,mpi_idt,mpi_itra1) |
---|
1104 | ! print*,'after dealloc, part1',myid |
---|
1105 | deallocate(mpi_uap,mpi_ucp,mpi_uzp) |
---|
1106 | ! print*,'after dealloc, part12',myid |
---|
1107 | deallocate(mpi_us,mpi_vs,mpi_ws,mpi_ztra1) |
---|
1108 | ! print*,'after dealloc, part2',myid |
---|
1109 | deallocate(mpi_xtra1,mpi_ytra1) |
---|
1110 | ! print*,'after dealloc, part3',myid |
---|
1111 | deallocate(mpi_cbt) |
---|
1112 | ! print*,'after dealloc, part4',myid |
---|
1113 | ! deallocate(mpi_xmass1,mpi_drydep1,mpi_nclass) |
---|
1114 | deallocate(mpi_xmass1,mpi_nclass) |
---|
1115 | ! print*,'after dealloc',myid |
---|
1116 | deallocate(dummyi2) |
---|
1117 | deallocate(dummyr2) |
---|
1118 | deallocate(dummyi22) |
---|
1119 | deallocate(dummyr22) |
---|
1120 | |
---|
1121 | endif |
---|
1122 | if (OHREA.eqv..TRUE.) then |
---|
1123 | deallocate(OH_field,OH_field_height) |
---|
1124 | endif |
---|
1125 | deallocate(gridunc) |
---|
1126 | deallocate(xpoint1,xpoint2,ypoint1,ypoint2,zpoint1,zpoint2,xmass) |
---|
1127 | deallocate(ireleasestart,ireleaseend,npart,kindz) |
---|
1128 | ! deallocate(xmasssave) |
---|
1129 | if (myid.eq.0) then |
---|
1130 | if (nested_output.eq.1) then |
---|
1131 | deallocate(orooutn, arean, volumen) |
---|
1132 | if (ldirect.gt.0) then |
---|
1133 | deallocate(griduncn,drygriduncn,wetgriduncn,drygriduncn2) |
---|
1134 | endif |
---|
1135 | endif |
---|
1136 | if (ldirect.gt.0) then |
---|
1137 | if (allocated(drygridunc)) deallocate(drygridunc) |
---|
1138 | if (allocated(wetgridunc)) deallocate(wetgridunc) |
---|
1139 | if (allocated(drygridunc2)) deallocate(drygridunc2) |
---|
1140 | if (allocated(drygriduncn2)) deallocate(drygriduncn2) |
---|
1141 | endif |
---|
1142 | deallocate(outheight,outheighthalf) |
---|
1143 | deallocate(oroout, area, volume) |
---|
1144 | endif |
---|
1145 | end subroutine timemanager_mpi |
---|
1146 | |
---|
1147 | |
---|
1148 | |
---|