Changeset 2193808 in flex_extract.git

Timestamp:

Dec 18, 2018, 4:00:28 PM (5 years ago)

Author:

anphi <anne.philipp@…>

Branches:

master, ctbto, dev

Children:

20577dc

Parents:

6067feb

Message:

added ensemble analysis/forecast retrievement option

Location:

python

Files:

• CONTROL_OD.glob.025 (added)
• CONTROL_ZAMG_SYNTH (added)
• CONTROL_ZAMG_SYNTH_ENAN (added)
• CONTROL_ZAMG_SYNTH_ENFO (added)
• CONTROL_ZAMG_SYNTH_ENFOCF (added)
• FlexpartTools.py (modified) (13 diffs)
• job.temp (modified) (1 diff)
• job.temp.o (modified) (1 diff)
• opposite.py (added)
• prepareFLEXPART.py (modified) (1 diff)

python/FlexpartTools.py

@@ -49 +49 @@
 from gribapi import *
 from GribTools import GribTools
+from opposite import opposite
 
 def interpret_args_and_control(*args,**kwargs):
@@ -411 +412 @@
 
     return sfeld
+
+def add_previousday_ifneeded(attrs):
+
+    if attrs['type']=='FC' and 'acc' not in attrs['target']:
+        steps=attrs['step'].split('/')
+        times=attrs['time'].split('/')
+        addpday=False
+        #for t in times:
+            #for s in steps:
+        if int(times[0])+int(steps[0])>23:
+            addpday=True
+        if(addpday):
+            dates=attrs['date'].split('/')
+            start_date=dates[0]
+            syear=int(start_date[:4])
+            smonth=int(start_date[4:6])
+            sday=int(start_date[6:])
+            start = datetime.date( year = syear, month = smonth, day = sday )
+            startm1=start- datetime.timedelta(days=1)
+
+            attrs['date']='/'.join([startm1.strftime("%Y%m%d")]+dates[1:])
+            print('CHANGED FC start date to '+startm1.strftime("%Y%m%d")+'to accomodate TIME='+times[0]+', STEP='+steps[0])
+
+    return
+
+
 
 class Control:
@@ -704 +731 @@
         del attrs['marsclass']
         attrs['class'] = mclass
+
+        add_previousday_ifneeded(attrs)
+
         target = attrs.get('target')
         if not int(self.public):
@@ -769 +799 @@
         self.types=dict()
         try:
-            if c.maxstep>len(c.type):    # Pure forecast mode
+            if c.maxstep>24: #len(c.type):    # Pure forecast mode
                 c.type=[c.type[0]] # AP changed this index from 1 to 0
                 c.step=['{:0>3}'.format(int(c.step[0]))]
@@ -804 +834 @@
 
 
-                if ((ty.upper() == 'AN' and mod(int(c.time[i]),int(c.dtime))==0) or \
+                if ((ty.upper() == 'AN' and mod(int(c.time[i]),int(c.dtime))==0 and int(c.step[i])==0) or \
                    (ty.upper() != 'AN' and mod(int(c.step[i]),int(c.dtime))==0 and \
                     mod(int(c.step[i]),int(c.dtime))==0) ) and \
@@ -1206 +1236 @@
                 p=subprocess.check_call(['grib_set','-s','edition=2,productDefinitionTemplateNumber=8',ofile,ofile+'_2'])
                 p=subprocess.check_call(['mv',ofile+'_2',ofile])
+        if c.debug==0:
+            inputfiles=glob.glob('*.grb')
+            for grb in inputfiles:
+                try:
+                    os.remove(grb)
+                except:
+                    pass
+        if c.stream=='ELDA':
+            opposite(self.inputdir+'/'+c.prefix)
+            for i in range(len(self.outputfilelist)):
+                if self.outputfilelist[i][-4:]!='N000' :
+                    j=int(self.outputfilelist[i][-3:])
+                    self.outputfilelist.append(self.outputfilelist[i][:-3]+'{:0>3}'.format(j+25))
 
         if int(c.ectrans)==1 and c.ecapi==False:
@@ -1276 +1319 @@
         wrfpars=toparamId('sp/mslp/skt/2t/10u/10v/2d/z/lsm/sst/ci/sd/stl1/stl2/stl3/stl4/swvl1/swvl2/swvl3/swvl4',table128)
 #        index_keys=["date","time","stepRange"]
-        index_keys=["date","time","step"]
+        if '/' in c.number:
+            index_keys=["number","date","time","step"]
+        else:
+            index_keys=["date","time","step"]
+
         indexfile=c.inputdir+"/date_time_stepRange.idx"
         silentremove(indexfile)
@@ -1333 +1380 @@
                     if timestamp<slimit or timestamp>elimit:
                         continue
+                else:
+                    if c.maxstep<24:
+                        if cdateH<c.start_date+'00':
+                            continue
+                        if cdateH>c.end_date+'23':
+                            continue
 
 
@@ -1432 +1485 @@
                 else:
                     suffix=cdateH[2:10]
+                try:
+                    numberindex=index_keys.index('number')
+                    if len(index_vals[numberindex])>1:
+                        suffix=suffix+'.N{:0>3}'.format(int(prod[numberindex]))
+                except:
+                    pass
 
                 fnout+=suffix
@@ -1460 +1519 @@
         grib_index_release(iid)
 
+        return
+
     def deacc_fluxes(self, inputfiles, c):
 
         table128=init128(c.flexextractdir+'/grib_templates/ecmwf_grib1_table_128')
         pars=toparamId(self.params['OG_acc_SL'][0],table128)
-        index_keys=["date","time","step"]
+        if '/' in c.number:
+            index_keys=["number","date","time","step"]
+        else:
+            index_keys=["date","time","step"]
         indexfile=c.inputdir+"/date_time_stepRange.idx"
         silentremove(indexfile)
@@ -1475 +1539 @@
         for key in index_keys:
             key_vals = grib_index_get(iid,key)
-
+            print(key_vals)
             l=[]
             for k in key_vals:
@@ -1521 +1585 @@
 
             fnout=c.inputdir+'/'
+            numbersuffix=''
+            try:
+                numberindex=index_keys.index('number')
+                if len(index_vals[numberindex])>1:
+                    numbersuffix='.N{:0>3}'.format(int(prod[numberindex]))
+            except:
+                pass
+
             if c.maxstep>12:
-                fnout+='flux'+sdate.strftime('%Y%m%d')+'.{:0>2}'.format(time/100)+'.{:0>3}'.format(step-2*int(c.dtime))
-                gnout=c.inputdir+'/flux'+sdate.strftime('%Y%m%d')+'.{:0>2}'.format(time/100)+'.{:0>3}'.format(step-int(c.dtime))
-                hnout=c.inputdir+'/flux'+sdate.strftime('%Y%m%d')+'.{:0>2}'.format(time/100)+'.{:0>3}'.format(step)
+                fnout+='flux'+sdate.strftime('%Y%m%d')+'.{:0>2}'.format(time/100)+'.{:0>3}'.format(step-2*int(c.dtime))+numbersuffix
+                gnout=c.inputdir+'/flux'+sdate.strftime('%Y%m%d')+'.{:0>2}'.format(time/100)+'.{:0>3}'.format(step-int(c.dtime))+numbersuffix
+                hnout=c.inputdir+'/flux'+sdate.strftime('%Y%m%d')+'.{:0>2}'.format(time/100)+'.{:0>3}'.format(step)+numbersuffix
                 g=open(gnout,'w')
                 h=open(hnout,'w')
             else:
-                fnout+='flux'+fdate.strftime('%Y%m%d%H')
-                gnout=c.inputdir+'/flux'+(fdate+datetime.timedelta(hours=int(c.dtime))).strftime('%Y%m%d%H')
-                hnout=c.inputdir+'/flux'+sdates.strftime('%Y%m%d%H')
+                fnout+='flux'+fdate.strftime('%Y%m%d%H')+numbersuffix
+                gnout=c.inputdir+'/flux'+(fdate+datetime.timedelta(hours=int(c.dtime))).strftime('%Y%m%d%H')+numbersuffix
+                hnout=c.inputdir+'/flux'+sdates.strftime('%Y%m%d%H')+numbersuffix
                 g=open(gnout,'w')
                 h=open(hnout,'w')
@@ -1556 +1628 @@
                     values=(reshape(values,(nj,ni))).flatten()/fak
                     vdp.append(values[:]) # save the accumulated values
-                    if c.marsclass.upper() == 'EA' or \
+                    if c.marsclass.upper() in ('EA') or \
                        step<=int(c.dtime):
                         svdp.append(values[:]/int(c.dtime))

python/job.temp

@@ -11 +11 @@
 #SBATCH --error=flex_ecmwf.%j.out
 #SBATCH --mail-type=FAIL
-#SBATCH --time=12:00:00
+#SBATCH --time=24:00:00
 
 ## CRAY specific batch requests

python/job.temp.o

@@ -11 +11 @@
 #SBATCH --error=flex_ecmwf.%j.out
 #SBATCH --mail-type=FAIL
-#SBATCH --time=12:00:00
+#SBATCH --time=24:00:00
 
 ## CRAY specific batch requests

python/prepareFLEXPART.py

@@ -37 +37 @@
 from GribTools import GribTools
 from FlexpartTools import EIFlexpart, Control,interpret_args_and_control, cleanup
+from opposite import opposite
 
 hostname=socket.gethostname()