wiki:FpInputMetMkavail

^up to FpInput

How to make an AVAILABLE file

There used to be a Fortran programme for automatic creation of the AVAILBLE file (which is read by FLEXPART to know which windfields are available). Thanks to John Burkhardt's efforts, it has been superseded by python programme with better functionality.

Find the latest version of mkAVAIL.py below and as an attachment (see link at end of page?).

#!/usr/bin/env python
2       
3       """
4       SYNOPSIS
5       
6         mk_AVAILABLE.py: [-s,--startdate] [-e,--enddate] [-m,--model /ECWMF/ERAI/EI/E5/EA/GFS] [-p, --path] [-a, --avpath] [-i, --interval] [-h] [-v,--verbose] [--version]
7       
8       DESCRIPTION
9       
10        mk_AVAIL is a script to create AVAILABLE files for FLEXPART windfields.
11      
12        Example usage:
13        %mk_AVAIL.py -s 200601 -e 200701 -m ECMWF -p . -h
14      
15        would create a file: AVAIALABLE that contained paths
16        of all files available in the current directory between 200601 and 200701.
17        The default start date is 197001 and the default end date is TODAY.
18       
19        adapted and simplified version / Petra Seibert
20        v.004: add ERA-Interim with EI as model, proper sorting of dates if year 2000 included, option to define a path for AVAILABLE / Petra Seibert 2015-10-21
21        v.005: check whether any files are found before opening AVAILABLE, if not issue warning and stop. / Petra Seibert 2015-12-07
22        v.006 PS 2018-04-24:   add ERA5 as model option
23        v.007 PS 2018-06-25:   correct bug introduced in v006: 2 spaces missing in output format
24       
25      QUESTIONS to
26      
27        JFB: <jfb@nilu.no>
28        version>=003: petra.seibert at boku.ac.at
29      
30      LICENSE
31      
32        This script follows creative commons usage.
33        Valid-License-Identifier: CC-BY-4.0
34      
35      VERSION
36      
37        $Id: 0.006 $
38      
39      """
40      
41      import sys
42      import os
43      import traceback
44      import optparse
45      import time
46      import datetime
47      import socket
48      import glob
49      import string
50      
51      
52      def main ():
53      
54        global options, args, models
55        version = '$Id: 0.006 $' 
56       
57        WIND_dir = '/xnilu_wrk/flex_wrk/WIND_FIELDS'
58       
59        # dict of dicts with model information
60        # timeindex: index in the filne name string where the time information starts
61        print  MODEL.keys()
62      
63        AVAIL_head = \
64        """DATE     TIME        FILENAME             SPECIFICATIONS\
65         \nYYYYMMDD HHMISS\
66        \n________ ______      __________________    __________________\n"""
67        start_date = options.startdate
68        sy = int(start_date[0:4])
69        sm = int(start_date[4:6])
70        end_date = options.enddate
71        ey = int(end_date[0:4])
72        em = int(end_date[4:6])
73       
74        #Get the directory information
75        M = MODEL[options.model]
76        prfx = M['prefix']
77        t = options.path
78        avpath = options.avpath
79       
80       
81        #Loop through the files in the directories
82       
83        warned = False
84        d = t #directory
85       
86        tind = M['timeindex'] #timestamp index
87        searchstr = os.path.join(t,prfx)
88        print 'searchstring:',searchstr
89        files = glob.glob(searchstr + '*')
90        if options.verbose: print len(files), ' wind-field files found'
91        dict_dates={}
92        for f in files:
93          if (f[0:2] == './'): f = f[2:] # PS - remove this part if present
94          fn = os.path.basename(f)
95          if fn[-1] != '*':
96            timestamp = fn[tind:]
97            year = int(timestamp[0:2])
98            if year < 58:
99              year += 2000
100           else:
101             year += 1900
102           dtstring = str(year)+' '+timestamp[2:9]
103           dict_dates[dtstring] = f
104       dates = sorted(dict_dates.items())
105       if len(dates) == 0:
106         print 'no files found with this search string'
107         print 'aborting. '
108         sys.exit(0)
109       else:
110         print 'found ',len(dates),'files'
111         #Prepare output files
112         fout = file(os.path.join(avpath,'AVAILABLE'),'w')
113         fout.write(AVAIL_head)
114     
115       for i,date in enumerate(dates): # go through dates in ascending order
116         f = date[1] # now we have the filename+path
117         fn = os.path.basename(f)
118         if fn[-1]!='*':
119           timestamp = fn[tind:]
120           year = int(timestamp[0:2])
121           if year < 58:
122             year += 2000
123           else:
124             year += 1900
125           month = int(timestamp[2:4])
126           day   = int(timestamp[4:6])
127           hour  = int(timestamp[6:8])
128           fileT = year*100 + int(month)
129     # PS: now check for irregular intervals     
130           date = datetime.datetime(year,month,day,hour)
131           if i == 2:
132             if options.timeint == '':
133               timeint = date - date1
134             else:
135               timeint = datetime.timedelta(0,3600*int(options.timeint)) 
136             if timeint != date - date1:
137               print 'WARNING - irregular interval',date - date1
138               print date1,f1,'\n',date, f,'\n'
139           elif i > 2:
140             if timeint != date - date1: 
141               print 'WARNING - irregular interval',date - date1
142               print date1,f1,'\n',date, f,'\n'
143             if options.timeint == '': timeint = date - date1
144           date1 = date 
145           f1 = f
146          
147           if i%5000 == 0: print 'progress:', i, 'of', len(dates), f
148          
149           if fileT >= sy*100 + sm and fileT <= ey*100 + em :
150     
151             relpath = os.path.relpath( os.path.dirname(f), avpath )
152             f = os.path.join(relpath,fn)
153     
154             if (f[0:2] == './'): f = f[2:] #  remove this part if present
155     
156             if len(f) > 18: #PS
157               if not warned:
158                 print 'WARNING: Flexpart can only read 18 chars in WF-name'
159                 print f, ' has ', len(f), ' characters!\n'
160                 warned = True
161     
162             #This is the fortran format: (i8,1x,i6,2(6x,a18))
163             string = "%s%s%s %s0000      %s    ON DISC\n" %\
164              (year,str(month).zfill(2),str(day).zfill(2),str(hour).zfill(2),f.ljust(18))
165             fout.write(string)
166     
167       print 'Done: ',i+1 # as i starts with 0
168       print 'Written:', os.path.join(avpath,'AVAILABLE')
169       fout.close()
170     
171     if __name__ == '__main__':
172       MODEL = {}
173       MODEL['ECMWF'] = {'prefix':'EN', 'timeindex':2}
174       MODEL['ERAI']  = {'prefix':'EI', 'timeindex':2}
175       MODEL['EI']  = {'prefix':'EI', 'timeindex':2}
176       MODEL['E5']  = {'prefix':'E5', 'timeindex':2}
177       MODEL['EA']  = {'prefix':'EA', 'timeindex':2}
178       MODEL['GFS']   = {'prefix':'GF', 'timeindex':2}
179      
180       models = '/'.join(MODEL.keys())
181      
182       try:
183         start_time = time.time()
184         today = datetime.datetime.now()
185     
186         parser = optparse.OptionParser(
187             formatter = optparse.TitledHelpFormatter(),
188             usage = globals()['__doc__'])
189     
190         parser.add_option ('-v', '--verbose', action = 'store_true',
191             default = False, help = 'verbose output')
192     
193         parser.add_option ('-s', '--startdate',
194             dest = 'startdate',
195             default = '197001',
196             help = 'startdate YYYYMM integer')
197     
198         parser.add_option ('-e', '--enddate',
199             # setting default as TODAY
200             dest = 'enddate',
201             default = str(today.year) + str(today.month).zfill(2),
202             #default = '200712',
203             help = 'enddate YYYYMM integer')
204     
205         parser.add_option ('-m', '--model',
206             dest = 'model',
207             default = 'ECMWF', help = models)
208     
209         parser.add_option ('-p', '--path',
210             dest = 'path',
211             default = '.', help = 'path to be searched for windfields. Escape or quote * and ? ')
212     
213         parser.add_option ('-a', '--avpath',
214             dest = 'avpath',
215             default = '.', help = 'path for AVAILABLE file ')
216     
217         parser.add_option ('-i', '--interval',
218             dest = 'timeint',
219             default = '', help = 'expected time interval in h. If omitted, show every change')
220     
221         (options, args) = parser.parse_args()
222     
223         #QUERY['modelType'] = options.model
224         #QUERY['start_date'] = options.startdate
225         #QUERY['end_date'] = options.enddate
226      
227         #if len(args) < 1:
228         #  parser.error ('missing argument')
229     
230         if options.verbose: print time.asctime()
231         exit_code = main()
232         if exit_code is None:
233           exit_code = 0
234         if options.verbose: print time.asctime()
235         if options.verbose: print 'TOTAL TIME IN MINUTES:',
236         if options.verbose: print (time.time() - start_time) / 60.0
237         sys.exit(exit_code)
238       except KeyboardInterrupt, e: # Ctrl-C
239         raise e
240       except SystemExit, e: # sys.exit()
241         raise e
242       except Exception, e:
243         print 'ERROR, UNEXPECTED EXCEPTION'
244         print str(e)
245         traceback.print_exc()
246         os._exit(1)
247        
248      
249     
250     # vim:set sr et
Last modified 95 seconds ago Last modified on Dec 13, 2018, 3:09:06 PM

Attachments (3)

Download all attachments as: .zip

hosted by ZAMG