source: flex_extract.git/python/Tools.py @ efdb01a

ctbtodev
Last change on this file since efdb01a was efdb01a, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

whole bunch of modifications due to new structure of ECMWFDATA, added basics of documentation, minor programming corrections
  • Property mode set to 100644
File size: 15.7 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#************************************************************************
4# TODO AP
5#AP
6# -
7#************************************************************************
8"""
9
10"""
11# ------------------------------------------------------------------------------
12# MODULES
13# ------------------------------------------------------------------------------
14from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
15import os
16import errno
17import sys
18import glob
19from numpy import *
20from gribapi import *
21import Control
22
23# ------------------------------------------------------------------------------
24# FUNCTIONS
25# ------------------------------------------------------------------------------
26
27def interpret_args_and_control():
28    '''
29    @Description:
30        Assigns the command line arguments and reads control file
31        content. Apply default values for non mentioned arguments.
32
33    @Input:
34        <nothing>
35
36    @Return:
37        args: instance of ArgumentParser
38            Contains the commandline arguments from script/program call.
39
40        c: instance of class Control
41            Contains all necessary information of a control file. The parameters
42            are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
43            NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
44            RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
45            SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
46            ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, DEBUG, INPUTDIR,
47            OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
48            For more information about format and content of the parameter see
49            documentation.
50
51    '''
52    parser = ArgumentParser(description='Retrieve FLEXPART input from \
53                            ECMWF MARS archive',
54                            formatter_class=ArgumentDefaultsHelpFormatter)
55
56    # the most important arguments
57    parser.add_argument("--start_date", dest="start_date",
58                        help="start date YYYYMMDD")
59    parser.add_argument("--end_date", dest="end_date",
60                        help="end_date YYYYMMDD")
61    parser.add_argument("--date_chunk", dest="date_chunk", default=None,
62                        help="# of days to be retrieved at once")
63
64    # some arguments that override the default in the control file
65    parser.add_argument("--basetime", dest="basetime",
66                        help="base such as 00/12 (for half day retrievals)")
67    parser.add_argument("--step", dest="step",
68                        help="steps such as 00/to/48")
69    parser.add_argument("--levelist", dest="levelist",
70                        help="Vertical levels to be retrieved, e.g. 30/to/60")
71    parser.add_argument("--area", dest="area",
72                        help="area defined as north/west/south/east")
73
74    # set the working directories
75    parser.add_argument("--inputdir", dest="inputdir", default=None,
76                        help="root directory for storing intermediate files")
77    parser.add_argument("--outputdir", dest="outputdir", default=None,
78                        help="root directory for storing output files")
79    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
80                        help="FLEXPART root directory (to find grib2flexpart \
81                        and COMMAND file)\n\ Normally ECMWFDATA resides in \
82                        the scripts directory of the FLEXPART distribution")
83
84    # this is only used by prepareFLEXPART.py to rerun a postprocessing step
85    parser.add_argument("--ppid", dest="ppid",
86                        help="Specify parent process id for \
87                        rerun of prepareFLEXPART")
88
89    # arguments for job submission to ECMWF, only needed by submit.py
90    parser.add_argument("--job_template", dest='job_template',
91                        default="job.temp",
92                        help="job template file for submission to ECMWF")
93    parser.add_argument("--queue", dest="queue",
94                        help="queue for submission to ECMWF \
95                        (e.g. ecgate or cca )")
96    parser.add_argument("--controlfile", dest="controlfile",
97                        default='CONTROL.temp',
98                        help="file with control parameters")
99    parser.add_argument("--debug", dest="debug", default=0,
100                        help="Debug mode - leave temporary files intact")
101
102    args = parser.parse_args()
103
104    # create instance of Control for specified controlfile
105    # and assign the parameters (and default values if necessary)
106    try:
107        c = Control.Control(args.controlfile)
108    except IOError:
109        try:
110            c = Control.Control(localpythonpath + args.controlfile)
111        except:
112            print('Could not read control file "' + args.controlfile + '"')
113            print('Either it does not exist or its syntax is wrong.')
114            print('Try "' + sys.argv[0].split('/')[-1] +
115                  ' -h" to print usage information')
116            exit(1)
117
118    # check for having at least a starting date
119    if  args.start_date is None and getattr(c, 'start_date') is None:
120        print('start_date specified neither in command line nor \
121               in control file ' + args.controlfile)
122        print('Try "' + sys.argv[0].split('/')[-1] +
123              ' -h" to print usage information')
124        exit(1)
125
126    # save all existing command line parameter to the Control instance
127    # if parameter is not specified through the command line or CONTROL file
128    # set default values
129    if args.start_date is not None:
130        c.start_date = args.start_date
131    if args.end_date is not None:
132        c.end_date = args.end_date
133    if c.end_date is None:
134        c.end_date = c.start_date
135    if args.date_chunk is not None:
136        c.date_chunk = args.date_chunk
137
138    if not hasattr(c, 'debug'):
139        c.debug = args.debug
140
141    if args.inputdir is None and args.outputdir is None:
142        c.inputdir = '../work'
143        c.outputdir = '../work'
144    else:
145        if args.inputdir is not None:
146            c.inputdir = args.inputdir
147        if args.outputdir is None:
148            c.outputdir = args.inputdir
149        if args.outputdir is not None:
150            c.outputdir = args.outputdir
151        if args.inputdir is None:
152            c.inputdir = args.outputdir
153
154    if hasattr(c, 'outputdir') is False and args.outputdir is None:
155        c.outputdir = c.inputdir
156    else:
157        if args.outputdir is not None:
158            c.outputdir = args.outputdir
159
160    if args.area is not None:
161        afloat = '.' in args.area
162        l = args.area.split('/')
163        if afloat:
164            for i in range(len(l)):
165                l[i] = str(int(float(l[i]) * 1000))
166        c.upper, c.left, c.lower, c.right = l
167
168    # NOTE: basetime activates the ''operational mode''
169    if args.basetime is not None:
170        c.basetime = args.basetime
171
172    if args.step is not None:
173        l = args.step.split('/')
174        if 'to' in args.step.lower():
175            if 'by' in args.step.lower():
176                ilist = arange(int(l[0]), int(l[2]) + 1, int(l[4]))
177                c.step = ['{:0>3}'.format(i) for i in ilist]
178            else:
179                myerror(None, args.step + ':\n' +
180                        'please use "by" as well if "to" is used')
181        else:
182            c.step = l
183
184    if args.levelist is not None:
185        c.levelist = args.levelist
186        if 'to' in c.levelist:
187            c.level = c.levelist.split('/')[2]
188        else:
189            c.level = c.levelist.split('/')[-1]
190
191    if args.flexpart_root_scripts is not None:
192        c.flexpart_root_scripts = args.flexpart_root_scripts
193
194    return args, c
195
196
197def cleanup(c):
198    '''
199    @Description:
200        Remove all files from intermediate directory
201        (inputdir from control file).
202
203    @Input:
204        c: instance of class Control
205            Contains all the parameters of control files, which are e.g.:
206            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
207            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
208            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
209            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
210            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
211            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
212            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
213
214            For more information about format and content of the parameter
215            see documentation.
216
217    @Return:
218        <nothing>
219    '''
220
221    print("cleanup")
222
223    cleanlist = glob.glob(c.inputdir + "/*")
224    for cl in cleanlist:
225        if c.prefix not in cl:
226            silentremove(cl)
227        if c.ecapi is False and (c.ectrans == '1' or c.ecstorage == '1'):
228            silentremove(cl)
229
230    print("Done")
231
232    return
233
234
235def myerror(c, message='ERROR'):
236    '''
237    @Description:
238        Prints a specified error message which can be passed to the function
239        before exiting the program.
240
241    @Input:
242        c: instance of class Control
243            Contains all the parameters of control files, which are e.g.:
244            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
245            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
246            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
247            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
248            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
249            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
250            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
251
252            For more information about format and content of the parameter
253            see documentation.
254
255        message: string, optional
256            Error message. Default value is "ERROR".
257
258    @Return:
259        <nothing>
260    '''
261    # uncomment if user wants email notification directly from python
262    #try:
263        #target = c.mailfail
264    #except AttributeError:
265        #target = os.getenv('USER')
266
267    #if(type(target) is not list):
268        #target = [target]
269
270    print(message)
271
272    # uncomment if user wants email notification directly from python
273    #for t in target:
274    #p = subprocess.Popen(['mail','-s ECMWFDATA v7.0 ERROR', os.path.expandvars(t)],
275    #                     stdin = subprocess.PIPE, stdout = subprocess.PIPE,
276    #                     stderr = subprocess.PIPE, bufsize = 1)
277    #tr = '\n'.join(traceback.format_stack())
278    #pout = p.communicate(input = message+'\n\n'+tr)[0]
279    #print 'Email sent to '+os.path.expandvars(t) # +' '+pout.decode()
280
281    exit(1)
282
283    return
284
285
286def normalexit(c, message='Done!'):
287    '''
288    @Description:
289        Prints a specific exit message which can be passed to the function.
290
291    @Input:
292        c: instance of class Control
293            Contains all the parameters of control files, which are e.g.:
294            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
295            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
296            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
297            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
298            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
299            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
300            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
301
302            For more information about format and content of the parameter
303            see documentation.
304
305        message: string, optional
306            Message for exiting program. Default value is "Done!".
307
308    @Return:
309        <nothing>
310
311    '''
312    # Uncomment if user wants notification directly from python
313    #try:
314        #target = c.mailops
315        #if(type(target) is not list):
316            #target = [target]
317        #for t in target:
318            #p = subprocess.Popen(['mail','-s ECMWFDATA v7.0 normal exit',
319            #                      os.path.expandvars(t)],
320            #                      stdin = subprocess.PIPE,
321            #                      stdout = subprocess.PIPE,
322            #                      stderr = subprocess.PIPE, bufsize = 1)
323            #pout = p.communicate(input = message+'\n\n')[0]
324            #print pout.decode()
325    #except:
326        #pass
327
328    print(message)
329
330    return
331
332
333def product(*args, **kwds):
334    '''
335    @Description:
336        This method is taken from an example at the ECMWF wiki website.
337        https://software.ecmwf.int/wiki/display/GRIB/index.py; 2018-03-16
338
339        This method combines the single characters of the passed arguments
340        with each other. So that each character of each argument value
341        will be combined with each character of the other arguments as a tuple.
342
343        Example:
344        product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
345        product(range(2), repeat = 3) --> 000 001 010 011 100 101 110 111
346
347    @Input:
348        *args: tuple
349            Positional arguments (arbitrary number).
350
351        **kwds: dictionary
352            Contains all the keyword arguments from *args.
353
354    @Return:
355        prod: tuple
356            Return will be done with "yield". A tuple of combined arguments.
357            See example in description above.
358    '''
359
360    pools = map(tuple, args) * kwds.get('repeat', 1)
361    result = [[]]
362    for pool in pools:
363        result = [x + [y] for x in result for y in pool]
364    for prod in result:
365        yield tuple(prod)
366
367    return
368
369
370def silentremove(filename):
371    '''
372    @Description:
373        Removes the file which name is passed to the function if
374        it exists. The function does not fail if the file does not
375        exist.
376
377    @Input:
378        filename: string
379            The name of the file to be removed without notification.
380
381    @Return:
382        <nothing>
383    '''
384    try:
385        os.remove(filename)
386    except OSError as e:
387        # this would be "except OSError, e:" before Python 2.6
388        if e.errno is not  errno.ENOENT:
389            # errno.ENOENT  =  no such file or directory
390            raise  # re-raise exception if a different error occured
391
392    return
393
394
395def init128(fn):
396    '''
397    @Description:
398        Opens and reads the grib file with table 128 information.
399
400    @Input:
401        fn: string
402            Path to file of ECMWF grib table number 128.
403
404    @Return:
405        table128: dictionary
406            Contains the ECMWF grib table 128 information.
407            The key is the parameter number and the value is the
408            short name of the parameter.
409    '''
410    table128 = dict()
411    with open(fn) as f:
412        fdata = f.read().split('\n')
413    for data in fdata:
414        if data[0] != '!':
415            table128[data[0:3]] = data[59:64].strip()
416
417    return table128
418
419
420def toparamId(pars, table):
421    '''
422    @Description:
423        Transform parameter names to parameter ids
424        with ECMWF grib table 128.
425
426    @Input:
427        pars: string
428            Addpar argument from control file in the format of
429            parameter names instead of ids. The parameter short
430            names are sepearted with "/" and they are passed as
431            one single string.
432
433        table: dictionary
434            Contains the ECMWF grib table 128 information.
435            The key is the parameter number and the value is the
436            short name of the parameter.
437
438    @Return:
439        ipar: list of integer
440            List of addpar parameters from control file transformed to
441            parameter ids in the format of integer.
442    '''
443    cpar = pars.upper().split('/')
444    ipar = []
445    for par in cpar:
446        found = False
447        for k, v in table.iteritems():
448            if par == k or par == v:
449                ipar.append(int(k))
450                found = True
451                break
452        if found is False:
453            print('Warning: par ' + par + ' not found in table 128')
454
455    return ipar
456
457def getListAsString(listobj):
458    '''
459    @Description:
460    '''
461    return ", ".join( str(l) for l in listobj)
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG