source: flex_extract.git/python/Tools.py @ 991df6a

ctbtodev
Last change on this file since 991df6a was 991df6a, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

finished documentation (except plot_retrieved)
  • Property mode set to 100644
File size: 17.4 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#************************************************************************
4# TODO AP
5# - check myerror
6# - check normalexit
7# - check getListAsString
8# - seperate args and control interpretation
9#************************************************************************
10#*******************************************************************************
11# @Author: Anne Philipp (University of Vienna)
12#
13# @Date: May 2018
14#
15# @Change History:
16#    October 2014 - Anne Fouilloux (University of Oslo)
17#        - created functions silentremove and product (taken from ECMWF)
18#
19#    November 2015 - Leopold Haimberger (University of Vienna)
20#        - created functions: interpret_args_and_control, cleanup
21#          myerror, normalexit, init128, toparamId
22#
23#    April 2018 - Anne Philipp (University of Vienna):
24#        - applied PEP8 style guide
25#        - added documentation
26#        - moved all functions from file FlexpartTools to this file Tools
27#        - added function getListAsString
28#
29# @License:
30#    (C) Copyright 2014-2018.
31#
32#    This software is licensed under the terms of the Apache Licence Version 2.0
33#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
34#
35# @Modul Description:
36#    This module contains a couple of helpful functions which are
37#    used in different places in flex_extract.
38#
39# @Module Content:
40#    - interpret_args_and_control
41#    - cleanup
42#    - myerror
43#    - normalexit
44#    - product
45#    - silentremove
46#    - init128
47#    - toparamId
48#    - getListAsString
49#
50#*******************************************************************************
51
52# ------------------------------------------------------------------------------
53# MODULES
54# ------------------------------------------------------------------------------
55import os
56import errno
57import sys
58import glob
59import traceback
60from numpy import *
61from gribapi import *
62from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
63
64# software specific class from flex_extract
65from ControlFile import ControlFile
66
67# ------------------------------------------------------------------------------
68# FUNCTIONS
69# ------------------------------------------------------------------------------
70
71def interpret_args_and_control():
72    '''
73    @Description:
74        Assigns the command line arguments and reads CONTROL file
75        content. Apply default values for non mentioned arguments.
76
77    @Input:
78        <nothing>
79
80    @Return:
81        args: instance of ArgumentParser
82            Contains the commandline arguments from script/program call.
83
84        c: instance of class ControlFile
85            Contains all necessary information of a CONTROL file. The parameters
86            are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
87            NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
88            RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
89            SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
90            ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, DEBUG, INPUTDIR,
91            OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
92            For more information about format and content of the parameter see
93            documentation.
94
95    '''
96    parser = ArgumentParser(description='Retrieve FLEXPART input from \
97                            ECMWF MARS archive',
98                            formatter_class=ArgumentDefaultsHelpFormatter)
99
100    # the most important arguments
101    parser.add_argument("--start_date", dest="start_date",
102                        help="start date YYYYMMDD")
103    parser.add_argument("--end_date", dest="end_date",
104                        help="end_date YYYYMMDD")
105    parser.add_argument("--date_chunk", dest="date_chunk", default=None,
106                        help="# of days to be retrieved at once")
107
108    # some arguments that override the default in the CONTROL file
109    parser.add_argument("--basetime", dest="basetime",
110                        help="base such as 00/12 (for half day retrievals)")
111    parser.add_argument("--step", dest="step",
112                        help="steps such as 00/to/48")
113    parser.add_argument("--levelist", dest="levelist",
114                        help="Vertical levels to be retrieved, e.g. 30/to/60")
115    parser.add_argument("--area", dest="area",
116                        help="area defined as north/west/south/east")
117
118    # set the working directories
119    parser.add_argument("--inputdir", dest="inputdir", default=None,
120                        help="root directory for storing intermediate files")
121    parser.add_argument("--outputdir", dest="outputdir", default=None,
122                        help="root directory for storing output files")
123    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
124                        help="FLEXPART root directory (to find grib2flexpart \
125                        and COMMAND file)\n\ Normally ECMWFDATA resides in \
126                        the scripts directory of the FLEXPART distribution")
127
128    # this is only used by prepareFLEXPART.py to rerun a postprocessing step
129    parser.add_argument("--ppid", dest="ppid",
130                        help="Specify parent process id for \
131                        rerun of prepareFLEXPART")
132
133    # arguments for job submission to ECMWF, only needed by submit.py
134    parser.add_argument("--job_template", dest='job_template',
135                        default="job.temp",
136                        help="job template file for submission to ECMWF")
137    parser.add_argument("--queue", dest="queue",
138                        help="queue for submission to ECMWF \
139                        (e.g. ecgate or cca )")
140    parser.add_argument("--controlfile", dest="controlfile",
141                        default='CONTROL.temp',
142                        help="file with CONTROL parameters")
143    parser.add_argument("--debug", dest="debug", default=0,
144                        help="Debug mode - leave temporary files intact")
145
146    args = parser.parse_args()
147
148    # create instance of ControlFile for specified controlfile
149    # and assign the parameters (and default values if necessary)
150    try:
151        c = ControlFile(args.controlfile)
152    except IOError:
153        try:
154            c = ControlFile(localpythonpath + args.controlfile)
155        except:
156            print('Could not read CONTROL file "' + args.controlfile + '"')
157            print('Either it does not exist or its syntax is wrong.')
158            print('Try "' + sys.argv[0].split('/')[-1] +
159                  ' -h" to print usage information')
160            exit(1)
161
162    # check for having at least a starting date
163    if  args.start_date is None and getattr(c, 'start_date') is None:
164        print('start_date specified neither in command line nor \
165               in CONTROL file ' + args.controlfile)
166        print('Try "' + sys.argv[0].split('/')[-1] +
167              ' -h" to print usage information')
168        exit(1)
169
170    # save all existing command line parameter to the ControlFile instance
171    # if parameter is not specified through the command line or CONTROL file
172    # set default values
173    if args.start_date is not None:
174        c.start_date = args.start_date
175    if args.end_date is not None:
176        c.end_date = args.end_date
177    if c.end_date is None:
178        c.end_date = c.start_date
179    if args.date_chunk is not None:
180        c.date_chunk = args.date_chunk
181
182    if not hasattr(c, 'debug'):
183        c.debug = args.debug
184
185    if args.inputdir is None and args.outputdir is None:
186        c.inputdir = '../work'
187        c.outputdir = '../work'
188    else:
189        if args.inputdir is not None:
190            c.inputdir = args.inputdir
191        if args.outputdir is None:
192            c.outputdir = args.inputdir
193        if args.outputdir is not None:
194            c.outputdir = args.outputdir
195        if args.inputdir is None:
196            c.inputdir = args.outputdir
197
198    if hasattr(c, 'outputdir') is False and args.outputdir is None:
199        c.outputdir = c.inputdir
200    else:
201        if args.outputdir is not None:
202            c.outputdir = args.outputdir
203
204    if args.area is not None:
205        afloat = '.' in args.area
206        l = args.area.split('/')
207        if afloat:
208            for i in range(len(l)):
209                l[i] = str(int(float(l[i]) * 1000))
210        c.upper, c.left, c.lower, c.right = l
211
212    # NOTE: basetime activates the ''operational mode''
213    if args.basetime is not None:
214        c.basetime = args.basetime
215
216    if args.step is not None:
217        l = args.step.split('/')
218        if 'to' in args.step.lower():
219            if 'by' in args.step.lower():
220                ilist = arange(int(l[0]), int(l[2]) + 1, int(l[4]))
221                c.step = ['{:0>3}'.format(i) for i in ilist]
222            else:
223                myerror(None, args.step + ':\n' +
224                        'please use "by" as well if "to" is used')
225        else:
226            c.step = l
227
228    if args.levelist is not None:
229        c.levelist = args.levelist
230        if 'to' in c.levelist:
231            c.level = c.levelist.split('/')[2]
232        else:
233            c.level = c.levelist.split('/')[-1]
234
235    if args.flexpart_root_scripts is not None:
236        c.flexpart_root_scripts = args.flexpart_root_scripts
237
238    return args, c
239
240
241def cleanup(c):
242    '''
243    @Description:
244        Remove all files from intermediate directory
245        (inputdir from CONTROL file).
246
247    @Input:
248        c: instance of class ControlFile
249            Contains all the parameters of CONTROL file, which are e.g.:
250            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
251            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
252            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
253            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
254            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
255            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
256            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
257
258            For more information about format and content of the parameter
259            see documentation.
260
261    @Return:
262        <nothing>
263    '''
264
265    print("cleanup")
266
267    cleanlist = glob.glob(c.inputdir + "/*")
268    for cl in cleanlist:
269        if c.prefix not in cl:
270            silentremove(cl)
271        if c.ecapi is False and (c.ectrans == '1' or c.ecstorage == '1'):
272            silentremove(cl)
273
274    print("Done")
275
276    return
277
278
279def myerror(c, message='ERROR'):
280    '''
281    @Description:
282        Prints a specified error message which can be passed to the function
283        before exiting the program.
284
285    @Input:
286        c: instance of class ControlFile
287            Contains all the parameters of CONTROL file, which are e.g.:
288            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
289            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
290            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
291            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
292            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
293            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
294            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
295
296            For more information about format and content of the parameter
297            see documentation.
298
299        message: string, optional
300            Error message. Default value is "ERROR".
301
302    @Return:
303        <nothing>
304    '''
305    # uncomment if user wants email notification directly from python
306    #try:
307        #target = c.mailfail
308    #except AttributeError:
309        #target = os.getenv('USER')
310
311    #if(type(target) is not list):
312        #target = [target]
313
314    print(message)
315
316    # uncomment if user wants email notification directly from python
317    #for t in target:
318    #p = subprocess.Popen(['mail','-s ECMWFDATA v7.0 ERROR', os.path.expandvars(t)],
319    #                     stdin = subprocess.PIPE, stdout = subprocess.PIPE,
320    #                     stderr = subprocess.PIPE, bufsize = 1)
321    #tr = '\n'.join(traceback.format_stack())
322    #pout = p.communicate(input = message+'\n\n'+tr)[0]
323    #print 'Email sent to '+os.path.expandvars(t) # +' '+pout.decode()
324
325    exit(1)
326
327    return
328
329
330def normalexit(c, message='Done!'):
331    '''
332    @Description:
333        Prints a specific exit message which can be passed to the function.
334
335    @Input:
336        c: instance of class ControlFile
337            Contains all the parameters of CONTROL file, which are e.g.:
338            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
339            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
340            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
341            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
342            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
343            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
344            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
345
346            For more information about format and content of the parameter
347            see documentation.
348
349        message: string, optional
350            Message for exiting program. Default value is "Done!".
351
352    @Return:
353        <nothing>
354
355    '''
356    # Uncomment if user wants notification directly from python
357    #try:
358        #target = c.mailops
359        #if(type(target) is not list):
360            #target = [target]
361        #for t in target:
362            #p = subprocess.Popen(['mail','-s ECMWFDATA v7.0 normal exit',
363            #                      os.path.expandvars(t)],
364            #                      stdin = subprocess.PIPE,
365            #                      stdout = subprocess.PIPE,
366            #                      stderr = subprocess.PIPE, bufsize = 1)
367            #pout = p.communicate(input = message+'\n\n')[0]
368            #print pout.decode()
369    #except:
370        #pass
371
372    print(message)
373
374    return
375
376
377def product(*args, **kwds):
378    '''
379    @Description:
380        This method is taken from an example at the ECMWF wiki website.
381        https://software.ecmwf.int/wiki/display/GRIB/index.py; 2018-03-16
382
383        This method combines the single characters of the passed arguments
384        with each other. So that each character of each argument value
385        will be combined with each character of the other arguments as a tuple.
386
387        Example:
388        product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
389        product(range(2), repeat = 3) --> 000 001 010 011 100 101 110 111
390
391    @Input:
392        *args: tuple
393            Positional arguments (arbitrary number).
394
395        **kwds: dictionary
396            Contains all the keyword arguments from *args.
397
398    @Return:
399        prod: tuple
400            Return will be done with "yield". A tuple of combined arguments.
401            See example in description above.
402    '''
403    pools = map(tuple, args) * kwds.get('repeat', 1)
404    result = [[]]
405    for pool in pools:
406        result = [x + [y] for x in result for y in pool]
407    for prod in result:
408        yield tuple(prod)
409
410    return
411
412
413def silentremove(filename):
414    '''
415    @Description:
416        If "filename" exists , it is removed.
417        The function does not fail if the file does not exist.
418
419    @Input:
420        filename: string
421            The name of the file to be removed without notification.
422
423    @Return:
424        <nothing>
425    '''
426    try:
427        os.remove(filename)
428    except OSError as e:
429        # this would be "except OSError, e:" before Python 2.6
430        if e.errno is not  errno.ENOENT:
431            # errno.ENOENT  =  no such file or directory
432            raise  # re-raise exception if a different error occured
433
434    return
435
436
437def init128(fn):
438    '''
439    @Description:
440        Opens and reads the grib file with table 128 information.
441
442    @Input:
443        fn: string
444            Path to file of ECMWF grib table number 128.
445
446    @Return:
447        table128: dictionary
448            Contains the ECMWF grib table 128 information.
449            The key is the parameter number and the value is the
450            short name of the parameter.
451    '''
452    table128 = dict()
453    with open(fn) as f:
454        fdata = f.read().split('\n')
455    for data in fdata:
456        if data[0] != '!':
457            table128[data[0:3]] = data[59:64].strip()
458
459    return table128
460
461
462def toparamId(pars, table):
463    '''
464    @Description:
465        Transform parameter names to parameter ids
466        with ECMWF grib table 128.
467
468    @Input:
469        pars: string
470            Addpar argument from CONTROL file in the format of
471            parameter names instead of ids. The parameter short
472            names are sepearted with "/" and they are passed as
473            one single string.
474
475        table: dictionary
476            Contains the ECMWF grib table 128 information.
477            The key is the parameter number and the value is the
478            short name of the parameter.
479
480    @Return:
481        ipar: list of integer
482            List of addpar parameters from CONTROL file transformed to
483            parameter ids in the format of integer.
484    '''
485    cpar = pars.upper().split('/')
486    ipar = []
487    for par in cpar:
488        found = False
489        for k, v in table.iteritems():
490            if par == k or par == v:
491                ipar.append(int(k))
492                found = True
493                break
494        if found is False:
495            print('Warning: par ' + par + ' not found in table 128')
496
497    return ipar
498
499def getListAsString(listObj):
500    '''
501    @Description:
502        Converts a list of arbitrary content into a single string.
503
504    @Input:
505        listObj: list
506            A list with arbitrary content.
507
508    @Return:
509        strOfList: string
510            The content of the list as a single string.
511    '''
512
513    strOfList = ", ".join( str(l) for l in listObj)
514
515    return strOfList
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG