source: flex_extract.git/python/tools.py @ e1228f3

ctbtodev
Last change on this file since e1228f3 was ff99eae, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

completed application of pep8 style guide and pylint investigations. added documentation almost everywhere

  • Property mode set to 100644
File size: 17.8 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#************************************************************************
4# ToDo AP
5# - check my_error
6# - check normal_exit
7# - check get_list_as_string
8# - seperate args and control interpretation
9#************************************************************************
10#*******************************************************************************
11# @Author: Anne Philipp (University of Vienna)
12#
13# @Date: May 2018
14#
15# @Change History:
16#    October 2014 - Anne Fouilloux (University of Oslo)
17#        - created functions silent_remove and product (taken from ECMWF)
18#
19#    November 2015 - Leopold Haimberger (University of Vienna)
20#        - created functions: interpret_args_and_control, clean_up
21#          my_error, normal_exit, init128, to_param_id
22#
23#    April 2018 - Anne Philipp (University of Vienna):
24#        - applied PEP8 style guide
25#        - added documentation
26#        - moved all functions from file Flexparttools to this file tools
27#        - added function get_list_as_string
28#
29# @License:
30#    (C) Copyright 2014-2018.
31#
32#    This software is licensed under the terms of the Apache Licence Version 2.0
33#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
34#
35# @Modul Description:
36#    This module contains a couple of helpful functions which are
37#    used in different places in flex_extract.
38#
39# @Module Content:
40#    - interpret_args_and_control
41#    - clean_up
42#    - my_error
43#    - normal_exit
44#    - product
45#    - silent_remove
46#    - init128
47#    - to_param_id
48#    - get_list_as_string
49#
50#*******************************************************************************
51
52# ------------------------------------------------------------------------------
53# MODULES
54# ------------------------------------------------------------------------------
55import os
56import errno
57import sys
58import glob
59import inspect
60import subprocess
61import traceback
62from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
63import numpy as np
64
65# software specific class from flex_extract
66from ControlFile import ControlFile
67
68# ------------------------------------------------------------------------------
69# FUNCTIONS
70# ------------------------------------------------------------------------------
71
72def interpret_args_and_control():
73    '''
74    @Description:
75        Assigns the command line arguments and reads CONTROL file
76        content. Apply default values for non mentioned arguments.
77
78    @Input:
79        <nothing>
80
81    @Return:
82        args: instance of ArgumentParser
83            Contains the commandline arguments from script/program call.
84
85        c: instance of class ControlFile
86            Contains all necessary information of a CONTROL file. The parameters
87            are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
88            NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
89            RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
90            SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
91            ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, DEBUG, INPUTDIR,
92            OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
93            For more information about format and content of the parameter see
94            documentation.
95
96    '''
97    parser = ArgumentParser(description='Retrieve FLEXPART input from \
98                            ECMWF MARS archive',
99                            formatter_class=ArgumentDefaultsHelpFormatter)
100
101    # the most important arguments
102    parser.add_argument("--start_date", dest="start_date",
103                        help="start date YYYYMMDD")
104    parser.add_argument("--end_date", dest="end_date",
105                        help="end_date YYYYMMDD")
106    parser.add_argument("--date_chunk", dest="date_chunk", default=None,
107                        help="# of days to be retrieved at once")
108
109    # some arguments that override the default in the CONTROL file
110    parser.add_argument("--basetime", dest="basetime",
111                        help="base such as 00/12 (for half day retrievals)")
112    parser.add_argument("--step", dest="step",
113                        help="steps such as 00/to/48")
114    parser.add_argument("--levelist", dest="levelist",
115                        help="Vertical levels to be retrieved, e.g. 30/to/60")
116    parser.add_argument("--area", dest="area",
117                        help="area defined as north/west/south/east")
118
119    # set the working directories
120    parser.add_argument("--inputdir", dest="inputdir", default=None,
121                        help="root directory for storing intermediate files")
122    parser.add_argument("--outputdir", dest="outputdir", default=None,
123                        help="root directory for storing output files")
124    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
125                        help="FLEXPART root directory (to find grib2flexpart \
126                        and COMMAND file)\n Normally ECMWFDATA resides in \
127                        the scripts directory of the FLEXPART distribution")
128
129    # this is only used by prepare_flexpart.py to rerun a postprocessing step
130    parser.add_argument("--ppid", dest="ppid",
131                        help="Specify parent process id for \
132                        rerun of prepare_flexpart")
133
134    # arguments for job submission to ECMWF, only needed by submit.py
135    parser.add_argument("--job_template", dest='job_template',
136                        default="job.temp",
137                        help="job template file for submission to ECMWF")
138    parser.add_argument("--queue", dest="queue",
139                        help="queue for submission to ECMWF \
140                        (e.g. ecgate or cca )")
141    parser.add_argument("--controlfile", dest="controlfile",
142                        default='CONTROL.temp',
143                        help="file with CONTROL parameters")
144    parser.add_argument("--debug", dest="debug", default=0,
145                        help="Debug mode - leave temporary files intact")
146
147    args = parser.parse_args()
148
149    # create instance of ControlFile for specified controlfile
150    # and assign the parameters (and default values if necessary)
151    try:
152        c = ControlFile(args.controlfile)
153    except IOError:
154        try:
155            LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
156                inspect.getfile(inspect.currentframe())))
157            c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile)
158        except IOError:
159            print 'Could not read CONTROL file "' + args.controlfile + '"'
160            print 'Either it does not exist or its syntax is wrong.'
161            print 'Try "' + sys.argv[0].split('/')[-1] + \
162                  ' -h" to print usage information'
163            exit(1)
164
165    # check for having at least a starting date
166    if  args.start_date is None and getattr(c, 'start_date') is None:
167        print 'start_date specified neither in command line nor \
168               in CONTROL file ' + args.controlfile
169        print 'Try "' + sys.argv[0].split('/')[-1] + \
170              ' -h" to print usage information'
171        exit(1)
172
173    # save all existing command line parameter to the ControlFile instance
174    # if parameter is not specified through the command line or CONTROL file
175    # set default values
176    if args.start_date is not None:
177        c.start_date = args.start_date
178    if args.end_date is not None:
179        c.end_date = args.end_date
180    if c.end_date is None:
181        c.end_date = c.start_date
182    if args.date_chunk is not None:
183        c.date_chunk = args.date_chunk
184
185    if not hasattr(c, 'debug'):
186        c.debug = args.debug
187
188    if args.inputdir is None and args.outputdir is None:
189        c.inputdir = '../work'
190        c.outputdir = '../work'
191    else:
192        if args.inputdir is not None:
193            c.inputdir = args.inputdir
194        if args.outputdir is None:
195            c.outputdir = args.inputdir
196        if args.outputdir is not None:
197            c.outputdir = args.outputdir
198        if args.inputdir is None:
199            c.inputdir = args.outputdir
200
201    if hasattr(c, 'outputdir') is False and args.outputdir is None:
202        c.outputdir = c.inputdir
203    else:
204        if args.outputdir is not None:
205            c.outputdir = args.outputdir
206
207    if args.area is not None:
208        afloat = '.' in args.area
209        l = args.area.split('/')
210        if afloat:
211            for i, item in enumerate(l):
212                item = str(int(float(item) * 1000))
213        c.upper, c.left, c.lower, c.right = l
214
215    # NOTE: basetime activates the ''operational mode''
216    if args.basetime is not None:
217        c.basetime = args.basetime
218
219    if args.step is not None:
220        l = args.step.split('/')
221        if 'to' in args.step.lower():
222            if 'by' in args.step.lower():
223                ilist = np.arange(int(l[0]), int(l[2]) + 1, int(l[4]))
224                c.step = ['{:0>3}'.format(i) for i in ilist]
225            else:
226                my_error(None, args.step + ':\n' +
227                         'please use "by" as well if "to" is used')
228        else:
229            c.step = l
230
231    if args.levelist is not None:
232        c.levelist = args.levelist
233        if 'to' in c.levelist:
234            c.level = c.levelist.split('/')[2]
235        else:
236            c.level = c.levelist.split('/')[-1]
237
238    if args.flexpart_root_scripts is not None:
239        c.flexpart_root_scripts = args.flexpart_root_scripts
240
241    return args, c
242
243
244def clean_up(c):
245    '''
246    @Description:
247        Remove all files from intermediate directory
248        (inputdir from CONTROL file).
249
250    @Input:
251        c: instance of class ControlFile
252            Contains all the parameters of CONTROL file, which are e.g.:
253            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
254            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
255            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
256            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
257            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
258            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
259            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
260
261            For more information about format and content of the parameter
262            see documentation.
263
264    @Return:
265        <nothing>
266    '''
267
268    print "clean_up"
269
270    cleanlist = glob.glob(c.inputdir + "/*")
271    for cl in cleanlist:
272        if c.prefix not in cl:
273            silent_remove(cl)
274        if c.ecapi is False and (c.ectrans == '1' or c.ecstorage == '1'):
275            silent_remove(cl)
276
277    print "Done"
278
279    return
280
281
282def my_error(c, message='ERROR'):
283    '''
284    @Description:
285        Prints a specified error message which can be passed to the function
286        before exiting the program.
287
288    @Input:
289        c: instance of class ControlFile
290            Contains all the parameters of CONTROL file, which are e.g.:
291            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
292            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
293            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
294            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
295            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
296            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
297            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
298
299            For more information about format and content of the parameter
300            see documentation.
301
302        message: string, optional
303            Error message. Default value is "ERROR".
304
305    @Return:
306        <nothing>
307    '''
308
309    print message
310
311    # comment if user does not want email notification directly from python
312    try:
313        target = []
314        target.extend(c.mailfail)
315    except AttributeError:
316        target = []
317        target.extend(os.getenv('USER'))
318
319    for t in target:
320        p = subprocess.Popen(['mail', '-s ECMWFDATA v7.0 ERROR',
321                              os.path.expandvars(t)],
322                             stdin=subprocess.PIPE,
323                             stdout=subprocess.PIPE,
324                             stderr=subprocess.PIPE,
325                             bufsize=1)
326        tr = '\n'.join(traceback.format_stack())
327        pout = p.communicate(input=message + '\n\n' + tr)[0]
328        print 'Email sent to ' + os.path.expandvars(t) + ' ' + pout.decode()
329
330    exit(1)
331
332    return
333
334
335def normal_exit(c, message='Done!'):
336    '''
337    @Description:
338        Prints a specific exit message which can be passed to the function.
339
340    @Input:
341        c: instance of class ControlFile
342            Contains all the parameters of CONTROL file, which are e.g.:
343            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
344            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
345            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
346            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
347            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
348            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
349            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
350
351            For more information about format and content of the parameter
352            see documentation.
353
354        message: string, optional
355            Message for exiting program. Default value is "Done!".
356
357    @Return:
358        <nothing>
359
360    '''
361    print message
362
363    # comment if user does not want notification directly from python
364    try:
365        target = []
366        target.extend(c.mailops)
367        for t in target:
368            p = subprocess.Popen(['mail', '-s ECMWFDATA v7.0 normal exit',
369                                  os.path.expandvars(t)],
370                                 stdin=subprocess.PIPE,
371                                 stdout=subprocess.PIPE,
372                                 stderr=subprocess.PIPE,
373                                 bufsize=1)
374            pout = p.communicate(input=message+'\n\n')[0]
375            print 'Email sent to ' + os.path.expandvars(t) + ' ' + pout.decode()
376    finally:
377        pass
378
379    return
380
381
382def product(*args, **kwds):
383    '''
384    @Description:
385        This method is taken from an example at the ECMWF wiki website.
386        https://software.ecmwf.int/wiki/display/GRIB/index.py; 2018-03-16
387
388        This method combines the single characters of the passed arguments
389        with each other. So that each character of each argument value
390        will be combined with each character of the other arguments as a tuple.
391
392        Example:
393        product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
394        product(range(2), repeat = 3) --> 000 001 010 011 100 101 110 111
395
396    @Input:
397        *args: tuple
398            Positional arguments (arbitrary number).
399
400        **kwds: dictionary
401            Contains all the keyword arguments from *args.
402
403    @Return:
404        prod: tuple
405            Return will be done with "yield". A tuple of combined arguments.
406            See example in description above.
407    '''
408    pools = map(tuple, args) * kwds.get('repeat', 1)
409    result = [[]]
410    for pool in pools:
411        result = [x + [y] for x in result for y in pool]
412    for prod in result:
413        yield tuple(prod)
414
415    return
416
417
418def silent_remove(filename):
419    '''
420    @Description:
421        If "filename" exists , it is removed.
422        The function does not fail if the file does not exist.
423
424    @Input:
425        filename: string
426            The name of the file to be removed without notification.
427
428    @Return:
429        <nothing>
430    '''
431    try:
432        os.remove(filename)
433    except OSError as e:
434        # this would be "except OSError, e:" before Python 2.6
435        if e.errno is not  errno.ENOENT:
436            # errno.ENOENT  =  no such file or directory
437            raise  # re-raise exception if a different error occured
438
439    return
440
441
442def init128(filepath):
443    '''
444    @Description:
445        Opens and reads the grib file with table 128 information.
446
447    @Input:
448        filepath: string
449            Path to file of ECMWF grib table number 128.
450
451    @Return:
452        table128: dictionary
453            Contains the ECMWF grib table 128 information.
454            The key is the parameter number and the value is the
455            short name of the parameter.
456    '''
457    table128 = dict()
458    with open(filepath) as f:
459        fdata = f.read().split('\n')
460    for data in fdata:
461        if data[0] != '!':
462            table128[data[0:3]] = data[59:64].strip()
463
464    return table128
465
466
467def to_param_id(pars, table):
468    '''
469    @Description:
470        Transform parameter names to parameter ids
471        with ECMWF grib table 128.
472
473    @Input:
474        pars: string
475            Addpar argument from CONTROL file in the format of
476            parameter names instead of ids. The parameter short
477            names are sepearted with "/" and they are passed as
478            one single string.
479
480        table: dictionary
481            Contains the ECMWF grib table 128 information.
482            The key is the parameter number and the value is the
483            short name of the parameter.
484
485    @Return:
486        ipar: list of integer
487            List of addpar parameters from CONTROL file transformed to
488            parameter ids in the format of integer.
489    '''
490    cpar = pars.upper().split('/')
491    ipar = []
492    for par in cpar:
493        for k, v in table.iteritems():
494            if par == k or par == v:
495                ipar.append(int(k))
496                break
497        else:
498            print 'Warning: par ' + par + ' not found in table 128'
499
500    return ipar
501
502def get_list_as_string(list_obj, concatenate_sign=', '):
503    '''
504    @Description:
505        Converts a list of arbitrary content into a single string.
506
507    @Input:
508        list_obj: list
509            A list with arbitrary content.
510
511        concatenate_sign: string, optional
512            A string which is used to concatenate the single
513            list elements. Default value is ", ".
514
515    @Return:
516        str_of_list: string
517            The content of the list as a single string.
518    '''
519
520    str_of_list = concatenate_sign.join(str(l) for l in list_obj)
521
522    return str_of_list
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG