source: flex_extract.git/source/python/mods/tools.py @ 27fe969

ctbtodev
Last change on this file since 27fe969 was 27fe969, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

put grib2flexpart part of code into its own function; changed function documentation of input controlfile parameter

  • Property mode set to 100644
File size: 16.7 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Philipp (University of Vienna)
5#
6# @Date: May 2018
7#
8# @Change History:
9#    October 2014 - Anne Fouilloux (University of Oslo)
10#        - created functions silent_remove and product (taken from ECMWF)
11#
12#    November 2015 - Leopold Haimberger (University of Vienna)
13#        - created functions: interpret_args_and_control, clean_up
14#          my_error, normal_exit, init128, to_param_id
15#
16#    April 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added documentation
19#        - moved all functions from file Flexparttools to this file tools
20#        - added function get_list_as_string
21#        - seperated args and control interpretation
22#
23# @License:
24#    (C) Copyright 2014-2018.
25#
26#    This software is licensed under the terms of the Apache Licence Version 2.0
27#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
28#
29# @Modul Description:
30#    This module contains a couple of helpful functions which are
31#    used in different places in flex_extract.
32#
33# @Module Content:
34#    - get_cmdline_arguments
35#    - clean_up
36#    - my_error
37#    - normal_exit
38#    - product
39#    - silent_remove
40#    - init128
41#    - to_param_id
42#    - get_list_as_string
43#    - make_dir
44#
45#*******************************************************************************
46
47# ------------------------------------------------------------------------------
48# MODULES
49# ------------------------------------------------------------------------------
50import os
51import errno
52import sys
53import glob
54import subprocess
55import traceback
56from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
57
58# ------------------------------------------------------------------------------
59# FUNCTIONS
60# ------------------------------------------------------------------------------
61
62def get_cmdline_arguments():
63    '''
64    @Description:
65        Decomposes the command line arguments and assigns them to variables.
66        Apply default values for non mentioned arguments.
67
68    @Input:
69        <nothing>
70
71    @Return:
72        args: instance of ArgumentParser
73            Contains the commandline arguments from script/program call.
74    '''
75
76    parser = ArgumentParser(description='Retrieve FLEXPART input from \
77                                ECMWF MARS archive',
78                            formatter_class=ArgumentDefaultsHelpFormatter)
79
80    # the most important arguments
81    parser.add_argument("--start_date", dest="start_date", default=None,
82                        help="start date YYYYMMDD")
83    parser.add_argument("--end_date", dest="end_date", default=None,
84                        help="end_date YYYYMMDD")
85    parser.add_argument("--date_chunk", dest="date_chunk", default=None,
86                        help="# of days to be retrieved at once")
87
88    # some arguments that override the default in the CONTROL file
89    parser.add_argument("--basetime", dest="basetime", default=None,
90                        help="base such as 00/12 (for half day retrievals)")
91    parser.add_argument("--step", dest="step", default=None,
92                        help="steps such as 00/to/48")
93    parser.add_argument("--levelist", dest="levelist", default=None,
94                        help="Vertical levels to be retrieved, e.g. 30/to/60")
95    parser.add_argument("--area", dest="area", default=None,
96                        help="area defined as north/west/south/east")
97
98    # set the working directories
99    parser.add_argument("--inputdir", dest="inputdir", default=None,
100                        help="root directory for storing intermediate files")
101    parser.add_argument("--outputdir", dest="outputdir", default=None,
102                        help="root directory for storing output files")
103    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
104                        default=None,
105                        help="FLEXPART root directory (to find grib2flexpart \
106                        and COMMAND file)\n Normally flex_extract resides in \
107                        the scripts directory of the FLEXPART distribution")
108
109    # this is only used by prepare_flexpart.py to rerun a postprocessing step
110    parser.add_argument("--ppid", dest="ppid", default=None,
111                        help="specify parent process id for \
112                        rerun of prepare_flexpart")
113
114    # arguments for job submission to ECMWF, only needed by submit.py
115    parser.add_argument("--job_template", dest='job_template',
116                        default="job.temp",
117                        help="job template file for submission to ECMWF")
118    parser.add_argument("--queue", dest="queue", default=None,
119                        help="queue for submission to ECMWF \
120                        (e.g. ecgate or cca )")
121    parser.add_argument("--controlfile", dest="controlfile",
122                        default='CONTROL.temp',
123                        help="file with CONTROL parameters")
124    parser.add_argument("--debug", dest="debug", default=None,
125                        help="debug mode - leave temporary files intact")
126    parser.add_argument("--request", dest="request", default=None,
127                        help="list all mars request in file mars_requests.dat \
128                        and skip submission to mars")
129
130    args = parser.parse_args()
131
132    return args
133
134def read_ecenv(filename):
135    '''
136    @Description:
137        Reads the file into a dictionary where the key values are the parameter
138        names.
139
140    @Input:
141        filename: string
142            Path to file where the ECMWV environment parameters are stored.
143
144    @Return:
145        envs: dict
146            Contains the environment parameter ecuid, ecgid, gateway
147            and destination for ECMWF server environments.
148    '''
149    envs= {}
150
151    with open(filename, 'r') as f:
152        for line in f:
153            data = line.strip().split()
154            envs[str(data[0])] = str(data[1])
155
156    return envs
157
158def clean_up(c):
159    '''
160    @Description:
161        Remove all files from intermediate directory
162        (inputdir from CONTROL file).
163
164    @Input:
165        c: instance of class ControlFile
166            Contains all the parameters of CONTROL file and
167            command line.
168            For more information about format and content of the parameter
169            see documentation.
170
171    @Return:
172        <nothing>
173    '''
174
175    print("clean_up")
176
177    cleanlist = glob.glob(c.inputdir + "/*")
178    for clist in cleanlist:
179        if c.prefix not in clist:
180            silent_remove(clist)
181        if c.ecapi is False and (c.ectrans == '1' or c.ecstorage == '1'):
182            silent_remove(clist)
183
184    print("Done")
185
186    return
187
188
189def my_error(users, message='ERROR'):
190    '''
191    @Description:
192        Prints a specified error message which can be passed to the function
193        before exiting the program.
194
195    @Input:
196        user: list of strings
197            Contains all email addresses which should be notified.
198            It might also contain just the ecmwf user name which wil trigger
199            mailing to the associated email address for this user.
200
201        message: string, optional
202            Error message. Default value is "ERROR".
203
204    @Return:
205        <nothing>
206    '''
207
208    print(message)
209
210    # comment if user does not want email notification directly from python
211    for user in users:
212        if '${USER}' in user:
213            user = os.getenv('USER')
214        try:
215            p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ERROR',
216                                  os.path.expandvars(user)],
217                                 stdin=subprocess.PIPE,
218                                 stdout=subprocess.PIPE,
219                                 stderr=subprocess.PIPE,
220                                 bufsize=1)
221            trace = '\n'.join(traceback.format_stack())
222            pout = p.communicate(input=message + '\n\n' + trace)[0]
223        except ValueError as e:
224            print('ERROR: ', e)
225            sys.exit('Email could not be sent!')
226        else:
227            print('Email sent to ' + os.path.expandvars(user) + ' ' +
228                  pout.decode())
229
230    sys.exit(1)
231
232    return
233
234
235def normal_exit(users, message='Done!'):
236    '''
237    @Description:
238        Prints a specific exit message which can be passed to the function.
239
240    @Input:
241        user: list of strings
242            Contains all email addresses which should be notified.
243            It might also contain just the ecmwf user name which wil trigger
244            mailing to the associated email address for this user.
245
246        message: string, optional
247            Message for exiting program. Default value is "Done!".
248
249    @Return:
250        <nothing>
251
252    '''
253    print(message)
254
255    # comment if user does not want notification directly from python
256    for user in users:
257        if '${USER}' in user:
258            user = os.getenv('USER')
259        try:
260            p = subprocess.Popen(['mail', '-s flex_extract_v7.1 normal exit',
261                                  os.path.expandvars(user)],
262                                 stdin=subprocess.PIPE,
263                                 stdout=subprocess.PIPE,
264                                 stderr=subprocess.PIPE,
265                                 bufsize=1)
266            pout = p.communicate(input=message+'\n\n')[0]
267        except ValueError as e:
268            print('ERROR: ', e)
269            print('Email could not be sent!')
270        else:
271            print('Email sent to ' + os.path.expandvars(user) + ' ' +
272                  pout.decode())
273
274    return
275
276
277def product(*args, **kwds):
278    '''
279    @Description:
280        This method is taken from an example at the ECMWF wiki website.
281        https://software.ecmwf.int/wiki/display/GRIB/index.py; 2018-03-16
282
283        This method combines the single characters of the passed arguments
284        with each other. So that each character of each argument value
285        will be combined with each character of the other arguments as a tuple.
286
287        Example:
288        product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
289        product(range(2), repeat = 3) --> 000 001 010 011 100 101 110 111
290
291    @Input:
292        *args: tuple
293            Positional arguments (arbitrary number).
294
295        **kwds: dictionary
296            Contains all the keyword arguments from *args.
297
298    @Return:
299        prod: tuple
300            Return will be done with "yield". A tuple of combined arguments.
301            See example in description above.
302    '''
303    pools = map(tuple, args) * kwds.get('repeat', 1)
304    result = [[]]
305    for pool in pools:
306        result = [x + [y] for x in result for y in pool]
307    for prod in result:
308        yield tuple(prod)
309
310    return
311
312
313def silent_remove(filename):
314    '''
315    @Description:
316        Remove file if it exists.
317        The function does not fail if the file does not exist.
318
319    @Input:
320        filename: string
321            The name of the file to be removed without notification.
322
323    @Return:
324        <nothing>
325    '''
326    try:
327        os.remove(filename)
328    except OSError as e:
329        if e.errno != errno.ENOENT:
330            # errno.ENOENT  =  no such file or directory
331            raise  # re-raise exception if a different error occured
332
333    return
334
335
336def init128(filepath):
337    '''
338    @Description:
339        Opens and reads the grib file with table 128 information.
340
341    @Input:
342        filepath: string
343            Path to file of ECMWF grib table number 128.
344
345    @Return:
346        table128: dictionary
347            Contains the ECMWF grib table 128 information.
348            The key is the parameter number and the value is the
349            short name of the parameter.
350    '''
351    table128 = dict()
352    with open(filepath) as f:
353        fdata = f.read().split('\n')
354    for data in fdata:
355        if data[0] != '!':
356            table128[data[0:3]] = data[59:64].strip()
357
358    return table128
359
360
361def to_param_id(pars, table):
362    '''
363    @Description:
364        Transform parameter names to parameter ids
365        with ECMWF grib table 128.
366
367    @Input:
368        pars: string
369            Addpar argument from CONTROL file in the format of
370            parameter names instead of ids. The parameter short
371            names are sepearted with "/" and they are passed as
372            one single string.
373
374        table: dictionary
375            Contains the ECMWF grib table 128 information.
376            The key is the parameter number and the value is the
377            short name of the parameter.
378
379    @Return:
380        ipar: list of integer
381            List of addpar parameters from CONTROL file transformed to
382            parameter ids in the format of integer.
383    '''
384    cpar = pars.upper().split('/')
385    ipar = []
386    for par in cpar:
387        for k, v in table.iteritems():
388            if par == k or par == v:
389                ipar.append(int(k))
390                break
391        else:
392            print('Warning: par ' + par + ' not found in table 128')
393
394    return ipar
395
396def get_list_as_string(list_obj, concatenate_sign=', '):
397    '''
398    @Description:
399        Converts a list of arbitrary content into a single string.
400
401    @Input:
402        list_obj: list
403            A list with arbitrary content.
404
405        concatenate_sign: string, optional
406            A string which is used to concatenate the single
407            list elements. Default value is ", ".
408
409    @Return:
410        str_of_list: string
411            The content of the list as a single string.
412    '''
413
414    str_of_list = concatenate_sign.join(str(l) for l in list_obj)
415
416    return str_of_list
417
418def make_dir(directory):
419    '''
420    @Description:
421        Creates a directory and gives a warning if the directory
422        already exists. The program stops only if there is another problem.
423
424    @Input:
425        directory: string
426            The directory name including the path which should be created.
427
428    @Return:
429        <nothing>
430    '''
431    try:
432        os.makedirs(directory)
433    except OSError as e:
434        if e.errno != errno.EEXIST:
435            # errno.EEXIST = directory already exists
436            raise # re-raise exception if a different error occured
437        else:
438            print('WARNING: Directory {0} already exists!'.format(directory))
439
440    return
441
442def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
443    '''
444    @Description:
445        Uses the ecaccess-file-put command to send a file to the ECMWF servers.
446
447        NOTE:
448        The return value is just for testing reasons. It does not have
449        to be used from the calling function since the whole error handling
450        is done in here.
451
452    @Input:
453        ecd: string
454            The path were the file is stored.
455
456        filename: string
457            The name of the file to send to the ECMWF server.
458
459        target: string
460            The target queue where the file should be sent to.
461
462        ecuid: string
463            The user id on ECMWF server.
464
465        ecgid: string
466            The group id on ECMWF server.
467
468    @Return:
469        rcode: string
470            Resulting code of command execution. If successful the string
471            will be empty.
472    '''
473
474    try:
475        rcode = subprocess.check_output(['ecaccess-file-put',
476                                          ecd + '/' + filename,
477                                          target + ':/home/ms/' +
478                                          ecgid + '/' + ecuid +
479                                          '/' + filename],
480                                         stderr=subprocess.STDOUT)
481    except subprocess.CalledProcessError as e:
482        print('... ERROR CODE:\n ... ' + str(e.returncode))
483        print('... ERROR MESSAGE:\n ... ' + str(e))
484
485        print('\n... Do you have a valid ecaccess certification key?')
486        sys.exit('... ECACCESS-FILE-PUT FAILED!')
487
488    return rcode
489
490def submit_job_to_ecserver(target, jobname):
491    '''
492    @Description:
493        Uses ecaccess-job-submit command to submit a job to the ECMWF server.
494
495        NOTE:
496        The return value is just for testing reasons. It does not have
497        to be used from the calling function since the whole error handling
498        is done in here.
499
500    @Input:
501        target: string
502            The target where the file should be sent to, e.g. the queue.
503
504        jobname: string
505            The name of the jobfile to be submitted to the ECMWF server.
506
507    @Return:
508        rcode: string
509            Resulting code of command execution. If successful the string
510            will contain an integer number, representing the id of the job
511            at the ecmwf server.
512    '''
513
514    try:
515        rcode = subprocess.check_output(['ecaccess-job-submit',
516                                         '-queueName', target,
517                                         jobname])
518    except subprocess.CalledProcessError as e:
519        print('... ERROR CODE:\n ... ' + str(e.returncode))
520        print('... ERROR MESSAGE:\n ... ' + str(e))
521
522
523        print('\n... Do you have a valid ecaccess certification key?')
524        sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
525
526    return rcode
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG