source: flex_extract.git/source/python/mods/tools.py @ 2d56c04

devfeature/makefilesorigin/task/language-editingtask/language-editing
Last change on this file since 2d56c04 was 2d56c04, checked in by Anne Philipp <anne.philipp@…>, 21 months ago

automatic detection of grid and area component formats (1/1000 or normal degree format)

  • Property mode set to 100644
File size: 19.5 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Philipp (University of Vienna)
5#
6# @Date: May 2018
7#
8# @Change History:
9#    October 2014 - Anne Fouilloux (University of Oslo)
10#        - created functions silent_remove and product (taken from ECMWF)
11#
12#    November 2015 - Leopold Haimberger (University of Vienna)
13#        - created functions: interpret_args_and_control, clean_up
14#          my_error, normal_exit, init128, to_param_id
15#
16#    April 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added documentation
19#        - moved all functions from file Flexparttools to this file tools
20#        - added function get_list_as_string
21#        - seperated args and control interpretation
22#
23# @License:
24#    (C) Copyright 2014-2018.
25#
26#    This software is licensed under the terms of the Apache Licence Version 2.0
27#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
28#
29# @Modul Description:
30#    This module contains a couple of helpful functions which are
31#    used in different places in flex_extract.
32#
33# @Module Content:
34#    - get_cmdline_arguments
35#    - clean_up
36#    - my_error
37#    - normal_exit
38#    - product
39#    - silent_remove
40#    - init128
41#    - to_param_id
42#    - get_list_as_string
43#    - make_dir
44#
45#*******************************************************************************
46
47# ------------------------------------------------------------------------------
48# MODULES
49# ------------------------------------------------------------------------------
50import os
51import errno
52import sys
53import glob
54import subprocess
55import traceback
56import exceptions
57from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
58
59# ------------------------------------------------------------------------------
60# FUNCTIONS
61# ------------------------------------------------------------------------------
62
63def none_or_str(value):
64    '''Converts the input string into pythons None-type if the string
65    contains string "None".
66
67    Parameters
68    ----------
69    value : :obj:`string`
70        String to be checked for the "None" word.
71
72    Return
73    ------
74    None or value:
75        Return depends on the content of the input value. If it was "None",
76        then the python type None is returned. Otherwise the string itself.
77    '''
78    if value == 'None':
79        return None
80    return value
81
82def none_or_int(value):
83    '''Converts the input string into pythons None-type if the string
84    contains string "None". Otherwise it is converted to an integer value.
85
86    Parameters
87    ----------
88    value : :obj:`string`
89        String to be checked for the "None" word.
90
91    Return
92    ------
93    None or int(value):
94        Return depends on the content of the input value. If it was "None",
95        then the python type None is returned. Otherwise the string is
96        converted into an integer value.
97    '''
98    if value == 'None':
99        return None
100    return int(value)
101
102def check_filepattern(filename):
103    '''
104    '''
105    if '.ksh' in filename:
106        return True
107    return False
108
109def get_cmdline_arguments():
110    '''Decomposes the command line arguments and assigns them to variables.
111    Apply default values for non mentioned arguments.
112
113    Parameters
114    ----------
115
116    Return
117    ------
118    args : :obj:`Namespace`
119        Contains the commandline arguments from script/program call.
120    '''
121
122    parser = ArgumentParser(description='Retrieve FLEXPART input from \
123                                ECMWF MARS archive',
124                            formatter_class=ArgumentDefaultsHelpFormatter)
125
126    # the most important arguments
127    parser.add_argument("--start_date", dest="start_date",
128                        type=none_or_str, default=None,
129                        help="start date YYYYMMDD")
130    parser.add_argument("--end_date", dest="end_date",
131                        type=none_or_str, default=None,
132                        help="end_date YYYYMMDD")
133    parser.add_argument("--date_chunk", dest="date_chunk",
134                        type=none_or_int, default=None,
135                        help="# of days to be retrieved at once")
136    parser.add_argument("--controlfile", dest="controlfile",
137                        type=none_or_str, default='CONTROL.temp',
138                        help="file with CONTROL parameters")
139
140    # parameter for extra output information
141    parser.add_argument("--debug", dest="debug",
142                        type=none_or_int, default=None,
143                        help="debug mode - leave temporary files intact")
144    parser.add_argument("--request", dest="request",
145                        type=none_or_int, default=None,
146                        help="list all mars request in file mars_requests.dat \
147                        and skip submission to mars")
148    parser.add_argument("--public", dest="public",
149                        type=none_or_int, default=None,
150                        help="public mode - retrieves the public datasets")
151
152    # some arguments that override the default in the CONTROL file
153    parser.add_argument("--basetime", dest="basetime",
154                        type=none_or_int, default=None,
155                        help="base such as 00 or 12 (for half day retrievals)")
156    parser.add_argument("--step", dest="step",
157                        type=none_or_str, default=None,
158                        help="steps such as 00/to/48")
159    parser.add_argument("--levelist", dest="levelist",
160                        type=none_or_str, default=None,
161                        help="Vertical levels to be retrieved, e.g. 30/to/60")
162    parser.add_argument("--area", dest="area",
163                        type=none_or_str, default=None,
164                        help="area defined as north/west/south/east")
165
166    # set the working directories
167    parser.add_argument("--inputdir", dest="inputdir",
168                        type=none_or_str, default=None,
169                        help="root directory for storing intermediate files")
170    parser.add_argument("--outputdir", dest="outputdir",
171                        type=none_or_str, default=None,
172                        help="root directory for storing output files")
173    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
174                        type=none_or_str, default=None,
175                        help="FLEXPART root directory (to find grib2flexpart \
176                        and COMMAND file)\n Normally flex_extract resides in \
177                        the scripts directory of the FLEXPART distribution")
178
179    # this is only used by prepare_flexpart.py to rerun a postprocessing step
180    parser.add_argument("--ppid", dest="ppid",
181                        type=none_or_str, default=None,
182                        help="specify parent process id for \
183                        rerun of prepare_flexpart")
184
185    # arguments for job submission to ECMWF, only needed by submit.py
186    parser.add_argument("--job_template", dest='job_template',
187                        type=none_or_str, default="job.temp",
188                        help="job template file for submission to ECMWF")
189    parser.add_argument("--queue", dest="queue",
190                        type=none_or_str, default=None,
191                        help="queue for submission to ECMWF \
192                        (e.g. ecgate or cca )")
193
194    args = parser.parse_args()
195
196    return args
197
198def read_ecenv(filepath):
199    '''Reads the file into a dictionary where the key values are the parameter
200    names.
201
202    Parameters
203    ----------
204    filepath : :obj:`string`
205        Path to file where the ECMWF environment parameters are stored.
206
207    Return
208    ------
209    envs : :obj:`dictionary`
210        Contains the environment parameter ecuid, ecgid, gateway
211        and destination for ECMWF server environments.
212    '''
213    envs= {}
214    try:
215        with open(filepath, 'r') as f:
216            for line in f:
217                data = line.strip().split()
218                envs[str(data[0])] = str(data[1])
219    except OSError as e:
220        print('... ERROR CODE: ' + str(e.errno))
221        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
222
223        sys.exit('\n... Error occured while trying to read ECMWF_ENV '
224                     'file: ' + str(filepath))
225
226    return envs
227
228def clean_up(c):
229    '''Remove files from the intermediate directory (inputdir).
230
231    It keeps the final FLEXPART input files if program runs without
232    ECMWF Api and keywords "ectrans" or "ecstorage" are set to "1".
233
234    Parameters
235    ----------
236    c : :obj:`ControlFile`
237        Contains all the parameters of CONTROL file and
238        command line.
239
240    Return
241    ------
242
243    '''
244
245    print("... clean inputdir!")
246
247    cleanlist = glob.glob(os.path.join(c.inputdir, "*"))
248
249    if cleanlist:
250        for element in cleanlist:
251            if c.prefix not in element:
252                silent_remove(element)
253            if c.ecapi is False and (c.ectrans == 1 or c.ecstorage == 1):
254                silent_remove(element)
255        print("... done!")
256    else:
257        print("... nothing to clean!")
258
259    return
260
261
262def my_error(users, message='ERROR'):
263    '''Prints a specified error message which can be passed to the function
264    before exiting the program.
265
266    Parameters
267    ----------
268    user : :obj:`list` of :obj:`string`
269        Contains all email addresses which should be notified.
270        It might also contain just the ecmwf user name which wil trigger
271        mailing to the associated email address for this user.
272
273    message : :obj:`string`, optional
274        Error message. Default value is "ERROR".
275
276    Return
277    ------
278
279    '''
280
281    trace = '\n'.join(traceback.format_stack())
282    full_message = message + '\n\n' + trace
283
284    print(full_message)
285
286    send_mail(users, 'ERROR', full_message)
287
288    sys.exit(1)
289
290    return
291
292
293def send_mail(users, success_mode, message):
294    '''Prints a specific exit message which can be passed to the function.
295
296    Parameters
297    ----------
298    users : :obj:`list` of :obj:`string`
299        Contains all email addresses which should be notified.
300        It might also contain just the ecmwf user name which wil trigger
301        mailing to the associated email address for this user.
302
303    success_mode : :obj:``string`
304        States the exit mode of the program to put into
305        the mail subject line.
306
307    message : :obj:`string`, optional
308        Message for exiting program. Default value is "Done!".
309
310    Return
311    ------
312
313    '''
314
315    for user in users:
316        if '${USER}' in user:
317            user = os.getenv('USER')
318        try:
319            p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ' +
320                                  success_mode, os.path.expandvars(user)],
321                                 stdin=subprocess.PIPE,
322                                 stdout=subprocess.PIPE,
323                                 stderr=subprocess.PIPE,
324                                 bufsize=1)
325            pout = p.communicate(input=message + '\n\n')[0]
326        except ValueError as e:
327            print('... ERROR: ' + str(e))
328            sys.exit('... Email could not be sent!')
329        except OSError as e:
330            print('... ERROR CODE: ' + str(e.errno))
331            print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
332            sys.exit('... Email could not be sent!')
333        else:
334            print('Email sent to ' + os.path.expandvars(user))
335
336    return
337
338
339def normal_exit(message='Done!'):
340    '''Prints a specific exit message which can be passed to the function.
341
342    Parameters
343    ----------
344    message : :obj:`string`, optional
345        Message for exiting program. Default value is "Done!".
346
347    Return
348    ------
349
350    '''
351
352    print(str(message))
353
354    return
355
356
357def product(*args, **kwds):
358    '''Creates combinations of all passed arguments.
359
360    This method combines the single characters of the passed arguments
361    with each other. So that each character of each argument value
362    will be combined with each character of the other arguments as a tuple.
363
364    Note
365    ----
366    This method is taken from an example at the ECMWF wiki website.
367    https://software.ecmwf.int/wiki/display/GRIB/index.py; 2018-03-16
368
369    Example
370    -------
371    product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
372
373    product(range(2), repeat = 3) --> 000 001 010 011 100 101 110 111
374
375    Parameters
376    ----------
377    \*args : :obj:`list` or :obj:`string`
378        Positional arguments (arbitrary number).
379
380    \*\*kwds : :obj:`dictionary`
381        Contains all the keyword arguments from \*args.
382
383    Return
384    ------
385    prod : :obj:`tuple`
386        Return will be done with "yield". A tuple of combined arguments.
387        See example in description above.
388    '''
389    try:
390        pools = map(tuple, args) * kwds.get('repeat', 1)
391        result = [[]]
392        for pool in pools:
393            result = [x + [y] for x in result for y in pool]
394        for prod in result:
395            yield tuple(prod)
396    except TypeError as e:
397        sys.exit('... PRODUCT GENERATION FAILED!')
398
399    return
400
401
402def silent_remove(filename):
403    '''Remove file if it exists.
404    The function does not fail if the file does not exist.
405
406    Parameters
407    ----------
408    filename : :obj:`string`
409        The name of the file to be removed without notification.
410
411    Return
412    ------
413
414    '''
415    try:
416        os.remove(filename)
417    except OSError as e:
418        # errno.ENOENT  =  no such file or directory
419        if e.errno == errno.ENOENT:
420            pass
421        else:
422            raise  # re-raise exception if a different error occured
423
424    return
425
426
427def init128(filepath):
428    '''Opens and reads the grib file with table 128 information.
429
430    Parameters
431    ----------
432    filepath : :obj:`string`
433        Path to file of ECMWF grib table number 128.
434
435    Return
436    ------
437    table128 : :obj:`dictionary`
438        Contains the ECMWF grib table 128 information.
439        The key is the parameter number and the value is the
440        short name of the parameter.
441    '''
442    table128 = dict()
443    try:
444        with open(filepath) as f:
445            fdata = f.read().split('\n')
446    except OSError as e:
447        print('... ERROR CODE: ' + str(e.errno))
448        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
449
450        sys.exit('\n... Error occured while trying to read parameter '
451                 'table file: ' + str(filepath))
452    else:
453        for data in fdata:
454            if data[0] != '!':
455                table128[data[0:3]] = data[59:64].strip()
456
457    return table128
458
459
460def to_param_id(pars, table):
461    '''Transform parameter names to parameter ids with ECMWF grib table 128.
462
463    Parameters
464    ----------
465    pars : :obj:`string`
466        Addpar argument from CONTROL file in the format of
467        parameter names instead of ids. The parameter short
468        names are sepearted with "/" and they are passed as
469        one single string.
470
471    table : :obj:`dictionary`
472        Contains the ECMWF grib table 128 information.
473        The key is the parameter number and the value is the
474        short name of the parameter.
475
476    Return
477    ------
478    ipar : :obj:`list` of :obj:`integer`
479        List of addpar parameters from CONTROL file transformed to
480        parameter ids in the format of integer.
481    '''
482    if not pars:
483        return []
484    if not isinstance(pars, str):
485        pars=str(pars)
486
487    cpar = pars.upper().split('/')
488    ipar = []
489    for par in cpar:
490        for k, v in table.iteritems():
491            if par == k or par == v:
492                ipar.append(int(k))
493                break
494        else:
495            print('Warning: par ' + par + ' not found in table 128')
496
497    return ipar
498
499def get_list_as_string(list_obj, concatenate_sign=', '):
500    '''Converts a list of arbitrary content into a single string.
501
502    Parameters
503    ----------
504    list_obj : :obj:`list`
505        A list with arbitrary content.
506
507    concatenate_sign : :obj:`string`, optional
508        A string which is used to concatenate the single
509        list elements. Default value is ", ".
510
511    Return
512    ------
513    str_of_list : :obj:`string`
514        The content of the list as a single string.
515    '''
516
517    if not isinstance(list_obj, list):
518        list_obj = list(list_obj)
519    str_of_list = concatenate_sign.join(str(l) for l in list_obj)
520
521    return str_of_list
522
523def make_dir(directory):
524    '''Creates a directory.
525
526    It gives a warning if the directory already exists and skips process.
527    The program stops only if there is another problem.
528
529    Parameters
530    ----------
531    directory : :obj:`string`
532        The path to directory which should be created.
533
534    Return
535    ------
536
537    '''
538    try:
539        os.makedirs(directory)
540    except OSError as e:
541        # errno.EEXIST = directory already exists
542        if e.errno == errno.EEXIST:
543            print('WARNING: Directory {0} already exists!'.format(directory))
544        else:
545            raise # re-raise exception if a different error occured
546
547    return
548
549def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
550    '''Uses the ecaccess-file-put command to send a file to the ECMWF servers.
551
552    Note
553    ----
554    The return value is just for testing reasons. It does not have
555    to be used from the calling function since the whole error handling
556    is done in here.
557
558    Parameters
559    ----------
560    ecd : :obj:`string`
561        The path were the file is stored.
562
563    filename : :obj:`string`
564        The name of the file to send to the ECMWF server.
565
566    target : :obj:`string`
567        The target queue where the file should be sent to.
568
569    ecuid : :obj:`string`
570        The user id on ECMWF server.
571
572    ecgid : :obj:`string`
573        The group id on ECMWF server.
574
575    Return
576    ------
577
578    '''
579
580    try:
581        subprocess.check_output(['ecaccess-file-put',
582                                 ecd + '/' + filename,
583                                 target + ':/home/ms/' +
584                                 ecgid + '/' + ecuid +
585                                 '/' + filename],
586                                stderr=subprocess.STDOUT)
587    except subprocess.CalledProcessError as e:
588        print('... ERROR CODE: ' + str(e.returncode))
589        print('... ERROR MESSAGE:\n \t ' + str(e))
590
591        print('\n... Do you have a valid ecaccess certification key?')
592        sys.exit('... ECACCESS-FILE-PUT FAILED!')
593    except OSError as e:
594        print('... ERROR CODE: ' + str(e.errno))
595        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
596
597        print('\n... Most likely the ECACCESS library is not available!')
598        sys.exit('... ECACCESS-FILE-PUT FAILED!')
599
600    return
601
602def submit_job_to_ecserver(target, jobname):
603    '''Uses ecaccess-job-submit command to submit a job to the ECMWF server.
604
605    Note
606    ----
607    The return value is just for testing reasons. It does not have
608    to be used from the calling function since the whole error handling
609    is done in here.
610
611    Parameters
612    ----------
613    target : :obj:`string`
614        The target where the file should be sent to, e.g. the queue.
615
616    jobname : :obj:`string`
617        The name of the jobfile to be submitted to the ECMWF server.
618
619    Return
620    ------
621    job_id : :obj:`int`
622        The id number of the job as a reference at the ecmwf server.
623    '''
624
625    try:
626        job_id = subprocess.check_output(['ecaccess-job-submit', '-queueName',
627                                          target, jobname])
628
629    except subprocess.CalledProcessError as e:
630        print('... ERROR CODE: ' + str(e.returncode))
631        print('... ERROR MESSAGE:\n \t ' + str(e))
632
633        print('\n... Do you have a valid ecaccess certification key?')
634        sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
635    except OSError as e:
636        print('... ERROR CODE: ' + str(e.errno))
637        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
638
639        print('\n... Most likely the ECACCESS library is not available!')
640        sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
641
642    return job_id
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG