source: flex_extract.git/source/python/mods/tools.py @ b2e95c3

ctbtodev
Last change on this file since b2e95c3 was b2e95c3, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

removed unused method
  • Property mode set to 100644
File size: 19.4 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Philipp (University of Vienna)
5#
6# @Date: May 2018
7#
8# @Change History:
9#    October 2014 - Anne Fouilloux (University of Oslo)
10#        - created functions silent_remove and product (taken from ECMWF)
11#
12#    November 2015 - Leopold Haimberger (University of Vienna)
13#        - created functions: interpret_args_and_control, clean_up
14#          my_error, normal_exit, init128, to_param_id
15#
16#    April 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added documentation
19#        - moved all functions from file Flexparttools to this file tools
20#        - added function get_list_as_string
21#        - seperated args and control interpretation
22#
23# @License:
24#    (C) Copyright 2014-2018.
25#
26#    This software is licensed under the terms of the Apache Licence Version 2.0
27#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
28#
29# @Modul Description:
30#    This module contains a couple of helpful functions which are
31#    used in different places in flex_extract.
32#
33# @Module Content:
34#    - get_cmdline_arguments
35#    - clean_up
36#    - my_error
37#    - normal_exit
38#    - product
39#    - silent_remove
40#    - init128
41#    - to_param_id
42#    - get_list_as_string
43#    - make_dir
44#
45#*******************************************************************************
46
47# ------------------------------------------------------------------------------
48# MODULES
49# ------------------------------------------------------------------------------
50import os
51import errno
52import sys
53import glob
54import subprocess
55import traceback
56import exceptions
57from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
58
59# ------------------------------------------------------------------------------
60# FUNCTIONS
61# ------------------------------------------------------------------------------
62
63def none_or_str(value):
64    '''Converts the input string into pythons None-type if the string
65    contains string "None".
66
67    Parameters
68    ----------
69    value : :obj:`string`
70        String to be checked for the "None" word.
71
72    Return
73    ------
74    None or value:
75        Return depends on the content of the input value. If it was "None",
76        then the python type None is returned. Otherwise the string itself.
77    '''
78    if value == 'None':
79        return None
80    return value
81
82def none_or_int(value):
83    '''Converts the input string into pythons None-type if the string
84    contains string "None". Otherwise it is converted to an integer value.
85
86    Parameters
87    ----------
88    value : :obj:`string`
89        String to be checked for the "None" word.
90
91    Return
92    ------
93    None or int(value):
94        Return depends on the content of the input value. If it was "None",
95        then the python type None is returned. Otherwise the string is
96        converted into an integer value.
97    '''
98    if value == 'None':
99        return None
100    return int(value)
101
102
103def get_cmdline_arguments():
104    '''Decomposes the command line arguments and assigns them to variables.
105    Apply default values for non mentioned arguments.
106
107    Parameters
108    ----------
109
110    Return
111    ------
112    args : :obj:`Namespace`
113        Contains the commandline arguments from script/program call.
114    '''
115
116    parser = ArgumentParser(description='Retrieve FLEXPART input from \
117                                ECMWF MARS archive',
118                            formatter_class=ArgumentDefaultsHelpFormatter)
119
120    # the most important arguments
121    parser.add_argument("--start_date", dest="start_date",
122                        type=none_or_str, default=None,
123                        help="start date YYYYMMDD")
124    parser.add_argument("--end_date", dest="end_date",
125                        type=none_or_str, default=None,
126                        help="end_date YYYYMMDD")
127    parser.add_argument("--date_chunk", dest="date_chunk",
128                        type=none_or_int, default=None,
129                        help="# of days to be retrieved at once")
130    parser.add_argument("--controlfile", dest="controlfile",
131                        type=none_or_str, default='CONTROL.temp',
132                        help="file with CONTROL parameters")
133
134    # parameter for extra output information
135    parser.add_argument("--debug", dest="debug",
136                        type=none_or_int, default=None,
137                        help="debug mode - leave temporary files intact")
138    parser.add_argument("--request", dest="request",
139                        type=none_or_int, default=None,
140                        help="list all mars request in file mars_requests.dat \
141                        and skip submission to mars")
142    parser.add_argument("--public", dest="public",
143                        type=none_or_int, default=None,
144                        help="public mode - retrieves the public datasets")
145
146    # some arguments that override the default in the CONTROL file
147    parser.add_argument("--basetime", dest="basetime",
148                        type=none_or_int, default=None,
149                        help="base such as 00 or 12 (for half day retrievals)")
150    parser.add_argument("--step", dest="step",
151                        type=none_or_str, default=None,
152                        help="steps such as 00/to/48")
153    parser.add_argument("--levelist", dest="levelist",
154                        type=none_or_str, default=None,
155                        help="Vertical levels to be retrieved, e.g. 30/to/60")
156    parser.add_argument("--area", dest="area",
157                        type=none_or_str, default=None,
158                        help="area defined as north/west/south/east")
159
160    # set the working directories
161    parser.add_argument("--inputdir", dest="inputdir",
162                        type=none_or_str, default=None,
163                        help="root directory for storing intermediate files")
164    parser.add_argument("--outputdir", dest="outputdir",
165                        type=none_or_str, default=None,
166                        help="root directory for storing output files")
167    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
168                        type=none_or_str, default=None,
169                        help="FLEXPART root directory (to find grib2flexpart \
170                        and COMMAND file)\n Normally flex_extract resides in \
171                        the scripts directory of the FLEXPART distribution")
172
173    # this is only used by prepare_flexpart.py to rerun a postprocessing step
174    parser.add_argument("--ppid", dest="ppid",
175                        type=none_or_str, default=None,
176                        help="specify parent process id for \
177                        rerun of prepare_flexpart")
178
179    # arguments for job submission to ECMWF, only needed by submit.py
180    parser.add_argument("--job_template", dest='job_template',
181                        type=none_or_str, default="job.temp",
182                        help="job template file for submission to ECMWF")
183    parser.add_argument("--queue", dest="queue",
184                        type=none_or_str, default=None,
185                        help="queue for submission to ECMWF \
186                        (e.g. ecgate or cca )")
187
188    args = parser.parse_args()
189
190    return args
191
192def read_ecenv(filepath):
193    '''Reads the file into a dictionary where the key values are the parameter
194    names.
195
196    Parameters
197    ----------
198    filepath : :obj:`string`
199        Path to file where the ECMWF environment parameters are stored.
200
201    Return
202    ------
203    envs : :obj:`dictionary`
204        Contains the environment parameter ecuid, ecgid, gateway
205        and destination for ECMWF server environments.
206    '''
207    envs= {}
208    try:
209        with open(filepath, 'r') as f:
210            for line in f:
211                data = line.strip().split()
212                envs[str(data[0])] = str(data[1])
213    except OSError as e:
214        print('... ERROR CODE: ' + str(e.errno))
215        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
216
217        sys.exit('\n... Error occured while trying to read ECMWF_ENV '
218                     'file: ' + str(filepath))
219
220    return envs
221
222def clean_up(c):
223    '''Remove files from the intermediate directory (inputdir).
224
225    It keeps the final FLEXPART input files if program runs without
226    ECMWF Api and keywords "ectrans" or "ecstorage" are set to "1".
227
228    Parameters
229    ----------
230    c : :obj:`ControlFile`
231        Contains all the parameters of CONTROL file and
232        command line.
233
234    Return
235    ------
236
237    '''
238
239    print("... clean inputdir!")
240
241    cleanlist = glob.glob(os.path.join(c.inputdir, "*"))
242
243    if cleanlist:
244        for element in cleanlist:
245            if c.prefix not in element:
246                silent_remove(element)
247            if c.ecapi is False and (c.ectrans == 1 or c.ecstorage == 1):
248                silent_remove(element)
249        print("... done!")
250    else:
251        print("... nothing to clean!")
252
253    return
254
255
256def my_error(users, message='ERROR'):
257    '''Prints a specified error message which can be passed to the function
258    before exiting the program.
259
260    Parameters
261    ----------
262    user : :obj:`list` of :obj:`string`
263        Contains all email addresses which should be notified.
264        It might also contain just the ecmwf user name which wil trigger
265        mailing to the associated email address for this user.
266
267    message : :obj:`string`, optional
268        Error message. Default value is "ERROR".
269
270    Return
271    ------
272
273    '''
274
275    trace = '\n'.join(traceback.format_stack())
276    full_message = message + '\n\n' + trace
277
278    print(full_message)
279
280    send_mail(users, 'ERROR', full_message)
281
282    sys.exit(1)
283
284    return
285
286
287def send_mail(users, success_mode, message):
288    '''Prints a specific exit message which can be passed to the function.
289
290    Parameters
291    ----------
292    users : :obj:`list` of :obj:`string`
293        Contains all email addresses which should be notified.
294        It might also contain just the ecmwf user name which wil trigger
295        mailing to the associated email address for this user.
296
297    success_mode : :obj:``string`
298        States the exit mode of the program to put into
299        the mail subject line.
300
301    message : :obj:`string`, optional
302        Message for exiting program. Default value is "Done!".
303
304    Return
305    ------
306
307    '''
308
309    for user in users:
310        if '${USER}' in user:
311            user = os.getenv('USER')
312        try:
313            p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ' +
314                                  success_mode, os.path.expandvars(user)],
315                                 stdin=subprocess.PIPE,
316                                 stdout=subprocess.PIPE,
317                                 stderr=subprocess.PIPE,
318                                 bufsize=1)
319            pout = p.communicate(input=message + '\n\n')[0]
320        except ValueError as e:
321            print('... ERROR: ' + str(e))
322            sys.exit('... Email could not be sent!')
323        except OSError as e:
324            print('... ERROR CODE: ' + str(e.errno))
325            print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
326            sys.exit('... Email could not be sent!')
327        else:
328            print('Email sent to ' + os.path.expandvars(user))
329
330    return
331
332
333def normal_exit(message='Done!'):
334    '''Prints a specific exit message which can be passed to the function.
335
336    Parameters
337    ----------
338    message : :obj:`string`, optional
339        Message for exiting program. Default value is "Done!".
340
341    Return
342    ------
343
344    '''
345
346    print(str(message))
347
348    return
349
350
351def product(*args, **kwds):
352    '''Creates combinations of all passed arguments.
353
354    This method combines the single characters of the passed arguments
355    with each other. So that each character of each argument value
356    will be combined with each character of the other arguments as a tuple.
357
358    Note
359    ----
360    This method is taken from an example at the ECMWF wiki website.
361    https://software.ecmwf.int/wiki/display/GRIB/index.py; 2018-03-16
362
363    Example
364    -------
365    product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
366
367    product(range(2), repeat = 3) --> 000 001 010 011 100 101 110 111
368
369    Parameters
370    ----------
371    \*args : :obj:`list` or :obj:`string`
372        Positional arguments (arbitrary number).
373
374    \*\*kwds : :obj:`dictionary`
375        Contains all the keyword arguments from \*args.
376
377    Return
378    ------
379    prod : :obj:`tuple`
380        Return will be done with "yield". A tuple of combined arguments.
381        See example in description above.
382    '''
383    try:
384        pools = map(tuple, args) * kwds.get('repeat', 1)
385        result = [[]]
386        for pool in pools:
387            result = [x + [y] for x in result for y in pool]
388        for prod in result:
389            yield tuple(prod)
390    except TypeError as e:
391        sys.exit('... PRODUCT GENERATION FAILED!')
392
393    return
394
395
396def silent_remove(filename):
397    '''Remove file if it exists.
398    The function does not fail if the file does not exist.
399
400    Parameters
401    ----------
402    filename : :obj:`string`
403        The name of the file to be removed without notification.
404
405    Return
406    ------
407
408    '''
409    try:
410        os.remove(filename)
411    except OSError as e:
412        # errno.ENOENT  =  no such file or directory
413        if e.errno == errno.ENOENT:
414            pass
415        else:
416            raise  # re-raise exception if a different error occured
417
418    return
419
420
421def init128(filepath):
422    '''Opens and reads the grib file with table 128 information.
423
424    Parameters
425    ----------
426    filepath : :obj:`string`
427        Path to file of ECMWF grib table number 128.
428
429    Return
430    ------
431    table128 : :obj:`dictionary`
432        Contains the ECMWF grib table 128 information.
433        The key is the parameter number and the value is the
434        short name of the parameter.
435    '''
436    table128 = dict()
437    try:
438        with open(filepath) as f:
439            fdata = f.read().split('\n')
440    except OSError as e:
441        print('... ERROR CODE: ' + str(e.errno))
442        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
443
444        sys.exit('\n... Error occured while trying to read parameter '
445                 'table file: ' + str(filepath))
446    else:
447        for data in fdata:
448            if data[0] != '!':
449                table128[data[0:3]] = data[59:64].strip()
450
451    return table128
452
453
454def to_param_id(pars, table):
455    '''Transform parameter names to parameter ids with ECMWF grib table 128.
456
457    Parameters
458    ----------
459    pars : :obj:`string`
460        Addpar argument from CONTROL file in the format of
461        parameter names instead of ids. The parameter short
462        names are sepearted with "/" and they are passed as
463        one single string.
464
465    table : :obj:`dictionary`
466        Contains the ECMWF grib table 128 information.
467        The key is the parameter number and the value is the
468        short name of the parameter.
469
470    Return
471    ------
472    ipar : :obj:`list` of :obj:`integer`
473        List of addpar parameters from CONTROL file transformed to
474        parameter ids in the format of integer.
475    '''
476    if not pars:
477        return []
478    if not isinstance(pars, str):
479        pars=str(pars)
480
481    cpar = pars.upper().split('/')
482    ipar = []
483    for par in cpar:
484        for k, v in table.iteritems():
485            if par == k or par == v:
486                ipar.append(int(k))
487                break
488        else:
489            print('Warning: par ' + par + ' not found in table 128')
490
491    return ipar
492
493def get_list_as_string(list_obj, concatenate_sign=', '):
494    '''Converts a list of arbitrary content into a single string.
495
496    Parameters
497    ----------
498    list_obj : :obj:`list`
499        A list with arbitrary content.
500
501    concatenate_sign : :obj:`string`, optional
502        A string which is used to concatenate the single
503        list elements. Default value is ", ".
504
505    Return
506    ------
507    str_of_list : :obj:`string`
508        The content of the list as a single string.
509    '''
510
511    if not isinstance(list_obj, list):
512        list_obj = list(list_obj)
513    str_of_list = concatenate_sign.join(str(l) for l in list_obj)
514
515    return str_of_list
516
517def make_dir(directory):
518    '''Creates a directory.
519
520    It gives a warning if the directory already exists and skips process.
521    The program stops only if there is another problem.
522
523    Parameters
524    ----------
525    directory : :obj:`string`
526        The path to directory which should be created.
527
528    Return
529    ------
530
531    '''
532    try:
533        os.makedirs(directory)
534    except OSError as e:
535        # errno.EEXIST = directory already exists
536        if e.errno == errno.EEXIST:
537            print('WARNING: Directory {0} already exists!'.format(directory))
538        else:
539            raise # re-raise exception if a different error occured
540
541    return
542
543def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
544    '''Uses the ecaccess-file-put command to send a file to the ECMWF servers.
545
546    Note
547    ----
548    The return value is just for testing reasons. It does not have
549    to be used from the calling function since the whole error handling
550    is done in here.
551
552    Parameters
553    ----------
554    ecd : :obj:`string`
555        The path were the file is stored.
556
557    filename : :obj:`string`
558        The name of the file to send to the ECMWF server.
559
560    target : :obj:`string`
561        The target queue where the file should be sent to.
562
563    ecuid : :obj:`string`
564        The user id on ECMWF server.
565
566    ecgid : :obj:`string`
567        The group id on ECMWF server.
568
569    Return
570    ------
571
572    '''
573
574    try:
575        subprocess.check_output(['ecaccess-file-put',
576                                 ecd + '/' + filename,
577                                 target + ':/home/ms/' +
578                                 ecgid + '/' + ecuid +
579                                 '/' + filename],
580                                stderr=subprocess.STDOUT)
581    except subprocess.CalledProcessError as e:
582        print('... ERROR CODE: ' + str(e.returncode))
583        print('... ERROR MESSAGE:\n \t ' + str(e))
584
585        print('\n... Do you have a valid ecaccess certification key?')
586        sys.exit('... ECACCESS-FILE-PUT FAILED!')
587    except OSError as e:
588        print('... ERROR CODE: ' + str(e.errno))
589        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
590
591        print('\n... Most likely the ECACCESS library is not available!')
592        sys.exit('... ECACCESS-FILE-PUT FAILED!')
593
594    return
595
596def submit_job_to_ecserver(target, jobname):
597    '''Uses ecaccess-job-submit command to submit a job to the ECMWF server.
598
599    Note
600    ----
601    The return value is just for testing reasons. It does not have
602    to be used from the calling function since the whole error handling
603    is done in here.
604
605    Parameters
606    ----------
607    target : :obj:`string`
608        The target where the file should be sent to, e.g. the queue.
609
610    jobname : :obj:`string`
611        The name of the jobfile to be submitted to the ECMWF server.
612
613    Return
614    ------
615    job_id : :obj:`int`
616        The id number of the job as a reference at the ecmwf server.
617    '''
618
619    try:
620        job_id = subprocess.check_output(['ecaccess-job-submit', '-queueName',
621                                          target, jobname])
622
623    except subprocess.CalledProcessError as e:
624        print('... ERROR CODE: ' + str(e.returncode))
625        print('... ERROR MESSAGE:\n \t ' + str(e))
626
627        print('\n... Do you have a valid ecaccess certification key?')
628        sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
629    except OSError as e:
630        print('... ERROR CODE: ' + str(e.errno))
631        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
632
633        print('\n... Most likely the ECACCESS library is not available!')
634        sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
635
636    return job_id
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG