source: flex_extract.git/source/python/mods/tools.py @ 5bad6ec

ctbtodev
Last change on this file since 5bad6ec was 5bad6ec, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

added possibility to extract public datasets via an logical public parameter
  • Property mode set to 100644
File size: 18.6 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Philipp (University of Vienna)
5#
6# @Date: May 2018
7#
8# @Change History:
9#    October 2014 - Anne Fouilloux (University of Oslo)
10#        - created functions silent_remove and product (taken from ECMWF)
11#
12#    November 2015 - Leopold Haimberger (University of Vienna)
13#        - created functions: interpret_args_and_control, clean_up
14#          my_error, normal_exit, init128, to_param_id
15#
16#    April 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added documentation
19#        - moved all functions from file Flexparttools to this file tools
20#        - added function get_list_as_string
21#        - seperated args and control interpretation
22#
23# @License:
24#    (C) Copyright 2014-2018.
25#
26#    This software is licensed under the terms of the Apache Licence Version 2.0
27#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
28#
29# @Modul Description:
30#    This module contains a couple of helpful functions which are
31#    used in different places in flex_extract.
32#
33# @Module Content:
34#    - get_cmdline_arguments
35#    - clean_up
36#    - my_error
37#    - normal_exit
38#    - product
39#    - silent_remove
40#    - init128
41#    - to_param_id
42#    - get_list_as_string
43#    - make_dir
44#
45#*******************************************************************************
46
47# ------------------------------------------------------------------------------
48# MODULES
49# ------------------------------------------------------------------------------
50import os
51import errno
52import sys
53import glob
54import subprocess
55import traceback
56from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
57
58# ------------------------------------------------------------------------------
59# FUNCTIONS
60# ------------------------------------------------------------------------------
61
62def none_or_str(value):
63    '''
64    @Description:
65        Converts the input string into pythons None-type if the string
66        contains "None".
67
68    @Input:
69        value: string
70            String to be checked for the "None" word.
71
72    @Return:
73        None or value:
74            Return depends on the content of the input value. If it was "None",
75            then the python type None is returned. Otherwise the string itself.
76    '''
77    if value == 'None':
78        return None
79    return value
80
81def none_or_int(value):
82    '''
83    @Description:
84        Converts the input string into pythons None-type if the string
85        contains "None". Otherwise it is converted to an integer value.
86
87    @Input:
88        value: string
89            String to be checked for the "None" word.
90
91    @Return:
92        None or int(value):
93            Return depends on the content of the input value. If it was "None",
94            then the python type None is returned. Otherwise the string is
95            converted into an integer value.
96    '''
97    if value == 'None':
98        return None
99    return int(value)
100
101def get_cmdline_arguments():
102    '''
103    @Description:
104        Decomposes the command line arguments and assigns them to variables.
105        Apply default values for non mentioned arguments.
106
107    @Input:
108        <nothing>
109
110    @Return:
111        args: instance of ArgumentParser
112            Contains the commandline arguments from script/program call.
113    '''
114
115    parser = ArgumentParser(description='Retrieve FLEXPART input from \
116                                ECMWF MARS archive',
117                            formatter_class=ArgumentDefaultsHelpFormatter)
118
119    # the most important arguments
120    parser.add_argument("--start_date", dest="start_date",
121                        type=none_or_str, default=None,
122                        help="start date YYYYMMDD")
123    parser.add_argument("--end_date", dest="end_date",
124                        type=none_or_str, default=None,
125                        help="end_date YYYYMMDD")
126    parser.add_argument("--date_chunk", dest="date_chunk",
127                        type=none_or_int, default=None,
128                        help="# of days to be retrieved at once")
129    parser.add_argument("--controlfile", dest="controlfile",
130                        type=none_or_str, default='CONTROL.temp',
131                        help="file with CONTROL parameters")
132
133    # parameter for extra output information
134    parser.add_argument("--debug", dest="debug",
135                        type=none_or_int, default=None,
136                        help="debug mode - leave temporary files intact")
137    parser.add_argument("--request", dest="request",
138                        type=none_or_int, default=None,
139                        help="list all mars request in file mars_requests.dat \
140                        and skip submission to mars")
141    parser.add_argument("--public", dest="public",
142                        type=none_or_int, default=None,
143                        help="public mode - retrieves the public datasets")
144
145    # some arguments that override the default in the CONTROL file
146    parser.add_argument("--basetime", dest="basetime",
147                        type=none_or_int, default=None,
148                        help="base such as 00 or 12 (for half day retrievals)")
149    parser.add_argument("--step", dest="step",
150                        type=none_or_str, default=None,
151                        help="steps such as 00/to/48")
152    parser.add_argument("--levelist", dest="levelist",
153                        type=none_or_str, default=None,
154                        help="Vertical levels to be retrieved, e.g. 30/to/60")
155    parser.add_argument("--area", dest="area",
156                        type=none_or_str, default=None,
157                        help="area defined as north/west/south/east")
158
159    # set the working directories
160    parser.add_argument("--inputdir", dest="inputdir",
161                        type=none_or_str, default=None,
162                        help="root directory for storing intermediate files")
163    parser.add_argument("--outputdir", dest="outputdir",
164                        type=none_or_str, default=None,
165                        help="root directory for storing output files")
166    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
167                        type=none_or_str, default=None,
168                        help="FLEXPART root directory (to find grib2flexpart \
169                        and COMMAND file)\n Normally flex_extract resides in \
170                        the scripts directory of the FLEXPART distribution")
171
172    # this is only used by prepare_flexpart.py to rerun a postprocessing step
173    parser.add_argument("--ppid", dest="ppid",
174                        type=none_or_int, default=None,
175                        help="specify parent process id for \
176                        rerun of prepare_flexpart")
177
178    # arguments for job submission to ECMWF, only needed by submit.py
179    parser.add_argument("--job_template", dest='job_template',
180                        type=none_or_str, default="job.temp",
181                        help="job template file for submission to ECMWF")
182    parser.add_argument("--queue", dest="queue",
183                        type=none_or_str, default=None,
184                        help="queue for submission to ECMWF \
185                        (e.g. ecgate or cca )")
186
187    args = parser.parse_args()
188
189    return args
190
191def read_ecenv(filename):
192    '''
193    @Description:
194        Reads the file into a dictionary where the key values are the parameter
195        names.
196
197    @Input:
198        filename: string
199            Path to file where the ECMWV environment parameters are stored.
200
201    @Return:
202        envs: dict
203            Contains the environment parameter ecuid, ecgid, gateway
204            and destination for ECMWF server environments.
205    '''
206    envs= {}
207
208    with open(filename, 'r') as f:
209        for line in f:
210            data = line.strip().split()
211            envs[str(data[0])] = str(data[1])
212
213    return envs
214
215def clean_up(c):
216    '''
217    @Description:
218        Remove all files from intermediate directory
219        (inputdir from CONTROL file).
220
221    @Input:
222        c: instance of class ControlFile
223            Contains all the parameters of CONTROL file and
224            command line.
225            For more information about format and content of the parameter
226            see documentation.
227
228    @Return:
229        <nothing>
230    '''
231
232    print("clean_up")
233
234    cleanlist = glob.glob(c.inputdir + "/*")
235    for clist in cleanlist:
236        if c.prefix not in clist:
237            silent_remove(clist)
238        if c.ecapi is False and (c.ectrans == '1' or c.ecstorage == '1'):
239            silent_remove(clist)
240
241    print("Done")
242
243    return
244
245
246def my_error(users, message='ERROR'):
247    '''
248    @Description:
249        Prints a specified error message which can be passed to the function
250        before exiting the program.
251
252    @Input:
253        user: list of strings
254            Contains all email addresses which should be notified.
255            It might also contain just the ecmwf user name which wil trigger
256            mailing to the associated email address for this user.
257
258        message: string, optional
259            Error message. Default value is "ERROR".
260
261    @Return:
262        <nothing>
263    '''
264
265    print(message)
266
267    # comment if user does not want email notification directly from python
268    for user in users:
269        if '${USER}' in user:
270            user = os.getenv('USER')
271        try:
272            p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ERROR',
273                                  os.path.expandvars(user)],
274                                 stdin=subprocess.PIPE,
275                                 stdout=subprocess.PIPE,
276                                 stderr=subprocess.PIPE,
277                                 bufsize=1)
278            trace = '\n'.join(traceback.format_stack())
279            pout = p.communicate(input=message + '\n\n' + trace)[0]
280        except ValueError as e:
281            print('ERROR: ', e)
282            sys.exit('Email could not be sent!')
283        else:
284            print('Email sent to ' + os.path.expandvars(user) + ' ' +
285                  pout.decode())
286
287    sys.exit(1)
288
289    return
290
291
292def normal_exit(users, message='Done!'):
293    '''
294    @Description:
295        Prints a specific exit message which can be passed to the function.
296
297    @Input:
298        user: list of strings
299            Contains all email addresses which should be notified.
300            It might also contain just the ecmwf user name which wil trigger
301            mailing to the associated email address for this user.
302
303        message: string, optional
304            Message for exiting program. Default value is "Done!".
305
306    @Return:
307        <nothing>
308
309    '''
310    print(message)
311
312    # comment if user does not want notification directly from python
313    for user in users:
314        if '${USER}' in user:
315            user = os.getenv('USER')
316        try:
317            p = subprocess.Popen(['mail', '-s flex_extract_v7.1 normal exit',
318                                  os.path.expandvars(user)],
319                                 stdin=subprocess.PIPE,
320                                 stdout=subprocess.PIPE,
321                                 stderr=subprocess.PIPE,
322                                 bufsize=1)
323            pout = p.communicate(input=message+'\n\n')[0]
324        except ValueError as e:
325            print('ERROR: ', e)
326            print('Email could not be sent!')
327        else:
328            print('Email sent to ' + os.path.expandvars(user) + ' ' +
329                  pout.decode())
330
331    return
332
333
334def product(*args, **kwds):
335    '''
336    @Description:
337        This method is taken from an example at the ECMWF wiki website.
338        https://software.ecmwf.int/wiki/display/GRIB/index.py; 2018-03-16
339
340        This method combines the single characters of the passed arguments
341        with each other. So that each character of each argument value
342        will be combined with each character of the other arguments as a tuple.
343
344        Example:
345        product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
346        product(range(2), repeat = 3) --> 000 001 010 011 100 101 110 111
347
348    @Input:
349        *args: tuple
350            Positional arguments (arbitrary number).
351
352        **kwds: dictionary
353            Contains all the keyword arguments from *args.
354
355    @Return:
356        prod: tuple
357            Return will be done with "yield". A tuple of combined arguments.
358            See example in description above.
359    '''
360    pools = map(tuple, args) * kwds.get('repeat', 1)
361    result = [[]]
362    for pool in pools:
363        result = [x + [y] for x in result for y in pool]
364    for prod in result:
365        yield tuple(prod)
366
367    return
368
369
370def silent_remove(filename):
371    '''
372    @Description:
373        Remove file if it exists.
374        The function does not fail if the file does not exist.
375
376    @Input:
377        filename: string
378            The name of the file to be removed without notification.
379
380    @Return:
381        <nothing>
382    '''
383    try:
384        os.remove(filename)
385    except OSError as e:
386        if e.errno != errno.ENOENT:
387            # errno.ENOENT  =  no such file or directory
388            raise  # re-raise exception if a different error occured
389
390    return
391
392
393def init128(filepath):
394    '''
395    @Description:
396        Opens and reads the grib file with table 128 information.
397
398    @Input:
399        filepath: string
400            Path to file of ECMWF grib table number 128.
401
402    @Return:
403        table128: dictionary
404            Contains the ECMWF grib table 128 information.
405            The key is the parameter number and the value is the
406            short name of the parameter.
407    '''
408    table128 = dict()
409    with open(filepath) as f:
410        fdata = f.read().split('\n')
411    for data in fdata:
412        if data[0] != '!':
413            table128[data[0:3]] = data[59:64].strip()
414
415    return table128
416
417
418def to_param_id(pars, table):
419    '''
420    @Description:
421        Transform parameter names to parameter ids
422        with ECMWF grib table 128.
423
424    @Input:
425        pars: string
426            Addpar argument from CONTROL file in the format of
427            parameter names instead of ids. The parameter short
428            names are sepearted with "/" and they are passed as
429            one single string.
430
431        table: dictionary
432            Contains the ECMWF grib table 128 information.
433            The key is the parameter number and the value is the
434            short name of the parameter.
435
436    @Return:
437        ipar: list of integer
438            List of addpar parameters from CONTROL file transformed to
439            parameter ids in the format of integer.
440    '''
441    cpar = pars.upper().split('/')
442    ipar = []
443    for par in cpar:
444        for k, v in table.iteritems():
445            if par == k or par == v:
446                ipar.append(int(k))
447                break
448        else:
449            print('Warning: par ' + par + ' not found in table 128')
450
451    return ipar
452
453def get_list_as_string(list_obj, concatenate_sign=', '):
454    '''
455    @Description:
456        Converts a list of arbitrary content into a single string.
457
458    @Input:
459        list_obj: list
460            A list with arbitrary content.
461
462        concatenate_sign: string, optional
463            A string which is used to concatenate the single
464            list elements. Default value is ", ".
465
466    @Return:
467        str_of_list: string
468            The content of the list as a single string.
469    '''
470
471    str_of_list = concatenate_sign.join(str(l) for l in list_obj)
472
473    return str_of_list
474
475def make_dir(directory):
476    '''
477    @Description:
478        Creates a directory and gives a warning if the directory
479        already exists. The program stops only if there is another problem.
480
481    @Input:
482        directory: string
483            The directory name including the path which should be created.
484
485    @Return:
486        <nothing>
487    '''
488    try:
489        os.makedirs(directory)
490    except OSError as e:
491        if e.errno != errno.EEXIST:
492            # errno.EEXIST = directory already exists
493            raise # re-raise exception if a different error occured
494        else:
495            print('WARNING: Directory {0} already exists!'.format(directory))
496
497    return
498
499def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
500    '''
501    @Description:
502        Uses the ecaccess-file-put command to send a file to the ECMWF servers.
503
504        NOTE:
505        The return value is just for testing reasons. It does not have
506        to be used from the calling function since the whole error handling
507        is done in here.
508
509    @Input:
510        ecd: string
511            The path were the file is stored.
512
513        filename: string
514            The name of the file to send to the ECMWF server.
515
516        target: string
517            The target queue where the file should be sent to.
518
519        ecuid: string
520            The user id on ECMWF server.
521
522        ecgid: string
523            The group id on ECMWF server.
524
525    @Return:
526        rcode: string
527            Resulting code of command execution. If successful the string
528            will be empty.
529    '''
530
531    try:
532        rcode = subprocess.check_output(['ecaccess-file-put',
533                                          ecd + '/' + filename,
534                                          target + ':/home/ms/' +
535                                          ecgid + '/' + ecuid +
536                                          '/' + filename],
537                                         stderr=subprocess.STDOUT)
538    except subprocess.CalledProcessError as e:
539        print('... ERROR CODE:\n ... ' + str(e.returncode))
540        print('... ERROR MESSAGE:\n ... ' + str(e))
541
542        print('\n... Do you have a valid ecaccess certification key?')
543        sys.exit('... ECACCESS-FILE-PUT FAILED!')
544
545    return rcode
546
547def submit_job_to_ecserver(target, jobname):
548    '''
549    @Description:
550        Uses ecaccess-job-submit command to submit a job to the ECMWF server.
551
552        NOTE:
553        The return value is just for testing reasons. It does not have
554        to be used from the calling function since the whole error handling
555        is done in here.
556
557    @Input:
558        target: string
559            The target where the file should be sent to, e.g. the queue.
560
561        jobname: string
562            The name of the jobfile to be submitted to the ECMWF server.
563
564    @Return:
565        rcode: string
566            Resulting code of command execution. If successful the string
567            will contain an integer number, representing the id of the job
568            at the ecmwf server.
569    '''
570
571    try:
572        rcode = subprocess.check_output(['ecaccess-job-submit',
573                                         '-queueName', target,
574                                         jobname])
575    except subprocess.CalledProcessError as e:
576        print('... ERROR CODE:\n ... ' + str(e.returncode))
577        print('... ERROR MESSAGE:\n ... ' + str(e))
578
579
580        print('\n... Do you have a valid ecaccess certification key?')
581        sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
582
583    return rcode
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG