source: flex_extract.git/source/python/mods/tools.py @ 79729d5

ctbtodev
Last change on this file since 79729d5 was 79729d5, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

switched from python2 to python3

  • Property mode set to 100644
File size: 26.2 KB
RevLine 
[79729d5]1#!/usr/bin/env python3
[efdb01a]2# -*- coding: utf-8 -*-
[991df6a]3#*******************************************************************************
4# @Author: Anne Philipp (University of Vienna)
5#
6# @Date: May 2018
7#
8# @Change History:
9#    October 2014 - Anne Fouilloux (University of Oslo)
[ff99eae]10#        - created functions silent_remove and product (taken from ECMWF)
[991df6a]11#
12#    November 2015 - Leopold Haimberger (University of Vienna)
[ff99eae]13#        - created functions: interpret_args_and_control, clean_up
14#          my_error, normal_exit, init128, to_param_id
[991df6a]15#
[6f951ca]16#    April - December 2018 - Anne Philipp (University of Vienna):
[991df6a]17#        - applied PEP8 style guide
18#        - added documentation
[6f951ca]19#        - moved all non class methods from former file Flexparttools in here
[54a8a01]20#        - seperated args and control interpretation
[6f951ca]21#        - added functions get_list_as_string, read_ecenv, send_mail, make_dir,
22#          put_file_to_ecserver, submit_job_to_ecserver, get_informations,
23#          get_dimensions, execute_subprocess, none_or_int, none_or_str
[991df6a]24#
25# @License:
[6f951ca]26#    (C) Copyright 2014-2019.
27#    Anne Philipp, Leopold Haimberger
[991df6a]28#
[6f951ca]29#    This work is licensed under the Creative Commons Attribution 4.0
30#    International License. To view a copy of this license, visit
31#    http://creativecommons.org/licenses/by/4.0/ or send a letter to
32#    Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
[991df6a]33#
[6f951ca]34# @Methods:
35#    none_or_str
36#    none_or_int
37#    get_cmdline_args
38#    read_ecenv
39#    clean_up
40#    my_error
41#    send_mail
42#    normal_exit
43#    product
44#    silent_remove
45#    init128
46#    to_param_id
47#    get_list_as_string
48#    make_dir
49#    put_file_to_ecserver
50#    submit_job_to_ecserver
51#    get_informations
52#    get_dimensions
53#    execute_subprocess
[991df6a]54#*******************************************************************************
[6f951ca]55'''This module contains a collection of diverse tasks within flex_extract.
56'''
[efdb01a]57
58# ------------------------------------------------------------------------------
59# MODULES
60# ------------------------------------------------------------------------------
61import os
62import errno
63import sys
64import glob
[ff99eae]65import subprocess
[991df6a]66import traceback
[f20af73]67try:
68    import exceptions
69except ImportError:
70    import builtins as exceptions
[095dc73]71from datetime import datetime
[991df6a]72from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
[efdb01a]73
[f20af73]74
75
[efdb01a]76# ------------------------------------------------------------------------------
[6f951ca]77# METHODS
[efdb01a]78# ------------------------------------------------------------------------------
79
[f20af73]80def setup_controldata():
81    '''Collects, stores and checks controlling arguments from command line,
82    CONTROL file and ECMWF_ENV file.
83
84    Parameters
85    ----------
86
87    Return
88    ------
89    c : ControlFile
90        Contains all the parameters of CONTROL file and
91        command line.
92
93    ppid : str
94        Parent process id.
95
96    queue : str
97        Name of queue for submission to ECMWF (e.g. ecgate or cca )
98
99    job_template : str
100        Name of the job template file for submission to ECMWF server.
101    '''
102    import _config
103    from classes.ControlFile import ControlFile
104
105    args = get_cmdline_args()
106    c = ControlFile(args.controlfile)
107    c.assign_args_to_control(args)
108    if os.path.isfile(_config.PATH_ECMWF_ENV):
109        env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
110        c.assign_envs_to_control(env_parameter)
111    c.check_conditions(args.queue)
112
113    return c, args.ppid, args.queue, args.job_template
114
[ca867de]115def none_or_str(value):
[274f9ef]116    '''Converts the input string into pythons None-type if the string
117    contains string "None".
118
119    Parameters
120    ----------
[6f951ca]121    value : str
[274f9ef]122        String to be checked for the "None" word.
123
124    Return
125    ------
126    None or value:
127        Return depends on the content of the input value. If it was "None",
128        then the python type None is returned. Otherwise the string itself.
[ca867de]129    '''
130    if value == 'None':
131        return None
132    return value
133
134def none_or_int(value):
[274f9ef]135    '''Converts the input string into pythons None-type if the string
136    contains string "None". Otherwise it is converted to an integer value.
137
138    Parameters
139    ----------
[6f951ca]140    value : str
[274f9ef]141        String to be checked for the "None" word.
142
143    Return
144    ------
145    None or int(value):
146        Return depends on the content of the input value. If it was "None",
147        then the python type None is returned. Otherwise the string is
148        converted into an integer value.
[ca867de]149    '''
150    if value == 'None':
151        return None
152    return int(value)
153
[9aefaad]154def get_cmdline_args():
[274f9ef]155    '''Decomposes the command line arguments and assigns them to variables.
156    Apply default values for non mentioned arguments.
[efdb01a]157
[274f9ef]158    Parameters
159    ----------
[efdb01a]160
[274f9ef]161    Return
162    ------
[6f951ca]163    args : Namespace
[274f9ef]164        Contains the commandline arguments from script/program call.
[efdb01a]165    '''
[54a8a01]166
[efdb01a]167    parser = ArgumentParser(description='Retrieve FLEXPART input from \
[54a8a01]168                                ECMWF MARS archive',
[efdb01a]169                            formatter_class=ArgumentDefaultsHelpFormatter)
170
[095dc73]171    # control parameters that override control file values
[ca867de]172    parser.add_argument("--start_date", dest="start_date",
173                        type=none_or_str, default=None,
[efdb01a]174                        help="start date YYYYMMDD")
[ca867de]175    parser.add_argument("--end_date", dest="end_date",
176                        type=none_or_str, default=None,
[efdb01a]177                        help="end_date YYYYMMDD")
[ca867de]178    parser.add_argument("--date_chunk", dest="date_chunk",
179                        type=none_or_int, default=None,
[efdb01a]180                        help="# of days to be retrieved at once")
[f2616a3]181    parser.add_argument("--job_chunk", dest="job_chunk",
182                        type=none_or_int, default=None,
183                        help="# of days to be retrieved within a single job")
[ca867de]184    parser.add_argument("--controlfile", dest="controlfile",
185                        type=none_or_str, default='CONTROL.temp',
186                        help="file with CONTROL parameters")
187    parser.add_argument("--basetime", dest="basetime",
188                        type=none_or_int, default=None,
[d4696e0]189                        help="base such as 0 or 12 (for half day retrievals)")
[ca867de]190    parser.add_argument("--step", dest="step",
191                        type=none_or_str, default=None,
[efdb01a]192                        help="steps such as 00/to/48")
[ca867de]193    parser.add_argument("--levelist", dest="levelist",
194                        type=none_or_str, default=None,
[efdb01a]195                        help="Vertical levels to be retrieved, e.g. 30/to/60")
[ca867de]196    parser.add_argument("--area", dest="area",
197                        type=none_or_str, default=None,
[efdb01a]198                        help="area defined as north/west/south/east")
199
[095dc73]200    # some switches
201    parser.add_argument("--debug", dest="debug",
202                        type=none_or_int, default=None,
203                        help="debug mode - leave temporary files intact")
[d4696e0]204    parser.add_argument("--oper", dest="oper",
205                        type=none_or_int, default=None,
206                        help="operational mode - prepares dates with \
207                        environment variables")
[095dc73]208    parser.add_argument("--request", dest="request",
209                        type=none_or_int, default=None,
210                        help="list all mars requests in file mars_requests.dat")
211    parser.add_argument("--public", dest="public",
212                        type=none_or_int, default=None,
213                        help="public mode - retrieves the public datasets")
214    parser.add_argument("--rrint", dest="rrint",
215                        type=none_or_int, default=None,
216                        help="select old or new precipitation interpolation \
217                        0 - old method\
218                        1 - new method (additional subgrid points)")
219
220    # set directories
[ca867de]221    parser.add_argument("--inputdir", dest="inputdir",
222                        type=none_or_str, default=None,
[efdb01a]223                        help="root directory for storing intermediate files")
[ca867de]224    parser.add_argument("--outputdir", dest="outputdir",
225                        type=none_or_str, default=None,
[efdb01a]226                        help="root directory for storing output files")
[2fef1f2]227    parser.add_argument("--flexpartdir", dest="flexpartdir",
[ca867de]228                        type=none_or_str, default=None,
[efdb01a]229                        help="FLEXPART root directory (to find grib2flexpart \
[54a8a01]230                        and COMMAND file)\n Normally flex_extract resides in \
[efdb01a]231                        the scripts directory of the FLEXPART distribution")
232
[ff99eae]233    # this is only used by prepare_flexpart.py to rerun a postprocessing step
[ca867de]234    parser.add_argument("--ppid", dest="ppid",
[786cfd6]235                        type=none_or_str, default=None,
[54a8a01]236                        help="specify parent process id for \
[ff99eae]237                        rerun of prepare_flexpart")
[efdb01a]238
239    # arguments for job submission to ECMWF, only needed by submit.py
240    parser.add_argument("--job_template", dest='job_template',
[ca867de]241                        type=none_or_str, default="job.temp",
[efdb01a]242                        help="job template file for submission to ECMWF")
[ca867de]243    parser.add_argument("--queue", dest="queue",
244                        type=none_or_str, default=None,
[efdb01a]245                        help="queue for submission to ECMWF \
246                        (e.g. ecgate or cca )")
247
248    args = parser.parse_args()
249
[54a8a01]250    return args
[efdb01a]251
[96e1533]252def read_ecenv(filepath):
[274f9ef]253    '''Reads the file into a dictionary where the key values are the parameter
254    names.
255
256    Parameters
257    ----------
[6f951ca]258    filepath : str
[274f9ef]259        Path to file where the ECMWF environment parameters are stored.
260
261    Return
262    ------
[6f951ca]263    envs : dict
[274f9ef]264        Contains the environment parameter ecuid, ecgid, gateway
265        and destination for ECMWF server environments.
[54a8a01]266    '''
267    envs= {}
[96e1533]268    try:
269        with open(filepath, 'r') as f:
270            for line in f:
271                data = line.strip().split()
272                envs[str(data[0])] = str(data[1])
273    except OSError as e:
274        print('... ERROR CODE: ' + str(e.errno))
275        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
[2fb99de]276
[96e1533]277        sys.exit('\n... Error occured while trying to read ECMWF_ENV '
278                     'file: ' + str(filepath))
[efdb01a]279
[54a8a01]280    return envs
[efdb01a]281
[ff99eae]282def clean_up(c):
[96e1533]283    '''Remove files from the intermediate directory (inputdir).
284
285    It keeps the final FLEXPART input files if program runs without
286    ECMWF Api and keywords "ectrans" or "ecstorage" are set to "1".
[274f9ef]287
288    Parameters
289    ----------
[6f951ca]290    c : ControlFile
[274f9ef]291        Contains all the parameters of CONTROL file and
292        command line.
293
294    Return
295    ------
296
[efdb01a]297    '''
298
[96e1533]299    print("... clean inputdir!")
[efdb01a]300
[f20af73]301    cleanlist = [file for file in glob.glob(os.path.join(c.inputdir, "*"))
302                 if not os.path.basename(file).startswith(c.prefix)]
[efdb01a]303
[96e1533]304    if cleanlist:
305        for element in cleanlist:
[f20af73]306            silent_remove(element)
[96e1533]307        print("... done!")
308    else:
309        print("... nothing to clean!")
[efdb01a]310
311    return
312
313
[f20af73]314def my_error(message='ERROR'):
[274f9ef]315    '''Prints a specified error message which can be passed to the function
316    before exiting the program.
317
318    Parameters
319    ----------
[6f951ca]320    message : str, optional
[274f9ef]321        Error message. Default value is "ERROR".
[efdb01a]322
[274f9ef]323    Return
324    ------
[efdb01a]325
326    '''
[ff99eae]327
[96e1533]328    trace = '\n'.join(traceback.format_stack())
329    full_message = message + '\n\n' + trace
[ff99eae]330
[96e1533]331    print(full_message)
[54a8a01]332
333    sys.exit(1)
[efdb01a]334
335    return
336
337
[96e1533]338def send_mail(users, success_mode, message):
[274f9ef]339    '''Prints a specific exit message which can be passed to the function.
[efdb01a]340
[274f9ef]341    Parameters
342    ----------
[6f951ca]343    users : list of str
[274f9ef]344        Contains all email addresses which should be notified.
345        It might also contain just the ecmwf user name which wil trigger
346        mailing to the associated email address for this user.
[efdb01a]347
[6f951ca]348    success_mode : str
[96e1533]349        States the exit mode of the program to put into
350        the mail subject line.
351
[6f951ca]352    message : str, optional
[274f9ef]353        Message for exiting program. Default value is "Done!".
[efdb01a]354
[274f9ef]355    Return
356    ------
[efdb01a]357
358    '''
[ff99eae]359
[54a8a01]360    for user in users:
361        if '${USER}' in user:
362            user = os.getenv('USER')
363        try:
[96e1533]364            p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ' +
365                                  success_mode, os.path.expandvars(user)],
[ff99eae]366                                 stdin=subprocess.PIPE,
367                                 stdout=subprocess.PIPE,
368                                 stderr=subprocess.PIPE,
369                                 bufsize=1)
[79729d5]370            pout = p.communicate(input=message.encode() + '\n\n')[0]
[54a8a01]371        except ValueError as e:
[96e1533]372            print('... ERROR: ' + str(e))
373            sys.exit('... Email could not be sent!')
374        except OSError as e:
375            print('... ERROR CODE: ' + str(e.errno))
376            print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
377            sys.exit('... Email could not be sent!')
[54a8a01]378        else:
[96e1533]379            print('Email sent to ' + os.path.expandvars(user))
380
381    return
382
383
384def normal_exit(message='Done!'):
385    '''Prints a specific exit message which can be passed to the function.
386
387    Parameters
388    ----------
[6f951ca]389    message : str, optional
[96e1533]390        Message for exiting program. Default value is "Done!".
391
392    Return
393    ------
394
395    '''
396
397    print(str(message))
[efdb01a]398
399    return
400
401
402def product(*args, **kwds):
[96e1533]403    '''Creates combinations of all passed arguments.
404
405    This method combines the single characters of the passed arguments
[274f9ef]406    with each other. So that each character of each argument value
407    will be combined with each character of the other arguments as a tuple.
408
409    Note
410    ----
411    This method is taken from an example at the ECMWF wiki website.
412    https://software.ecmwf.int/wiki/display/GRIB/index.py; 2018-03-16
413
414    Example
415    -------
416    product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
417
418    product(range(2), repeat = 3) --> 000 001 010 011 100 101 110 111
419
420    Parameters
421    ----------
[6f951ca]422    \*args : list or str
[274f9ef]423        Positional arguments (arbitrary number).
424
[6f951ca]425    \*\*kwds : dict
[274f9ef]426        Contains all the keyword arguments from \*args.
427
428    Return
429    ------
430    prod : :obj:`tuple`
431        Return will be done with "yield". A tuple of combined arguments.
432        See example in description above.
[efdb01a]433    '''
[96e1533]434    try:
[79729d5]435        pools = [tuple(arg) for arg in args] * kwds.get('repeat', 1)
[96e1533]436        result = [[]]
437        for pool in pools:
438            result = [x + [y] for x in result for y in pool]
439        for prod in result:
440            yield tuple(prod)
441    except TypeError as e:
442        sys.exit('... PRODUCT GENERATION FAILED!')
[efdb01a]443
444    return
445
446
[ff99eae]447def silent_remove(filename):
[274f9ef]448    '''Remove file if it exists.
449    The function does not fail if the file does not exist.
450
451    Parameters
452    ----------
[6f951ca]453    filename : str
[274f9ef]454        The name of the file to be removed without notification.
[efdb01a]455
[274f9ef]456    Return
457    ------
[efdb01a]458
459    '''
460    try:
461        os.remove(filename)
462    except OSError as e:
[96e1533]463        # errno.ENOENT  =  no such file or directory
464        if e.errno == errno.ENOENT:
465            pass
466        else:
[efdb01a]467            raise  # re-raise exception if a different error occured
468
469    return
470
471
[ff99eae]472def init128(filepath):
[274f9ef]473    '''Opens and reads the grib file with table 128 information.
474
475    Parameters
476    ----------
[6f951ca]477    filepath : str
[274f9ef]478        Path to file of ECMWF grib table number 128.
479
480    Return
481    ------
[6f951ca]482    table128 : dict
[274f9ef]483        Contains the ECMWF grib table 128 information.
484        The key is the parameter number and the value is the
485        short name of the parameter.
[efdb01a]486    '''
487    table128 = dict()
[96e1533]488    try:
489        with open(filepath) as f:
490            fdata = f.read().split('\n')
491    except OSError as e:
492        print('... ERROR CODE: ' + str(e.errno))
493        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
494
495        sys.exit('\n... Error occured while trying to read parameter '
496                 'table file: ' + str(filepath))
497    else:
498        for data in fdata:
[f20af73]499            if data != '' and data[0] != '!':
500                table128[data[0:3]] = data[59:65].strip()
[efdb01a]501
502    return table128
503
504
[ff99eae]505def to_param_id(pars, table):
[274f9ef]506    '''Transform parameter names to parameter ids with ECMWF grib table 128.
507
508    Parameters
509    ----------
[6f951ca]510    pars : str
[274f9ef]511        Addpar argument from CONTROL file in the format of
512        parameter names instead of ids. The parameter short
513        names are sepearted with "/" and they are passed as
514        one single string.
515
[6f951ca]516    table : dict
[274f9ef]517        Contains the ECMWF grib table 128 information.
518        The key is the parameter number and the value is the
519        short name of the parameter.
520
521    Return
522    ------
[6f951ca]523    ipar : list of int
[274f9ef]524        List of addpar parameters from CONTROL file transformed to
525        parameter ids in the format of integer.
[efdb01a]526    '''
[96e1533]527    if not pars:
528        return []
529    if not isinstance(pars, str):
530        pars=str(pars)
531
[efdb01a]532    cpar = pars.upper().split('/')
533    ipar = []
534    for par in cpar:
[79729d5]535        for k, v in table.items():
[efdb01a]536            if par == k or par == v:
537                ipar.append(int(k))
538                break
[ff99eae]539        else:
[2fb99de]540            print('Warning: par ' + par + ' not found in table 128')
[efdb01a]541
542    return ipar
543
[f20af73]544def to_param_id_with_tablenumber(pars, table):
545    '''Transform parameter names to parameter ids and add table id.
546
547    Conversion with ECMWF grib table 128.
548
549    Parameters
550    ----------
551    pars : str
552        Addpar argument from CONTROL file in the format of
553        parameter names instead of ids. The parameter short
554        names are sepearted with "/" and they are passed as
555        one single string.
556
557    table : dict
558        Contains the ECMWF grib table 128 information.
559        The key is the parameter number and the value is the
560        short name of the parameter.
561
562    Return
563    ------
564    spar : str
565        List of addpar parameters from CONTROL file transformed to
566        parameter ids in the format of integer.
567    '''
568    if not pars:
569        return []
570    if not isinstance(pars, str):
571        pars=str(pars)
572
573    cpar = pars.upper().split('/')
574    spar = []
575    for par in cpar:
[79729d5]576        for k, v in table.items():
[f20af73]577            if par == k or par == v:
578                spar.append(k + '.128')
579                break
580        else:
581            print('\n\n\t\tWarning: par ' + par + ' not found in table 128\n\n')
582
583    return '/'.join(spar)
584
[ff99eae]585def get_list_as_string(list_obj, concatenate_sign=', '):
[274f9ef]586    '''Converts a list of arbitrary content into a single string.
[991df6a]587
[274f9ef]588    Parameters
589    ----------
[6f951ca]590    list_obj : list of *
[274f9ef]591        A list with arbitrary content.
[991df6a]592
[6f951ca]593    concatenate_sign : str, optional
[274f9ef]594        A string which is used to concatenate the single
595        list elements. Default value is ", ".
[ff99eae]596
[274f9ef]597    Return
598    ------
[6f951ca]599    str_of_list : str
[274f9ef]600        The content of the list as a single string.
[efdb01a]601    '''
[991df6a]602
[96e1533]603    if not isinstance(list_obj, list):
604        list_obj = list(list_obj)
[ff99eae]605    str_of_list = concatenate_sign.join(str(l) for l in list_obj)
[991df6a]606
[ff99eae]607    return str_of_list
[54a8a01]608
609def make_dir(directory):
[96e1533]610    '''Creates a directory.
611
612    It gives a warning if the directory already exists and skips process.
613    The program stops only if there is another problem.
[54a8a01]614
[274f9ef]615    Parameters
616    ----------
[6f951ca]617    directory : str
[96e1533]618        The path to directory which should be created.
[274f9ef]619
620    Return
621    ------
[54a8a01]622
623    '''
624    try:
625        os.makedirs(directory)
626    except OSError as e:
[96e1533]627        # errno.EEXIST = directory already exists
628        if e.errno == errno.EEXIST:
[2fb99de]629            print('WARNING: Directory {0} already exists!'.format(directory))
[96e1533]630        else:
631            raise # re-raise exception if a different error occured
[54a8a01]632
633    return
634
635def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
[274f9ef]636    '''Uses the ecaccess-file-put command to send a file to the ECMWF servers.
[97e09f4]637
[274f9ef]638    Note
639    ----
640    The return value is just for testing reasons. It does not have
641    to be used from the calling function since the whole error handling
642    is done in here.
[54a8a01]643
[274f9ef]644    Parameters
645    ----------
[6f951ca]646    ecd : str
[274f9ef]647        The path were the file is stored.
[54a8a01]648
[6f951ca]649    filename : str
[274f9ef]650        The name of the file to send to the ECMWF server.
[54a8a01]651
[6f951ca]652    target : str
[274f9ef]653        The target queue where the file should be sent to.
[54a8a01]654
[6f951ca]655    ecuid : str
[274f9ef]656        The user id on ECMWF server.
[54a8a01]657
[6f951ca]658    ecgid : str
[274f9ef]659        The group id on ECMWF server.
[54a8a01]660
[274f9ef]661    Return
662    ------
[96e1533]663
[54a8a01]664    '''
665
666    try:
[96e1533]667        subprocess.check_output(['ecaccess-file-put',
668                                 ecd + '/' + filename,
669                                 target + ':/home/ms/' +
670                                 ecgid + '/' + ecuid +
671                                 '/' + filename],
672                                stderr=subprocess.STDOUT)
[54a8a01]673    except subprocess.CalledProcessError as e:
[96e1533]674        print('... ERROR CODE: ' + str(e.returncode))
675        print('... ERROR MESSAGE:\n \t ' + str(e))
[54a8a01]676
[2fb99de]677        print('\n... Do you have a valid ecaccess certification key?')
[97e09f4]678        sys.exit('... ECACCESS-FILE-PUT FAILED!')
[96e1533]679    except OSError as e:
680        print('... ERROR CODE: ' + str(e.errno))
681        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
682
683        print('\n... Most likely the ECACCESS library is not available!')
684        sys.exit('... ECACCESS-FILE-PUT FAILED!')
[97e09f4]685
[96e1533]686    return
[54a8a01]687
[b1d07c9]688def submit_job_to_ecserver(target, jobname):
[274f9ef]689    '''Uses ecaccess-job-submit command to submit a job to the ECMWF server.
690
691    Note
692    ----
693    The return value is just for testing reasons. It does not have
694    to be used from the calling function since the whole error handling
695    is done in here.
696
697    Parameters
698    ----------
[6f951ca]699    target : str
[274f9ef]700        The target where the file should be sent to, e.g. the queue.
701
[6f951ca]702    jobname : str
[274f9ef]703        The name of the jobfile to be submitted to the ECMWF server.
704
705    Return
706    ------
[6f951ca]707    job_id : int
[96e1533]708        The id number of the job as a reference at the ecmwf server.
[54a8a01]709    '''
710
711    try:
[96e1533]712        job_id = subprocess.check_output(['ecaccess-job-submit', '-queueName',
713                                          target, jobname])
[2fb99de]714
[96e1533]715    except subprocess.CalledProcessError as e:
716        print('... ERROR CODE: ' + str(e.returncode))
717        print('... ERROR MESSAGE:\n \t ' + str(e))
[97e09f4]718
[2fb99de]719        print('\n... Do you have a valid ecaccess certification key?')
[54a8a01]720        sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
[96e1533]721    except OSError as e:
722        print('... ERROR CODE: ' + str(e.errno))
723        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
724
725        print('\n... Most likely the ECACCESS library is not available!')
726        sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
[54a8a01]727
[79729d5]728    return job_id.decode()
[095dc73]729
730
731def get_informations(filename):
732    '''Gets basic information from an example grib file.
733
734    These information are important for later use and the
735    initialization of numpy arrays for data storing.
736
737    Parameters
738    ----------
[6f951ca]739    filename : str
[095dc73]740            Name of the file which will be opened to extract basic information.
741
742    Return
743    ------
[6f951ca]744    data : dict
[095dc73]745        Contains basic informations of the ECMWF grib files, e.g.
746        'Ni', 'Nj', 'latitudeOfFirstGridPointInDegrees',
747        'longitudeOfFirstGridPointInDegrees', 'latitudeOfLastGridPointInDegrees',
748        'longitudeOfLastGridPointInDegrees', 'jDirectionIncrementInDegrees',
749        'iDirectionIncrementInDegrees', 'missingValue'
750    '''
[2fef1f2]751    from eccodes import codes_grib_new_from_file, codes_get, codes_release
[095dc73]752
753    data = {}
754
755    # --- open file ---
756    print("Opening file for getting information data --- %s" % filename)
757    with open(filename) as f:
758        # load first message from file
759        gid = codes_grib_new_from_file(f)
760
761        # information needed from grib message
762        keys = [
763                'Ni',
764                'Nj',
765                'latitudeOfFirstGridPointInDegrees',
766                'longitudeOfFirstGridPointInDegrees',
767                'latitudeOfLastGridPointInDegrees',
768                'longitudeOfLastGridPointInDegrees',
769                'jDirectionIncrementInDegrees',
770                'iDirectionIncrementInDegrees',
771                'missingValue',
772               ]
773
774        print('\nInformations are: ')
775        for key in keys:
776            # Get the value of the key in a grib message.
777            data[key] = codes_get(gid,key)
778            print("%s = %s" % (key,data[key]))
779
780        # Free the memory for the message referred as gribid.
781        codes_release(gid)
782
783    return data
784
785
[2fef1f2]786def get_dimensions(info, purefc, dtime, index_vals, start_date, end_date):
[095dc73]787    '''This function specifies the correct dimensions for x, y and t.
788
789    Parameters
790    ----------
[6f951ca]791    info : dict
[095dc73]792        Contains basic informations of the ECMWF grib files, e.g.
793        'Ni', 'Nj', 'latitudeOfFirstGridPointInDegrees',
794        'longitudeOfFirstGridPointInDegrees', 'latitudeOfLastGridPointInDegrees',
795        'longitudeOfLastGridPointInDegrees', 'jDirectionIncrementInDegrees',
796        'iDirectionIncrementInDegrees', 'missingValue'
797
[6f951ca]798    purefc : int
[2fef1f2]799        Switch for definition of pure forecast mode or not.
800
[6f951ca]801    dtime : str
[2fef1f2]802        Time step in hours.
803
[6f951ca]804    index_vals : list of list of str
[2fef1f2]805        Contains the values from the keys used for a distinct selection
806        of grib messages in processing  the grib files.
807        Content looks like e.g.:
808        index_vals[0]: ('20171106', '20171107', '20171108') ; date
809        index_vals[1]: ('0', '1200', '1800', '600') ; time
810        index_vals[2]: ('0', '12', '3', '6', '9') ; stepRange
811
[6f951ca]812    start_date : str
[2fef1f2]813        The start date of the retrieval job.
814
[6f951ca]815    end_date : str
[2fef1f2]816        The end date of the retrieval job.
817
[095dc73]818    Return
819    ------
[6f951ca]820    (ix, jy, it) : tuple of int
[095dc73]821        Dimension in x-direction, y-direction and in time.
822    '''
823
824    ix = info['Ni']
825
826    jy = info['Nj']
827
[2fef1f2]828    if not purefc:
[79729d5]829        it = ((end_date - start_date).days + 1) * 24 // int(dtime)
[2fef1f2]830    else:
831        # #no of step * #no of times * #no of days
832        it = len(index_vals[2]) * len(index_vals[1]) * len(index_vals[0])
[095dc73]833
834    return (ix, jy, it)
[bf48c8a]835
836
837def execute_subprocess(cmd_list, error_msg='SUBPROCESS FAILED!'):
838    '''Executes a command line instruction via a subprocess.
839
840    Error handling is done if an error occures.
841
842    Parameters
843    ----------
[6f951ca]844    cmd_list : list of str
[bf48c8a]845        A list of the components for the command line execution. Each
846        list entry is a single part of the command which is seperated from
847        the rest by a blank space.
848        E.g. ['mv', file1, file2]
849
850    Return
851    ------
[6f951ca]852    error_msg : str, optional
[bf48c8a]853        The possible error message if the subprocess failed.
854        By default it will just tell "SUBPROCESS FAILED!".
855    '''
856
857    try:
858        subprocess.check_call(cmd_list)
859    except subprocess.CalledProcessError as e:
860        print('... ERROR CODE: ' + str(e.returncode))
861        print('... ERROR MESSAGE:\n \t ' + str(e))
862
863        sys.exit('... ' + error_msg)
864    except OSError as e:
865        print('... ERROR CODE: ' + str(e.errno))
866        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
867
868        sys.exit('... ' + error_msg)
869
870    return
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG