Changeset 54a8a01 in flex_extract.git for python/tools.py


Ignore:
Timestamp:
Aug 31, 2018, 7:50:37 AM (6 years ago)
Author:
Anne Philipp <anne.philipp@…>
Branches:
master, ctbto, dev
Children:
597d4d1
Parents:
e1228f3
Message:

restructuring, documentations and bug fixes

File:
1 edited

Legend:

Unmodified
Added
Removed

  • python/tools.py

    rff99eae r54a8a01  
    11#!/usr/bin/env python
    22# -*- coding: utf-8 -*-
    3 #************************************************************************
    4 # ToDo AP
    5 # - check my_error
    6 # - check normal_exit
    7 # - check get_list_as_string
    8 # - seperate args and control interpretation
    9 #************************************************************************
    103#*******************************************************************************
    114# @Author: Anne Philipp (University of Vienna)
    … …  
    2619#        - moved all functions from file Flexparttools to this file tools
    2720#        - added function get_list_as_string
     21#        - seperated args and control interpretation
    2822#
    2923# @License:
    … …  
    3832#
    3933# @Module Content:
    40 #    - interpret_args_and_control
     34#    - get_cmdline_arguments
    4135#    - clean_up
    4236#    - my_error
    … …  
    4741#    - to_param_id
    4842#    - get_list_as_string
     43#    - make_dir
    4944#
    5045#*******************************************************************************
    … …  
    5752import sys
    5853import glob
    59 import inspect
    6054import subprocess
    6155import traceback
    6256from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
    63 import numpy as np
    64 
    65 # software specific class from flex_extract
    66 from ControlFile import ControlFile
    6757
    6858# ------------------------------------------------------------------------------
    … …  
    7060# ------------------------------------------------------------------------------
    7161
    72 def interpret_args_and_control():
    73     '''
    74     @Description:
    75         Assigns the command line arguments and reads CONTROL file
    76         content. Apply default values for non mentioned arguments.
     62def get_cmdline_arguments():
     63    '''
     64    @Description:
     65        Decomposes the command line arguments and assigns them to variables.
     66        Apply default values for non mentioned arguments.
    7767
    7868    @Input:
    … …  
    8272        args: instance of ArgumentParser
    8373            Contains the commandline arguments from script/program call.
    84 
    85         c: instance of class ControlFile
    86             Contains all necessary information of a CONTROL file. The parameters
    87             are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
    88             NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
    89             RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
    90             SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
    91             ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, DEBUG, INPUTDIR,
    92             OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
    93             For more information about format and content of the parameter see
    94             documentation.
    95 
    96     '''
     74    '''
     75
    9776    parser = ArgumentParser(description='Retrieve FLEXPART input from \
    98                             ECMWF MARS archive',
     77                                ECMWF MARS archive',
    9978                            formatter_class=ArgumentDefaultsHelpFormatter)
    10079
    10180    # the most important arguments
    102     parser.add_argument("--start_date", dest="start_date",
     81    parser.add_argument("--start_date", dest="start_date", default=None,
    10382                        help="start date YYYYMMDD")
    104     parser.add_argument("--end_date", dest="end_date",
     83    parser.add_argument("--end_date", dest="end_date", default=None,
    10584                        help="end_date YYYYMMDD")
    10685    parser.add_argument("--date_chunk", dest="date_chunk", default=None,
    … …  
    10887
    10988    # some arguments that override the default in the CONTROL file
    110     parser.add_argument("--basetime", dest="basetime",
     89    parser.add_argument("--basetime", dest="basetime", default=None,
    11190                        help="base such as 00/12 (for half day retrievals)")
    112     parser.add_argument("--step", dest="step",
     91    parser.add_argument("--step", dest="step", default=None,
    11392                        help="steps such as 00/to/48")
    114     parser.add_argument("--levelist", dest="levelist",
     93    parser.add_argument("--levelist", dest="levelist", default=None,
    11594                        help="Vertical levels to be retrieved, e.g. 30/to/60")
    116     parser.add_argument("--area", dest="area",
     95    parser.add_argument("--area", dest="area", default=None,
    11796                        help="area defined as north/west/south/east")
    11897
    … …  
    123102                        help="root directory for storing output files")
    124103    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
     104                        default=None,
    125105                        help="FLEXPART root directory (to find grib2flexpart \
    126                         and COMMAND file)\n Normally ECMWFDATA resides in \
     106                        and COMMAND file)\n Normally flex_extract resides in \
    127107                        the scripts directory of the FLEXPART distribution")
    128108
    129109    # this is only used by prepare_flexpart.py to rerun a postprocessing step
    130     parser.add_argument("--ppid", dest="ppid",
    131                         help="Specify parent process id for \
     110    parser.add_argument("--ppid", dest="ppid", default=None,
     111                        help="specify parent process id for \
    132112                        rerun of prepare_flexpart")
    133113
    … …  
    136116                        default="job.temp",
    137117                        help="job template file for submission to ECMWF")
    138     parser.add_argument("--queue", dest="queue",
     118    parser.add_argument("--queue", dest="queue", default=None,
    139119                        help="queue for submission to ECMWF \
    140120                        (e.g. ecgate or cca )")
    … …  
    142122                        default='CONTROL.temp',
    143123                        help="file with CONTROL parameters")
    144     parser.add_argument("--debug", dest="debug", default=0,
    145                         help="Debug mode - leave temporary files intact")
     124    parser.add_argument("--debug", dest="debug", default=None,
     125                        help="debug mode - leave temporary files intact")
    146126
    147127    args = parser.parse_args()
    148128
    149     # create instance of ControlFile for specified controlfile
    150     # and assign the parameters (and default values if necessary)
    151     try:
    152         c = ControlFile(args.controlfile)
    153     except IOError:
    154         try:
    155             LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
    156                 inspect.getfile(inspect.currentframe())))
    157             c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile)
    158         except IOError:
    159             print 'Could not read CONTROL file "' + args.controlfile + '"'
    160             print 'Either it does not exist or its syntax is wrong.'
    161             print 'Try "' + sys.argv[0].split('/')[-1] + \
    162                   ' -h" to print usage information'
    163             exit(1)
    164 
    165     # check for having at least a starting date
    166     if  args.start_date is None and getattr(c, 'start_date') is None:
    167         print 'start_date specified neither in command line nor \
    168                in CONTROL file ' + args.controlfile
    169         print 'Try "' + sys.argv[0].split('/')[-1] + \
    170               ' -h" to print usage information'
    171         exit(1)
    172 
    173     # save all existing command line parameter to the ControlFile instance
    174     # if parameter is not specified through the command line or CONTROL file
    175     # set default values
    176     if args.start_date is not None:
    177         c.start_date = args.start_date
    178     if args.end_date is not None:
    179         c.end_date = args.end_date
    180     if c.end_date is None:
    181         c.end_date = c.start_date
    182     if args.date_chunk is not None:
    183         c.date_chunk = args.date_chunk
    184 
    185     if not hasattr(c, 'debug'):
    186         c.debug = args.debug
    187 
    188     if args.inputdir is None and args.outputdir is None:
    189         c.inputdir = '../work'
    190         c.outputdir = '../work'
    191     else:
    192         if args.inputdir is not None:
    193             c.inputdir = args.inputdir
    194         if args.outputdir is None:
    195             c.outputdir = args.inputdir
    196         if args.outputdir is not None:
    197             c.outputdir = args.outputdir
    198         if args.inputdir is None:
    199             c.inputdir = args.outputdir
    200 
    201     if hasattr(c, 'outputdir') is False and args.outputdir is None:
    202         c.outputdir = c.inputdir
    203     else:
    204         if args.outputdir is not None:
    205             c.outputdir = args.outputdir
    206 
    207     if args.area is not None:
    208         afloat = '.' in args.area
    209         l = args.area.split('/')
    210         if afloat:
    211             for i, item in enumerate(l):
    212                 item = str(int(float(item) * 1000))
    213         c.upper, c.left, c.lower, c.right = l
    214 
    215     # NOTE: basetime activates the ''operational mode''
    216     if args.basetime is not None:
    217         c.basetime = args.basetime
    218 
    219     if args.step is not None:
    220         l = args.step.split('/')
    221         if 'to' in args.step.lower():
    222             if 'by' in args.step.lower():
    223                 ilist = np.arange(int(l[0]), int(l[2]) + 1, int(l[4]))
    224                 c.step = ['{:0>3}'.format(i) for i in ilist]
    225             else:
    226                 my_error(None, args.step + ':\n' +
    227                          'please use "by" as well if "to" is used')
    228         else:
    229             c.step = l
    230 
    231     if args.levelist is not None:
    232         c.levelist = args.levelist
    233         if 'to' in c.levelist:
    234             c.level = c.levelist.split('/')[2]
    235         else:
    236             c.level = c.levelist.split('/')[-1]
    237 
    238     if args.flexpart_root_scripts is not None:
    239         c.flexpart_root_scripts = args.flexpart_root_scripts
    240 
    241     return args, c
    242 
     129    return args
     130
     131def read_ecenv(filename):
     132    '''
     133    @Description:
     134        Reads the file into a dictionary where the key values are the parameter
     135        names.
     136
     137    @Input:
     138        filename: string
     139            Name of file where the ECMWV environment parameters are stored.
     140
     141    @Return:
     142        envs: dict
     143    '''
     144    envs= {}
     145    print filename
     146    with open(filename, 'r') as f:
     147        for line in f:
     148            data = line.strip().split()
     149            envs[str(data[0])] = str(data[1])
     150
     151    return envs
    243152
    244153def clean_up(c):
    … …  
    269178
    270179    cleanlist = glob.glob(c.inputdir + "/*")
    271     for cl in cleanlist:
    272         if c.prefix not in cl:
    273             silent_remove(cl)
     180    for clist in cleanlist:
     181        if c.prefix not in clist:
     182            silent_remove(clist)
    274183        if c.ecapi is False and (c.ectrans == '1' or c.ecstorage == '1'):
    275             silent_remove(cl)
     184            silent_remove(clist)
    276185
    277186    print "Done"
    … …  
    280189
    281190
    282 def my_error(c, message='ERROR'):
     191def my_error(users, message='ERROR'):
    283192    '''
    284193    @Description:
    … …  
    287196
    288197    @Input:
    289         c: instance of class ControlFile
    290             Contains all the parameters of CONTROL file, which are e.g.:
    291             DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
    292             STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
    293             LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
    294             OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
    295             ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
    296             MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
    297             DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
    298 
    299             For more information about format and content of the parameter
    300             see documentation.
     198        user: list of strings
     199            Contains all email addresses which should be notified.
     200            It might also contain just the ecmwf user name which wil trigger
     201            mailing to the associated email address for this user.
    301202
    302203        message: string, optional
    … …  
    310211
    311212    # comment if user does not want email notification directly from python
    312     try:
    313         target = []
    314         target.extend(c.mailfail)
    315     except AttributeError:
    316         target = []
    317         target.extend(os.getenv('USER'))
    318 
    319     for t in target:
    320         p = subprocess.Popen(['mail', '-s ECMWFDATA v7.0 ERROR',
    321                               os.path.expandvars(t)],
    322                              stdin=subprocess.PIPE,
    323                              stdout=subprocess.PIPE,
    324                              stderr=subprocess.PIPE,
    325                              bufsize=1)
    326         tr = '\n'.join(traceback.format_stack())
    327         pout = p.communicate(input=message + '\n\n' + tr)[0]
    328         print 'Email sent to ' + os.path.expandvars(t) + ' ' + pout.decode()
    329 
    330     exit(1)
    331 
    332     return
    333 
    334 
    335 def normal_exit(c, message='Done!'):
     213    for user in users:
     214        if '${USER}' in user:
     215            user = os.getenv('USER')
     216        try:
     217            p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ERROR',
     218                                  os.path.expandvars(user)],
     219                                 stdin=subprocess.PIPE,
     220                                 stdout=subprocess.PIPE,
     221                                 stderr=subprocess.PIPE,
     222                                 bufsize=1)
     223            trace = '\n'.join(traceback.format_stack())
     224            pout = p.communicate(input=message + '\n\n' + trace)[0]
     225        except ValueError as e:
     226            print 'ERROR: ', e
     227            sys.exit('Email could not be sent!')
     228        else:
     229            print 'Email sent to ' + os.path.expandvars(user) + ' ' + \
     230                  pout.decode()
     231
     232    sys.exit(1)
     233
     234    return
     235
     236
     237def normal_exit(users, message='Done!'):
    336238    '''
    337239    @Description:
    … …  
    339241
    340242    @Input:
    341         c: instance of class ControlFile
    342             Contains all the parameters of CONTROL file, which are e.g.:
    343             DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
    344             STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
    345             LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
    346             OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
    347             ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
    348             MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
    349             DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
    350 
    351             For more information about format and content of the parameter
    352             see documentation.
     243        user: list of strings
     244            Contains all email addresses which should be notified.
     245            It might also contain just the ecmwf user name which wil trigger
     246            mailing to the associated email address for this user.
    353247
    354248        message: string, optional
    … …  
    362256
    363257    # comment if user does not want notification directly from python
    364     try:
    365         target = []
    366         target.extend(c.mailops)
    367         for t in target:
    368             p = subprocess.Popen(['mail', '-s ECMWFDATA v7.0 normal exit',
    369                                   os.path.expandvars(t)],
     258    for user in users:
     259        if '${USER}' in user:
     260            user = os.getenv('USER')
     261        try:
     262            p = subprocess.Popen(['mail', '-s flex_extract_v7.1 normal exit',
     263                                  os.path.expandvars(user)],
    370264                                 stdin=subprocess.PIPE,
    371265                                 stdout=subprocess.PIPE,
    … …  
    373267                                 bufsize=1)
    374268            pout = p.communicate(input=message+'\n\n')[0]
    375             print 'Email sent to ' + os.path.expandvars(t) + ' ' + pout.decode()
    376     finally:
    377         pass
     269        except ValueError as e:
     270            print 'ERROR: ', e
     271            print 'Email could not be sent!'
     272        else:
     273            print 'Email sent to ' + os.path.expandvars(user) + ' ' + \
     274                  pout.decode()
    378275
    379276    return
    … …  
    433330    except OSError as e:
    434331        # this would be "except OSError, e:" before Python 2.6
    435         if e.errno is not errno.ENOENT:
     332        if e.errno != errno.ENOENT:
    436333            # errno.ENOENT  =  no such file or directory
    437334            raise  # re-raise exception if a different error occured
    … …  
    521418
    522419    return str_of_list
     420
     421def make_dir(directory):
     422    '''
     423    @Description:
     424        Creates a directory and gives a warning if the directory
     425        already exists. The program stops only if there is another problem.
     426
     427    @Input:
     428        directory: string
     429            The directory path which should be created.
     430
     431    @Return:
     432        <nothing>
     433    '''
     434    try:
     435        os.makedirs(directory)
     436    except OSError as e:
     437        if e.errno != errno.EEXIST:
     438            # errno.EEXIST = directory already exists
     439            raise # re-raise exception if a different error occured
     440        else:
     441            print 'WARNING: Directory {0} already exists!'.format(directory)
     442
     443    return
     444
     445def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
     446    '''
     447    @Description:
     448        Uses the ecaccess command to send a file to the ECMWF servers.
     449        Catches and prints the error if it failed.
     450
     451    @Input:
     452        ecd: string
     453            The path were the file is to be stored.
     454
     455        filename: string
     456            The name of the file to send to the ECMWF server.
     457
     458        target: string
     459            The target where the file should be sent to, e.g. the queue.
     460
     461        ecuid: string
     462            The user id on ECMWF server.
     463
     464        ecgid: string
     465            The group id on ECMWF server.
     466
     467    @Return:
     468        <nothing>
     469    '''
     470
     471    try:
     472        subprocess.check_call(['ecaccess-file-put',
     473                               ecd + '../' + filename,
     474                               target + ':/home/ms/' +
     475                               ecgid + '/' + ecuid +
     476                               '/' + filename])
     477    except subprocess.CalledProcessError as e:
     478        print 'ERROR:'
     479        print e
     480        sys.exit('ecaccess-file-put failed!\n' + \
     481                 'Probably the eccert key has expired.')
     482
     483    return
     484
     485def submit_job_to_ecserver(ecd, target, jobname):
     486    '''
     487    @Description:
     488        Uses ecaccess to submit a job to the ECMWF server.
     489        Catches and prints the error if one arise.
     490
     491    @Input:
     492        ecd: string
     493            The path were the file is to be stored.
     494
     495        target: string
     496            The target where the file should be sent to, e.g. the queue.
     497
     498        jobname: string
     499            The name of the jobfile to be submitted to the ECMWF server.
     500
     501    @Return:
     502        <nothing>
     503    '''
     504
     505    try:
     506        subprocess.check_call(['ecaccess-job-submit',
     507                               '-queueName', target,
     508                               jobname])
     509    except subprocess.CalledProcessError as e:
     510        print '... ERROR CODE: ', e.returncode
     511        sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
     512
     513    return
Note: See TracChangeset for help on using the changeset viewer.
hosted by ZAMG