Changeset 96e1533 in flex_extract.git for source/python/mods/tools.py


Ignore:
Timestamp:
Nov 10, 2018, 9:37:36 PM (5 years ago)
Author:
Anne Philipp <anne.philipp@…>
Branches:
master, ctbto, dev
Children:
1abf820
Parents:
8500ba1
Message:

redefined test data dir and completed unittests for tools module

File:
1 edited

Legend:

Unmodified
Added
Removed

  • source/python/mods/tools.py

    r274f9ef r96e1533  
    5454import subprocess
    5555import traceback
     56import exceptions
    5657from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
    5758
    … …  
    188189    return args
    189190
    190 def read_ecenv(filename):
     191def read_ecenv(filepath):
    191192    '''Reads the file into a dictionary where the key values are the parameter
    192193    names.
    … …  
    194195    Parameters
    195196    ----------
    196     filename : :obj:`string`
     197    filepath : :obj:`string`
    197198        Path to file where the ECMWF environment parameters are stored.
    198199
    … …  
    204205    '''
    205206    envs= {}
    206 
    207     with open(filename, 'r') as f:
    208         for line in f:
    209             data = line.strip().split()
    210             envs[str(data[0])] = str(data[1])
     207    try:
     208        with open(filepath, 'r') as f:
     209            for line in f:
     210                data = line.strip().split()
     211                envs[str(data[0])] = str(data[1])
     212    except OSError as e:
     213        print('... ERROR CODE: ' + str(e.errno))
     214        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
     215
     216        sys.exit('\n... Error occured while trying to read ECMWF_ENV '
     217                     'file: ' + str(filepath))
    211218
    212219    return envs
    213220
    214221def clean_up(c):
    215     '''Remove all files from intermediate directory (inputdir).
     222    '''Remove files from the intermediate directory (inputdir).
     223
     224    It keeps the final FLEXPART input files if program runs without
     225    ECMWF Api and keywords "ectrans" or "ecstorage" are set to "1".
    216226
    217227    Parameters
    … …  
    226236    '''
    227237
    228     print("clean_up")
    229 
    230     cleanlist = glob.glob(c.inputdir + "/*")
    231     for clist in cleanlist:
    232         if c.prefix not in clist:
    233             silent_remove(clist)
    234         if c.ecapi is False and (c.ectrans == '1' or c.ecstorage == '1'):
    235             silent_remove(clist)
    236 
    237     print("Done")
     238    print("... clean inputdir!")
     239
     240    cleanlist = glob.glob(os.path.join(c.inputdir, "*"))
     241
     242    if cleanlist:
     243        for element in cleanlist:
     244            if c.prefix not in element:
     245                silent_remove(element)
     246            if c.ecapi is False and (c.ectrans == 1 or c.ecstorage == 1):
     247                silent_remove(element)
     248        print("... done!")
     249    else:
     250        print("... nothing to clean!")
    238251
    239252    return
    … …  
    259272    '''
    260273
    261     print(message)
    262 
    263     # comment if user does not want email notification directly from python
     274    trace = '\n'.join(traceback.format_stack())
     275    full_message = message + '\n\n' + trace
     276
     277    print(full_message)
     278
     279    send_mail(users, 'ERROR', full_message)
     280
     281    sys.exit(1)
     282
     283    return
     284
     285
     286def send_mail(users, success_mode, message):
     287    '''Prints a specific exit message which can be passed to the function.
     288
     289    Parameters
     290    ----------
     291    users : :obj:`list` of :obj:`string`
     292        Contains all email addresses which should be notified.
     293        It might also contain just the ecmwf user name which wil trigger
     294        mailing to the associated email address for this user.
     295
     296    success_mode : :obj:``string`
     297        States the exit mode of the program to put into
     298        the mail subject line.
     299
     300    message : :obj:`string`, optional
     301        Message for exiting program. Default value is "Done!".
     302
     303    Return
     304    ------
     305
     306    '''
     307
    264308    for user in users:
    265309        if '${USER}' in user:
    266310            user = os.getenv('USER')
    267311        try:
    268             p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ERROR',
    269                                   os.path.expandvars(user)],
     312            p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ' +
     313                                  success_mode, os.path.expandvars(user)],
    270314                                 stdin=subprocess.PIPE,
    271315                                 stdout=subprocess.PIPE,
    272316                                 stderr=subprocess.PIPE,
    273317                                 bufsize=1)
    274             trace = '\n'.join(traceback.format_stack())
    275             pout = p.communicate(input=message + '\n\n' + trace)[0]
     318            pout = p.communicate(input=message + '\n\n')[0]
    276319        except ValueError as e:
    277             print('ERROR: ', e)
    278             sys.exit('Email could not be sent!')
     320            print('... ERROR: ' + str(e))
     321            sys.exit('... Email could not be sent!')
     322        except OSError as e:
     323            print('... ERROR CODE: ' + str(e.errno))
     324            print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
     325            sys.exit('... Email could not be sent!')
    279326        else:
    280             print('Email sent to ' + os.path.expandvars(user) + ' ' +
    281                   pout.decode())
    282 
    283     sys.exit(1)
    284 
    285     return
    286 
    287 
    288 def normal_exit(users, message='Done!'):
     327            print('Email sent to ' + os.path.expandvars(user))
     328
     329    return
     330
     331
     332def normal_exit(message='Done!'):
    289333    '''Prints a specific exit message which can be passed to the function.
    290334
    291335    Parameters
    292336    ----------
    293     user : :obj:`list` of :obj:`string`
    294         Contains all email addresses which should be notified.
    295         It might also contain just the ecmwf user name which wil trigger
    296         mailing to the associated email address for this user.
    297 
    298337    message : :obj:`string`, optional
    299338        Message for exiting program. Default value is "Done!".
    … …  
    303342
    304343    '''
    305     print(message)
    306 
    307     # comment if user does not want notification directly from python
    308     for user in users:
    309         if '${USER}' in user:
    310             user = os.getenv('USER')
    311         try:
    312             p = subprocess.Popen(['mail', '-s flex_extract_v7.1 normal exit',
    313                                   os.path.expandvars(user)],
    314                                  stdin=subprocess.PIPE,
    315                                  stdout=subprocess.PIPE,
    316                                  stderr=subprocess.PIPE,
    317                                  bufsize=1)
    318             pout = p.communicate(input=message+'\n\n')[0]
    319         except ValueError as e:
    320             print('ERROR: ', e)
    321             print('Email could not be sent!')
    322         else:
    323             print('Email sent to ' + os.path.expandvars(user) + ' ' +
    324                   pout.decode())
     344
     345    print(str(message))
    325346
    326347    return
    … …  
    328349
    329350def product(*args, **kwds):
    330     '''This method combines the single characters of the passed arguments
     351    '''Creates combinations of all passed arguments.
     352
     353    This method combines the single characters of the passed arguments
    331354    with each other. So that each character of each argument value
    332355    will be combined with each character of the other arguments as a tuple.
    … …  
    345368    Parameters
    346369    ----------
    347     \*args : :obj:`tuple`
     370    \*args : :obj:`list` or :obj:`string`
    348371        Positional arguments (arbitrary number).
    349372
    … …  
    357380        See example in description above.
    358381    '''
    359     pools = map(tuple, args) * kwds.get('repeat', 1)
    360     result = [[]]
    361     for pool in pools:
    362         result = [x + [y] for x in result for y in pool]
    363     for prod in result:
    364         yield tuple(prod)
     382    try:
     383        pools = map(tuple, args) * kwds.get('repeat', 1)
     384        result = [[]]
     385        for pool in pools:
     386            result = [x + [y] for x in result for y in pool]
     387        for prod in result:
     388            yield tuple(prod)
     389    except TypeError as e:
     390        sys.exit('... PRODUCT GENERATION FAILED!')
    365391
    366392    return
    … …  
    383409        os.remove(filename)
    384410    except OSError as e:
    385         if e.errno != errno.ENOENT:
    386             # errno.ENOENT  =  no such file or directory
     411        # errno.ENOENT  =  no such file or directory
     412        if e.errno == errno.ENOENT:
     413            pass
     414        else:
    387415            raise  # re-raise exception if a different error occured
    388416
    … …  
    406434    '''
    407435    table128 = dict()
    408     with open(filepath) as f:
    409         fdata = f.read().split('\n')
    410     for data in fdata:
    411         if data[0] != '!':
    412             table128[data[0:3]] = data[59:64].strip()
     436    try:
     437        with open(filepath) as f:
     438            fdata = f.read().split('\n')
     439    except OSError as e:
     440        print('... ERROR CODE: ' + str(e.errno))
     441        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
     442
     443        sys.exit('\n... Error occured while trying to read parameter '
     444                 'table file: ' + str(filepath))
     445    else:
     446        for data in fdata:
     447            if data[0] != '!':
     448                table128[data[0:3]] = data[59:64].strip()
    413449
    414450    return table128
    … …  
    437473        parameter ids in the format of integer.
    438474    '''
     475    if not pars:
     476        return []
     477    if not isinstance(pars, str):
     478        pars=str(pars)
     479
    439480    cpar = pars.upper().split('/')
    440481    ipar = []
    … …  
    467508    '''
    468509
     510    if not isinstance(list_obj, list):
     511        list_obj = list(list_obj)
    469512    str_of_list = concatenate_sign.join(str(l) for l in list_obj)
    470513
    … …  
    472515
    473516def make_dir(directory):
    474     '''Creates a directory and gives a warning if the directory
    475     already exists. The program stops only if there is another problem.
     517    '''Creates a directory.
     518
     519    It gives a warning if the directory already exists and skips process.
     520    The program stops only if there is another problem.
    476521
    477522    Parameters
    478523    ----------
    479524    directory : :obj:`string`
    480         The directory name including the path which should be created.
     525        The path to directory which should be created.
    481526
    482527    Return
    … …  
    487532        os.makedirs(directory)
    488533    except OSError as e:
    489         if e.errno != errno.EEXIST:
    490             # errno.EEXIST = directory already exists
     534        # errno.EEXIST = directory already exists
     535        if e.errno == errno.EEXIST:
     536            print('WARNING: Directory {0} already exists!'.format(directory))
     537        else:
    491538            raise # re-raise exception if a different error occured
    492         else:
    493             print('WARNING: Directory {0} already exists!'.format(directory))
    494539
    495540    return
    … …  
    523568    Return
    524569    ------
    525     rcode : :obj:`string`
    526         Resulting code of command execution. If successful the string
    527         will be empty.
     570
    528571    '''
    529572
    530573    try:
    531         rcode = subprocess.check_output(['ecaccess-file-put',
    532                                           ecd + '/' + filename,
    533                                           target + ':/home/ms/' +
    534                                           ecgid + '/' + ecuid +
    535                                           '/' + filename],
    536                                          stderr=subprocess.STDOUT)
     574        subprocess.check_output(['ecaccess-file-put',
     575                                 ecd + '/' + filename,
     576                                 target + ':/home/ms/' +
     577                                 ecgid + '/' + ecuid +
     578                                 '/' + filename],
     579                                stderr=subprocess.STDOUT)
    537580    except subprocess.CalledProcessError as e:
    538         print('... ERROR CODE:\n ... ' + str(e.returncode))
    539         print('... ERROR MESSAGE:\n ... ' + str(e))
     581        print('... ERROR CODE: ' + str(e.returncode))
     582        print('... ERROR MESSAGE:\n \t ' + str(e))
    540583
    541584        print('\n... Do you have a valid ecaccess certification key?')
    542585        sys.exit('... ECACCESS-FILE-PUT FAILED!')
    543 
    544     return rcode
     586    except OSError as e:
     587        print('... ERROR CODE: ' + str(e.errno))
     588        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
     589
     590        print('\n... Most likely the ECACCESS library is not available!')
     591        sys.exit('... ECACCESS-FILE-PUT FAILED!')
     592
     593    return
    545594
    546595def submit_job_to_ecserver(target, jobname):
    … …  
    563612    Return
    564613    ------
    565     rcode : :obj:`string`
    566         Resulting code of command execution. If successful the string
    567         will contain an integer number, representing the id of the job
    568         at the ecmwf server.
     614    job_id : :obj:`int`
     615        The id number of the job as a reference at the ecmwf server.
    569616    '''
    570617
    571618    try:
    572         rcode = subprocess.check_output(['ecaccess-job-submit',
    573                                          '-queueName', target,
    574                                          jobname])
     619        job_id = subprocess.check_output(['ecaccess-job-submit', '-queueName',
     620                                          target, jobname])
     621
    575622    except subprocess.CalledProcessError as e:
    576         print('... ERROR CODE:\n ... ' + str(e.returncode))
    577         print('... ERROR MESSAGE:\n ... ' + str(e))
    578 
     623        print('... ERROR CODE: ' + str(e.returncode))
     624        print('... ERROR MESSAGE:\n \t ' + str(e))
    579625
    580626        print('\n... Do you have a valid ecaccess certification key?')
    581627        sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
    582 
    583     return rcode
     628    except OSError as e:
     629        print('... ERROR CODE: ' + str(e.errno))
     630        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
     631
     632        print('\n... Most likely the ECACCESS library is not available!')
     633        sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
     634
     635    return job_id
Note: See TracChangeset for help on using the changeset viewer.
hosted by ZAMG