Changeset 45fc9b4 in flex_extract.git


Ignore:
Timestamp:
Dec 13, 2018, 11:44:05 PM (5 years ago)
Author:
Anne Philipp <anne.philipp@…>
Branches:
master, ctbto, dev
Children:
83b8e3c
Parents:
c2c5948
Message:

outsourced all of the necessary checks on control variables into own module functions

File:
1 edited

Legend:

Unmodified
Added
Removed
  • source/python/mods/checks.py

    ra55ac71 r45fc9b4  
    2626# ------------------------------------------------------------------------------
    2727
     28import os
    2829import _config
     30import exceptions
     31from tools import my_error, silent_remove
     32from datetime import datetime
    2933# ------------------------------------------------------------------------------
    3034# FUNCTIONS
    3135# ------------------------------------------------------------------------------
    3236
     37def check_logicals_type(c, logicals):
     38    '''Check that the logical variables have correct type integer.
     39
     40    Parameters
     41    ----------
     42    c : :obj:`ControlFile`
     43        Contains all the parameters of CONTROL file and
     44        command line.
     45
     46    logicals : :obj:`list` of (:obj:`string` or :obj:`integer`)
     47        Names of the switches that are used to control the flow of the
     48        program.
     49
     50    Return
     51    ------
     52
     53    '''
     54
     55    for var in logicals:
     56        if not isinstance(getattr(c, var), int):
     57            setattr(c, var, int(getattr(c, var)))
     58
     59    return
    3360
    3461def check_grid(grid):
     
    82109    ----------
    83110    grid : :obj:`string`
    84         Contains grid information
     111        Contains grid information.
     112
     113    area : :obj:`string`
     114        Contains area informtion.
     115
     116    upper : :obj:`string`
     117        The northern most latitude.
     118
     119    lower : :obj:`string`
     120        The souther most latitude.
     121
     122    left : :obj:`string`
     123        The western most longitude.
     124
     125    right : :obj:`string`
     126        The eastern most longiude.
    85127
    86128    Return
     
    99141
    100142    # determine area format
    101     if (abs(float(upper) / 1000.) >= 0.05 and
    102         abs(float(lower) / 1000.) >= 0.05 and
    103         abs(float(left) / 1000.) >= 0.05 and
    104         abs(float(right) / 1000.) >= 0.05):
     143    if ((abs(float(upper) / 10000.) >= 0.01 or float(upper) / 1000. == 0. ) and
     144        (abs(float(lower) / 10000.) >= 0.01 or float(lower) / 1000. == 0. ) and
     145        (abs(float(left) / 10000.) >= 0.01 or float(left) / 1000. == 0. ) and
     146        (abs(float(right) / 10000.) >= 0.01 or float(right) / 1000. == 0.)):
    105147        # area is defined in 1/1000 degrees; old format
    106148        area = '{}/{}/{}/{}'.format(float(upper) / 1000.,
     
    108150                                    float(lower) / 1000.,
    109151                                    float(right) / 1000.)
    110     elif (abs(float(upper) / 1000.) < 0.05 and
    111           abs(float(lower) / 1000.) < 0.05 and
    112           abs(float(left) / 1000.) < 0.05 and
    113           abs(float(right) / 1000.) < 0.05):
     152    elif (abs(float(upper) / 10000.) < 0.05 and
     153          abs(float(lower) / 10000.) < 0.05 and
     154          abs(float(left) / 10000.) < 0.05 and
     155          abs(float(right) / 10000.) < 0.05):
    114156        # area is already in new format
    115157        area = '{}/{}/{}/{}'.format(float(upper),
     
    200242
    201243
    202 def check_purefc(type):
     244def check_purefc(ftype):
    203245    '''Check for a pure forecast mode.
    204246
    205247    Parameters
    206248    ----------
    207     type : :obj:`list` of :obj:`string`
     249    ftype : :obj:`list` of :obj:`string`
    208250        List of field types.
    209251
     
    215257    '''
    216258
    217     if 'AN' not in type and '4V' not in type:
     259    if 'AN' not in ftype and '4V' not in ftype:
    218260        # pure forecast
    219         return True
    220 
    221     return False
    222 
    223 
    224 def check_():
    225     '''
    226 
    227     Parameters
    228     ----------
    229     par : :obj:``
    230         ...
    231 
    232     Return
    233     ------
    234 
    235     '''
     261        return 1
     262
     263    return 0
     264
     265
     266def check_step(step, mailfail):
     267    '''Checks on step format and convert into a list of steps.
     268
     269    If the steps were defined with "to" and "by" they are converted into
     270    a list of steps. If the steps were set in a string, it is
     271    converted into a list.
     272
     273    Parameters
     274    ----------
     275    step : :obj:`list` of :obj:`string` or :obj:`string`
     276        Specifies the forecast time step from forecast base time.
     277        Valid values are hours (HH) from forecast base time.
     278
     279    mailfail : :obj:`list` of :obj:``string`
     280        Contains all email addresses which should be notified.
     281        It might also contain just the ecmwf user name which will trigger
     282        mailing to the associated email address for this user.
     283
     284    Return
     285    ------
     286    step : :obj:`list` of :obj:`string`
     287        List of forecast steps in format e.g. [001, 002, ...]
     288    '''
     289
     290    if '/' in step:
     291        steps = step.split('/')
     292        if 'to' in step.lower() and 'by' in step.lower():
     293            ilist = np.arange(int(steps[0]),
     294                              int(steps[2]) + 1,
     295                              int(steps[4]))
     296            step = ['{:0>3}'.format(i) for i in ilist]
     297        elif 'to' in step.lower() and 'by' not in step.lower():
     298            my_error(mailfail, step + ':\n' +
     299                     'if "to" is used in steps parameter, '
     300                     'please use "by" as well')
     301        else:
     302            step = steps
     303
     304    if not isinstance(step, list):
     305        step = [step]
     306
     307    return step
     308
     309def check_type(ftype, steps):
     310    '''Check if type variable is of type list and if analysis field has
     311    forecast step 0.
     312
     313    Parameters
     314    ----------
     315    ftype : :obj:`list` of :obj:`string` or :obj:`string`
     316        List of field types.
     317
     318    steps : :obj:`string`
     319        Specifies the forecast time step from forecast base time.
     320        Valid values are hours (HH) from forecast base time.
     321
     322    Return
     323    ------
     324    ftype : :obj:`list` of :obj:`string`
     325        List of field types.
     326    '''
     327    if not isinstance(ftype, list):
     328        ftype = [ftype]
     329
     330    for i, val in enumerate(ftype):
     331        if ftype[i] == 'AN' and int(steps[i]) != 0:
     332            print('Analysis retrievals must have STEP = 0 (now set to 0)')
     333            ftype[i] = 0
     334
     335    return ftype
     336
     337def check_time(ftime):
     338    '''Check if time variable is of type list. Otherwise convert to list.
     339
     340    Parameters
     341    ----------
     342    ftime : :obj:`list` of :obj:`string` or :obj:`string`
     343        The time in hours of the field.
     344
     345    Return
     346    ------
     347    ftime : :obj:`list` of :obj:`string`
     348        The time in hours of the field.
     349    '''
     350    if not isinstance(ftime, list):
     351        ftime = [ftime]
     352
     353    return ftime
     354
     355def check_len_type_time_step(ftype, ftime, steps, maxstep, purefc):
     356    '''Check if
     357
     358    Parameters
     359    ----------
     360    ftype : :obj:`list` of :obj:`string`
     361        List of field types.
     362
     363    ftime : :obj:`list` of :obj:`string` or :obj:`string`
     364        The time in hours of the field.
     365
     366    steps : :obj:`string`
     367        Specifies the forecast time step from forecast base time.
     368        Valid values are hours (HH) from forecast base time.
     369
     370    maxstep : :obj:`integer`
     371        The maximum forecast time step in hours from the forecast base time.
     372        This is the maximum step for non flux (accumulated) forecast data.
     373
     374    purefc : :obj:`integer`
     375        Switch for definition of pure forecast mode or not.
     376
     377    Return
     378    ------
     379    ftype : :obj:`list` of :obj:`string`
     380        List of field types.
     381
     382    ftime : :obj:`list` of :obj:`string`
     383        The time in hours of the field.
     384
     385    steps : :obj:`string`
     386        Specifies the forecast time step from forecast base time.
     387        Valid values are hours (HH) from forecast base time.
     388    '''
     389    if not (len(ftype) == len(ftime) == len(steps)):
     390        raise ValueError('ERROR: The number of field types, times and steps '
     391                         'are not the same! Please check the setting in the '
     392                         'CONTROL file!')
     393
     394    # if pure forecast is selected and only one field type/time is set
     395    # prepare a complete list of type/time/step combination upto maxstep
     396    if len(ftype) == 1 and purefc:
     397        ftype = []
     398        steps = []
     399        ftime = []
     400        for i in range(0, maxstep + 1):
     401            ftype.append(ftype[0])
     402            steps.append('{:0>3}'.format(i))
     403            ftime.append(ftime[0])
     404
     405    return ftype, ftime, steps
     406
     407def check_mail(mail):
     408    '''Check the string of mail addresses, seperate them and convert to a list.
     409
     410    Parameters
     411    ----------
     412    mail : :obj:`list` of :obj:`string` or :obj:`string`
     413        Contains email addresses for notifications.
     414        It might also contain just the ecmwf user name which will trigger
     415        mailing to the associated email address for this user.
     416
     417    Return
     418    ------
     419    mail : :obj:`list` of :obj:``string`
     420        Contains email addresses for notifications.
     421        It might also contain just the ecmwf user name which will trigger
     422        mailing to the associated email address for this user.
     423
     424    '''
     425    if not isinstance(mail, list):
     426        if ',' in mail:
     427            mail = mail.split(',')
     428        elif ' ' in mail:
     429            mail = mail.split()
     430        else:
     431            mail = [mail]
     432
     433    return mail
     434
     435def check_queue(queue, gateway, destination, ecuid, ecgid):
     436    '''Check if the necessary ECMWF parameters are set if the queue is
     437    one of the QUEUES_LIST (in _config).
     438
     439    Parameters
     440    ----------
     441    queue : :obj:`string`
     442        Name of the queue if submitted to the ECMWF servers.
     443        Used to check if ecuid, ecgid, gateway and destination
     444        are set correctly and are not empty.
     445
     446    gateway : :obj:`string`
     447        The address of the gateway server.
     448
     449    destination : :obj:`string`
     450        The name of the destination of the gateway server for data
     451        transfer through ectrans. E.g. name@genericSftp
     452
     453    ecuid : :obj:`string`
     454        ECMWF user id.
     455
     456    ecgid : :obj:`string`
     457        ECMWF group id.
     458
     459    Return
     460    ------
     461
     462    '''
     463    if queue in _config.QUEUES_LIST and \
     464        not gateway or not destination or \
     465        not ecuid or not ecgid:
     466        raise ValueError('\nEnvironment variables GATEWAY, DESTINATION, ECUID '
     467                         'and ECGID were not set properly! \n '
     468                         'Please check for existence of file "ECMWF_ENV" '
     469                         'in the run directory!')
    236470    return
     471
     472def check_pathes(idir, odir, fpdir, fedir):
     473    '''Check if output and flexpart pathes are set.
     474
     475    Parameters
     476    ----------
     477    idir : :obj:`string`
     478        Path to the temporary directory for MARS retrieval data.
     479
     480    odir : :obj:`string`
     481        Path to the final output directory where the FLEXPART input files
     482        will be stored.
     483
     484    fpdir : :obj:`string`
     485        Path to FLEXPART root directory.
     486
     487    fedir : :obj:`string`
     488        Path to flex_extract root directory.
     489
     490    Return
     491    ------
     492    odir : :obj:`string`
     493        Path to the final output directory where the FLEXPART input files
     494        will be stored.
     495
     496    fpdir : :obj:`string`
     497        Path to FLEXPART root directory.
     498
     499    '''
     500    if not fpdir:
     501        fpdir = fedir
     502
     503    if not odir:
     504        odir = idir
     505
     506    return odir, fpdir
     507
     508def check_dates(start, end):
     509    '''Checks if there is at least a start date for a one day retrieval.
     510
     511    Checks if end date lies after start date and end date is set.
     512
     513    Parameters
     514    ----------
     515    start : :obj:`string`
     516        The start date of the retrieval job.
     517
     518    end : :obj:`string`
     519        The end date of the retrieval job.
     520
     521    Return
     522    ------
     523    start : :obj:`string`
     524        The start date of the retrieval job.
     525
     526    end : :obj:`string`
     527        The end date of the retrieval job.
     528
     529    '''
     530    # check for having at least a starting date
     531    # otherwise program is not allowed to run
     532    if not start:
     533        raise ValueError('start_date was neither specified in command line nor '
     534                         'in CONTROL file.\n'
     535                         'Try "{} -h" to print usage information'
     536                         .format(sys.argv[0].split('/')[-1]) )
     537
     538    # retrieve just one day if end_date isn't set
     539    if not end:
     540        end = start
     541
     542    dstart = datetime.strptime(start, '%Y%m%d')
     543    dend = datetime.strptime(end, '%Y%m%d')
     544    if dstart > dend:
     545        raise ValueError('ERROR: Start date is after end date! \n'
     546                         'Please adapt the dates in CONTROL file or '
     547                         'command line! (start={}; end={})'.format(start, end))
     548
     549    return start, end
     550
     551def check_maxstep(maxstep, steps):
     552    '''Convert maxstep into integer if it is already given. Otherwise, select
     553    maxstep by going through the steps list.
     554
     555    Parameters
     556    ----------
     557    maxstep : :obj:`string`
     558        The maximum forecast time step in hours from the forecast base time.
     559        This is the maximum step for non flux (accumulated) forecast data.
     560
     561    steps : :obj:`string`
     562        Specifies the forecast time step from forecast base time.
     563        Valid values are hours (HH) from forecast base time.
     564
     565    Return
     566    ------
     567    maxstep : :obj:`integer`
     568        The maximum forecast time step in hours from the forecast base time.
     569        This is the maximum step for non flux (accumulated) forecast data.
     570
     571    '''
     572    # if maxstep wasn't provided
     573    # search for it in the "step" parameter
     574    if not maxstep:
     575        maxstep = 0
     576        for s in steps:
     577            if int(s) > maxstep:
     578                maxstep = int(s)
     579    else:
     580        maxstep = int(maxstep)
     581
     582    return maxstep
     583
     584def check_basetime(basetime):
     585    '''Check if basetime is set and contains one of the two
     586    possible values (0, 12).
     587
     588    Parameters
     589    ----------
     590    basetime : :obj:``
     591        The time for a half day retrieval. The 12 hours upfront are to be
     592        retrieved.
     593
     594    Return
     595    ------
     596
     597    '''
     598    if basetime:
     599        if int(basetime) != 0 and int(basetime) != 12:
     600            raise ValueError('ERROR: Basetime has an invalid value '
     601                             '-> {}'.format(str(basetime)))
     602    return
     603
     604def check_request(request, marsfile):
     605    '''Check if there is an old mars request file and remove it.
     606
     607    Parameters
     608    ----------
     609    request : :obj:`integer`
     610        Selects the mode of retrieval.
     611        0: Retrieves the data from ECMWF.
     612        1: Prints the mars requests to an output file.
     613        2: Retrieves the data and prints the mars request.
     614
     615    marsfile : :obj:`string`
     616        Path to the mars request file.
     617
     618    Return
     619    ------
     620
     621    '''
     622    if request != 0:
     623        if os.path.isfile(marsfile):
     624            silent_remove(marsfile)
     625    return
     626
     627def check_public(public, dataset):
     628    '''Check wether the dataset parameter is set for a
     629    public data set retrieval.
     630
     631    Parameters
     632    ----------
     633    public : :obj:`ìnteger`
     634        Specifies if public data are to be retrieved or not.
     635
     636    dataset : :obj:`string`
     637        Specific name which identifies the public dataset.
     638
     639    Return
     640    ------
     641
     642    '''
     643    if public and not dataset:
     644        raise ValueError('ERROR: If public mars data wants to be retrieved, '
     645                         'the "dataset"-parameter has to be set too!')
     646    return
     647
     648def check_acctype(acctype, ftype):
     649    '''Guarantees that the accumulation field type is set.
     650
     651    If not set, it is derivated as in the old method (TYPE[1]).
     652
     653    Parameters
     654    ----------
     655    acctype : :obj:`string`
     656        The field type for the accumulated forecast fields.
     657
     658    ftype : :obj:`list` of :obj:`string`
     659        List of field types.
     660
     661    Return
     662    ------
     663    acctype : :obj:`string`
     664        The field type for the accumulated forecast fields.
     665    '''
     666    if not acctype:
     667        print('... Control parameter ACCTYPE was not defined.')
     668        try:
     669            if len(ftype) == 1 and ftype[0] != 'AN':
     670                print('Use same field type as for the non-flux fields.')
     671                acctype = ftype[0]
     672            elif len(ftype) > 1 and ftype[1] != 'AN':
     673                print('Use old setting by using TYPE[1] for flux forecast!')
     674                acctype = ftype[1]
     675        except:
     676            raise ValueError('ERROR: Accumulation field type could not be set!')
     677    else:
     678        if acctype.upper() == 'AN':
     679            raise ValueError('ERROR: Accumulation forecast fields can not be '
     680                             'of type "analysis"!')
     681    return acctype
     682
     683
     684def check_acctime(acctime, acctype, purefc):
     685    '''Guarantees that the accumulation forecast times were set.
     686
     687    If it is not set, it is tried to set the value fore some of the
     688    most commonly used data sets. Otherwise it raises an error.
     689
     690    Parameters
     691    ----------
     692    acctime : :obj:`string`
     693        The starting time from the accumulated forecasts.
     694
     695    acctype : :obj:`string`
     696        The field type for the accumulated forecast fields.
     697
     698    purefc : :obj:`integer`
     699        Switch for definition of pure forecast mode or not.
     700
     701    Return
     702    ------
     703    acctime : :obj:`string`
     704        The starting time from the accumulated forecasts.
     705    '''
     706    if not acctime:
     707        print('... Control parameter ACCTIME was not defined.')
     708        print('... Value will be set depending on field type: '
     709              '\t\t EA=06/18\n\t\t EI/OD=00/12\n\t\t  EP=18')
     710        if acctype.upper() == 'EA': # Era 5
     711            acctime = '06/18'
     712        elif acctype.upper() == 'EI': # Era-Interim
     713            acctime = '00/12'
     714        elif acctype.upper() == 'EP': # CERA
     715            acctime = '18'
     716        elif acctype.upper() == 'OD' and not purefc: # On-demand operational
     717            acctime = '00/12'
     718        else:
     719            raise ValueError('ERROR: Accumulation forecast time can not '
     720                             'automatically be derived!')
     721    return acctime
     722
     723def check_accmaxstep(accmaxstep, acctype, purefc, maxstep):
     724    '''Guarantees that the accumulation forecast step were set.
     725
     726    Parameters
     727    ----------
     728    accmaxstep : :obj:`string`
     729        The maximum forecast step for the accumulated forecast fields.
     730
     731    acctype : :obj:`string`
     732        The field type for the accumulated forecast fields.
     733
     734    purefc : :obj:`integer`
     735        Switch for definition of pure forecast mode or not.
     736
     737    maxstep : :obj:`string`
     738        The maximum forecast time step in hours from the forecast base time.
     739        This is the maximum step for non flux (accumulated) forecast data.
     740
     741    Return
     742    ------
     743    accmaxstep : :obj:`string`
     744        The maximum forecast step for the accumulated forecast fields.
     745    '''
     746    if not accmaxstep:
     747        print('... Control parameter ACCMAXSTEP was not defined.')
     748        print('... Value will be set depending on field type/time: '
     749              '\t\t EA/EI/OD=12\n\t\t  EP=24')
     750        if acctype.upper() in ['EA', 'EI', 'OD'] and not purefc:
     751            # Era 5, Era-Interim, On-demand operational
     752            accmaxstep = '12'
     753        elif acctype.upper() == 'EP': # CERA
     754            accmaxstep = '18'
     755        elif purefc and accmaxstep != maxstep:
     756            accmaxstep = maxstep
     757            print('... For pure forecast mode, the accumulated forecast must '
     758                  'have the same maxstep as the normal forecast fields!\n'
     759                  '\t\t Accmaxstep was set to maxstep!')
     760        else:
     761            raise ValueError('ERROR: Accumulation forecast step can not '
     762                             'automatically be derived!')
     763    else:
     764        if purefc and int(accmaxstep) != int(maxstep):
     765            accmaxstep = maxstep
     766            print('... For pure forecast mode, the accumulated forecast must '
     767                          'have the same maxstep as the normal forecast fields!\n'
     768                          '\t\t Accmaxstep was set to maxstep!')
     769    return accmaxstep
     770
     771def check_addpar(addpar):
     772    '''Check that addpar has correct format of additional parameters in
     773    a single string, so that it can be easily appended to the hard coded
     774    parameters that are retrieved in any case.
     775
     776    Parameters
     777    ----------
     778    addpar : :obj:`string` or :obj:'list' of :obj:'string'
     779        List of additional parameters to be retrieved.
     780
     781    Return
     782    ------
     783    addpar : :obj:'string'
     784        List of additional parameters to be retrieved.
     785    '''
     786
     787    if addpar and isinstance(addpar, str):
     788        if '/' in addpar:
     789            parlist = addpar.split('/')
     790            parlist = [p for p in parlist if p is not '']
     791        else:
     792            parlist = [addpar]
     793
     794        addpar = '/' + '/'.join(parlist)
     795
     796    return addpar
     797
Note: See TracChangeset for help on using the changeset viewer.
hosted by ZAMG