Changeset 6275f7b in flex_extract.git


Ignore:
Timestamp:
Jul 29, 2019, 8:44:41 AM (5 years ago)
Author:
Anne Philipp <anne.philipp@…>
Branches:
master, ctbto, dev
Children:
433d1eb
Parents:
bc27d19
Message:

completed testcase for mars request comparison

Location:
test/Regression/Mars_request
Files:
70 added
6 deleted
5 edited

Legend:

Unmodified
Added
Removed
  • test/Regression/Mars_request/7.1/EA5.highres_mr.csv

    rd096177 r6275f7b  
    11request_number, accuracy, area, dataset, date, expver, gaussian, grid, levelist, levtype, marsclass, number, param, repres, resol, step, stream, target, time, type
    2 1, 24, 60.0/-100.0/0.0/60.0, None, 20181029/to/20181031, 1, , 0.25/0.25, 1, SFC, EA, OFF, LSP/CP/SSHF/EWSS/NSSS/SSR, , 799, 1/to/12/by/1, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//FCOG_acc_SL.20181029.100726.100727.grb, 06/18, FC
    3 1, 24, 60.0/-100.0/0.0/60.0, None, 20181030/to/20181030, 1, , 0.25/0.25, 60/to/137, ML, EA, OFF, T/Q/U/V/77, , 799, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__ML.20181030.100726.100727.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
    4 2, 24, 60.0/-100.0/0.0/60.0, None, 20181030, 1, , 0.25/0.25, 1, SFC, EA, OFF, 160/27/28/244, , 799, 000, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//OG_OROLSM__SL.20181030.100726.100727.grb, 00, AN
    5 3, 24, 60.0/-100.0/0.0/60.0, None, 20181030/to/20181030, 1, , OFF, 1, ML, EA, OFF, LNSP, , 799, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANSH__SL.20181030.100726.100727.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
    6 4, 24, 60.0/-100.0/0.0/60.0, None, 20181030/to/20181030, 1, , 0.25/0.25, 1, SFC, EA, OFF, 141/151/164/165/166/167/168/129/172/186/187/188/235/139/39, , 799, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__SL.20181030.100726.100727.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
     21, 24, 60.0/-100.0/0.0/60.0, None, 20181029/to/20181031, 1, , 0.25/0.25, 1, SFC, EA, OFF, 142.128/143.128/146.128/180.128/181.128/176.128, , 799, 1/to/12/by/1, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//FCOG_acc_SL.20181029.135553.135554.grb, 06/18, FC
     31, 24, 60.0/-100.0/0.0/60.0, None, 20181030, 1, , 0.25/0.25, 1, SFC, EA, OFF, 160.128/027.128/028.128/244.128, , 799, 000, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//OG_OROLSM__SL.20181030.135553.135554.grb, 00, AN
     42, 24, 60.0/-100.0/0.0/60.0, None, 20181030/to/20181030, 1, , 0.25/0.25, 60/to/137, ML, EA, OFF, 130.128/133.128/131.128/132.128/077.128, , 799, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__ML.20181030.135553.135554.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
     53, 24, 60.0/-100.0/0.0/60.0, None, 20181030/to/20181030, 1, , 0.25/0.25, 1, SFC, EA, OFF, 141.128/151.128/164.128/165.128/166.128/167.128/168.128/129.128/172.128/186.128/187.128/188.128/235.128/139.128/039.128, , 799, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__SL.20181030.135553.135554.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
     64, 24, 60.0/-100.0/0.0/60.0, None, 20181030/to/20181030, 1, , OFF, 1, ML, EA, OFF, 152.128, , 799, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANSH__SL.20181030.135553.135554.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
  • test/Regression/Mars_request/7.1/EA5.public_mr.csv

    rd096177 r6275f7b  
    11request_number, accuracy, area, dataset, date, expver, gaussian, grid, levelist, levtype, marsclass, number, param, repres, resol, step, stream, target, time, type
    2 1, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181029/to/20181031, 1, , 1.0/1.0, 1, SFC, EA, OFF, LSP/CP/SSHF/EWSS/NSSS/SSR, , 213, 1/to/12/by/1, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//FCOG_acc_SL.20181029.100842.100843.grb, 06/18, FC
    3 1, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181030/to/20181030, 1, , 1.0/1.0, 1/to/137, ML, EA, OFF, T/Q/U/V/77, , 213, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__ML.20181030.100842.100843.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
    4 2, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181030, 1, , 1.0/1.0, 1, SFC, EA, OFF, 160/27/28/244, , 213, 000, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//OG_OROLSM__SL.20181030.100842.100843.grb, 00, AN
    5 3, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181030/to/20181030, 1, , OFF, 1, ML, EA, OFF, LNSP, , 213, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANSH__SL.20181030.100842.100843.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
    6 4, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181030/to/20181030, 1, , 1.0/1.0, 1, SFC, EA, OFF, 141/151/164/165/166/167/168/129/172/186/187/188/235/139/39, , 213, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__SL.20181030.100842.100843.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
     21, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181029/to/20181031, 1, , 1.0/1.0, 1, SFC, EA, OFF, 142.128/143.128/146.128/180.128/181.128/176.128, , 213, 1/to/12/by/1, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//FCOG_acc_SL.20181029.135612.135613.grb, 06/18, FC
     31, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181030, 1, , 1.0/1.0, 1, SFC, EA, OFF, 160.128/027.128/028.128/244.128, , 213, 000, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//OG_OROLSM__SL.20181030.135612.135613.grb, 00, AN
     42, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181030/to/20181030, 1, , 1.0/1.0, 1/to/137, ML, EA, OFF, 130.128/133.128/131.128/132.128/077.128, , 213, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__ML.20181030.135612.135613.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
     53, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181030/to/20181030, 1, , 1.0/1.0, 1, SFC, EA, OFF, 141.128/151.128/164.128/165.128/166.128/167.128/168.128/129.128/172.128/186.128/187.128/188.128/235.128/139.128/039.128, , 213, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__SL.20181030.135612.135613.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
     64, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181030/to/20181030, 1, , OFF, 1, ML, EA, OFF, 152.128, , 213, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANSH__SL.20181030.135612.135613.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
  • test/Regression/Mars_request/7.1/EA5_mr.csv

    rd096177 r6275f7b  
    11request_number, accuracy, area, dataset, date, expver, gaussian, grid, levelist, levtype, marsclass, number, param, repres, resol, step, stream, target, time, type
    2 1, 16, 75.0/-25.0/10.0/60.0, None, 20181029/to/20181031, 1, , 1.0/1.0, 1, SFC, EA, OFF, LSP/CP/SSHF/EWSS/NSSS/SSR, , 159, 1/to/12/by/1, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//FCOG_acc_SL.20181029.100785.100786.grb, 06/18, FC
    3 1, 16, 75.0/-25.0/10.0/60.0, None, 20181030/to/20181030, 1, , 1.0/1.0, 130/to/137, ML, EA, OFF, T/Q/U/V/77, , 159, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__ML.20181030.100785.100786.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
    4 2, 16, 75.0/-25.0/10.0/60.0, None, 20181030, 1, , 1.0/1.0, 1, SFC, EA, OFF, 160/27/28/244, , 159, 000, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//OG_OROLSM__SL.20181030.100785.100786.grb, 00, AN
    5 3, 16, 75.0/-25.0/10.0/60.0, None, 20181030/to/20181030, 1, , OFF, 1, ML, EA, OFF, LNSP, , 159, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANSH__SL.20181030.100785.100786.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
    6 4, 16, 75.0/-25.0/10.0/60.0, None, 20181030/to/20181030, 1, , 1.0/1.0, 1, SFC, EA, OFF, 141/151/164/165/166/167/168/129/172/186/187/188/235/139/39, , 159, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__SL.20181030.100785.100786.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
     21, 16, 75.0/-25.0/10.0/60.0, None, 20181029/to/20181031, 1, , 1.0/1.0, 1, SFC, EA, OFF, 142.128/143.128/146.128/180.128/181.128/176.128, , 159, 1/to/12/by/1, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//FCOG_acc_SL.20181029.135671.135672.grb, 06/18, FC
     31, 16, 75.0/-25.0/10.0/60.0, None, 20181030, 1, , 1.0/1.0, 1, SFC, EA, OFF, 160.128/027.128/028.128/244.128, , 159, 000, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//OG_OROLSM__SL.20181030.135671.135672.grb, 00, AN
     42, 16, 75.0/-25.0/10.0/60.0, None, 20181030/to/20181030, 1, , 1.0/1.0, 130/to/137, ML, EA, OFF, 130.128/133.128/131.128/132.128/077.128, , 159, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__ML.20181030.135671.135672.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
     53, 16, 75.0/-25.0/10.0/60.0, None, 20181030/to/20181030, 1, , 1.0/1.0, 1, SFC, EA, OFF, 141.128/151.128/164.128/165.128/166.128/167.128/168.128/129.128/172.128/186.128/187.128/188.128/235.128/139.128/039.128, , 159, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__SL.20181030.135671.135672.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
     64, 16, 75.0/-25.0/10.0/60.0, None, 20181030/to/20181030, 1, , OFF, 1, ML, EA, OFF, 152.128, , 159, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANSH__SL.20181030.135671.135672.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN
  • test/Regression/Mars_request/run_local.sh

    r2f5ca80 r6275f7b  
    3030END_DATE=None
    3131DATE_CHUNK=None
     32JOB_CHUNK=None
    3233BASETIME=None
    3334STEP=None
     
    3637INPUTDIR='./'${1}'/'
    3738OUTPUTDIR=None
    38 FLEXPART_ROOT_SCRIPTS=None
    3939PP_ID=None
    4040JOB_TEMPLATE=''
     
    6666  parameterlist+=" --date_chunk=$DATE_CHUNK"
    6767fi
     68if [ -n "$JOB_CHUNK" ]; then
     69  parameterlist+=" --job_chunk=$JOB_CHUNK"
     70fi
    6871if [ -n "$BASETIME" ]; then
    6972  parameterlist+=" --basetime=$BASETIME"
     
    8386if [ -n "$OUTPUTDIR" ]; then
    8487  parameterlist+=" --outputdir=$OUTPUTDIR"
    85 fi
    86 if [ -n "$FLEXPART_ROOT_SCRIPTS" ]; then
    87   parameterlist+=" --flexpart_root_scripts=$FLEXPART_ROOT_SCRIPTS"
    8888fi
    8989if [ -n "$PP_ID" ]; then
  • test/Regression/Mars_request/test_cmp_mars_requests.py

    r2f5ca80 r6275f7b  
    2626Example
    2727-------
    28     python test_cmp_mars_requests.py 7.0.3 7.1
     28    python test_cmp_mars_requests.py 7.0.4 7.1
    2929"""
    3030
     
    4141sys.path.append('../../../source/python')
    4242import _config
     43from  mods.tools import init128
    4344
    4445# ------------------------------------------------------------------------------
     
    129130    columns = list(mr_new.columns.values)
    130131    del columns[columns.index('target')]
     132    mr_new = trim_all_columns(mr_new)
     133    mr_old = trim_all_columns(mr_old)
    131134    for col in columns:
    132135        if mr_new[col].equals(mr_old[col]):
     
    134137        else:
    135138            err_msg += 'Unconsistency happend to be in column: ' + col + '\n'
     139            print mr_new[col]
     140            print mr_old[col]
    136141            return False
    137142    return lresult
     143
     144
     145def trim_all_columns(df):
     146    """
     147    Trim whitespace from ends of each value across all series in dataframe
     148    """
     149    trim_strings = lambda x: x.strip() if isinstance(x, str) else x
     150    return df.applymap(trim_strings)
     151
     152def convert_param_numbers(mr_old):
     153    """
     154    Convert the numbers parameter into integers with 3 digits.
     155    """
     156
     157    if str(mr_old).strip() != "OFF" and mr_old != None and '/' in str(mr_old) :
     158        numbers = mr_old.split('/')
     159        number = str(int(numbers[0])).zfill(3)+'/TO/'+str(int(numbers[2])).zfill(3)
     160
     161        return number
     162
     163    return mr_old
     164
     165def convert_param_step(mr_old):
     166    """
     167    For pure forecast with steps greater than 23 hours, the older versions
     168    writes out a list of steps instead with the syntax 'to' and 'by'.
     169    e.g. 000/003/006/009/012/015/018/021/024/027/030/033/036
     170   
     171    Convert this to 0/to/36/by/3
     172    """
     173
     174    #if 'to' in str(mr_old) and 'by' in str(mr_old):
     175    #    steps = mr_old.split('/')
     176    #    step = []
     177    #    for i in range(int(steps[0]),int(steps[2]),int(steps[4])):
     178    #        step.append(str(int(i)).zfill(2))
     179    #    return '/'.join(step)
     180   
     181    if not mr_old.isdigit() and 'to' not in mr_old.lower():
     182        if int(mr_old.split('/')[-1]) > 23:
     183   
     184            steps = mr_old.split('/')
     185            dtime = int(steps[1]) - int(steps[0])
     186           
     187            nsteps = str(int(steps[1]))+'/to/'+str(int(steps[-1]))+'/by/'+str(int(dtime))
     188            return nsteps
     189           
     190   
     191    return mr_old
     192
     193def to_param_id(pars, table):
     194    '''Transform parameter names to parameter ids with ECMWF grib table 128.
     195
     196    Parameters
     197    ----------
     198    pars : str
     199        Addpar argument from CONTROL file in the format of
     200        parameter names instead of ids. The parameter short
     201        names are sepearted with "/" and they are passed as
     202        one single string.
     203
     204    table : dict
     205        Contains the ECMWF grib table 128 information.
     206        The key is the parameter number and the value is the
     207        short name of the parameter.
     208
     209    Return
     210    ------
     211    ipar : list of int
     212        List of addpar parameters from CONTROL file transformed to
     213        parameter ids in the format of integer.
     214    '''
     215    if not pars:
     216        return []
     217    if not isinstance(pars, str):
     218        pars=str(pars)
     219
     220    cpar = pars.upper().split('/')
     221    spar = []
     222    for par in cpar:
     223        par = par.strip()
     224        for k, v in table.items():
     225            if par.isdigit():
     226                par = str(int(par)).zfill(3)
     227            if par == k or par == v:
     228                spar.append(k + '.128')
     229                break
     230        else:
     231            print('\n\Warning: par ' + par + ' not found in table 128\n\n”')
     232
     233    return '/'.join(str(l) for l in spar)
     234
    138235
    139236
     
    143240    control_path = 'Controls'
    144241    log_path = 'Log'
    145     old_dir = sys.argv[1] # e.g. '7.0.3'
     242    old_dir = sys.argv[1] # e.g. '7.0.4'
    146243    new_dir = sys.argv[2] # e.g. '7.1'
    147244    mr_filename = 'mars_requests.csv'
     
    183280        mr_old.columns = mr_old.columns.str.strip()
    184281        mr_new.columns = mr_new.columns.str.strip()
     282
     283        # convert names in old to ids
     284        table128 = init128(_config.PATH_GRIBTABLE)
     285        #print mr_old['param']
     286
     287        # some format corrections are necessary to compare older versions with 7.1
     288        mr_old['param'] = mr_old['param'].apply(to_param_id, args=[table128])
     289        mr_old['number'] = mr_old['number'].apply(convert_param_numbers) 
     290        if '142' in mr_old.ix[0,'param']: # if flux request
     291            mr_old.ix[0,'step'] = convert_param_step(mr_old.ix[0,'step'])
     292
     293        print 'Results: ', c
    185294
    186295        # do tests on mr files
     
    201310                    f.write('... \t' + err_msg + '\n')
    202311
     312        exit
     313
    203314    # exit with success or error status
    204315    if lfinal:
Note: See TracChangeset for help on using the changeset viewer.
hosted by ZAMG