Changeset 6275f7b in flex_extract.git
- Timestamp:
- Jul 29, 2019, 8:44:41 AM (5 years ago)
- Branches:
- master, ctbto, dev
- Children:
- 433d1eb
- Parents:
- bc27d19
- Location:
- test/Regression/Mars_request
- Files:
-
- 70 added
- 6 deleted
- 5 edited
Legend:
- Unmodified
- Added
- Removed
-
test/Regression/Mars_request/7.1/EA5.highres_mr.csv
rd096177 r6275f7b 1 1 request_number, accuracy, area, dataset, date, expver, gaussian, grid, levelist, levtype, marsclass, number, param, repres, resol, step, stream, target, time, type 2 1, 24, 60.0/-100.0/0.0/60.0, None, 20181029/to/20181031, 1, , 0.25/0.25, 1, SFC, EA, OFF, LSP/CP/SSHF/EWSS/NSSS/SSR, , 799, 1/to/12/by/1, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//FCOG_acc_SL.20181029.100726.100727.grb, 06/18, FC3 1, 24, 60.0/-100.0/0.0/60.0, None, 20181030 /to/20181030, 1, , 0.25/0.25, 60/to/137, ML, EA, OFF, T/Q/U/V/77, , 799, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__ML.20181030.100726.100727.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN4 2, 24, 60.0/-100.0/0.0/60.0, None, 20181030 , 1, , 0.25/0.25, 1, SFC, EA, OFF, 160/27/28/244, , 799, 000, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//OG_OROLSM__SL.20181030.100726.100727.grb, 00, AN5 3, 24, 60.0/-100.0/0.0/60.0, None, 20181030/to/20181030, 1, , OFF, 1, ML, EA, OFF, LNSP, , 799, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANSH__SL.20181030.100726.100727.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN6 4, 24, 60.0/-100.0/0.0/60.0, None, 20181030/to/20181030, 1, , 0.25/0.25, 1, SFC, EA, OFF, 141/151/164/165/166/167/168/129/172/186/187/188/235/139/39, , 799, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__SL.20181030.100726.100727.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN2 1, 24, 60.0/-100.0/0.0/60.0, None, 20181029/to/20181031, 1, , 0.25/0.25, 1, SFC, EA, OFF, 142.128/143.128/146.128/180.128/181.128/176.128, , 799, 1/to/12/by/1, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//FCOG_acc_SL.20181029.135553.135554.grb, 06/18, FC 3 1, 24, 60.0/-100.0/0.0/60.0, None, 20181030, 1, , 0.25/0.25, 1, SFC, EA, OFF, 160.128/027.128/028.128/244.128, , 799, 000, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//OG_OROLSM__SL.20181030.135553.135554.grb, 00, AN 4 2, 24, 60.0/-100.0/0.0/60.0, None, 20181030/to/20181030, 1, , 0.25/0.25, 60/to/137, ML, EA, OFF, 130.128/133.128/131.128/132.128/077.128, , 799, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__ML.20181030.135553.135554.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN 5 3, 24, 60.0/-100.0/0.0/60.0, None, 20181030/to/20181030, 1, , 0.25/0.25, 1, SFC, EA, OFF, 141.128/151.128/164.128/165.128/166.128/167.128/168.128/129.128/172.128/186.128/187.128/188.128/235.128/139.128/039.128, , 799, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__SL.20181030.135553.135554.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN 6 4, 24, 60.0/-100.0/0.0/60.0, None, 20181030/to/20181030, 1, , OFF, 1, ML, EA, OFF, 152.128, , 799, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANSH__SL.20181030.135553.135554.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN -
test/Regression/Mars_request/7.1/EA5.public_mr.csv
rd096177 r6275f7b 1 1 request_number, accuracy, area, dataset, date, expver, gaussian, grid, levelist, levtype, marsclass, number, param, repres, resol, step, stream, target, time, type 2 1, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181029/to/20181031, 1, , 1.0/1.0, 1, SFC, EA, OFF, LSP/CP/SSHF/EWSS/NSSS/SSR, , 213, 1/to/12/by/1, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//FCOG_acc_SL.20181029.100842.100843.grb, 06/18, FC3 1, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181030 /to/20181030, 1, , 1.0/1.0, 1/to/137, ML, EA, OFF, T/Q/U/V/77, , 213, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__ML.20181030.100842.100843.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN4 2, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181030 , 1, , 1.0/1.0, 1, SFC, EA, OFF, 160/27/28/244, , 213, 000, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//OG_OROLSM__SL.20181030.100842.100843.grb, 00, AN5 3, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181030/to/20181030, 1, , OFF, 1, ML, EA, OFF, LNSP, , 213, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANSH__SL.20181030.100842.100843.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN6 4, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181030/to/20181030, 1, , 1.0/1.0, 1, SFC, EA, OFF, 141/151/164/165/166/167/168/129/172/186/187/188/235/139/39, , 213, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__SL.20181030.100842.100843.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN2 1, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181029/to/20181031, 1, , 1.0/1.0, 1, SFC, EA, OFF, 142.128/143.128/146.128/180.128/181.128/176.128, , 213, 1/to/12/by/1, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//FCOG_acc_SL.20181029.135612.135613.grb, 06/18, FC 3 1, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181030, 1, , 1.0/1.0, 1, SFC, EA, OFF, 160.128/027.128/028.128/244.128, , 213, 000, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//OG_OROLSM__SL.20181030.135612.135613.grb, 00, AN 4 2, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181030/to/20181030, 1, , 1.0/1.0, 1/to/137, ML, EA, OFF, 130.128/133.128/131.128/132.128/077.128, , 213, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__ML.20181030.135612.135613.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN 5 3, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181030/to/20181030, 1, , 1.0/1.0, 1, SFC, EA, OFF, 141.128/151.128/164.128/165.128/166.128/167.128/168.128/129.128/172.128/186.128/187.128/188.128/235.128/139.128/039.128, , 213, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__SL.20181030.135612.135613.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN 6 4, 24, 75.0/-15.0/30.0/45.0, ERA5, 20181030/to/20181030, 1, , OFF, 1, ML, EA, OFF, 152.128, , 213, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANSH__SL.20181030.135612.135613.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN -
test/Regression/Mars_request/7.1/EA5_mr.csv
rd096177 r6275f7b 1 1 request_number, accuracy, area, dataset, date, expver, gaussian, grid, levelist, levtype, marsclass, number, param, repres, resol, step, stream, target, time, type 2 1, 16, 75.0/-25.0/10.0/60.0, None, 20181029/to/20181031, 1, , 1.0/1.0, 1, SFC, EA, OFF, LSP/CP/SSHF/EWSS/NSSS/SSR, , 159, 1/to/12/by/1, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//FCOG_acc_SL.20181029.100785.100786.grb, 06/18, FC3 1, 16, 75.0/-25.0/10.0/60.0, None, 20181030 /to/20181030, 1, , 1.0/1.0, 130/to/137, ML, EA, OFF, T/Q/U/V/77, , 159, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__ML.20181030.100785.100786.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN4 2, 16, 75.0/-25.0/10.0/60.0, None, 20181030 , 1, , 1.0/1.0, 1, SFC, EA, OFF, 160/27/28/244, , 159, 000, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//OG_OROLSM__SL.20181030.100785.100786.grb, 00, AN5 3, 16, 75.0/-25.0/10.0/60.0, None, 20181030/to/20181030, 1, , OFF, 1, ML, EA, OFF, LNSP, , 159, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANSH__SL.20181030.100785.100786.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN6 4, 16, 75.0/-25.0/10.0/60.0, None, 20181030/to/20181030, 1, , 1.0/1.0, 1, SFC, EA, OFF, 141/151/164/165/166/167/168/129/172/186/187/188/235/139/39, , 159, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__SL.20181030.100785.100786.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN2 1, 16, 75.0/-25.0/10.0/60.0, None, 20181029/to/20181031, 1, , 1.0/1.0, 1, SFC, EA, OFF, 142.128/143.128/146.128/180.128/181.128/176.128, , 159, 1/to/12/by/1, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//FCOG_acc_SL.20181029.135671.135672.grb, 06/18, FC 3 1, 16, 75.0/-25.0/10.0/60.0, None, 20181030, 1, , 1.0/1.0, 1, SFC, EA, OFF, 160.128/027.128/028.128/244.128, , 159, 000, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//OG_OROLSM__SL.20181030.135671.135672.grb, 00, AN 4 2, 16, 75.0/-25.0/10.0/60.0, None, 20181030/to/20181030, 1, , 1.0/1.0, 130/to/137, ML, EA, OFF, 130.128/133.128/131.128/132.128/077.128, , 159, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__ML.20181030.135671.135672.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN 5 3, 16, 75.0/-25.0/10.0/60.0, None, 20181030/to/20181030, 1, , 1.0/1.0, 1, SFC, EA, OFF, 141.128/151.128/164.128/165.128/166.128/167.128/168.128/129.128/172.128/186.128/187.128/188.128/235.128/139.128/039.128, , 159, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANOG__SL.20181030.135671.135672.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN 6 4, 16, 75.0/-25.0/10.0/60.0, None, 20181030/to/20181030, 1, , OFF, 1, ML, EA, OFF, 152.128, , 159, 00, OPER, /raid60/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/test/Regression/Mars_request/./7.1//ANSH__SL.20181030.135671.135672.grb, 00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23, AN -
test/Regression/Mars_request/run_local.sh
r2f5ca80 r6275f7b 30 30 END_DATE=None 31 31 DATE_CHUNK=None 32 JOB_CHUNK=None 32 33 BASETIME=None 33 34 STEP=None … … 36 37 INPUTDIR='./'${1}'/' 37 38 OUTPUTDIR=None 38 FLEXPART_ROOT_SCRIPTS=None39 39 PP_ID=None 40 40 JOB_TEMPLATE='' … … 66 66 parameterlist+=" --date_chunk=$DATE_CHUNK" 67 67 fi 68 if [ -n "$JOB_CHUNK" ]; then 69 parameterlist+=" --job_chunk=$JOB_CHUNK" 70 fi 68 71 if [ -n "$BASETIME" ]; then 69 72 parameterlist+=" --basetime=$BASETIME" … … 83 86 if [ -n "$OUTPUTDIR" ]; then 84 87 parameterlist+=" --outputdir=$OUTPUTDIR" 85 fi86 if [ -n "$FLEXPART_ROOT_SCRIPTS" ]; then87 parameterlist+=" --flexpart_root_scripts=$FLEXPART_ROOT_SCRIPTS"88 88 fi 89 89 if [ -n "$PP_ID" ]; then -
test/Regression/Mars_request/test_cmp_mars_requests.py
r2f5ca80 r6275f7b 26 26 Example 27 27 ------- 28 python test_cmp_mars_requests.py 7.0. 37.128 python test_cmp_mars_requests.py 7.0.4 7.1 29 29 """ 30 30 … … 41 41 sys.path.append('../../../source/python') 42 42 import _config 43 from mods.tools import init128 43 44 44 45 # ------------------------------------------------------------------------------ … … 129 130 columns = list(mr_new.columns.values) 130 131 del columns[columns.index('target')] 132 mr_new = trim_all_columns(mr_new) 133 mr_old = trim_all_columns(mr_old) 131 134 for col in columns: 132 135 if mr_new[col].equals(mr_old[col]): … … 134 137 else: 135 138 err_msg += 'Unconsistency happend to be in column: ' + col + '\n' 139 print mr_new[col] 140 print mr_old[col] 136 141 return False 137 142 return lresult 143 144 145 def trim_all_columns(df): 146 """ 147 Trim whitespace from ends of each value across all series in dataframe 148 """ 149 trim_strings = lambda x: x.strip() if isinstance(x, str) else x 150 return df.applymap(trim_strings) 151 152 def convert_param_numbers(mr_old): 153 """ 154 Convert the numbers parameter into integers with 3 digits. 155 """ 156 157 if str(mr_old).strip() != "OFF" and mr_old != None and '/' in str(mr_old) : 158 numbers = mr_old.split('/') 159 number = str(int(numbers[0])).zfill(3)+'/TO/'+str(int(numbers[2])).zfill(3) 160 161 return number 162 163 return mr_old 164 165 def convert_param_step(mr_old): 166 """ 167 For pure forecast with steps greater than 23 hours, the older versions 168 writes out a list of steps instead with the syntax 'to' and 'by'. 169 e.g. 000/003/006/009/012/015/018/021/024/027/030/033/036 170 171 Convert this to 0/to/36/by/3 172 """ 173 174 #if 'to' in str(mr_old) and 'by' in str(mr_old): 175 # steps = mr_old.split('/') 176 # step = [] 177 # for i in range(int(steps[0]),int(steps[2]),int(steps[4])): 178 # step.append(str(int(i)).zfill(2)) 179 # return '/'.join(step) 180 181 if not mr_old.isdigit() and 'to' not in mr_old.lower(): 182 if int(mr_old.split('/')[-1]) > 23: 183 184 steps = mr_old.split('/') 185 dtime = int(steps[1]) - int(steps[0]) 186 187 nsteps = str(int(steps[1]))+'/to/'+str(int(steps[-1]))+'/by/'+str(int(dtime)) 188 return nsteps 189 190 191 return mr_old 192 193 def to_param_id(pars, table): 194 '''Transform parameter names to parameter ids with ECMWF grib table 128. 195 196 Parameters 197 ---------- 198 pars : str 199 Addpar argument from CONTROL file in the format of 200 parameter names instead of ids. The parameter short 201 names are sepearted with "/" and they are passed as 202 one single string. 203 204 table : dict 205 Contains the ECMWF grib table 128 information. 206 The key is the parameter number and the value is the 207 short name of the parameter. 208 209 Return 210 ------ 211 ipar : list of int 212 List of addpar parameters from CONTROL file transformed to 213 parameter ids in the format of integer. 214 ''' 215 if not pars: 216 return [] 217 if not isinstance(pars, str): 218 pars=str(pars) 219 220 cpar = pars.upper().split('/') 221 spar = [] 222 for par in cpar: 223 par = par.strip() 224 for k, v in table.items(): 225 if par.isdigit(): 226 par = str(int(par)).zfill(3) 227 if par == k or par == v: 228 spar.append(k + '.128') 229 break 230 else: 231 print('\n\Warning: par ' + par + ' not found in table 128\n\n”') 232 233 return '/'.join(str(l) for l in spar) 234 138 235 139 236 … … 143 240 control_path = 'Controls' 144 241 log_path = 'Log' 145 old_dir = sys.argv[1] # e.g. '7.0. 3'242 old_dir = sys.argv[1] # e.g. '7.0.4' 146 243 new_dir = sys.argv[2] # e.g. '7.1' 147 244 mr_filename = 'mars_requests.csv' … … 183 280 mr_old.columns = mr_old.columns.str.strip() 184 281 mr_new.columns = mr_new.columns.str.strip() 282 283 # convert names in old to ids 284 table128 = init128(_config.PATH_GRIBTABLE) 285 #print mr_old['param'] 286 287 # some format corrections are necessary to compare older versions with 7.1 288 mr_old['param'] = mr_old['param'].apply(to_param_id, args=[table128]) 289 mr_old['number'] = mr_old['number'].apply(convert_param_numbers) 290 if '142' in mr_old.ix[0,'param']: # if flux request 291 mr_old.ix[0,'step'] = convert_param_step(mr_old.ix[0,'step']) 292 293 print 'Results: ', c 185 294 186 295 # do tests on mr files … … 201 310 f.write('... \t' + err_msg + '\n') 202 311 312 exit 313 203 314 # exit with success or error status 204 315 if lfinal:
Note: See TracChangeset
for help on using the changeset viewer.