- Timestamp:
- Aug 31, 2018, 7:50:37 AM (6 years ago)
- Branches:
- master, ctbto, dev
- Children:
- 597d4d1
- Parents:
- e1228f3
- Location:
- python
- Files:
-
- 5 added
- 2 deleted
- 23 edited
Legend:
- Unmodified
- Added
- Removed
-
python/CONTROL.temp
rccab809 r54a8a01 1 DAY1 1 DAY1 20130501 2 2 DAY2 3 3 DTIME 3 -
python/CONTROL_FC
rd69b677 r54a8a01 1 DAY1 201311072 DAY2 201311083 DTIME 11 DAY1 2 DAY2 3 DTIME 3 4 4 M_TYPE FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC 5 5 M_TIME 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 6 6 M_STEP 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 7 ACCTYPE FC 8 ACCTIME 00 9 ACCMAXSTEP 36 7 10 M_MAXSTEP 36 8 11 M_LEVEL 137 … … 13 16 M_EXPVER 1 14 17 M_GRID 1000 15 M_LEFT - 17900016 M_LOWER -9000017 M_UPPER 9000018 M_RIGHT 18000018 M_LEFT -25000 19 M_LOWER 10000 20 M_UPPER 60000 21 M_RIGHT 60000 19 22 M_RESOL 159 20 M_GAUSS 021 M_ACCURACY 1622 M_OMEGA 023 M_OMEGADIFF 024 23 M_ETA 1 25 M_ETADIFF 026 24 M_DPDETA 1 27 25 M_SMOOTH 0 28 M_FORMAT GRIB129 26 M_ADDPAR /186/187/188/235/139/39 30 27 M_WRF 1 31 28 M_CWC 1 32 29 PREFIX EE 33 ECSTORAGE 1 34 ECTRANS 0 35 ECFSDIR ectmp:/${USER}/econdemand/ 36 MAILOPS ${USER} 37 MAILFAIL ${USER} 30 ECTRANS 1 38 31 EOF -
python/CONTROL_PF.temp
rd69b677 r54a8a01 34 34 MAILOPS ${USER} 35 35 MAILFAIL ${USER} 36 FLEXPARTDIR ${HOME}/ ECMWFDATA7.0/flexpart_code36 FLEXPARTDIR ${HOME}/flex_extract_v7.1/flexpart_code 37 37 GRIB2FLEXPART grib2flexpart 38 38 EOF -
python/ControlFile.py
re1228f3 r54a8a01 1 1 #!/usr/bin/env python 2 2 # -*- coding: utf-8 -*- 3 #************************************************************************4 # ToDo AP5 # - write a test class6 #************************************************************************7 3 #******************************************************************************* 8 4 # @Author: Leopold Haimberger (University of Vienna) … … 19 15 # self-explanation naming 20 16 # - outsource of class ControlFile 17 # - initialisation of class attributes ( to avoid high number of 18 # conditional statements and set default values ) 19 # - divided assignment of attributes and the check of conditions 20 # - outsourced the commandline argument assignments to control attributes 21 21 # 22 22 # @License: … … 36 36 # @Class Content: 37 37 # - __init__ 38 # - __read_controlfile__ 38 39 # - __str__ 40 # - assign_args_to_control 41 # - assign_envs_to_control 42 # - check_conditions 43 # - check_install_conditions 39 44 # - to_list 40 45 # 41 46 # @Class Attributes: 42 # - start_date 43 # - end_date 44 # - accuracy 45 # - omega 46 # - cwc 47 # - omegadiff 48 # - etadiff 49 # - level 50 # - levelist 51 # - step 52 # - maxstep 53 # - prefix 54 # - makefile 55 # - basetime 56 # - date_chunk 57 # - grib2flexpart 58 # - exedir 59 # - flexpart_root_scripts 60 # - ecmwfdatadir 47 # 61 48 # 62 49 #******************************************************************************* … … 66 53 # ------------------------------------------------------------------------------ 67 54 import os 55 import sys 68 56 import inspect 69 57 … … 92 80 ''' 93 81 @Description: 94 Initialises the instance of ControlFile class and defines and 95 assign all CONTROL file variables. Set default values if 96 parameter was not in CONTROL file. 82 Initialises the instance of ControlFile class and defines 83 all class attributes with default values. Afterwards calls 84 function __read_controlfile__ to read parameter from 85 Control file. 97 86 98 87 @Input: … … 106 95 <nothing> 107 96 ''' 97 98 # list of all possible class attributes and their default values 99 self.controlfile = filename 100 self.start_date = None 101 self.end_date = None 102 self.date_chunk = 3 103 self.dtime = None 104 self.basetime = None 105 self.maxstep = None 106 self.type = None 107 self.time = None 108 self.step = None 109 self.marsclass = None 110 self.stream = None 111 self.number = 'OFF' 112 self.expver = None 113 self.grid = None 114 self.area = '' 115 self.left = None 116 self.lower = None 117 self.upper = None 118 self.right = None 119 self.level = None 120 self.levelist = None 121 self.resol = None 122 self.gauss = 0 123 self.accuracy = 24 124 self.omega = 0 125 self.omegadiff = 0 126 self.eta = 0 127 self.etadiff = 0 128 self.etapar = 77 129 self.dpdeta = 1 130 self.smooth = 0 131 self.format = 'GRIB1' 132 self.addpar = None 133 self.prefix = 'EN' 134 self.cwc = 0 135 self.wrf = 0 136 self.ecfsdir = 'ectmp:/${USER}/econdemand/' 137 self.mailfail = ['${USER}'] 138 self.mailops = ['${USER}'] 139 self.grib2flexpart = 0 140 self.ecstorage = 0 141 self.ectrans = 0 142 self.inputdir = '../work' 143 self.outputdir = self.inputdir 144 self.ecmwfdatadir = None 145 self.exedir = None 146 self.flexpart_root_scripts = None 147 self.makefile = None 148 self.destination = None 149 self.gateway = None 150 self.ecuid = None 151 self.ecgid = None 152 self.install_target = None 153 self.debug = 0 154 155 self.__read_controlfile__() 156 157 return 158 159 def __read_controlfile__(self): 160 ''' 161 @Description: 162 Read CONTROL file and assign all CONTROL file variables. 163 164 @Input: 165 self: instance of ControlFile class 166 Description see class documentation. 167 168 @Return: 169 <nothing> 170 ''' 108 171 from tools import my_error 109 172 110 173 # read whole CONTROL file 111 with open( filename) as f:174 with open(self.controlfile) as f: 112 175 fdata = f.read().split('\n') 113 176 114 177 # go through every line and store parameter 115 # as class variable116 178 for ldata in fdata: 117 179 data = ldata.split() … … 145 207 data[1] = data[1][:i] + var + data[1][k+1:] 146 208 else: 147 my_error(None, 'Could not find variable ' + 148 data[1][j+1:k] + ' while reading ' + 149 filename) 209 my_error(self.mailfail, 210 'Could not find variable ' 211 + data[1][j+1:k] + ' while reading ' + 212 self.controlfile) 150 213 setattr(self, data[0].lower() + '_expanded', data[1]) 151 214 else: … … 159 222 pass 160 223 161 # check a couple of necessary attributes if they contain values 162 # otherwise set default values 163 if not hasattr(self, 'start_date'): 164 self.start_date = None 165 if not hasattr(self, 'end_date'): 224 # script directory 225 self.ecmwfdatadir = os.path.dirname(os.path.abspath(inspect.getfile( 226 inspect.currentframe()))) + '/../' 227 228 # Fortran source directory 229 self.exedir = self.ecmwfdatadir + 'src/' 230 231 return 232 233 def __str__(self): 234 ''' 235 @Description: 236 Prepares a string which have all the ControlFile 237 class attributes with its associated values. 238 Each attribute is printed in one line and in 239 alphabetical order. 240 241 Example: 242 'age': 10 243 'color': 'Spotted' 244 'kids': 0 245 'legs': 2 246 'name': 'Dog' 247 'smell': 'Alot' 248 249 @Input: 250 self: instance of ControlFile class 251 Description see class documentation. 252 253 @Return: 254 string of ControlFile class attributes with their values 255 ''' 256 import collections 257 258 attrs = vars(self) 259 attrs = collections.OrderedDict(sorted(attrs.items())) 260 261 return '\n'.join("%s: %s" % item for item in attrs.items()) 262 263 def assign_args_to_control(self, args): 264 ''' 265 @Description: 266 Overwrites the existing ControlFile instance attributes with 267 the command line arguments. 268 269 @Input: 270 self: instance of ControlFile class 271 Description see class documentation. 272 273 args: instance of ArgumentParser 274 Contains the commandline arguments from script/program call. 275 276 @Return: 277 <nothing> 278 ''' 279 280 # get dictionary of command line parameters and eliminate all 281 # parameters which are None (were not specified) 282 args_dict = vars(args) 283 arguments = {k : args_dict[k] for k in args_dict 284 if args_dict[k] != None} 285 286 # assign all passed command line arguments to ControlFile instance 287 for k, v in arguments.iteritems(): 288 setattr(self, str(k), v) 289 290 return 291 292 def assign_envs_to_control(self, envs): 293 ''' 294 @Description: 295 Assigns the ECMWF environment parameter. 296 297 @Input: 298 envs: dict of strings 299 Contains the ECMWF environment parameternames "ECUID", "ECGID", 300 "DESTINATION" and "GATEWAY" with its corresponding values. 301 They were read from the file "ECMWF_ENV". 302 303 @Return: 304 <nothing> 305 ''' 306 307 for k, v in envs.iteritems(): 308 setattr(self, str(k).lower(), str(v)) 309 310 return 311 312 def check_conditions(self): 313 ''' 314 @Description: 315 Checks a couple of necessary attributes and conditions, 316 such as if they exist and contain values. 317 Otherwise set default values. 318 319 @Input: 320 self: instance of ControlFile class 321 Description see class documentation. 322 323 @Return: 324 <nothing> 325 ''' 326 from tools import my_error 327 import numpy as np 328 329 # check for having at least a starting date 330 # otherwise program is not allowed to run 331 if self.start_date is None: 332 print 'start_date specified neither in command line nor ' + \ 333 'in CONTROL file ' + self.controlfile 334 print 'Try "' + sys.argv[0].split('/')[-1] + \ 335 ' -h" to print usage information' 336 sys.exit(1) 337 338 # retrieve just one day if end_date isn't set 339 if self.end_date is None: 166 340 self.end_date = self.start_date 167 if not hasattr(self, 'accuracy'): 168 self.accuracy = 24 169 if not hasattr(self, 'omega'): 170 self.omega = '0' 171 if not hasattr(self, 'cwc'): 172 self.cwc = '0' 173 if not hasattr(self, 'omegadiff'): 174 self.omegadiff = '0' 175 if not hasattr(self, 'etadiff'): 176 self.etadiff = '0' 177 if not hasattr(self, 'levelist'): 178 if not hasattr(self, 'level'): 179 print 'Warning: neither levelist nor level \ 180 specified in CONTROL file' 341 342 # assure consistency of levelist and level 343 if self.levelist is None: 344 if self.level is None: 345 print 'Warning: neither levelist nor level ' + \ 346 'specified in CONTROL file' 347 sys.exit(1) 181 348 else: 182 349 self.levelist = '1/to/' + self.level 183 350 else: 184 if 'to' in self.levelist :351 if 'to' in self.levelist.lower(): 185 352 self.level = self.levelist.split('/')[2] 186 353 else: 187 354 self.level = self.levelist.split('/')[-1] 188 355 189 if not hasattr(self, 'maxstep'): 190 # find out maximum step 356 # if area was provided at command line 357 # decompse area into its 4 components 358 if self.area: 359 afloat = '.' in self.area 360 l = self.area.split('/') 361 if afloat: 362 for i, item in enumerate(l): 363 item = str(int(float(item) * 1000)) 364 self.upper, self.left, self.lower, self.right = l 365 366 # prepare step for correct usage 367 if '/' in self.step: 368 l = self.step.split('/') 369 if 'to' in self.step.lower(): 370 if 'by' in self.step.lower(): 371 ilist = np.arange(int(l[0]), int(l[2]) + 1, int(l[4])) 372 self.step = ['{:0>3}'.format(i) for i in ilist] 373 else: 374 my_error(self.mailfail, self.step + ':\n' + 375 'if "to" is used, please use "by" as well') 376 else: 377 self.step = l 378 379 # if maxstep wasn't provided 380 # search for it in the "step" parameter 381 if self.maxstep is None: 191 382 self.maxstep = 0 192 383 for s in self.step: … … 196 387 self.maxstep = int(self.maxstep) 197 388 198 if not hasattr(self, 'prefix'): 199 self.prefix = 'EN' 200 if not hasattr(self, 'makefile'): 201 self.makefile = None 202 if not hasattr(self, 'basetime'): 203 self.basetime = None 204 if not hasattr(self, 'date_chunk'): 205 self.date_chunk = '3' 206 if not hasattr(self, 'grib2flexpart'): 207 self.grib2flexpart = '0' 208 209 # script directory 210 self.ecmwfdatadir = os.path.dirname(os.path.abspath(inspect.getfile( 211 inspect.currentframe()))) + '/../' 212 # Fortran source directory 213 self.exedir = self.ecmwfdatadir + 'src/' 214 215 # FLEXPART directory 216 if not hasattr(self, 'flexpart_root_scripts'): 389 # set root scripts since it is needed later on 390 if not self.flexpart_root_scripts: 217 391 self.flexpart_root_scripts = self.ecmwfdatadir 218 392 393 if not isinstance(self.mailfail, list): 394 if ',' in self.mailfail: 395 self.mailfail = self.mailfail.split(',') 396 elif ' ' in self.mailfail: 397 self.mailfail = self.mailfail.split() 398 else: 399 self.mailfail = [self.mailfail] 400 401 if not isinstance(self.mailops, list): 402 if ',' in self.mailops: 403 self.mailops = self.mailops.split(',') 404 elif ' ' in self.mailops: 405 self.mailops = self.mailops.split() 406 else: 407 self.mailops = [self.mailops] 408 409 if not self.gateway or not self.destination or \ 410 not self.ecuid or not self.ecgid: 411 print '\nEnvironment variables GATWAY, DESTINATION, ECUID and ' + \ 412 'ECGID were not set properly!' 413 print 'Please check for excistence of file "ECMWF_ENV" in the ' + \ 414 'python directory!' 415 sys.exit(1) 416 219 417 return 220 418 221 def __str__(self): 222 ''' 223 @Description: 224 Prepares a single string with all the comma seperated ControlFile 225 class attributes including their values. 226 227 Example: 228 {'kids': 0, 'name': 'Dog', 'color': 'Spotted', 229 'age': 10, 'legs': 2, 'smell': 'Alot'} 230 231 @Input: 232 self: instance of ControlFile class 233 Description see class documentation. 234 235 @Return: 236 string of ControlFile class attributes with their values 237 ''' 238 239 attrs = vars(self) 240 241 return ', '.join("%s: %s" % item for item in attrs.items()) 419 def check_install_conditions(self): 420 ''' 421 @Description: 422 Checks a couple of necessary attributes and conditions 423 for the installation such as if they exist and contain values. 424 Otherwise set default values. 425 426 @Input: 427 self: instance of ControlFile class 428 Description see class documentation. 429 430 @Return: 431 <nothing> 432 ''' 433 434 if self.install_target and \ 435 self.install_target not in ['local', 'ecgate', 'cca']: 436 print 'ERROR: unknown or missing installation target ' 437 print 'target: ', self.install_target 438 print 'please specify correct installation target \ 439 (local | ecgate | cca)' 440 print 'use -h or --help for help' 441 sys.exit(1) 442 443 if self.install_target and self.install_target != 'local': 444 if not self.ecgid or not self.ecuid or \ 445 not self.gateway or not self.destination: 446 print 'Please enter your ECMWF user id and group id as well as \ 447 the \nname of the local gateway and the ectrans \ 448 destination ' 449 print 'with command line options --ecuid --ecgid \ 450 --gateway --destination' 451 print 'Try "' + sys.argv[0].split('/')[-1] + \ 452 ' -h" to print usage information' 453 print 'Please consult ecaccess documentation or ECMWF user \ 454 support for further details' 455 sys.exit(1) 456 457 if not self.flexpart_root_scripts: 458 self.flexpart_root_scripts = '${HOME}' 459 else: 460 self.flexpart_root_scripts = self.flexpart_root_scripts 461 else: 462 if not self.flexpart_root_scripts: 463 self.flexpart_root_scripts = '../' 464 465 if not self.makefile: 466 self.makefile = 'Makefile.gfortran' 467 468 return 242 469 243 470 def to_list(self): … … 259 486 ''' 260 487 261 attrs = vars(self) 488 import collections 489 490 attrs = collections.OrderedDict(sorted(vars(self).items())) 491 262 492 l = list() 263 493 … … 283 513 return sorted(l) 284 514 285 # def to_dict(self):286 # '''287 288 # '''289 # parameters_dict = vars(self)290 291 # # remove unneeded parameter292 # parameters_dict.pop('_expanded', None)293 # parameters_dict.pop('exedir', None)294 # parameters_dict.pop('flexpart_root_scripts', None)295 # parameters_dict.pop('ecmwfdatadir', None)296 297 # parameters_dict_str = {}298 # for key, value in parameters_dict.iteritems():299 # if isinstance(value, list):300 # parameters_dict_str[str(key)] = get_list_as_string(value, ' ')301 # else:302 # parameters_dict_str[str(key)] = str(value)303 304 # return parameters_dict_str -
python/EcFlexpart.py
re1228f3 r54a8a01 1 1 #!/usr/bin/env python 2 2 # -*- coding: utf-8 -*- 3 #************************************************************************4 # ToDo AP5 # - specifiy file header documentation6 # - add class description in header information7 # - apply classtests8 # - add references to ECMWF specific software packages9 # - add describtion of deacc_fluxes10 # - change name of func deacc ( weil disagg )11 # - add desc of retrieve function12 #************************************************************************13 3 #******************************************************************************* 14 4 # @Author: Anne Fouilloux (University of Oslo) … … 151 141 152 142 fluxes: boolean, optional 153 Decides if athe flux parameter settings are stored or143 Decides if the flux parameter settings are stored or 154 144 the rest of the parameter list. 155 145 Default value is False. … … 175 165 self.dtime = c.dtime 176 166 i = 0 177 if fluxes is True and c.maxstep <24:167 if fluxes and c.maxstep <= 24: 178 168 # no forecast beyond one day is needed! 179 169 # Thus, prepare flux data manually as usual … … 192 182 btlist = [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 0] 193 183 194 if i % int(c.dtime) == 0 and c.maxstep > 24 or i in btlist:184 if i % int(c.dtime) == 0 and (i in btlist or c.maxstep > 24): 195 185 196 186 if ty not in self.types.keys(): … … 207 197 self.types[ty]['steps'] += st 208 198 i += 1 209 210 199 211 200 self.marsclass = c.marsclass … … 304 293 '{}'.format((int(self.resol) + 1) / 2)] 305 294 306 if c.omega == '1':295 if hasattr(c, 'omega') and c.omega == '1': 307 296 self.params['OG__ML'][0] += '/W' 308 297 309 298 # add cloud water content if necessary 310 if c.cwc == '1':299 if hasattr(c, 'cwc') and c.cwc == '1': 311 300 self.params['OG__ML'][0] += '/CLWC/CIWC' 312 301 313 302 # add vorticity and geopotential height for WRF if necessary 314 if c.wrf == '1':303 if hasattr(c, 'wrf') and c.wrf == '1': 315 304 self.params['OG__ML'][0] += '/Z/VO' 316 305 if '/D' not in self.params['OG__ML'][0]: … … 740 729 c.gateway, '-remote', c.destination, 741 730 '-source', ofile]) 742 print('ectrans:', p)731 #print('ectrans:', p) 743 732 744 733 if int(c.ecstorage) == 1 and c.ecapi is False: … … 890 879 891 880 for prod in product(*index_vals): 892 # flag for Fortran program CONVERT2 , initially False881 # flag for Fortran program CONVERT2 and file merging 893 882 convertFlag = False 894 883 print 'current prod: ', prod … … 906 895 # prepare some date and time parameter before reading the data 907 896 if gid is not None: 908 # Fortran program CONVERT2 is only done if gid at this time is 909 # not None, therefore save information in convertFlag 897 # Combine all temporary data files into final grib file if 898 # gid is at least one time not None. Therefore set convertFlag 899 # to save information. The fortran program CONVERT2 is also 900 # only done if convertFlag is True 910 901 convertFlag = True 911 902 # remove old fort.* files and open new ones 903 # they are just valid for a single product 912 904 for k, f in fdict.iteritems(): 913 905 silent_remove(c.inputdir + "/fort." + k) … … 922 914 '%Y%m%d%H') 923 915 timestamp += timedelta(hours=int(step)) 924 925 916 cdateH = datetime.strftime(timestamp, '%Y%m%d%H') 926 917 … … 966 957 levtype = grib_get(gid, 'typeOfLevel') 967 958 if paramId == 133 and gridtype == 'reduced_gg': 968 # Relativehumidity (Q.grb) is used as a template only959 # Specific humidity (Q.grb) is used as a template only 969 960 # so we need the first we "meet" 970 961 with open(c.inputdir + '/fort.18', 'w') as fout: … … 1028 1019 not available for this type or date/time\n' 1029 1020 print 'Check parameters CLASS, TYPE, STREAM, START_DATE\n' 1030 my_error(c , 'fort.21 is empty while parameter eta is set\1031 1021 my_error(c.mailfail, 'fort.21 is empty while parameter eta \ 1022 is set to 1 in CONTROL file') 1032 1023 1033 1024 # create the corresponding output file fort.15 … … 1061 1052 with open(fnout, 'wb') as fout: 1062 1053 for f in flist: 1063 shutil.copyfileobj(open(c.inputdir + '/' + f, 'rb'), fout) 1054 shutil.copyfileobj( 1055 open(c.inputdir + '/' + f, 'rb'), fout) 1064 1056 1065 1057 if c.omega == '1': … … 1068 1060 open(c.inputdir + '/fort.25', 'rb'), fout) 1069 1061 1070 if c.wrf == '1':1062 if hasattr(c, 'wrf') and c.wrf == '1': 1071 1063 fwrf.close() 1072 1064 … … 1307 1299 grib_index_release(iid) 1308 1300 1309 exit()1310 1301 return -
python/GribTools.py
re1228f3 r54a8a01 309 309 iid = grib_index_new_from_file(filename, index_keys) 310 310 else: 311 print 'in else zweig'312 311 grib_index_add_file(iid, filename) 313 312 -
python/MarsRetrieval.py
rff99eae r54a8a01 79 79 ''' 80 80 81 def __init__(self, server, marsclass="ei", dtype="", levtype="",81 def __init__(self, server, marsclass="ei", type="", levtype="", 82 82 levelist="", repres="", date="", resol="", stream="", 83 83 area="", time="", step="", expver="1", number="", … … 107 107 Default is the ERA-Interim dataset "ei". 108 108 109 dtype: string, optional109 type: string, optional 110 110 Determines the type of fields to be retrieved. 111 111 Selects between observations, images or fields. … … 286 286 self.server = server 287 287 self.marsclass = marsclass 288 self. dtype = dtype288 self.type = type 289 289 self.levtype = levtype 290 290 self.levelist = levelist -
python/UioFiles.py
rff99eae r54a8a01 34 34 # @Class Content: 35 35 # - __init__ 36 # - list_files 36 # - __str__ 37 # - __list_files__ 37 38 # - delete_files 38 39 # … … 51 52 # software specific module from flex_extract 52 53 #import profiling 53 from tools import silent_remove 54 from tools import silent_remove, get_list_as_string 54 55 55 56 # ------------------------------------------------------------------------------ … … 66 67 # CLASS FUNCTIONS 67 68 # -------------------------------------------------------------------------- 68 def __init__(self, pat tern):69 def __init__(self, path, pattern): 69 70 ''' 70 71 @Description: … … 75 76 Description see class documentation. 76 77 78 path: string 79 Directory where to list the files. 80 77 81 pattern: string 78 82 Regular expression pattern. For example: '*.grb' … … 82 86 ''' 83 87 88 self.path = path 84 89 self.pattern = pattern 85 90 self.files = None 91 92 self.__list_files__(self.path) 86 93 87 94 return 88 95 89 96 #@profiling.timefn 90 def list_files(self, path, callid=0):97 def __list_files__(self, path, callid=0): 91 98 ''' 92 99 @Description: … … 99 106 100 107 path: string 101 Directory where to listthe files.108 Path to the files. 102 109 103 110 callid: integer … … 129 136 if subdirs: 130 137 for subdir in subdirs: 131 self. list_files(os.path.join(path, subdir), callid=1)138 self.__list_files__(os.path.join(path, subdir), callid=1) 132 139 133 140 return 141 142 def __str__(self): 143 ''' 144 @Description: 145 Converts the list of files into a single string. 146 The entries are sepereated by "," sign. 147 148 @Input: 149 self: instance of UioFiles 150 Description see class documentation. 151 152 @Return: 153 files_string: string 154 The content of the list as a single string. 155 ''' 156 157 filenames = [os.path.basename(f) for f in self.files] 158 files_string = get_list_as_string(filenames, concatenate_sign=', ') 159 160 return files_string 134 161 135 162 def delete_files(self): -
python/compilejob.ksh
r991df6a r54a8a01 5 5 # start with sbatch NAME_OF_THIS_FILE directly on machine 6 6 7 #SBATCH --workdir=/scratch/ms/ spatlh00/lh07 #SBATCH --workdir=/scratch/ms/at/km4a 8 8 #SBATCH --qos=normal 9 9 #SBATCH --job-name=flex_ecmwf … … 33 33 module load grib_api/1.14.5 34 34 module load emos/437-r64 35 export FLEXPART_ROOT_SCRIPTS=$HOME 36 # export ECMWFDATA=$FLEXPART_ROOT/ECMWFDATA$VERSION 37 # export PYTHONPATH=$ECMWFDATA/python 38 # export PATH=${PATH}:$ECMWFDATA/python 39 export MAKEFILE=Makefile.gfortran 35 export FLEXPART_ROOT_SCRIPTS=${HOME} 36 export MAKEFILE=Makefile.gfortran 40 37 ;; 41 38 *cca*) … … 49 46 export GROUP=`echo $HOME | awk -F / '{print $4}'` 50 47 export SCRATCH=/scratch/ms/${GROUP}/${USER} 51 export FLEXPART_ROOT_SCRIPTS=$HOME 52 # export ECMWFDATA=$FLEXPART_ROOT/ECMWFDATA$VERSION 53 # export PYTHONPATH=$ECMWFDATA/python 54 # export PATH=${PATH}:$ECMWFDATA/python 55 export MAKEFILE=Makefile.CRAY 48 export FLEXPART_ROOT_SCRIPTS=${HOME} 49 export MAKEFILE=Makefile.gfortran 56 50 ;; 57 51 esac 58 52 59 mkdir -p $FLEXPART_ROOT_SCRIPTS/ ECMWFDATA$VERSION60 cd $FLEXPART_ROOT_SCRIPTS/ ECMWFDATA$VERSION # if FLEXPART_ROOT is not set this means cd to the home directory61 tar -xvf $HOME/ ECMWFDATA$VERSION.tar53 mkdir -p $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION 54 cd $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION # if FLEXPART_ROOT is not set this means cd to the home directory 55 tar -xvf $HOME/flex_extract_v$VERSION.tar 62 56 cd src 63 57 \rm *.o *.mod CONVERT2 -
python/compilejob.temp
refdb01a r54a8a01 34 34 module load emos/437-r64 35 35 export FLEXPART_ROOT_SCRIPTS= 36 # export ECMWFDATA=$FLEXPART_ROOT/ECMWFDATA$VERSION37 # export PYTHONPATH=$ECMWFDATA/python38 # export PATH=${PATH}:$ECMWFDATA/python39 36 export MAKEFILE=Makefile.gfortran 40 37 ;; … … 50 47 export SCRATCH=/scratch/ms/${GROUP}/${USER} 51 48 export FLEXPART_ROOT_SCRIPTS= 52 # export ECMWFDATA=$FLEXPART_ROOT/ECMWFDATA$VERSION53 # export PYTHONPATH=$ECMWFDATA/python54 # export PATH=${PATH}:$ECMWFDATA/python55 49 export MAKEFILE=Makefile.CRAY 56 50 ;; 57 51 esac 58 52 59 mkdir -p $FLEXPART_ROOT_SCRIPTS/ ECMWFDATA$VERSION60 cd $FLEXPART_ROOT_SCRIPTS/ ECMWFDATA$VERSION # if FLEXPART_ROOT is not set this means cd to the home directory61 tar -xvf $HOME/ ECMWFDATA$VERSION.tar53 mkdir -p $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION 54 cd $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION # if FLEXPART_ROOT is not set this means cd to the home directory 55 tar -xvf $HOME/flex_extract_v$VERSION.tar 62 56 cd src 63 57 \rm *.o *.mod CONVERT2 -
python/disaggregation.py
rff99eae r54a8a01 1 1 #!/usr/bin/env python 2 2 # -*- coding: utf-8 -*- 3 #************************************************************************4 # ToDo AP5 # - check alist of size 4 ?6 # - write a test, IMPORTANT7 #************************************************************************8 3 #******************************************************************************* 9 4 # @Author: Anne Philipp (University of Vienna) … … 12 7 # 13 8 # @Change History: 9 # 14 10 # November 2015 - Leopold Haimberger (University of Vienna): 15 11 # - migration of the methods dapoly and darain from Fortran … … 29 25 # 30 26 # @Module Description: 31 # disaggregation regationof deaccumulated flux data from an ECMWF model FG field.27 # disaggregation of deaccumulated flux data from an ECMWF model FG field. 32 28 # Initially the flux data to be concerned are: 33 29 # - large-scale precipitation -
python/get_mars_data.py
rff99eae r54a8a01 22 22 # online documentation) 23 23 # - use of UIFiles class for file selection and deletion 24 24 # 25 25 # 26 26 # @License: … … 39 39 # - main 40 40 # - get_mars_data 41 # - do_retrievement 41 42 # 42 43 #******************************************************************************* … … 56 57 57 58 # software specific classes and modules from flex_extract 58 from tools import my_error, normal_exit, interpret_args_and_control59 from tools import my_error, normal_exit, get_cmdline_arguments, read_ecenv 59 60 from EcFlexpart import EcFlexpart 60 61 from UioFiles import UioFiles … … 82 83 <nothing> 83 84 ''' 84 args, c = interpret_args_and_control() 85 86 args = get_cmdline_arguments() 87 88 try: 89 c = ControlFile(args.controlfile) 90 except IOError: 91 try: 92 c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile) 93 except IOError: 94 print 'Could not read CONTROL file "' + args.controlfile + '"' 95 print 'Either it does not exist or its syntax is wrong.' 96 print 'Try "' + sys.argv[0].split('/')[-1] + \ 97 ' -h" to print usage information' 98 sys.exit(1) 99 100 env_parameter = read_ecenv(c.ecmwfdatadir + 'python/ECMWF_ENV') 101 c.assign_args_to_control(args, env_parameter) 102 c.assign_envs_to_control(env_parameter) 103 c.check_conditions() 104 85 105 get_mars_data(c) 86 normal_exit(c )106 normal_exit(c.mailfail, 'Done!') 87 107 88 108 return … … 129 149 print 'ecapi: ', c.ecapi 130 150 151 # basetime geht rückwärts 152 153 # if basetime 00 154 # dann wird von 12 am vortag bis 00 am start tag geholt 155 # aber ohne 12 selbst sondern 12 + step 156 157 # if basetime 12 158 # dann wird von 00 + step bis 12 am start tag geholt 159 160 # purer forecast wird vorwärts bestimmt. 161 # purer forecast mode ist dann wenn größer 24 stunden 162 # wie kann das noch festgestellt werden ???? 163 # nur FC und steps mehr als 24 ? 164 # die einzige problematik beim reinen forecast ist die benennung der files! 165 # also sobald es Tagesüberschneidungen gibt 166 # allerdings ist das relevant und ersichtlich an den NICHT FLUSS DATEN 167 168 131 169 # set start date of retrieval period 132 170 start = datetime.date(year=int(c.start_date[:4]), … … 134 172 day=int(c.start_date[6:])) 135 173 startm1 = start - datetime.timedelta(days=1) 136 if c.basetime == '00':137 start = startm1138 174 139 175 # set end date of retrieval period … … 141 177 month=int(c.end_date[4:6]), 142 178 day=int(c.end_date[6:])) 179 180 # set time period for one single retrieval 181 datechunk = datetime.timedelta(days=int(c.date_chunk)) 182 183 if c.basetime == '00': 184 start = startm1 185 143 186 if c.basetime == '00' or c.basetime == '12': 187 # endp1 = end + datetime.timedelta(days=1) 188 endp1 = end 189 else: 190 # endp1 = end + datetime.timedelta(days=2) 144 191 endp1 = end + datetime.timedelta(days=1) 145 else:146 endp1 = end + datetime.timedelta(days=2)147 148 # set time period of one single retrieval149 datechunk = datetime.timedelta(days=int(c.date_chunk))150 192 151 193 # -------------- flux data ------------------------------------------------ 152 194 print 'removing old flux content of ' + c.inputdir 153 tobecleaned = UioFiles( '*_acc_*.' + str(os.getppid()) + '.*.grb')154 tobecleaned.list_files(c.inputdir)195 tobecleaned = UioFiles(c.inputdir, 196 '*_acc_*.' + str(os.getppid()) + '.*.grb') 155 197 tobecleaned.delete_files() 156 198 157 # if forecast for maximum one day (upto 2 3h) are to be retrieved,199 # if forecast for maximum one day (upto 24h) are to be retrieved, 158 200 # collect accumulation data (flux data) 159 201 # with additional days in the beginning and at the end 160 202 # (used for complete disaggregation of original period) 161 if c.maxstep < 24: 162 day = startm1 163 while day < endp1: 164 # retrieve MARS data for the whole period 165 flexpart = EcFlexpart(c, fluxes=True) 166 tmpday = day + datechunk - datetime.timedelta(days=1) 167 if tmpday < endp1: 168 dates = day.strftime("%Y%m%d") + "/to/" + \ 169 tmpday.strftime("%Y%m%d") 170 else: 171 dates = day.strftime("%Y%m%d") + "/to/" + \ 172 end.strftime("%Y%m%d") 173 174 print "retrieve " + dates + " in dir " + c.inputdir 175 176 try: 177 flexpart.retrieve(server, dates, c.inputdir) 178 except IOError: 179 my_error(c, 'MARS request failed') 180 181 day += datechunk 203 if c.maxstep <= 24: 204 do_retrievement(c, server, startm1, endp1, datechunk, fluxes=True) 182 205 183 206 # if forecast data longer than 24h are to be retrieved, … … 187 210 # exact time period with boundary conditions) 188 211 else: 189 day = start 190 while day <= end: 191 # retrieve MARS data for the whole period 192 flexpart = EcFlexpart(c, fluxes=True) 193 tmpday = day + datechunk - datetime.timedelta(days=1) 194 if tmpday < end: 195 dates = day.strftime("%Y%m%d") + "/to/" + \ 196 tmpday.trftime("%Y%m%d") 197 else: 198 dates = day.strftime("%Y%m%d") + "/to/" + \ 199 end.strftime("%Y%m%d") 200 201 print "retrieve " + dates + " in dir " + c.inputdir 202 203 try: 204 flexpart.retrieve(server, dates, c.inputdir) 205 except IOError: 206 my_error(c, 'MARS request failed') 207 208 day += datechunk 212 do_retrievement(c, server, start, end, datechunk, fluxes=True) 209 213 210 214 # -------------- non flux data -------------------------------------------- 211 215 print 'removing old non flux content of ' + c.inputdir 212 tobecleaned = UioFiles( '*__*.' + str(os.getppid()) + '.*.grb')213 tobecleaned.list_files(c.inputdir)216 tobecleaned = UioFiles(c.inputdir, 217 '*__*.' + str(os.getppid()) + '.*.grb') 214 218 tobecleaned.delete_files() 219 220 do_retrievement(c, server, start, end, datechunk, fluxes=False) 221 222 return 223 224 def do_retrievement(c, server, start, end, delta_t, fluxes=False): 225 ''' 226 @Description: 227 Divides the complete retrieval period in smaller chunks and 228 retrieves the data from MARS. 229 230 @Input: 231 c: instance of ControlFile 232 Contains all the parameters of CONTROL file, which are e.g.: 233 DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME, 234 STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT, 235 LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY, 236 OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT, 237 ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR, 238 MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME 239 DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS 240 241 For more information about format and content of the parameter 242 see documentation. 243 244 server: instance of ECMWFService 245 The server connection to ECMWF 246 247 start: instance of datetime 248 The start date of the retrieval. 249 250 end: instance of datetime 251 The end date of the retrieval. 252 253 delta_t: instance of datetime 254 Delta_t +1 is the maximal time period of a single 255 retrieval. 256 257 fluxes: boolean, optional 258 Decides if the flux parameters are to be retrieved or 259 the rest of the parameter list. 260 Default value is False. 261 262 @Return: 263 <nothing> 264 ''' 265 266 # since actual day also counts as one day, 267 # we only need to add datechunk - 1 days to retrieval 268 # for a period 269 delta_t_m1 = delta_t - datetime.timedelta(days=1) 215 270 216 271 day = start 217 272 while day <= end: 218 # retrieve all non flux MARS data for the whole period 219 flexpart = EcFlexpart(c, fluxes=False) 220 tmpday = day + datechunk - datetime.timedelta(days=1) 273 flexpart = EcFlexpart(c, fluxes) 274 tmpday = day + delta_t_m1 221 275 if tmpday < end: 222 276 dates = day.strftime("%Y%m%d") + "/to/" + \ … … 231 285 flexpart.retrieve(server, dates, c.inputdir) 232 286 except IOError: 233 my_error(c , 'MARS request failed')234 235 day += d atechunk287 my_error(c.mailfail, 'MARS request failed') 288 289 day += delta_t 236 290 237 291 return -
python/install.py
rff99eae r54a8a01 1 1 #!/usr/bin/env python 2 2 # -*- coding: utf-8 -*- 3 #************************************************************************4 # ToDo AP5 # - create a class Installation and divide installation in 3 subdefs for6 # ecgate, local and cca seperatly7 # - Change History ist nicht angepasst ans File! Original geben lassen8 #************************************************************************9 3 #******************************************************************************* 10 4 # @Author: Leopold Haimberger (University of Vienna) … … 37 31 # @Program Content: 38 32 # - main 39 # - install_args_and_control33 # - get_install_cmdline_arguments 40 34 # - install_via_gateway 35 # - mk_tarball 36 # - mk_env_vars 37 # - mk_compilejob 38 # - mk_job_template 39 # - delete_convert_build 40 # - make_convert_build 41 41 # 42 42 #******************************************************************************* … … 47 47 import os 48 48 import sys 49 import glob50 49 import subprocess 51 50 import inspect … … 54 53 # software specific classes and modules from flex_extract 55 54 from ControlFile import ControlFile 55 from UioFiles import UioFiles 56 from tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver 56 57 57 58 # add path to pythonpath so that python finds its buddies … … 61 62 sys.path.append(LOCAL_PYTHON_PATH) 62 63 64 _VERSION_STR = '7.1' 65 63 66 # ------------------------------------------------------------------------------ 64 67 # FUNCTIONS … … 78 81 79 82 os.chdir(LOCAL_PYTHON_PATH) 80 args, c = install_args_and_control() 81 82 if args.install_target is not None: 83 install_via_gateway(c, args.install_target) 84 else: 85 print 'Please specify installation target (local|ecgate|cca)' 86 print 'use -h or --help for help' 87 88 sys.exit() 89 90 return 91 92 93 def install_args_and_control(): 94 ''' 95 @Description: 96 Assigns the command line arguments for installation and reads 97 CONTROL file content. Apply default values for non mentioned arguments. 83 args = get_install_cmdline_arguments() 84 85 try: 86 c = ControlFile(args.controlfile) 87 except IOError: 88 print 'Could not read CONTROL file "' + args.controlfile + '"' 89 print 'Either it does not exist or its syntax is wrong.' 90 print 'Try "' + sys.argv[0].split('/')[-1] + \ 91 ' -h" to print usage information' 92 exit(1) 93 94 c.assign_args_to_control(args) 95 c.check_install_conditions() 96 97 install_via_gateway(c) 98 99 return 100 101 def get_install_cmdline_arguments(): 102 ''' 103 @Description: 104 Decomposes the command line arguments and assigns them to variables. 105 Apply default values for non mentioned arguments. 98 106 99 107 @Input: … … 103 111 args: instance of ArgumentParser 104 112 Contains the commandline arguments from script/program call. 105 113 ''' 114 parser = ArgumentParser(description='Install flex_extract software locally or \ 115 on ECMWF machines', 116 formatter_class=ArgumentDefaultsHelpFormatter) 117 118 parser.add_argument('--target', dest='install_target', default=None, 119 help="Valid targets: local | ecgate | cca , \ 120 the latter two are at ECMWF") 121 parser.add_argument("--makefile", dest="makefile", default=None, 122 help='Name of Makefile to use for compiling CONVERT2') 123 parser.add_argument("--ecuid", dest="ecuid", default=None, 124 help='user id at ECMWF') 125 parser.add_argument("--ecgid", dest="ecgid", default=None, 126 help='group id at ECMWF') 127 parser.add_argument("--gateway", dest="gateway", default=None, 128 help='name of local gateway server') 129 parser.add_argument("--destination", dest="destination", default=None, 130 help='ecaccess destination, e.g. leo@genericSftp') 131 132 parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts", 133 default=None, help="FLEXPART root directory on ECMWF \ 134 servers (to find grib2flexpart and COMMAND file)\n\ 135 Normally flex_extract resides in the scripts directory \ 136 of the FLEXPART distribution, thus the:") 137 138 # arguments for job submission to ECMWF, only needed by submit.py 139 parser.add_argument("--job_template", dest='job_template', 140 default="job.temp.o", 141 help="job template file for submission to ECMWF") 142 143 parser.add_argument("--controlfile", dest="controlfile", 144 default='CONTROL.temp', 145 help="file with CONTROL parameters") 146 147 args = parser.parse_args() 148 149 return args 150 151 152 def install_via_gateway(c): 153 ''' 154 @Description: 155 Perform the actual installation on local machine or prepare data 156 transfer to remote gate and submit a job script which will 157 install everything on the remote gate. 158 159 @Input: 106 160 c: instance of class ControlFile 107 161 Contains all necessary information of a CONTROL file. The parameters … … 113 167 For more information about format and content of the parameter see 114 168 documentation. 115 ''' 116 parser = ArgumentParser(description='Install ECMWFDATA software locally or \ 117 on ECMWF machines', 118 formatter_class=ArgumentDefaultsHelpFormatter) 119 120 parser.add_argument('--target', dest='install_target', 121 help="Valid targets: local | ecgate | cca , \ 122 the latter two are at ECMWF") 123 parser.add_argument("--makefile", dest="makefile", 124 help='Name of Makefile to use for compiling CONVERT2') 125 parser.add_argument("--ecuid", dest="ecuid", 126 help='user id at ECMWF') 127 parser.add_argument("--ecgid", dest="ecgid", 128 help='group id at ECMWF') 129 parser.add_argument("--gateway", dest="gateway", 130 help='name of local gateway server') 131 parser.add_argument("--destination", dest="destination", 132 help='ecaccess destination, e.g. leo@genericSftp') 133 134 parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts", 135 help="FLEXPART root directory on ECMWF servers \ 136 (to find grib2flexpart and COMMAND file)\n\ 137 Normally ECMWFDATA resides in the scripts directory \ 138 of the FLEXPART distribution, thus the:") 139 140 # arguments for job submission to ECMWF, only needed by submit.py 141 parser.add_argument("--job_template", dest='job_template', 142 default="job.temp.o", 143 help="job template file for submission to ECMWF") 144 145 parser.add_argument("--controlfile", dest="controlfile", 146 default='CONTROL.temp', 147 help="file with CONTROL parameters") 148 149 args = parser.parse_args() 150 151 try: 152 c = ControlFile(args.controlfile) 153 except IOError: 154 print 'Could not read CONTROL file "' + args.controlfile + '"' 155 print 'Either it does not exist or its syntax is wrong.' 156 print 'Try "' + sys.argv[0].split('/')[-1] + \ 157 ' -h" to print usage information' 158 exit(1) 159 160 if args.install_target != 'local': 161 if args.ecgid is None or args.ecuid is None or args.gateway is None \ 162 or args.destination is None: 163 print 'Please enter your ECMWF user id and group id as well as \ 164 the \nname of the local gateway and the ectrans \ 165 destination ' 166 print 'with command line options --ecuid --ecgid \ 167 --gateway --destination' 168 print 'Try "' + sys.argv[0].split('/')[-1] + \ 169 ' -h" to print usage information' 170 print 'Please consult ecaccess documentation or ECMWF user support \ 171 for further details' 172 sys.exit(1) 173 else: 174 c.ecuid = args.ecuid 175 c.ecgid = args.ecgid 176 c.gateway = args.gateway 177 c.destination = args.destination 178 179 if args.makefile: 180 c.makefile = args.makefile 181 182 if args.install_target == 'local': 183 if args.flexpart_root_scripts is None: 184 c.flexpart_root_scripts = '../' 185 else: 186 c.flexpart_root_scripts = args.flexpart_root_scripts 187 188 if args.install_target != 'local': 189 if args.flexpart_root_scripts is None: 190 c.ec_flexpart_root_scripts = '${HOME}' 191 else: 192 c.ec_flexpart_root_scripts = args.flexpart_root_scripts 193 194 return args, c 195 196 197 def install_via_gateway(c, target): 198 ''' 199 @Description: 200 Perform the actual installation on local machine or prepare data 201 transfer to remote gate and submit a job script which will 202 install everything on the remote gate. 203 204 @Input: 205 c: instance of class ControlFile 206 Contains all necessary information of a CONTROL file. The parameters 207 are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM, 208 NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST, 209 RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, 210 SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, 211 ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR 212 For more information about format and content of the parameter see 213 documentation. 214 215 target: string 216 The target where the installation should be processed. 217 E.g. "local", "ecgate" or "cca" 218 219 @Return: 220 <nothing> 221 ''' 169 170 @Return: 171 <nothing> 172 ''' 173 222 174 ecd = c.ecmwfdatadir 223 template = ecd + 'python/compilejob.temp' 224 job = ecd + 'python/compilejob.ksh' 225 fo = open(job, 'w') 226 #AP could do with open(template) as f, open(job, 'w') as fo: 227 #AP or nested with statements 228 with open(template) as f: 229 fdata = f.read().split('\n') 230 for data in fdata: 231 if 'MAKEFILE=' in data: 232 if c.makefile is not None: 233 data = 'export MAKEFILE=' + c.makefile 234 if 'FLEXPART_ROOT_SCRIPTS=' in data: 235 if c.flexpart_root_scripts != '../': 236 data = 'export FLEXPART_ROOT_SCRIPTS=' + \ 237 c.flexpart_root_scripts 238 else: 239 data = 'export FLEXPART_ROOT_SCRIPTS=$HOME' 240 if target.lower() != 'local': 241 if '--workdir' in data: 242 data = '#SBATCH --workdir=/scratch/ms/' + c.ecgid + \ 243 '/' + c.ecuid 244 if '##PBS -o' in data: 245 data = '##PBS -o /scratch/ms/' + c.ecgid + '/' + c.ecuid + \ 246 'flex_ecmwf.$Jobname.$Job_ID.out' 247 if 'FLEXPART_ROOT_SCRIPTS=' in data: 248 if c.ec_flexpart_root_scripts != '../': 249 data = 'export FLEXPART_ROOT_SCRIPTS=' + \ 250 c.ec_flexpart_root_scripts 251 else: 252 data = 'export FLEXPART_ROOT_SCRIPTS=$HOME' 253 fo.write(data + '\n') 254 f.close() 255 fo.close() 256 257 if target.lower() != 'local': 258 template = ecd + 'python/job.temp.o' 259 #AP hier eventuell Zeile für Zeile lesen und dann if Entscheidung 260 with open(template) as f: 261 fdata = f.read().split('\n') 262 f.close() 263 fo = open(template[:-2], 'w') 264 for data in fdata: 265 if '--workdir' in data: 266 data = '#SBATCH --workdir=/scratch/ms/' + c.ecgid + \ 267 '/' + c.ecuid 268 if '##PBS -o' in data: 269 data = '##PBS -o /scratch/ms/' + c.ecgid + '/' + \ 270 c.ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out' 271 if 'export PATH=${PATH}:' in data: 272 data += c.ec_flexpart_root_scripts + '/ECMWFDATA7.1/python' 273 if 'cat>>' in data or 'cat >>' in data: 274 i = data.index('>') 275 fo.write(data[:i] + data[i+1:] + '\n') 276 fo.write('GATEWAY ' + c.gateway + '\n') 277 fo.write('DESTINATION ' + c.destination + '\n') 278 fo.write('EOF\n') 279 280 fo.write(data + '\n') 281 fo.close() 282 283 job = ecd + 'python/ECMWF_ENV' 284 with open(job, 'w') as fo: 285 fo.write('ECUID ' + c.ecuid + '\n') 286 fo.write('ECGID ' + c.ecgid + '\n') 287 fo.write('GATEWAY ' + c.gateway + '\n') 288 fo.write('DESTINATION ' + c.destination + '\n') 289 fo.close() 290 291 if target.lower() == 'local': 292 # compile CONVERT2 293 if c.flexpart_root_scripts is None or c.flexpart_root_scripts == '../': 294 print 'Warning: FLEXPART_ROOT_SCRIPTS has not been specified' 295 print 'Only CONVERT2 will be compiled in ' + ecd + '/../src' 296 else: 175 tarball_name = 'flex_extract_v' + _VERSION_STR + '.tar' 176 target_dir = 'flex_extract_v' + _VERSION_STR 177 fortran_executable = 'CONVERT2' 178 179 if c.install_target.lower() != 'local': 180 181 mk_compilejob(ecd + 'python/compilejob.temp', c.makefile, 182 c.install_target, c.ecuid, c.ecgid, 183 c.flexpart_root_scripts) 184 185 mk_job_template(ecd + 'python/job.temp.o', c.ecuid, c.ecgid, c.gateway, 186 c.destination, c.flexpart_root_scripts) 187 188 mk_env_vars(ecd, c.ecuid, c.ecgid, c.gateway, c.destination) 189 190 #os.chdir('/') 191 192 mk_tarball(ecd, tarball_name) 193 194 put_file_to_ecserver(ecd, tarball_name, c.install_target, 195 c.ecuid, c.ecgid) 196 197 submit_job_to_ecserver(ecd + '/python/', c.install_target, 198 'compilejob.ksh') 199 200 print 'job compilation script has been submitted to ecgate for ' + \ 201 'installation in ' + c.flexpart_root_scripts + \ 202 '/' + target_dir 203 print 'You should get an email with subject flexcompile within ' + \ 204 'the next few minutes!' 205 206 else: #local 207 if not c.flexpart_root_scripts or c.flexpart_root_scripts == '../': 208 print 'WARNING: FLEXPART_ROOT_SCRIPTS has not been specified' 209 print 'There will be only the compilation of ' + \ 210 ' in ' + ecd + '/src' 211 os.chdir(ecd + '/src') 212 else: # creates the target working directory for flex_extract 297 213 c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser( 298 214 c.flexpart_root_scripts)) 299 215 if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts): 300 216 os.chdir('/') 301 p = subprocess.check_call(['tar', '-cvf', 302 ecd + '../ECMWFDATA7.1.tar', 303 ecd + 'python', 304 ecd + 'grib_templates', 305 ecd + 'src']) 306 try: 307 os.makedirs(c.flexpart_root_scripts + '/ECMWFDATA7.1') 308 finally: 309 pass 310 os.chdir(c.flexpart_root_scripts + '/ECMWFDATA7.1') 311 p = subprocess.check_call(['tar', '-xvf', 312 ecd + '../ECMWFDATA7.1.tar']) 313 os.chdir(c.flexpart_root_scripts + '/ECMWFDATA7.1/src') 314 315 os.chdir('../src') 316 print(('install ECMWFDATA7.1 software on ' + target + ' in directory ' 317 + os.getcwd())) 318 if c.makefile is None: 319 makefile = 'Makefile.local.ifort' 320 else: 321 makefile = c.makefile 322 flist = glob.glob('*.mod') + glob.glob('*.o') 323 if flist: 324 p = subprocess.check_call(['rm'] + flist) 325 try: 326 print 'Using makefile: ' + makefile 327 p = subprocess.check_call(['make', '-f', makefile]) 328 p = subprocess.check_call(['ls', '-l', 'CONVERT2']) 329 except subprocess.CalledProcessError as e: 330 print 'compile failed with the following error:' 331 print e.output 332 print 'please edit ' + makefile + \ 217 mk_tarball(ecd, tarball_name) 218 make_dir(c.flexpart_root_scripts + '/' + target_dir) 219 os.chdir(c.flexpart_root_scripts + '/' + target_dir) 220 print 'Untar ...' 221 subprocess.check_call(['tar', '-xvf', 222 ecd + '../' + tarball_name]) 223 os.chdir(c.flexpart_root_scripts + '/' + target_dir + '/src') 224 225 # Create Fortran executable - CONVERT2 226 print 'Install ' + target_dir + ' software on ' + \ 227 c.install_target + ' in directory ' + \ 228 os.path.abspath(os.getcwd() + '/../') + '\n' 229 230 delete_convert_build('') 231 make_convert_build('', c.makefile, fortran_executable) 232 233 return 234 235 def mk_tarball(ecd, tarname): 236 ''' 237 @Description: 238 Creates a tarball from all files which need to be sent to the 239 installation directory. 240 It does not matter if this is local or remote. 241 Collects all python files, the Fortran source and makefiles, 242 the ECMWF_ENV file, the CONTROL files as well as 243 the korn shell and template files. 244 245 @Input: 246 ecd: string 247 The path were the file is to be stored. 248 249 tarname: string 250 The name of the file to send to the ECMWF server. 251 252 @Return: 253 <nothing> 254 ''' 255 256 print 'Create tarball ...' 257 try: 258 subprocess.check_call(['tar -cvf '+ 259 ecd + '../' + tarname + ' ' + 260 ecd + 'python/*py ' + 261 ecd + 'python/CONTROL* ' + 262 ecd + 'python/*ksh ' + 263 ecd + 'python/*temp* ' + 264 ecd + 'python/ECMWF_ENV ' + 265 ecd + 'grib_templates ' + 266 ecd + 'src/*.f ' + 267 ecd + 'src/*.f90 ' + 268 ecd + 'src/*.h ' + 269 ecd + 'src/Makefile*'], shell=True) 270 except subprocess.CalledProcessError as e: 271 print 'ERROR:' 272 print e.output 273 sys.exit('could not make installation tar ball!') 274 275 return 276 277 def mk_env_vars(ecd, ecuid, ecgid, gateway, destination): 278 ''' 279 @Description: 280 Creates a file named ECMWF_ENV which contains the 281 necessary environmental variables at ECMWF servers. 282 283 @Input: 284 ecd: string 285 The path were the file is to be stored. 286 287 ecuid: string 288 The user id on ECMWF server. 289 290 ecgid: string 291 The group id on ECMWF server. 292 293 gateway: string 294 The gateway server the user is using. 295 296 destination: string 297 The remote destination which is used to transfer files 298 from ECMWF server to local gateway server. 299 300 @Return: 301 <nothing> 302 ''' 303 304 with open(ecd + 'python/ECMWF_ENV', 'w') as fo: 305 fo.write('ECUID ' + ecuid + '\n') 306 fo.write('ECGID ' + ecgid + '\n') 307 fo.write('GATEWAY ' + gateway + '\n') 308 fo.write('DESTINATION ' + destination + '\n') 309 310 return 311 312 def mk_compilejob(template, makefile, target, ecuid, ecgid, fp_root): 313 ''' 314 @Description: 315 Modifies the original job template file so that it is specified 316 for the user and the environment were it will be applied. Result 317 is stored in a new file "job.temp" in the python directory. 318 319 @Input: 320 template: string 321 File which contains the original text for the job template. 322 It must contain the complete path to the file. 323 324 makefile: string 325 Name of the makefile which should be used to compile FORTRAN 326 CONVERT2 program. 327 328 target: string 329 The target where the installation should be done, e.g. the queue. 330 331 ecuid: string 332 The user id on ECMWF server. 333 334 ecgid: string 335 The group id on ECMWF server. 336 337 fp_root: string 338 Path to the root directory of FLEXPART environment or flex_extract 339 environment. 340 341 @Return: 342 <nothing> 343 ''' 344 345 with open(template) as f: 346 fdata = f.read().split('\n') 347 348 with open(template[:-4] + 'ksh', 'w') as fo: 349 for data in fdata: 350 if 'MAKEFILE=' in data: 351 data = 'export MAKEFILE=' + makefile 352 elif 'FLEXPART_ROOT_SCRIPTS=' in data: 353 if fp_root != '../': 354 data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root 355 else: 356 data = 'export FLEXPART_ROOT_SCRIPTS=$HOME' 357 elif target.lower() != 'local': 358 if '--workdir' in data: 359 data = '#SBATCH --workdir=/scratch/ms/' + \ 360 ecgid + '/' + ecuid 361 elif '##PBS -o' in data: 362 data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \ 363 'flex_ecmwf.$Jobname.$Job_ID.out' 364 elif 'FLEXPART_ROOT_SCRIPTS=' in data: 365 if fp_root != '../': 366 data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root 367 else: 368 data = 'export FLEXPART_ROOT_SCRIPTS=$HOME' 369 fo.write(data + '\n') 370 371 return 372 373 def mk_job_template(template, ecuid, ecgid, gateway, destination, fp_root): 374 ''' 375 @Description: 376 Modifies the original job template file so that it is specified 377 for the user and the environment were it will be applied. Result 378 is stored in a new file "job.temp" in the python directory. 379 380 @Input: 381 template: string 382 File which contains the original text for the job template. 383 It must contain the complete path to the file. 384 385 ecuid: string 386 The user id on ECMWF server. 387 388 ecgid: string 389 The group id on ECMWF server. 390 391 gateway: string 392 The gateway server the user is using. 393 394 destination: string 395 The remote destination which is used to transfer files 396 from ECMWF server to local gateway server. 397 398 fp_root: string 399 Path to the root directory of FLEXPART environment or flex_extract 400 environment. 401 402 @Return: 403 <nothing> 404 ''' 405 406 with open(template) as f: 407 fdata = f.read().split('\n') 408 409 with open(template[:-2], 'w') as fo: 410 for data in fdata: 411 if '--workdir' in data: 412 data = '#SBATCH --workdir=/scratch/ms/' + ecgid + \ 413 '/' + ecuid 414 elif '##PBS -o' in data: 415 data = '##PBS -o /scratch/ms/' + ecgid + '/' + \ 416 ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out' 417 elif 'export PATH=${PATH}:' in data: 418 data += fp_root + '/flex_extract_v7.1/python' 419 420 fo.write(data + '\n') 421 return 422 423 def delete_convert_build(ecd): 424 ''' 425 @Description: 426 Clean up the Fortran source directory and remove all 427 build files (e.g. *.o, *.mod and CONVERT2) 428 429 @Input: 430 ecd: string 431 The path to the Fortran program. 432 433 @Return: 434 <nothing> 435 ''' 436 437 modfiles = UioFiles(ecd, '*.mod') 438 objfiles = UioFiles(ecd, '*.o') 439 exefile = UioFiles(ecd, 'CONVERT2') 440 441 modfiles.delete_files() 442 objfiles.delete_files() 443 exefile.delete_files() 444 445 return 446 447 def make_convert_build(ecd, makefile, f_executable): 448 ''' 449 @Description: 450 Compiles the Fortran code and generates the executable. 451 452 @Input: 453 ecd: string 454 The path were the file is to be stored. 455 456 makefile: string 457 The name of the makefile which should be used. 458 459 f_executable: string 460 The name of the executable the Fortran program generates after 461 compilation. 462 463 @Return: 464 <nothing> 465 ''' 466 467 try: 468 print 'Using makefile: ' + makefile 469 p = subprocess.Popen(['make', '-f', ecd + makefile], 470 stdin=subprocess.PIPE, 471 stdout=subprocess.PIPE, 472 stderr=subprocess.PIPE, 473 bufsize=1) 474 pout, perr = p.communicate() 475 print pout 476 if p.returncode != 0: 477 print perr 478 print 'Please edit ' + makefile + \ 333 479 ' or try another Makefile in the src directory.' 334 print ' most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB\335 480 print 'Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB ' \ 481 'and EMOSLIB must be adapted.' 336 482 print 'Available Makefiles:' 337 print glob.glob('Makefile*') 338 elif target.lower() == 'ecgate': 339 os.chdir('/') 340 p = subprocess.check_call(['tar', '-cvf', 341 ecd + '../ECMWFDATA7.1.tar', 342 ecd + 'python', 343 ecd + 'grib_templates', 344 ecd + 'src']) 345 try: 346 p = subprocess.check_call(['ecaccess-file-put', 347 ecd + '../ECMWFDATA7.1.tar', 348 'ecgate:/home/ms/' + c.ecgid + '/' + 349 c.ecuid + '/ECMWFDATA7.1.tar']) 350 except subprocess.CalledProcessError as e: 351 print 'ecaccess-file-put failed! \ 352 Probably the eccert key has expired.' 353 exit(1) 354 355 try: 356 p = subprocess.check_call(['ecaccess-job-submit', 357 '-queueName', 358 target, 359 ecd + 'python/compilejob.ksh']) 360 print 'compilejob.ksh has been submitted to ecgate for \ 361 installation in ' + c.ec_flexpart_root_scripts + \ 362 '/ECMWFDATA7.1' 363 print 'You should get an email with subject flexcompile within \ 364 the next few minutes' 365 except subprocess.CalledProcessError as e: 366 print 'ecaccess-job-submit failed!' 367 exit(1) 368 369 elif target.lower() == 'cca': 370 os.chdir('/') 371 p = subprocess.check_call(['tar', '-cvf', 372 ecd + '../ECMWFDATA7.1.tar', 373 ecd + 'python', 374 ecd + 'grib_templates', 375 ecd + 'src']) 376 try: 377 p = subprocess.check_call(['ecaccess-file-put', 378 ecd + '../ECMWFDATA7.1.tar', 379 'cca:/home/ms/' + c.ecgid + '/' + 380 c.ecuid + '/ECMWFDATA7.1.tar']) 381 except subprocess.CalledProcessError as e: 382 print 'ecaccess-file-put failed! \ 383 Probably the eccert key has expired.' 384 exit(1) 385 386 try: 387 p = subprocess.check_call(['ecaccess-job-submit', 388 '-queueName', 389 target, 390 ecd + 'python/compilejob.ksh']) 391 print 'compilejob.ksh has been submitted to cca for installation in ' +\ 392 c.ec_flexpart_root_scripts + '/ECMWFDATA7.1' 393 print 'You should get an email with subject flexcompile \ 394 within the next few minutes' 395 except subprocess.CalledProcessError as e: 396 print 'ecaccess-job-submit failed!' 397 exit(1) 398 483 print UioFiles('.', 'Makefile*') 484 sys.exit('Compilation failed!') 485 except ValueError as e: 486 print 'ERROR: Makefile call failed:' 487 print e 399 488 else: 400 print 'ERROR: unknown installation target ', target 401 print 'Valid targets: ecgate, cca, local' 489 subprocess.check_call(['ls', '-l', ecd + f_executable]) 402 490 403 491 return -
python/job.ksh
rccab809 r54a8a01 33 33 module load grib_api/1.14.5 34 34 module load emos/437-r64 35 # export ECMWFDATA=$HOME/ECMWFDATA$VERSION 36 # export PYTHONPATH=$ECMWFDATA/python 37 export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python 35 export PATH=${PATH}:${HOME}/flex_extract_v7.1/python 38 36 ;; 39 37 *cca*) … … 43 41 module load python 44 42 export SCRATCH=$TMPDIR 45 # export ECMWFDATA=$HOME/ECMWFDATA$VERSION 46 # export PYTHONPATH=$ECMWFDATA/python 47 export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python 43 export PATH=${PATH}:${HOME}/flex_extract_v7.1/python 48 44 ;; 49 # *)50 # export ECMWFDATA=$HOME/ECMWFDATA$VERSION51 # export PATH=/opt/anaconda/bin:$ECMWFDATA/python:${PATH}52 # export PYTHONPATH=/opt/anaconda/lib/python2.7/site-packages/grib_api:$ECMWFDATA/python53 # export SCRATCH=$ECMWFDATA/python54 # which python55 # ;;56 45 esac 57 46 … … 63 52 64 53 cat >$CONTROL<<EOF 65 GATEWAY srvx8.img.univie.ac.at 66 DESTINATION annep@genericSftp 67 accuracy 16 54 accuracy 24 68 55 addpar 186 187 188 235 139 39 56 area 69 57 basetime None 58 controlfile CONTROL.test 70 59 cwc 0 71 60 date_chunk 3 72 debug 1 61 debug 0 62 destination annep@genericSftp 73 63 dpdeta 1 74 64 dtime 3 75 65 ecfsdir ectmp:/${USER}/econdemand/ 66 ecgid at 76 67 ecstorage 0 77 68 ectrans 1 78 end_date 20160809 69 ecuid km4a 70 end_date 20000101 79 71 eta 0 80 72 etadiff 0 73 etapar 77 81 74 expver 1 82 75 format GRIB1 76 gateway srvx8.img.univie.ac.at 83 77 gauss 1 84 78 grib2flexpart 0 85 79 grid 5000 86 80 inputdir ../work 81 install_target None 82 job_template job.temp 87 83 left -15000 88 84 level 60 89 85 levelist 55/to/60 90 86 lower 30000 91 mailfail ${USER} 92 mailops ${USER} 87 mailfail ${USER} 88 mailops ${USER} 93 89 makefile None 94 90 marsclass EI … … 98 94 omegadiff 0 99 95 outputdir ../work 100 prefix EI 96 prefix EItest_ 97 queue ecgate 101 98 resol 63 102 99 right 45000 103 100 smooth 0 104 start_date 20 160809101 start_date 20000101 105 102 step 00 01 02 03 04 05 00 07 08 09 10 11 00 01 02 03 04 05 00 07 08 09 10 11 106 103 stream OPER … … 108 105 type AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC 109 106 upper 75000 107 wrf 0 110 108 EOF 111 109 -
python/job.temp
r991df6a r54a8a01 33 33 module load grib_api/1.14.5 34 34 module load emos/437-r64 35 # export ECMWFDATA=$HOME/ECMWFDATA$VERSION 36 # export PYTHONPATH=$ECMWFDATA/python 37 export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python 35 export PATH=${PATH}:${HOME}/flex_extract_v7.1/python 38 36 ;; 39 37 *cca*) … … 43 41 module load python 44 42 export SCRATCH=$TMPDIR 45 # export ECMWFDATA=$HOME/ECMWFDATA$VERSION 46 # export PYTHONPATH=$ECMWFDATA/python 47 export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python 43 export PATH=${PATH}:${HOME}/flex_extract_v7.1/python 48 44 ;; 49 # *)50 # export ECMWFDATA=$HOME/ECMWFDATA$VERSION51 # export PATH=/opt/anaconda/bin:$ECMWFDATA/python:${PATH}52 # export PYTHONPATH=/opt/anaconda/lib/python2.7/site-packages/grib_api:$ECMWFDATA/python53 # export SCRATCH=$ECMWFDATA/python54 # which python55 # ;;56 45 esac 57 46 … … 63 52 64 53 cat >$CONTROL<<EOF 65 GATEWAY srvx8.img.univie.ac.at66 DESTINATION annep@genericSftp67 EOF68 cat >>$CONTROL<<EOF69 54 EOF 70 55 -
python/job.temp.o
refdb01a r54a8a01 33 33 module load grib_api/1.14.5 34 34 module load emos/437-r64 35 # export ECMWFDATA=$HOME/ECMWFDATA$VERSION36 # export PYTHONPATH=$ECMWFDATA/python37 35 export PATH=${PATH}: 38 36 ;; … … 43 41 module load python 44 42 export SCRATCH=$TMPDIR 45 # export ECMWFDATA=$HOME/ECMWFDATA$VERSION46 # export PYTHONPATH=$ECMWFDATA/python47 43 export PATH=${PATH}: 48 44 ;; 49 # *)50 # export ECMWFDATA=$HOME/ECMWFDATA$VERSION51 # export PATH=/opt/anaconda/bin:$ECMWFDATA/python:${PATH}52 # export PYTHONPATH=/opt/anaconda/lib/python2.7/site-packages/grib_api:$ECMWFDATA/python53 # export SCRATCH=$ECMWFDATA/python54 # which python55 # ;;56 45 esac 57 46 … … 62 51 export CONTROL=CONTROL 63 52 64 cat > >$CONTROL<<EOF53 cat >$CONTROL<<EOF 65 54 EOF 66 55 -
python/joboper.ksh
rccab809 r54a8a01 33 33 module load grib_api/1.14.5 34 34 module load emos/437-r64 35 # export ECMWFDATA=$HOME/ECMWFDATA$VERSION 36 # export PYTHONPATH=$ECMWFDATA/python 37 export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python 35 export PATH=${PATH}:${HOME}/flex_extract_v7.1/python 38 36 ;; 39 37 *cca*) … … 43 41 module load python 44 42 export SCRATCH=$TMPDIR 45 # export ECMWFDATA=$HOME/ECMWFDATA$VERSION 46 # export PYTHONPATH=$ECMWFDATA/python 47 export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python 43 export PATH=${PATH}:${HOME}/flex_extract_v7.1/python 48 44 ;; 49 # *)50 # export ECMWFDATA=$HOME/ECMWFDATA$VERSION51 # export PATH=/opt/anaconda/bin:$ECMWFDATA/python:${PATH}52 # export PYTHONPATH=/opt/anaconda/lib/python2.7/site-packages/grib_api:$ECMWFDATA/python53 # export SCRATCH=$ECMWFDATA/python54 # which python55 # ;;56 45 esac 57 46 … … 67 56 accuracy 16 68 57 addpar 186 187 188 235 139 39 58 area 59 base_time ${MSJ_BASETIME} 69 60 basetime None 61 controlfile CONTROL.temp 70 62 cwc 0 71 63 date_chunk 3 72 64 debug 1 65 destination None 73 66 dpdeta 1 74 67 dtime 3 75 68 ecfsdir ectmp:/${USER}/econdemand/ 69 ecgid None 76 70 ecstorage 0 77 71 ectrans 1 78 start_date ${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY} 72 ecuid None 73 end_date ${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY} 79 74 eta 0 80 75 etadiff 0 76 etapar 77 81 77 expver 1 82 78 format GRIB1 79 gateway None 83 80 gauss 1 84 81 grib2flexpart 0 85 82 grid 5000 86 83 inputdir ../work 84 install_target None 85 job_template job.temp 87 86 left -15000 88 87 level 60 89 88 levelist 55/to/60 90 89 lower 30000 91 mailfail ${USER} 92 mailops ${USER} 90 mailfail ${USER} 91 mailops ${USER} 93 92 makefile None 94 93 marsclass EI … … 99 98 outputdir ../work 100 99 prefix EI 100 queue ecgate 101 101 resol 63 102 102 right 45000 … … 108 108 type AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC 109 109 upper 75000 110 wrf 0 110 111 EOF 111 112 -
python/plot_retrieved.py
rff99eae r54a8a01 1 1 #!/usr/bin/env python 2 2 # -*- coding: utf-8 -*- 3 #************************************************************************4 # ToDo AP5 # - documentation der Funktionen6 # - docu der progam functionality7 # - apply pep88 #************************************************************************9 3 #******************************************************************************* 10 4 # @Author: Leopold Haimberger (University of Vienna) … … 34 28 # - main 35 29 # - get_basics 30 # - get_files_per_date 36 31 # - plot_retrieved 37 32 # - plot_timeseries … … 219 214 c.area = np.asarray(c.area) 220 215 221 files = UioFiles(c.prefix+'*') 222 files.list_files(c.inputdir) 216 files = UioFiles(c.inputdir, c.prefix+'*') 223 217 ifiles = get_files_per_date(files.files, datelist) 224 218 ifiles.sort() … … 622 616 help="FLEXPART root directory (to find \ 623 617 'grib2flexpart and COMMAND file)\n \ 624 Normally ECMWFDATAresides in the scripts directory \618 Normally flex_extract resides in the scripts directory \ 625 619 of the FLEXPART distribution") 626 620 -
python/prepare_flexpart.py
rff99eae r54a8a01 1 1 #!/usr/bin/env python 2 2 # -*- coding: utf-8 -*- 3 #************************************************************************4 # ToDo AP5 # - wieso start=startm1 wenn basetime = 0 ? wenn die fluxes nicht mehr6 # relevant sind? verstehe ich nicht7 #************************************************************************8 3 #******************************************************************************* 9 4 # @Author: Anne Fouilloux (University of Oslo) … … 62 57 import sys 63 58 import socket 59 import _config 64 60 65 61 # software specific classes and modules from flex_extract 66 62 from UioFiles import UioFiles 67 from tools import interpret_args_and_control, clean_up63 from tools import clean_up, get_cmdline_arguments, read_ecenv 68 64 from EcFlexpart import EcFlexpart 69 65 … … 81 77 sys.path.append(LOCAL_PYTHON_PATH) 82 78 79 83 80 # ------------------------------------------------------------------------------ 84 81 # FUNCTION … … 97 94 <nothing> 98 95 ''' 99 args, c = interpret_args_and_control() 100 prepare_flexpart(args, c) 96 97 args = get_cmdline_arguments() 98 99 try: 100 c = ControlFile(args.controlfile) 101 except IOError: 102 try: 103 c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile) 104 except IOError: 105 print 'Could not read CONTROL file "' + args.controlfile + '"' 106 print 'Either it does not exist or its syntax is wrong.' 107 print 'Try "' + sys.argv[0].split('/')[-1] + \ 108 ' -h" to print usage information' 109 sys.exit(1) 110 111 env_parameter = read_ecenv(c.ecmwfdatadir + 'python/ECMWF_ENV') 112 c.assign_args_to_control(args, env_parameter) 113 c.assign_envs_to_control(env_parameter) 114 c.check_conditions() 115 prepare_flexpart(args.ppid, c) 101 116 102 117 return 103 118 104 def prepare_flexpart( args, c):119 def prepare_flexpart(ppid, c): 105 120 ''' 106 121 @Description: … … 112 127 113 128 @Input: 114 args: instance of ArgumentParser 115 Contains the commandline arguments from script/program call. 129 ppid: int 130 Contains the ppid number of the current ECMWF job. If it is called 131 from this script, it is "None". 116 132 117 133 c: instance of class ControlFile … … 132 148 ''' 133 149 134 if not args.ppid:150 if not ppid: 135 151 c.ppid = str(os.getppid()) 136 152 else: 137 c.ppid = args.ppid153 c.ppid = ppid 138 154 139 155 c.ecapi = ecapi … … 148 164 day=int(c.end_date[6:])) 149 165 150 # to deaccumulate the fluxes correctly 151 # one day ahead of the start date and 152 # one day after the end date is needed 153 startm1 = start - datetime.timedelta(days=1) 154 # endp1 = end + datetime.timedelta(days=1) 155 156 # get all files with flux data to be deaccumulated 157 inputfiles = UioFiles('*OG_acc_SL*.' + c.ppid + '.*') 158 inputfiles.list_files(c.inputdir) 166 # assign starting date minus 1 day 167 # since for basetime 00 we need the 12 hours upfront 168 # (the day before from 12 UTC to current day 00 UTC) 169 if c.basetime == '00': 170 start = start - datetime.timedelta(days=1) 171 172 print 'Prepare ' + start.strftime("%Y%m%d") + \ 173 "/to/" + end.strftime("%Y%m%d") 159 174 160 175 # create output dir if necessary 161 176 if not os.path.exists(c.outputdir): 162 177 os.makedirs(c.outputdir) 178 179 # get all files with flux data to be deaccumulated 180 inputfiles = UioFiles(c.inputdir, '*OG_acc_SL*.' + c.ppid + '.*') 163 181 164 182 # deaccumulate the flux data … … 167 185 flexpart.deacc_fluxes(inputfiles, c) 168 186 169 print 'Prepare ' + start.strftime("%Y%m%d") + \170 "/to/" + end.strftime("%Y%m%d")171 172 187 # get a list of all files from the root inputdir 173 inputfiles = UioFiles('????__??.*' + c.ppid + '.*') 174 inputfiles.list_files(c.inputdir) 175 176 # produce FLEXPART-ready GRIB files and 177 # process GRIB files - 188 inputfiles = UioFiles(c.inputdir, '????__??.*' + c.ppid + '.*') 189 190 # produce FLEXPART-ready GRIB files and process them - 178 191 # copy/transfer/interpolate them or make them GRIB2 179 if c.basetime == '00':180 start = startm1181 182 192 flexpart = EcFlexpart(c, fluxes=False) 183 193 flexpart.create(inputfiles, c) … … 187 197 # otherwise delete temporary files 188 198 if int(c.debug) != 0: 189 print ' Temporary files left intact'199 print '\nTemporary files left intact' 190 200 else: 191 201 clean_up(c) -
python/profiling.py
rff99eae r54a8a01 3 3 #************************************************************************ 4 4 # ToDo AP 5 # - check oflicense of book content5 # - check license of book content 6 6 #************************************************************************ 7 7 #******************************************************************************* -
python/submit.py
rff99eae r54a8a01 44 44 import subprocess 45 45 import inspect 46 import collections 46 47 47 48 # software specific classes and modules from flex_extract 48 from tools import interpret_args_and_control, normal_exit 49 import _config 50 from tools import normal_exit, get_cmdline_arguments, submit_job_to_ecserver, \ 51 read_ecenv 49 52 from get_mars_data import get_mars_data 50 53 from prepare_flexpart import prepare_flexpart 51 52 # add path to pythonpath so that python finds its buddies 53 LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath( 54 inspect.getfile(inspect.currentframe()))) 55 if LOCAL_PYTHON_PATH not in sys.path: 56 sys.path.append(LOCAL_PYTHON_PATH) 54 from ControlFile import ControlFile 57 55 58 56 # ------------------------------------------------------------------------------ … … 76 74 77 75 called_from_dir = os.getcwd() 78 args, c = interpret_args_and_control() 76 77 args = get_cmdline_arguments() 78 79 try: 80 c = ControlFile(args.controlfile) 81 except IOError: 82 try: 83 c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile) 84 except IOError: 85 print 'Could not read CONTROL file "' + args.controlfile + '"' 86 print 'Either it does not exist or its syntax is wrong.' 87 print 'Try "' + sys.argv[0].split('/')[-1] + \ 88 ' -h" to print usage information' 89 sys.exit(1) 90 91 env_parameter = read_ecenv(c.ecmwfdatadir + 'python/ECMWF_ENV') 92 c.assign_args_to_control(args) 93 c.assign_envs_to_control(env_parameter) 94 c.check_conditions() 79 95 80 96 # on local side 97 # on ECMWF server this would be the local side 81 98 if args.queue is None: 82 99 if c.inputdir[0] != '/': … … 84 101 if c.outputdir[0] != '/': 85 102 c.outputdir = os.path.join(called_from_dir, c.outputdir) 86 get_mars_data( args,c)87 prepare_flexpart(args , c)88 normal_exit(c )103 get_mars_data(c) 104 prepare_flexpart(args.ppid, c) 105 normal_exit(c.mailfail, 'Done!') 89 106 # on ECMWF server 90 107 else: … … 125 142 ''' 126 143 127 # read template file and split from newline signs144 # read template file and get index for CONTROL input 128 145 with open(jtemplate) as f: 129 146 lftext = f.read().split('\n') 130 147 insert_point = lftext.index('EOF') 131 148 132 # put all parameters of ControlFile instance into a list 133 clist = c.to_list() # ondemand 134 colist = [] # operational 135 mt = 0 149 if not c.basetime: 150 # --------- create on demand job script ------------------------------------ 151 if c.maxstep > 24: 152 print '---- Pure forecast mode! ----' 153 else: 154 print '---- On-demand mode! ----' 155 job_file = jtemplate[:-4] + 'ksh' 156 clist = c.to_list() 136 157 137 for elem in clist: 138 if 'maxstep' in elem: 139 mt = int(elem.split(' ')[1]) 158 lftextondemand = lftext[:insert_point] + clist + lftext[insert_point:] 140 159 141 for elem in clist: 142 if 'start_date' in elem: 143 elem = 'start_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}' 144 if 'end_date' in elem: 145 elem = 'end_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}' 146 if 'base_time' in elem: 147 elem = 'base_time ' + '${MSJ_BASETIME}' 148 if 'time' in elem and mt > 24: 149 elem = 'time ' + '${MSJ_BASETIME} {MSJ_BASETIME}' 150 colist.append(elem) 160 with open(job_file, 'w') as f: 161 f.write('\n'.join(lftextondemand)) 151 162 152 lftextondemand = lftext[:insert_point] + clist + lftext[insert_point + 2:] 153 lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:] 163 submit_job_to_ecserver('', queue, job_file) 154 164 155 with open('job.ksh', 'w') as h: 156 h.write('\n'.join(lftextondemand)) 165 else: 166 # --------- create operational job script ---------------------------------- 167 print '---- Operational mode! ----' 168 job_file = jtemplate[:-5] + 'oper.ksh' 169 #colist = [] 157 170 158 with open('joboper.ksh', 'w') as h: 159 h.write('\n'.join(lftextoper)) 171 if c.maxstep: 172 mt = int(c.maxstep) 173 else: 174 mt = 0 160 175 161 # submit job script to queue 162 try: 163 p = subprocess.check_call(['ecaccess-job-submit', '-queueName', 164 queue, 'job.ksh']) 165 except subprocess.CalledProcessError as e: 166 print 'ecaccess-job-submit failed!' 167 print 'Error Message: ' 168 print e.output 169 exit(1) 176 c.start_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}' 177 c.end_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}' 178 c.base_time = '${MSJ_BASETIME}' 179 if mt > 24: 180 c.time = '${MSJ_BASETIME} {MSJ_BASETIME}' 170 181 182 colist = c.to_list() 183 184 lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:] 185 186 with open(job_file, 'w') as f: 187 f.write('\n'.join(lftextoper)) 188 189 submit_job_to_ecserver('', queue, job_file) 190 191 # -------------------------------------------------------------------------- 171 192 print 'You should get an email with subject flex.hostname.pid' 172 193 -
python/test_suite.py
rff99eae r54a8a01 1 1 #!/usr/bin/env python 2 2 # -*- coding: utf-8 -*- 3 #************************************************************************4 # ToDo AP5 # - provide more tests6 # - provide more documentation7 #************************************************************************8 9 3 #******************************************************************************* 10 4 # @Author: Leopold Haimberger (University of Vienna) … … 25 19 # 26 20 # @Program Functionality: 27 # This script triggers the ECMWFDATAtest suite. Call with21 # This script triggers the flex_extract test suite. Call with 28 22 # test_suite.py [test group] 29 23 # -
python/tools.py
rff99eae r54a8a01 1 1 #!/usr/bin/env python 2 2 # -*- coding: utf-8 -*- 3 #************************************************************************4 # ToDo AP5 # - check my_error6 # - check normal_exit7 # - check get_list_as_string8 # - seperate args and control interpretation9 #************************************************************************10 3 #******************************************************************************* 11 4 # @Author: Anne Philipp (University of Vienna) … … 26 19 # - moved all functions from file Flexparttools to this file tools 27 20 # - added function get_list_as_string 21 # - seperated args and control interpretation 28 22 # 29 23 # @License: … … 38 32 # 39 33 # @Module Content: 40 # - interpret_args_and_control34 # - get_cmdline_arguments 41 35 # - clean_up 42 36 # - my_error … … 47 41 # - to_param_id 48 42 # - get_list_as_string 43 # - make_dir 49 44 # 50 45 #******************************************************************************* … … 57 52 import sys 58 53 import glob 59 import inspect60 54 import subprocess 61 55 import traceback 62 56 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter 63 import numpy as np64 65 # software specific class from flex_extract66 from ControlFile import ControlFile67 57 68 58 # ------------------------------------------------------------------------------ … … 70 60 # ------------------------------------------------------------------------------ 71 61 72 def interpret_args_and_control():73 ''' 74 @Description: 75 Assigns the command line arguments and reads CONTROL file76 content.Apply default values for non mentioned arguments.62 def get_cmdline_arguments(): 63 ''' 64 @Description: 65 Decomposes the command line arguments and assigns them to variables. 66 Apply default values for non mentioned arguments. 77 67 78 68 @Input: … … 82 72 args: instance of ArgumentParser 83 73 Contains the commandline arguments from script/program call. 84 85 c: instance of class ControlFile 86 Contains all necessary information of a CONTROL file. The parameters 87 are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM, 88 NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST, 89 RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, 90 SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, 91 ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, DEBUG, INPUTDIR, 92 OUTPUTDIR, FLEXPART_ROOT_SCRIPTS 93 For more information about format and content of the parameter see 94 documentation. 95 96 ''' 74 ''' 75 97 76 parser = ArgumentParser(description='Retrieve FLEXPART input from \ 98 ECMWF MARS archive',77 ECMWF MARS archive', 99 78 formatter_class=ArgumentDefaultsHelpFormatter) 100 79 101 80 # the most important arguments 102 parser.add_argument("--start_date", dest="start_date", 81 parser.add_argument("--start_date", dest="start_date", default=None, 103 82 help="start date YYYYMMDD") 104 parser.add_argument("--end_date", dest="end_date", 83 parser.add_argument("--end_date", dest="end_date", default=None, 105 84 help="end_date YYYYMMDD") 106 85 parser.add_argument("--date_chunk", dest="date_chunk", default=None, … … 108 87 109 88 # some arguments that override the default in the CONTROL file 110 parser.add_argument("--basetime", dest="basetime", 89 parser.add_argument("--basetime", dest="basetime", default=None, 111 90 help="base such as 00/12 (for half day retrievals)") 112 parser.add_argument("--step", dest="step", 91 parser.add_argument("--step", dest="step", default=None, 113 92 help="steps such as 00/to/48") 114 parser.add_argument("--levelist", dest="levelist", 93 parser.add_argument("--levelist", dest="levelist", default=None, 115 94 help="Vertical levels to be retrieved, e.g. 30/to/60") 116 parser.add_argument("--area", dest="area", 95 parser.add_argument("--area", dest="area", default=None, 117 96 help="area defined as north/west/south/east") 118 97 … … 123 102 help="root directory for storing output files") 124 103 parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts", 104 default=None, 125 105 help="FLEXPART root directory (to find grib2flexpart \ 126 and COMMAND file)\n Normally ECMWFDATAresides in \106 and COMMAND file)\n Normally flex_extract resides in \ 127 107 the scripts directory of the FLEXPART distribution") 128 108 129 109 # this is only used by prepare_flexpart.py to rerun a postprocessing step 130 parser.add_argument("--ppid", dest="ppid", 131 help=" Specify parent process id for \110 parser.add_argument("--ppid", dest="ppid", default=None, 111 help="specify parent process id for \ 132 112 rerun of prepare_flexpart") 133 113 … … 136 116 default="job.temp", 137 117 help="job template file for submission to ECMWF") 138 parser.add_argument("--queue", dest="queue", 118 parser.add_argument("--queue", dest="queue", default=None, 139 119 help="queue for submission to ECMWF \ 140 120 (e.g. ecgate or cca )") … … 142 122 default='CONTROL.temp', 143 123 help="file with CONTROL parameters") 144 parser.add_argument("--debug", dest="debug", default= 0,145 help=" Debug mode - leave temporary files intact")124 parser.add_argument("--debug", dest="debug", default=None, 125 help="debug mode - leave temporary files intact") 146 126 147 127 args = parser.parse_args() 148 128 149 # create instance of ControlFile for specified controlfile 150 # and assign the parameters (and default values if necessary) 151 try: 152 c = ControlFile(args.controlfile) 153 except IOError: 154 try: 155 LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath( 156 inspect.getfile(inspect.currentframe()))) 157 c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile) 158 except IOError: 159 print 'Could not read CONTROL file "' + args.controlfile + '"' 160 print 'Either it does not exist or its syntax is wrong.' 161 print 'Try "' + sys.argv[0].split('/')[-1] + \ 162 ' -h" to print usage information' 163 exit(1) 164 165 # check for having at least a starting date 166 if args.start_date is None and getattr(c, 'start_date') is None: 167 print 'start_date specified neither in command line nor \ 168 in CONTROL file ' + args.controlfile 169 print 'Try "' + sys.argv[0].split('/')[-1] + \ 170 ' -h" to print usage information' 171 exit(1) 172 173 # save all existing command line parameter to the ControlFile instance 174 # if parameter is not specified through the command line or CONTROL file 175 # set default values 176 if args.start_date is not None: 177 c.start_date = args.start_date 178 if args.end_date is not None: 179 c.end_date = args.end_date 180 if c.end_date is None: 181 c.end_date = c.start_date 182 if args.date_chunk is not None: 183 c.date_chunk = args.date_chunk 184 185 if not hasattr(c, 'debug'): 186 c.debug = args.debug 187 188 if args.inputdir is None and args.outputdir is None: 189 c.inputdir = '../work' 190 c.outputdir = '../work' 191 else: 192 if args.inputdir is not None: 193 c.inputdir = args.inputdir 194 if args.outputdir is None: 195 c.outputdir = args.inputdir 196 if args.outputdir is not None: 197 c.outputdir = args.outputdir 198 if args.inputdir is None: 199 c.inputdir = args.outputdir 200 201 if hasattr(c, 'outputdir') is False and args.outputdir is None: 202 c.outputdir = c.inputdir 203 else: 204 if args.outputdir is not None: 205 c.outputdir = args.outputdir 206 207 if args.area is not None: 208 afloat = '.' in args.area 209 l = args.area.split('/') 210 if afloat: 211 for i, item in enumerate(l): 212 item = str(int(float(item) * 1000)) 213 c.upper, c.left, c.lower, c.right = l 214 215 # NOTE: basetime activates the ''operational mode'' 216 if args.basetime is not None: 217 c.basetime = args.basetime 218 219 if args.step is not None: 220 l = args.step.split('/') 221 if 'to' in args.step.lower(): 222 if 'by' in args.step.lower(): 223 ilist = np.arange(int(l[0]), int(l[2]) + 1, int(l[4])) 224 c.step = ['{:0>3}'.format(i) for i in ilist] 225 else: 226 my_error(None, args.step + ':\n' + 227 'please use "by" as well if "to" is used') 228 else: 229 c.step = l 230 231 if args.levelist is not None: 232 c.levelist = args.levelist 233 if 'to' in c.levelist: 234 c.level = c.levelist.split('/')[2] 235 else: 236 c.level = c.levelist.split('/')[-1] 237 238 if args.flexpart_root_scripts is not None: 239 c.flexpart_root_scripts = args.flexpart_root_scripts 240 241 return args, c 242 129 return args 130 131 def read_ecenv(filename): 132 ''' 133 @Description: 134 Reads the file into a dictionary where the key values are the parameter 135 names. 136 137 @Input: 138 filename: string 139 Name of file where the ECMWV environment parameters are stored. 140 141 @Return: 142 envs: dict 143 ''' 144 envs= {} 145 print filename 146 with open(filename, 'r') as f: 147 for line in f: 148 data = line.strip().split() 149 envs[str(data[0])] = str(data[1]) 150 151 return envs 243 152 244 153 def clean_up(c): … … 269 178 270 179 cleanlist = glob.glob(c.inputdir + "/*") 271 for cl in cleanlist:272 if c.prefix not in cl :273 silent_remove(cl )180 for clist in cleanlist: 181 if c.prefix not in clist: 182 silent_remove(clist) 274 183 if c.ecapi is False and (c.ectrans == '1' or c.ecstorage == '1'): 275 silent_remove(cl )184 silent_remove(clist) 276 185 277 186 print "Done" … … 280 189 281 190 282 def my_error( c, message='ERROR'):191 def my_error(users, message='ERROR'): 283 192 ''' 284 193 @Description: … … 287 196 288 197 @Input: 289 c: instance of class ControlFile 290 Contains all the parameters of CONTROL file, which are e.g.: 291 DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME, 292 STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT, 293 LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY, 294 OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT, 295 ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR, 296 MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME 297 DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS 298 299 For more information about format and content of the parameter 300 see documentation. 198 user: list of strings 199 Contains all email addresses which should be notified. 200 It might also contain just the ecmwf user name which wil trigger 201 mailing to the associated email address for this user. 301 202 302 203 message: string, optional … … 310 211 311 212 # comment if user does not want email notification directly from python 312 try: 313 target = [] 314 target.extend(c.mailfail) 315 except AttributeError: 316 target = [] 317 target.extend(os.getenv('USER')) 318 319 for t in target: 320 p = subprocess.Popen(['mail', '-s ECMWFDATA v7.0 ERROR', 321 os.path.expandvars(t)], 322 stdin=subprocess.PIPE, 323 stdout=subprocess.PIPE, 324 stderr=subprocess.PIPE, 325 bufsize=1) 326 tr = '\n'.join(traceback.format_stack()) 327 pout = p.communicate(input=message + '\n\n' + tr)[0] 328 print 'Email sent to ' + os.path.expandvars(t) + ' ' + pout.decode() 329 330 exit(1) 331 332 return 333 334 335 def normal_exit(c, message='Done!'): 213 for user in users: 214 if '${USER}' in user: 215 user = os.getenv('USER') 216 try: 217 p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ERROR', 218 os.path.expandvars(user)], 219 stdin=subprocess.PIPE, 220 stdout=subprocess.PIPE, 221 stderr=subprocess.PIPE, 222 bufsize=1) 223 trace = '\n'.join(traceback.format_stack()) 224 pout = p.communicate(input=message + '\n\n' + trace)[0] 225 except ValueError as e: 226 print 'ERROR: ', e 227 sys.exit('Email could not be sent!') 228 else: 229 print 'Email sent to ' + os.path.expandvars(user) + ' ' + \ 230 pout.decode() 231 232 sys.exit(1) 233 234 return 235 236 237 def normal_exit(users, message='Done!'): 336 238 ''' 337 239 @Description: … … 339 241 340 242 @Input: 341 c: instance of class ControlFile 342 Contains all the parameters of CONTROL file, which are e.g.: 343 DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME, 344 STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT, 345 LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY, 346 OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT, 347 ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR, 348 MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME 349 DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS 350 351 For more information about format and content of the parameter 352 see documentation. 243 user: list of strings 244 Contains all email addresses which should be notified. 245 It might also contain just the ecmwf user name which wil trigger 246 mailing to the associated email address for this user. 353 247 354 248 message: string, optional … … 362 256 363 257 # comment if user does not want notification directly from python 364 try:365 target = []366 target.extend(c.mailops)367 for t in target:368 p = subprocess.Popen(['mail', '-s ECMWFDATA v7.0normal exit',369 os.path.expandvars( t)],258 for user in users: 259 if '${USER}' in user: 260 user = os.getenv('USER') 261 try: 262 p = subprocess.Popen(['mail', '-s flex_extract_v7.1 normal exit', 263 os.path.expandvars(user)], 370 264 stdin=subprocess.PIPE, 371 265 stdout=subprocess.PIPE, … … 373 267 bufsize=1) 374 268 pout = p.communicate(input=message+'\n\n')[0] 375 print 'Email sent to ' + os.path.expandvars(t) + ' ' + pout.decode() 376 finally: 377 pass 269 except ValueError as e: 270 print 'ERROR: ', e 271 print 'Email could not be sent!' 272 else: 273 print 'Email sent to ' + os.path.expandvars(user) + ' ' + \ 274 pout.decode() 378 275 379 276 return … … 433 330 except OSError as e: 434 331 # this would be "except OSError, e:" before Python 2.6 435 if e.errno is noterrno.ENOENT:332 if e.errno != errno.ENOENT: 436 333 # errno.ENOENT = no such file or directory 437 334 raise # re-raise exception if a different error occured … … 521 418 522 419 return str_of_list 420 421 def make_dir(directory): 422 ''' 423 @Description: 424 Creates a directory and gives a warning if the directory 425 already exists. The program stops only if there is another problem. 426 427 @Input: 428 directory: string 429 The directory path which should be created. 430 431 @Return: 432 <nothing> 433 ''' 434 try: 435 os.makedirs(directory) 436 except OSError as e: 437 if e.errno != errno.EEXIST: 438 # errno.EEXIST = directory already exists 439 raise # re-raise exception if a different error occured 440 else: 441 print 'WARNING: Directory {0} already exists!'.format(directory) 442 443 return 444 445 def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid): 446 ''' 447 @Description: 448 Uses the ecaccess command to send a file to the ECMWF servers. 449 Catches and prints the error if it failed. 450 451 @Input: 452 ecd: string 453 The path were the file is to be stored. 454 455 filename: string 456 The name of the file to send to the ECMWF server. 457 458 target: string 459 The target where the file should be sent to, e.g. the queue. 460 461 ecuid: string 462 The user id on ECMWF server. 463 464 ecgid: string 465 The group id on ECMWF server. 466 467 @Return: 468 <nothing> 469 ''' 470 471 try: 472 subprocess.check_call(['ecaccess-file-put', 473 ecd + '../' + filename, 474 target + ':/home/ms/' + 475 ecgid + '/' + ecuid + 476 '/' + filename]) 477 except subprocess.CalledProcessError as e: 478 print 'ERROR:' 479 print e 480 sys.exit('ecaccess-file-put failed!\n' + \ 481 'Probably the eccert key has expired.') 482 483 return 484 485 def submit_job_to_ecserver(ecd, target, jobname): 486 ''' 487 @Description: 488 Uses ecaccess to submit a job to the ECMWF server. 489 Catches and prints the error if one arise. 490 491 @Input: 492 ecd: string 493 The path were the file is to be stored. 494 495 target: string 496 The target where the file should be sent to, e.g. the queue. 497 498 jobname: string 499 The name of the jobfile to be submitted to the ECMWF server. 500 501 @Return: 502 <nothing> 503 ''' 504 505 try: 506 subprocess.check_call(['ecaccess-job-submit', 507 '-queueName', target, 508 jobname]) 509 except subprocess.CalledProcessError as e: 510 print '... ERROR CODE: ', e.returncode 511 sys.exit('... ECACCESS-JOB-SUBMIT FAILED!') 512 513 return
Note: See TracChangeset
for help on using the changeset viewer.