Changeset 96e1533 in flex_extract.git for source/python
- Timestamp:
- Nov 10, 2018, 9:37:36 PM (5 years ago)
- Branches:
- master, ctbto, dev
- Children:
- 1abf820
- Parents:
- 8500ba1
- Location:
- source/python
- Files:
-
- 6 edited
-
_config.py (modified) (2 diffs)
-
classes/ControlFile.py (modified) (1 diff)
-
classes/EcFlexpart.py (modified) (3 diffs)
-
mods/get_mars_data.py (modified) (2 diffs)
-
mods/tools.py (modified) (18 diffs)
-
submit.py (modified) (3 diffs)
Legend:
- Unmodified
- Added
- Removed
-
source/python/_config.py
r36a79af r96e1533 30 30 _VERSION_STR = '7.1' 31 31 32 QUEUES_LIST = ['ecgate', 'cca' ]32 QUEUES_LIST = ['ecgate', 'cca', 'ccb'] 33 33 34 34 # ------------------------------------------------------------------------------ … … 78 78 PATH_JOBSCRIPTS = os.path.join(PATH_RUN_DIR, 'jobscripts') 79 79 PATH_FORTRAN_SRC = os.path.join(PATH_SOURCES, 'fortran') 80 PATH_ TEST_DIR= os.path.join(PATH_SOURCES, 'pythontest')80 PATH_PYTHONTEST_SRC = os.path.join(PATH_SOURCES, 'pythontest') 81 81 PATH_INPUT_DIR = os.path.join(PATH_RUN_DIR, INPUT_DIRNAME_DEFAULT) 82 82 if os.getenv('CONTROL'): -
source/python/classes/ControlFile.py
r274f9ef r96e1533 148 148 self.request = 0 149 149 self.public = 0 150 self.ecapi = None 150 151 151 152 self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff', -
source/python/classes/EcFlexpart.py
r274f9ef r96e1533 755 755 # index_vals[2]: ('0', '12', '3', '6', '9') ; stepRange 756 756 757 # initialize dictionaries 757 758 valsdict = {} 758 759 svalsdict = {} 759 # stepsdict = {}760 760 for p in pars: 761 761 valsdict[str(p)] = [] 762 762 svalsdict[str(p)] = [] 763 # stepsdict[str(p)] = []764 765 print('maxstep: ', c.maxstep)766 763 767 764 # "product" genereates each possible combination between the … … 828 825 vdp = valsdict[cparamId] 829 826 svdp = svalsdict[cparamId] 830 # sd = stepsdict[cparamId]831 827 832 828 if cparamId == '142' or cparamId == '143': … … 1194 1190 1195 1191 if c.format.lower() == 'grib2': 1196 p = subprocess.check_call(['grib_set', '-s', 'edition=2, \1197 productDefinitionTemplateNumber=8',1192 p = subprocess.check_call(['grib_set', '-s', 'edition=2,', 1193 'productDefinitionTemplateNumber=8', 1198 1194 ofile, ofile + '_2']) 1199 1195 p = subprocess.check_call(['mv', ofile + '_2', ofile]) -
source/python/mods/get_mars_data.py
r274f9ef r96e1533 42 42 # 43 43 #******************************************************************************* 44 44 """ToDo: Name of litte program 45 46 ToDo: Add desccription 47 48 ToDo: Add Conditions 49 50 This script requires that `...` be installed within the Python 51 environment you are running this script in. 52 53 This file can also be imported as a module and contains the following 54 functions: 55 56 * get_mars_data - 57 * do_retrievement - 58 * main - the main function of the script 59 """ 45 60 # ------------------------------------------------------------------------------ 46 61 # MODULES … … 91 106 92 107 get_mars_data(c) 93 normal_exit(c.mail fail, 'Done!')108 normal_exit(c.mailops, c.queue, 'Done!') 94 109 95 110 return -
source/python/mods/tools.py
r274f9ef r96e1533 54 54 import subprocess 55 55 import traceback 56 import exceptions 56 57 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter 57 58 … … 188 189 return args 189 190 190 def read_ecenv(file name):191 def read_ecenv(filepath): 191 192 '''Reads the file into a dictionary where the key values are the parameter 192 193 names. … … 194 195 Parameters 195 196 ---------- 196 file name: :obj:`string`197 filepath : :obj:`string` 197 198 Path to file where the ECMWF environment parameters are stored. 198 199 … … 204 205 ''' 205 206 envs= {} 206 207 with open(filename, 'r') as f: 208 for line in f: 209 data = line.strip().split() 210 envs[str(data[0])] = str(data[1]) 207 try: 208 with open(filepath, 'r') as f: 209 for line in f: 210 data = line.strip().split() 211 envs[str(data[0])] = str(data[1]) 212 except OSError as e: 213 print('... ERROR CODE: ' + str(e.errno)) 214 print('... ERROR MESSAGE:\n \t ' + str(e.strerror)) 215 216 sys.exit('\n... Error occured while trying to read ECMWF_ENV ' 217 'file: ' + str(filepath)) 211 218 212 219 return envs 213 220 214 221 def clean_up(c): 215 '''Remove all files from intermediate directory (inputdir). 222 '''Remove files from the intermediate directory (inputdir). 223 224 It keeps the final FLEXPART input files if program runs without 225 ECMWF Api and keywords "ectrans" or "ecstorage" are set to "1". 216 226 217 227 Parameters … … 226 236 ''' 227 237 228 print("clean_up") 229 230 cleanlist = glob.glob(c.inputdir + "/*") 231 for clist in cleanlist: 232 if c.prefix not in clist: 233 silent_remove(clist) 234 if c.ecapi is False and (c.ectrans == '1' or c.ecstorage == '1'): 235 silent_remove(clist) 236 237 print("Done") 238 print("... clean inputdir!") 239 240 cleanlist = glob.glob(os.path.join(c.inputdir, "*")) 241 242 if cleanlist: 243 for element in cleanlist: 244 if c.prefix not in element: 245 silent_remove(element) 246 if c.ecapi is False and (c.ectrans == 1 or c.ecstorage == 1): 247 silent_remove(element) 248 print("... done!") 249 else: 250 print("... nothing to clean!") 238 251 239 252 return … … 259 272 ''' 260 273 261 print(message) 262 263 # comment if user does not want email notification directly from python 274 trace = '\n'.join(traceback.format_stack()) 275 full_message = message + '\n\n' + trace 276 277 print(full_message) 278 279 send_mail(users, 'ERROR', full_message) 280 281 sys.exit(1) 282 283 return 284 285 286 def send_mail(users, success_mode, message): 287 '''Prints a specific exit message which can be passed to the function. 288 289 Parameters 290 ---------- 291 users : :obj:`list` of :obj:`string` 292 Contains all email addresses which should be notified. 293 It might also contain just the ecmwf user name which wil trigger 294 mailing to the associated email address for this user. 295 296 success_mode : :obj:``string` 297 States the exit mode of the program to put into 298 the mail subject line. 299 300 message : :obj:`string`, optional 301 Message for exiting program. Default value is "Done!". 302 303 Return 304 ------ 305 306 ''' 307 264 308 for user in users: 265 309 if '${USER}' in user: 266 310 user = os.getenv('USER') 267 311 try: 268 p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ERROR',269 os.path.expandvars(user)],312 p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ' + 313 success_mode, os.path.expandvars(user)], 270 314 stdin=subprocess.PIPE, 271 315 stdout=subprocess.PIPE, 272 316 stderr=subprocess.PIPE, 273 317 bufsize=1) 274 trace = '\n'.join(traceback.format_stack()) 275 pout = p.communicate(input=message + '\n\n' + trace)[0] 318 pout = p.communicate(input=message + '\n\n')[0] 276 319 except ValueError as e: 277 print('ERROR: ', e) 278 sys.exit('Email could not be sent!') 320 print('... ERROR: ' + str(e)) 321 sys.exit('... Email could not be sent!') 322 except OSError as e: 323 print('... ERROR CODE: ' + str(e.errno)) 324 print('... ERROR MESSAGE:\n \t ' + str(e.strerror)) 325 sys.exit('... Email could not be sent!') 279 326 else: 280 print('Email sent to ' + os.path.expandvars(user) + ' ' + 281 pout.decode()) 282 283 sys.exit(1) 284 285 return 286 287 288 def normal_exit(users, message='Done!'): 327 print('Email sent to ' + os.path.expandvars(user)) 328 329 return 330 331 332 def normal_exit(message='Done!'): 289 333 '''Prints a specific exit message which can be passed to the function. 290 334 291 335 Parameters 292 336 ---------- 293 user : :obj:`list` of :obj:`string`294 Contains all email addresses which should be notified.295 It might also contain just the ecmwf user name which wil trigger296 mailing to the associated email address for this user.297 298 337 message : :obj:`string`, optional 299 338 Message for exiting program. Default value is "Done!". … … 303 342 304 343 ''' 305 print(message) 306 307 # comment if user does not want notification directly from python 308 for user in users: 309 if '${USER}' in user: 310 user = os.getenv('USER') 311 try: 312 p = subprocess.Popen(['mail', '-s flex_extract_v7.1 normal exit', 313 os.path.expandvars(user)], 314 stdin=subprocess.PIPE, 315 stdout=subprocess.PIPE, 316 stderr=subprocess.PIPE, 317 bufsize=1) 318 pout = p.communicate(input=message+'\n\n')[0] 319 except ValueError as e: 320 print('ERROR: ', e) 321 print('Email could not be sent!') 322 else: 323 print('Email sent to ' + os.path.expandvars(user) + ' ' + 324 pout.decode()) 344 345 print(str(message)) 325 346 326 347 return … … 328 349 329 350 def product(*args, **kwds): 330 '''This method combines the single characters of the passed arguments 351 '''Creates combinations of all passed arguments. 352 353 This method combines the single characters of the passed arguments 331 354 with each other. So that each character of each argument value 332 355 will be combined with each character of the other arguments as a tuple. … … 345 368 Parameters 346 369 ---------- 347 \*args : :obj:` tuple`370 \*args : :obj:`list` or :obj:`string` 348 371 Positional arguments (arbitrary number). 349 372 … … 357 380 See example in description above. 358 381 ''' 359 pools = map(tuple, args) * kwds.get('repeat', 1) 360 result = [[]] 361 for pool in pools: 362 result = [x + [y] for x in result for y in pool] 363 for prod in result: 364 yield tuple(prod) 382 try: 383 pools = map(tuple, args) * kwds.get('repeat', 1) 384 result = [[]] 385 for pool in pools: 386 result = [x + [y] for x in result for y in pool] 387 for prod in result: 388 yield tuple(prod) 389 except TypeError as e: 390 sys.exit('... PRODUCT GENERATION FAILED!') 365 391 366 392 return … … 383 409 os.remove(filename) 384 410 except OSError as e: 385 if e.errno != errno.ENOENT: 386 # errno.ENOENT = no such file or directory 411 # errno.ENOENT = no such file or directory 412 if e.errno == errno.ENOENT: 413 pass 414 else: 387 415 raise # re-raise exception if a different error occured 388 416 … … 406 434 ''' 407 435 table128 = dict() 408 with open(filepath) as f: 409 fdata = f.read().split('\n') 410 for data in fdata: 411 if data[0] != '!': 412 table128[data[0:3]] = data[59:64].strip() 436 try: 437 with open(filepath) as f: 438 fdata = f.read().split('\n') 439 except OSError as e: 440 print('... ERROR CODE: ' + str(e.errno)) 441 print('... ERROR MESSAGE:\n \t ' + str(e.strerror)) 442 443 sys.exit('\n... Error occured while trying to read parameter ' 444 'table file: ' + str(filepath)) 445 else: 446 for data in fdata: 447 if data[0] != '!': 448 table128[data[0:3]] = data[59:64].strip() 413 449 414 450 return table128 … … 437 473 parameter ids in the format of integer. 438 474 ''' 475 if not pars: 476 return [] 477 if not isinstance(pars, str): 478 pars=str(pars) 479 439 480 cpar = pars.upper().split('/') 440 481 ipar = [] … … 467 508 ''' 468 509 510 if not isinstance(list_obj, list): 511 list_obj = list(list_obj) 469 512 str_of_list = concatenate_sign.join(str(l) for l in list_obj) 470 513 … … 472 515 473 516 def make_dir(directory): 474 '''Creates a directory and gives a warning if the directory 475 already exists. The program stops only if there is another problem. 517 '''Creates a directory. 518 519 It gives a warning if the directory already exists and skips process. 520 The program stops only if there is another problem. 476 521 477 522 Parameters 478 523 ---------- 479 524 directory : :obj:`string` 480 The directory name including the pathwhich should be created.525 The path to directory which should be created. 481 526 482 527 Return … … 487 532 os.makedirs(directory) 488 533 except OSError as e: 489 if e.errno != errno.EEXIST: 490 # errno.EEXIST = directory already exists 534 # errno.EEXIST = directory already exists 535 if e.errno == errno.EEXIST: 536 print('WARNING: Directory {0} already exists!'.format(directory)) 537 else: 491 538 raise # re-raise exception if a different error occured 492 else:493 print('WARNING: Directory {0} already exists!'.format(directory))494 539 495 540 return … … 523 568 Return 524 569 ------ 525 rcode : :obj:`string` 526 Resulting code of command execution. If successful the string 527 will be empty. 570 528 571 ''' 529 572 530 573 try: 531 rcode =subprocess.check_output(['ecaccess-file-put',532 533 534 535 536 574 subprocess.check_output(['ecaccess-file-put', 575 ecd + '/' + filename, 576 target + ':/home/ms/' + 577 ecgid + '/' + ecuid + 578 '/' + filename], 579 stderr=subprocess.STDOUT) 537 580 except subprocess.CalledProcessError as e: 538 print('... ERROR CODE: \n ...' + str(e.returncode))539 print('... ERROR MESSAGE:\n ...' + str(e))581 print('... ERROR CODE: ' + str(e.returncode)) 582 print('... ERROR MESSAGE:\n \t ' + str(e)) 540 583 541 584 print('\n... Do you have a valid ecaccess certification key?') 542 585 sys.exit('... ECACCESS-FILE-PUT FAILED!') 543 544 return rcode 586 except OSError as e: 587 print('... ERROR CODE: ' + str(e.errno)) 588 print('... ERROR MESSAGE:\n \t ' + str(e.strerror)) 589 590 print('\n... Most likely the ECACCESS library is not available!') 591 sys.exit('... ECACCESS-FILE-PUT FAILED!') 592 593 return 545 594 546 595 def submit_job_to_ecserver(target, jobname): … … 563 612 Return 564 613 ------ 565 rcode : :obj:`string` 566 Resulting code of command execution. If successful the string 567 will contain an integer number, representing the id of the job 568 at the ecmwf server. 614 job_id : :obj:`int` 615 The id number of the job as a reference at the ecmwf server. 569 616 ''' 570 617 571 618 try: 572 rcode = subprocess.check_output(['ecaccess-job-submit',573 '-queueName', target,574 jobname]) 619 job_id = subprocess.check_output(['ecaccess-job-submit', '-queueName', 620 target, jobname]) 621 575 622 except subprocess.CalledProcessError as e: 576 print('... ERROR CODE:\n ... ' + str(e.returncode)) 577 print('... ERROR MESSAGE:\n ... ' + str(e)) 578 623 print('... ERROR CODE: ' + str(e.returncode)) 624 print('... ERROR MESSAGE:\n \t ' + str(e)) 579 625 580 626 print('\n... Do you have a valid ecaccess certification key?') 581 627 sys.exit('... ECACCESS-JOB-SUBMIT FAILED!') 582 583 return rcode 628 except OSError as e: 629 print('... ERROR CODE: ' + str(e.errno)) 630 print('... ERROR MESSAGE:\n \t ' + str(e.strerror)) 631 632 print('\n... Most likely the ECACCESS library is not available!') 633 sys.exit('... ECACCESS-JOB-SUBMIT FAILED!') 634 635 return job_id -
source/python/submit.py
r274f9ef r96e1533 94 94 if c.request == 0 or c.request == 2: 95 95 prepare_flexpart(args.ppid, c) 96 normal_exit(c.mailfail, 'FLEX_EXTRACT IS DONE!')96 exit_message = 'FLEX_EXTRACT IS DONE!' 97 97 else: 98 normal_exit(c.mailfail, 'PRINTING MARS_REQUESTS DONE!')99 # send files to ECMWF server and install there98 exit_message = 'PRINTING MARS_REQUESTS DONE!' 99 # send files to ECMWF server 100 100 else: 101 101 submit(args.job_template, c, args.queue) 102 103 normal_exit(c.mailops, c.queue, exit_message) 102 104 103 105 return … … 147 149 f.write('\n'.join(lftextondemand)) 148 150 149 submit_job_to_ecserver(queue, job_file)151 job_id = submit_job_to_ecserver(queue, job_file) 150 152 151 153 else: … … 173 175 f.write('\n'.join(lftextoper)) 174 176 175 submit_job_to_ecserver(queue, job_file)177 job_id = submit_job_to_ecserver(queue, job_file) 176 178 177 179 # -------------------------------------------------------------------------- 180 print('The job id is: ' + str(job_id.strip())) 178 181 print('You should get an email with subject flex.hostname.pid') 179 182
Note: See TracChangeset
for help on using the changeset viewer.
