- Timestamp:
- Nov 10, 2018, 9:37:36 PM (5 years ago)
- Branches:
- master, ctbto, dev
- Children:
- 1abf820
- Parents:
- 8500ba1
- Location:
- source
- Files:
-
- 2 added
- 8 edited
Legend:
- Unmodified
- Added
- Removed
-
source/python/_config.py
r36a79af r96e1533 30 30 _VERSION_STR = '7.1' 31 31 32 QUEUES_LIST = ['ecgate', 'cca' ]32 QUEUES_LIST = ['ecgate', 'cca', 'ccb'] 33 33 34 34 # ------------------------------------------------------------------------------ … … 78 78 PATH_JOBSCRIPTS = os.path.join(PATH_RUN_DIR, 'jobscripts') 79 79 PATH_FORTRAN_SRC = os.path.join(PATH_SOURCES, 'fortran') 80 PATH_ TEST_DIR= os.path.join(PATH_SOURCES, 'pythontest')80 PATH_PYTHONTEST_SRC = os.path.join(PATH_SOURCES, 'pythontest') 81 81 PATH_INPUT_DIR = os.path.join(PATH_RUN_DIR, INPUT_DIRNAME_DEFAULT) 82 82 if os.getenv('CONTROL'): -
source/python/classes/ControlFile.py
r274f9ef r96e1533 148 148 self.request = 0 149 149 self.public = 0 150 self.ecapi = None 150 151 151 152 self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff', -
source/python/classes/EcFlexpart.py
r274f9ef r96e1533 755 755 # index_vals[2]: ('0', '12', '3', '6', '9') ; stepRange 756 756 757 # initialize dictionaries 757 758 valsdict = {} 758 759 svalsdict = {} 759 # stepsdict = {}760 760 for p in pars: 761 761 valsdict[str(p)] = [] 762 762 svalsdict[str(p)] = [] 763 # stepsdict[str(p)] = []764 765 print('maxstep: ', c.maxstep)766 763 767 764 # "product" genereates each possible combination between the … … 828 825 vdp = valsdict[cparamId] 829 826 svdp = svalsdict[cparamId] 830 # sd = stepsdict[cparamId]831 827 832 828 if cparamId == '142' or cparamId == '143': … … 1194 1190 1195 1191 if c.format.lower() == 'grib2': 1196 p = subprocess.check_call(['grib_set', '-s', 'edition=2, \1197 productDefinitionTemplateNumber=8',1192 p = subprocess.check_call(['grib_set', '-s', 'edition=2,', 1193 'productDefinitionTemplateNumber=8', 1198 1194 ofile, ofile + '_2']) 1199 1195 p = subprocess.check_call(['mv', ofile + '_2', ofile]) -
source/python/mods/get_mars_data.py
r274f9ef r96e1533 42 42 # 43 43 #******************************************************************************* 44 44 """ToDo: Name of litte program 45 46 ToDo: Add desccription 47 48 ToDo: Add Conditions 49 50 This script requires that `...` be installed within the Python 51 environment you are running this script in. 52 53 This file can also be imported as a module and contains the following 54 functions: 55 56 * get_mars_data - 57 * do_retrievement - 58 * main - the main function of the script 59 """ 45 60 # ------------------------------------------------------------------------------ 46 61 # MODULES … … 91 106 92 107 get_mars_data(c) 93 normal_exit(c.mail fail, 'Done!')108 normal_exit(c.mailops, c.queue, 'Done!') 94 109 95 110 return -
source/python/mods/tools.py
r274f9ef r96e1533 54 54 import subprocess 55 55 import traceback 56 import exceptions 56 57 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter 57 58 … … 188 189 return args 189 190 190 def read_ecenv(file name):191 def read_ecenv(filepath): 191 192 '''Reads the file into a dictionary where the key values are the parameter 192 193 names. … … 194 195 Parameters 195 196 ---------- 196 file name: :obj:`string`197 filepath : :obj:`string` 197 198 Path to file where the ECMWF environment parameters are stored. 198 199 … … 204 205 ''' 205 206 envs= {} 206 207 with open(filename, 'r') as f: 208 for line in f: 209 data = line.strip().split() 210 envs[str(data[0])] = str(data[1]) 207 try: 208 with open(filepath, 'r') as f: 209 for line in f: 210 data = line.strip().split() 211 envs[str(data[0])] = str(data[1]) 212 except OSError as e: 213 print('... ERROR CODE: ' + str(e.errno)) 214 print('... ERROR MESSAGE:\n \t ' + str(e.strerror)) 215 216 sys.exit('\n... Error occured while trying to read ECMWF_ENV ' 217 'file: ' + str(filepath)) 211 218 212 219 return envs 213 220 214 221 def clean_up(c): 215 '''Remove all files from intermediate directory (inputdir). 222 '''Remove files from the intermediate directory (inputdir). 223 224 It keeps the final FLEXPART input files if program runs without 225 ECMWF Api and keywords "ectrans" or "ecstorage" are set to "1". 216 226 217 227 Parameters … … 226 236 ''' 227 237 228 print("clean_up") 229 230 cleanlist = glob.glob(c.inputdir + "/*") 231 for clist in cleanlist: 232 if c.prefix not in clist: 233 silent_remove(clist) 234 if c.ecapi is False and (c.ectrans == '1' or c.ecstorage == '1'): 235 silent_remove(clist) 236 237 print("Done") 238 print("... clean inputdir!") 239 240 cleanlist = glob.glob(os.path.join(c.inputdir, "*")) 241 242 if cleanlist: 243 for element in cleanlist: 244 if c.prefix not in element: 245 silent_remove(element) 246 if c.ecapi is False and (c.ectrans == 1 or c.ecstorage == 1): 247 silent_remove(element) 248 print("... done!") 249 else: 250 print("... nothing to clean!") 238 251 239 252 return … … 259 272 ''' 260 273 261 print(message) 262 263 # comment if user does not want email notification directly from python 274 trace = '\n'.join(traceback.format_stack()) 275 full_message = message + '\n\n' + trace 276 277 print(full_message) 278 279 send_mail(users, 'ERROR', full_message) 280 281 sys.exit(1) 282 283 return 284 285 286 def send_mail(users, success_mode, message): 287 '''Prints a specific exit message which can be passed to the function. 288 289 Parameters 290 ---------- 291 users : :obj:`list` of :obj:`string` 292 Contains all email addresses which should be notified. 293 It might also contain just the ecmwf user name which wil trigger 294 mailing to the associated email address for this user. 295 296 success_mode : :obj:``string` 297 States the exit mode of the program to put into 298 the mail subject line. 299 300 message : :obj:`string`, optional 301 Message for exiting program. Default value is "Done!". 302 303 Return 304 ------ 305 306 ''' 307 264 308 for user in users: 265 309 if '${USER}' in user: 266 310 user = os.getenv('USER') 267 311 try: 268 p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ERROR',269 os.path.expandvars(user)],312 p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ' + 313 success_mode, os.path.expandvars(user)], 270 314 stdin=subprocess.PIPE, 271 315 stdout=subprocess.PIPE, 272 316 stderr=subprocess.PIPE, 273 317 bufsize=1) 274 trace = '\n'.join(traceback.format_stack()) 275 pout = p.communicate(input=message + '\n\n' + trace)[0] 318 pout = p.communicate(input=message + '\n\n')[0] 276 319 except ValueError as e: 277 print('ERROR: ', e) 278 sys.exit('Email could not be sent!') 320 print('... ERROR: ' + str(e)) 321 sys.exit('... Email could not be sent!') 322 except OSError as e: 323 print('... ERROR CODE: ' + str(e.errno)) 324 print('... ERROR MESSAGE:\n \t ' + str(e.strerror)) 325 sys.exit('... Email could not be sent!') 279 326 else: 280 print('Email sent to ' + os.path.expandvars(user) + ' ' + 281 pout.decode()) 282 283 sys.exit(1) 284 285 return 286 287 288 def normal_exit(users, message='Done!'): 327 print('Email sent to ' + os.path.expandvars(user)) 328 329 return 330 331 332 def normal_exit(message='Done!'): 289 333 '''Prints a specific exit message which can be passed to the function. 290 334 291 335 Parameters 292 336 ---------- 293 user : :obj:`list` of :obj:`string`294 Contains all email addresses which should be notified.295 It might also contain just the ecmwf user name which wil trigger296 mailing to the associated email address for this user.297 298 337 message : :obj:`string`, optional 299 338 Message for exiting program. Default value is "Done!". … … 303 342 304 343 ''' 305 print(message) 306 307 # comment if user does not want notification directly from python 308 for user in users: 309 if '${USER}' in user: 310 user = os.getenv('USER') 311 try: 312 p = subprocess.Popen(['mail', '-s flex_extract_v7.1 normal exit', 313 os.path.expandvars(user)], 314 stdin=subprocess.PIPE, 315 stdout=subprocess.PIPE, 316 stderr=subprocess.PIPE, 317 bufsize=1) 318 pout = p.communicate(input=message+'\n\n')[0] 319 except ValueError as e: 320 print('ERROR: ', e) 321 print('Email could not be sent!') 322 else: 323 print('Email sent to ' + os.path.expandvars(user) + ' ' + 324 pout.decode()) 344 345 print(str(message)) 325 346 326 347 return … … 328 349 329 350 def product(*args, **kwds): 330 '''This method combines the single characters of the passed arguments 351 '''Creates combinations of all passed arguments. 352 353 This method combines the single characters of the passed arguments 331 354 with each other. So that each character of each argument value 332 355 will be combined with each character of the other arguments as a tuple. … … 345 368 Parameters 346 369 ---------- 347 \*args : :obj:` tuple`370 \*args : :obj:`list` or :obj:`string` 348 371 Positional arguments (arbitrary number). 349 372 … … 357 380 See example in description above. 358 381 ''' 359 pools = map(tuple, args) * kwds.get('repeat', 1) 360 result = [[]] 361 for pool in pools: 362 result = [x + [y] for x in result for y in pool] 363 for prod in result: 364 yield tuple(prod) 382 try: 383 pools = map(tuple, args) * kwds.get('repeat', 1) 384 result = [[]] 385 for pool in pools: 386 result = [x + [y] for x in result for y in pool] 387 for prod in result: 388 yield tuple(prod) 389 except TypeError as e: 390 sys.exit('... PRODUCT GENERATION FAILED!') 365 391 366 392 return … … 383 409 os.remove(filename) 384 410 except OSError as e: 385 if e.errno != errno.ENOENT: 386 # errno.ENOENT = no such file or directory 411 # errno.ENOENT = no such file or directory 412 if e.errno == errno.ENOENT: 413 pass 414 else: 387 415 raise # re-raise exception if a different error occured 388 416 … … 406 434 ''' 407 435 table128 = dict() 408 with open(filepath) as f: 409 fdata = f.read().split('\n') 410 for data in fdata: 411 if data[0] != '!': 412 table128[data[0:3]] = data[59:64].strip() 436 try: 437 with open(filepath) as f: 438 fdata = f.read().split('\n') 439 except OSError as e: 440 print('... ERROR CODE: ' + str(e.errno)) 441 print('... ERROR MESSAGE:\n \t ' + str(e.strerror)) 442 443 sys.exit('\n... Error occured while trying to read parameter ' 444 'table file: ' + str(filepath)) 445 else: 446 for data in fdata: 447 if data[0] != '!': 448 table128[data[0:3]] = data[59:64].strip() 413 449 414 450 return table128 … … 437 473 parameter ids in the format of integer. 438 474 ''' 475 if not pars: 476 return [] 477 if not isinstance(pars, str): 478 pars=str(pars) 479 439 480 cpar = pars.upper().split('/') 440 481 ipar = [] … … 467 508 ''' 468 509 510 if not isinstance(list_obj, list): 511 list_obj = list(list_obj) 469 512 str_of_list = concatenate_sign.join(str(l) for l in list_obj) 470 513 … … 472 515 473 516 def make_dir(directory): 474 '''Creates a directory and gives a warning if the directory 475 already exists. The program stops only if there is another problem. 517 '''Creates a directory. 518 519 It gives a warning if the directory already exists and skips process. 520 The program stops only if there is another problem. 476 521 477 522 Parameters 478 523 ---------- 479 524 directory : :obj:`string` 480 The directory name including the pathwhich should be created.525 The path to directory which should be created. 481 526 482 527 Return … … 487 532 os.makedirs(directory) 488 533 except OSError as e: 489 if e.errno != errno.EEXIST: 490 # errno.EEXIST = directory already exists 534 # errno.EEXIST = directory already exists 535 if e.errno == errno.EEXIST: 536 print('WARNING: Directory {0} already exists!'.format(directory)) 537 else: 491 538 raise # re-raise exception if a different error occured 492 else:493 print('WARNING: Directory {0} already exists!'.format(directory))494 539 495 540 return … … 523 568 Return 524 569 ------ 525 rcode : :obj:`string` 526 Resulting code of command execution. If successful the string 527 will be empty. 570 528 571 ''' 529 572 530 573 try: 531 rcode =subprocess.check_output(['ecaccess-file-put',532 533 534 535 536 574 subprocess.check_output(['ecaccess-file-put', 575 ecd + '/' + filename, 576 target + ':/home/ms/' + 577 ecgid + '/' + ecuid + 578 '/' + filename], 579 stderr=subprocess.STDOUT) 537 580 except subprocess.CalledProcessError as e: 538 print('... ERROR CODE: \n ...' + str(e.returncode))539 print('... ERROR MESSAGE:\n ...' + str(e))581 print('... ERROR CODE: ' + str(e.returncode)) 582 print('... ERROR MESSAGE:\n \t ' + str(e)) 540 583 541 584 print('\n... Do you have a valid ecaccess certification key?') 542 585 sys.exit('... ECACCESS-FILE-PUT FAILED!') 543 544 return rcode 586 except OSError as e: 587 print('... ERROR CODE: ' + str(e.errno)) 588 print('... ERROR MESSAGE:\n \t ' + str(e.strerror)) 589 590 print('\n... Most likely the ECACCESS library is not available!') 591 sys.exit('... ECACCESS-FILE-PUT FAILED!') 592 593 return 545 594 546 595 def submit_job_to_ecserver(target, jobname): … … 563 612 Return 564 613 ------ 565 rcode : :obj:`string` 566 Resulting code of command execution. If successful the string 567 will contain an integer number, representing the id of the job 568 at the ecmwf server. 614 job_id : :obj:`int` 615 The id number of the job as a reference at the ecmwf server. 569 616 ''' 570 617 571 618 try: 572 rcode = subprocess.check_output(['ecaccess-job-submit',573 '-queueName', target,574 jobname]) 619 job_id = subprocess.check_output(['ecaccess-job-submit', '-queueName', 620 target, jobname]) 621 575 622 except subprocess.CalledProcessError as e: 576 print('... ERROR CODE:\n ... ' + str(e.returncode)) 577 print('... ERROR MESSAGE:\n ... ' + str(e)) 578 623 print('... ERROR CODE: ' + str(e.returncode)) 624 print('... ERROR MESSAGE:\n \t ' + str(e)) 579 625 580 626 print('\n... Do you have a valid ecaccess certification key?') 581 627 sys.exit('... ECACCESS-JOB-SUBMIT FAILED!') 582 583 return rcode 628 except OSError as e: 629 print('... ERROR CODE: ' + str(e.errno)) 630 print('... ERROR MESSAGE:\n \t ' + str(e.strerror)) 631 632 print('\n... Most likely the ECACCESS library is not available!') 633 sys.exit('... ECACCESS-JOB-SUBMIT FAILED!') 634 635 return job_id -
source/python/submit.py
r274f9ef r96e1533 94 94 if c.request == 0 or c.request == 2: 95 95 prepare_flexpart(args.ppid, c) 96 normal_exit(c.mailfail, 'FLEX_EXTRACT IS DONE!')96 exit_message = 'FLEX_EXTRACT IS DONE!' 97 97 else: 98 normal_exit(c.mailfail, 'PRINTING MARS_REQUESTS DONE!')99 # send files to ECMWF server and install there98 exit_message = 'PRINTING MARS_REQUESTS DONE!' 99 # send files to ECMWF server 100 100 else: 101 101 submit(args.job_template, c, args.queue) 102 103 normal_exit(c.mailops, c.queue, exit_message) 102 104 103 105 return … … 147 149 f.write('\n'.join(lftextondemand)) 148 150 149 submit_job_to_ecserver(queue, job_file)151 job_id = submit_job_to_ecserver(queue, job_file) 150 152 151 153 else: … … 173 175 f.write('\n'.join(lftextoper)) 174 176 175 submit_job_to_ecserver(queue, job_file)177 job_id = submit_job_to_ecserver(queue, job_file) 176 178 177 179 # -------------------------------------------------------------------------- 180 print('The job id is: ' + str(job_id.strip())) 178 181 print('You should get an email with subject flex.hostname.pid') 179 182 -
source/pythontest/TestTools.py
r25b14be r96e1533 1 #!/ usr/bin/envpython1 #!/opt/anaconda/bin/python 2 2 # -*- coding: utf-8 -*- 3 4 # for the gateway tests, the env vars of ECUID and ECGID have to be set upfront 3 5 4 6 import os 5 7 import sys 8 import errno 9 from exceptions import OSError 6 10 import subprocess 7 11 import pipes 8 12 import pytest 9 10 sys.path.append('../python') 13 from mock import patch, call 14 #from mock import PropertyMock 15 16 11 17 import _config 12 from mods.tools import (get_cmdline_arguments, read_ecenv, clean_up, my_error, 13 normal_exit, product, silent_remove, init128, 14 to_param_id, get_list_as_string, make_dir, 18 import _config_test 19 from classes.ControlFile import ControlFile 20 from mods.tools import (none_or_str, none_or_int, get_cmdline_arguments, 21 read_ecenv, clean_up, my_error, send_mail, 22 normal_exit, product, silent_remove, 23 init128, to_param_id, get_list_as_string, make_dir, 15 24 put_file_to_ecserver, submit_job_to_ecserver) 16 25 17 18 class TestTools(): 26 class TestTools(object): 19 27 ''' 20 28 ''' 21 29 22 def test_get_cmdline_arguments(self): 30 def setup_method(self): 31 self.testdir = _config_test.PATH_TEST_DIR 32 self.testfilesdir = _config_test.PATH_TESTFILES_DIR 33 self.c = ControlFile(self.testdir+'/Controls/CONTROL.test') 34 35 def test_nonestr_none_or_int(self): 36 assert None == none_or_int('None') 37 38 def test_intstr_none_or_int(self): 39 assert 42 == none_or_int('42') 40 41 def test_nonestr_none_or_str(self): 42 assert None == none_or_str('None') 43 44 def test_anystr_none_or_str(self): 45 assert 'test' == none_or_str('test') 46 47 def test_default_get_cmdline_arguments(self): 48 cmd_dict_control = {'start_date':None, 49 'end_date':None, 50 'date_chunk':None, 51 'basetime':None, 52 'step':None, 53 'levelist':None, 54 'area':None, 55 'inputdir':None, 56 'outputdir':None, 57 'flexpart_root_scripts':None, 58 'ppid':None, 59 'job_template':'job.temp', 60 'queue':None, 61 'controlfile':'CONTROL.temp', 62 'debug':None, 63 'public':None, 64 'request':None} 65 66 sys.argv = ['dummy.py'] 67 68 results = get_cmdline_arguments() 69 70 assert cmd_dict_control == vars(results) 71 72 73 def test_input_get_cmdline_arguments(self): 23 74 cmd_dict_control = {'start_date':'20180101', 24 75 'end_date':'20180101', 25 'date_chunk': '3',26 'basetime': '12',76 'date_chunk':3, 77 'basetime':12, 27 78 'step':'1', 28 79 'levelist':'1/to/10', 29 80 'area':'50/10/60/20', 30 81 'inputdir':'../work', 31 'outputdir': '../work',82 'outputdir':None, 32 83 'flexpart_root_scripts':'../', 33 'ppid': '1234',84 'ppid':1234, 34 85 'job_template':'job.sh', 35 86 'queue':'ecgate', 36 87 'controlfile':'CONTROL.WORK', 37 'debug':'1', 38 'request':'0'} 88 'debug':1, 89 'public':None, 90 'request':0} 39 91 40 92 sys.argv = ['dummy.py', … … 47 99 '--area=50/10/60/20', 48 100 '--inputdir=../work', 49 '--outputdir= ../work',101 '--outputdir=None', 50 102 '--flexpart_root_scripts=../', 51 103 '--ppid=1234', … … 54 106 '--controlfile=CONTROL.WORK', 55 107 '--debug=1', 108 '--public=None', 56 109 '--request=0'] 57 110 … … 60 113 assert cmd_dict_control == vars(results) 61 114 62 def test_ init128(self):115 def test_success_init128(self): 63 116 table128 = init128(_config.PATH_GRIBTABLE) 64 expected = {'078': 'TCLW', '130': 'T', '034': 'SST'}117 expected_sample = {'078': 'TCLW', '130': 'T', '034': 'SST'} 65 118 # check a sample of parameters which must have been read in 66 result = all((k in table128 and table128[k]==v) for k,v in expected.iteritems()) 67 assert result == True 68 69 def test_to_param_id(self): 119 assert all((k in table128 and table128[k]==v) 120 for k,v in expected_sample.iteritems()) 121 122 @patch('__builtin__.open', side_effect=[OSError(errno.EEXIST)]) 123 def test_fail_open_init128(self, mock_openfile): 124 with pytest.raises(SystemExit): 125 table128 = init128(_config.PATH_GRIBTABLE) 126 127 @pytest.mark.parametrize( 128 'ipar_str, opar_listint', 129 [('SP/LSP/SSHF', [134, 142, 146]), 130 ('T', [130]), 131 ('', []), 132 (None, []), 133 ('testtest', []), 134 ('130/142', [130, 142]), 135 (130, [130]), 136 (50.56, [])]) 137 def test_to_param_id(self, ipar_str, opar_listint): 70 138 table128 = init128(_config.PATH_GRIBTABLE) 71 pars = to_param_id("T/SP/LSP/SSHF", table128) 72 for par in pars: 73 assert par in [130, 134, 142, 146] 74 75 def test_my_error(self): 76 with pytest.raises(SystemExit) as pytest_wrapped_e: 77 my_error(['${USER}', 'anne.philipp@univie.ac.at'], 'Failed!') 78 assert pytest_wrapped_e.type == SystemExit 79 assert pytest_wrapped_e.value.code == 1 80 81 def test_read_ecenv(self): 82 envs_ref = {'ECUID': 'km4a', 83 'ECGID': 'at', 84 'GATEWAY': 'srvx8.img.univie.ac.at', 85 'DESTINATION': 'annep@genericSftp' 139 ipars = to_param_id(ipar_str, table128) 140 assert sorted(ipars) == sorted(opar_listint) 141 142 @patch('traceback.format_stack', return_value='empty trace') 143 @patch('mods.tools.send_mail', return_value=0) 144 def test_success_my_error(self, mock_mail, mock_trace, capfd): 145 with pytest.raises(SystemExit): 146 my_error(['any_user'], 'Failed!') 147 out, err = capfd.readouterr() 148 assert out == "Failed!\n\nempty_trace\n" 149 150 @patch('subprocess.Popen') 151 @patch('os.getenv', return_value='user') 152 @patch('os.path.expandvars', return_value='user') 153 def test_success_userenv_twouser_send_mail(self, mock_os, mock_env, mock_popen, capfd): 154 mock_popen.return_value = subprocess.Popen(["echo", "Hello Test!"], 155 stdout=subprocess.PIPE) 156 send_mail(['${USER}', 'any_user'], 'ERROR', message='error mail') 157 out, err = capfd.readouterr() 158 assert out == b'Email sent to user\nEmail sent to user\n' 159 160 @patch('subprocess.Popen') 161 @patch('os.path.expandvars', return_value='any_user') 162 def test_success_send_mail(self, mock_os, mock_popen, capfd): 163 mock_popen.return_value = subprocess.Popen(["echo", "Hello Test!"], 164 stdout=subprocess.PIPE) 165 send_mail(['any-user'], 'ERROR', message='error mail') 166 out, err = capfd.readouterr() 167 assert out == b'Email sent to any_user\n' 168 169 @patch('subprocess.Popen', side_effect=[ValueError, OSError]) 170 @patch('os.path.expandvars', return_value='any_user') 171 def test_fail_valueerror_send_mail(self, mock_osvar, mock_popen): 172 with pytest.raises(SystemExit): # ValueError 173 send_mail(['any-user'], 'ERROR', message='error mail') 174 with pytest.raises(SystemExit): # OSError 175 send_mail(['any-user'], 'ERROR', message='error mail') 176 177 def test_success_read_ecenv(self): 178 envs_ref = {'ECUID': 'testuser', 179 'ECGID': 'testgroup', 180 'GATEWAY': 'gateway.test.ac.at', 181 'DESTINATION': 'user@destination' 86 182 } 87 envs = read_ecenv( os.getcwd() + '/TestData/ECMWF_ENV')183 envs = read_ecenv(self.testfilesdir + '/ECMWF_ENV.test') 88 184 89 185 assert envs_ref == envs 90 186 91 def test_clean_up(self): 92 93 assert True 94 95 def test_normal_exit(self): 96 assert True 97 98 def test_product(self): 99 assert True 100 101 def test_success_silent_remove(self, capfd): 102 testfile = 'testfile.test' 187 @patch('__builtin__.open', side_effect=[OSError(errno.EPERM)]) 188 def test_fail_read_ecenv(self, mock_open): 189 with pytest.raises(SystemExit): 190 read_ecenv('any_file') 191 192 @patch('glob.glob', return_value=[]) 193 @patch('mods.tools.silent_remove') 194 def test_empty_clean_up(self, mock_rm, mock_clean): 195 clean_up(self.c) 196 mock_rm.assert_not_called() 197 198 @patch('glob.glob', return_value=['any_file','EIfile']) 199 @patch('os.remove', return_value=0) 200 def test_success_clean_up(self, mock_rm, mock_glob): 201 # ectrans=0; ecstorage=0; ecapi=None; prefix not in filename 202 clean_up(self.c) 203 mock_rm.assert_has_calls([call('any_file'), call('EIfile')]) 204 mock_rm.reset_mock() 205 206 # ectrans=0; ecstorage=0; ecapi=False; prefix in filename 207 self.c.prefix = 'EI' 208 self.c.ecapi = False 209 clean_up(self.c) 210 mock_rm.assert_has_calls([call('any_file')]) 211 mock_rm.reset_mock() 212 213 # ectrans=0; ecstorage=0; ecapi=True; prefix in filename 214 self.c.prefix = 'EI' 215 self.c.ecapi = True 216 clean_up(self.c) 217 mock_rm.assert_has_calls([call('any_file')]) 218 mock_rm.reset_mock() 219 220 # ectrans=1; ecstorage=0; ecapi=True; prefix in filename 221 self.c.prefix = 'EI' 222 self.c.ecapi = True 223 self.c.ectrans = 1 224 clean_up(self.c) 225 mock_rm.assert_has_calls([call('any_file')]) 226 mock_rm.reset_mock() 227 228 # ectrans=1; ecstorage=0; ecapi=False; prefix in filename 229 self.c.prefix = 'EI' 230 self.c.ecapi = False 231 self.c.ectrans = 1 232 clean_up(self.c) 233 mock_rm.assert_has_calls([call('any_file'), call('EIfile')]) 234 mock_rm.reset_mock() 235 236 # ectrans=1; ecstorage=1; ecapi=False; prefix in filename 237 self.c.prefix = 'EI' 238 self.c.ecapi = False 239 self.c.ectrans = 1 240 self.c.ecstorage = 1 241 clean_up(self.c) 242 mock_rm.assert_has_calls([call('any_file'), call('EIfile')]) 243 mock_rm.reset_mock() 244 245 def test_default_normal_exit(self, capfd): 246 normal_exit() 247 out, err = capfd.readouterr() 248 assert out == 'Done!\n' 249 250 def test_message_normal_exit(self, capfd): 251 normal_exit('Hi there!') 252 out, err = capfd.readouterr() 253 assert out == 'Hi there!\n' 254 255 def test_int_normal_exit(self, capfd): 256 normal_exit(42) 257 out, err = capfd.readouterr() 258 assert out == '42\n' 259 260 @pytest.mark.parametrize( 261 'input1, input2, output_list', 262 [('ABC','xy',[('A','x'),('A','y'),('B','x'),('B','y'),('C','x'),('C','y')]), 263 (range(1), range(1), [(0,0),(0,1),(1,0),(1,1)])]) 264 def test_success_product(self, input1, input2, output_list): 265 index = 0 266 for prod in product(input1, input2): 267 assert isinstance(prod, tuple) 268 assert prod == output_list[index] 269 index += 1 270 271 @pytest.mark.parametrize( 272 'input1, input2, output_list', 273 [(1,1,(1,1))]) 274 def test_fail_product(self, input1, input2, output_list): 275 index = 0 276 with pytest.raises(SystemExit): 277 for prod in product(input1, input2): 278 assert isinstance(prod, tuple) 279 assert prod == output_list[index] 280 index += 1 281 282 def test_success_silent_remove(self): 283 testfile = self.testfilesdir + 'test.txt' 103 284 open(testfile, 'w').close() 104 285 silent_remove(testfile) 105 out, err = capfd.readouterr()106 286 assert os.path.isfile(testfile) == False 107 assert out == '' 108 109 def test_failnotexist_silent_remove(self, capfd): 110 testfile = 'testfile.test' 111 silent_remove(testfile) 112 out, err = capfd.readouterr() 113 assert os.path.isfile(testfile) == False 114 assert out == '' 115 116 @pytest.mark.skip(reason="no way of currently testing this") 117 def test_failany_silent_remove(self): 118 testfile = 'testfileany.test' 287 288 @patch('os.remove', side_effect=OSError(errno.ENOENT)) 289 def test_fail_notexist_silent_remove(self, mock_rm): 119 290 with pytest.raises(OSError) as pytest_wrapped_e: 120 silent_remove(testfile) 121 #out, err = capfd.readouterr() 122 #assert os.path.isfile(testfile) == False 123 #assert out == '' 124 125 def test_success_get_list_as_string(self): 126 list_object = [1, 2, 3, '...', 'testlist'] 127 list_as_string = '1, 2, 3, ..., testlist' 128 assert list_as_string == get_list_as_string(list_object) 129 130 @pytest.mark.skip(reason="no way of currently testing this") 131 def test_fail_get_list_as_string(self): 132 list_object = [1, 2, 3, '...', 'testlist'] 133 list_as_string = '1, 2, 3, ..., testlist' 134 with pytest.raises(Exception) as pytest_wrapped_e: 135 result = get_list_as_string(list_object) 136 assert result == list_as_string 137 138 def test_warningexist_make_dir(self, capfd): 139 testdir = 'TestData' 140 make_dir(testdir) 141 out, err = capfd.readouterr() 142 assert out.strip() == 'WARNING: Directory {0} already exists!'.format(testdir) 143 144 def test_failany_make_dir(self): 145 testdir = '/test' # force a permission denied error 291 silent_remove('any_dir') 292 assert pytest_wrapped_e.e.errno == errno.ENOENT 293 294 @patch('os.remove', side_effect=OSError(errno.EEXIST)) 295 def test_fail_any_silent_remove(self, mock_rm): 296 with pytest.raises(OSError): 297 silent_remove('any_dir') 298 299 @pytest.mark.parametrize( 300 'input_list, output_list', 301 [([],''), 302 ([1, 2, 3.5, '...', 'testlist'], '1, 2, 3.5, ..., testlist'), 303 ('2', '2')]) 304 def test_success_get_list_as_string(self, input_list, output_list): 305 assert output_list == get_list_as_string(input_list) 306 307 @patch('os.makedirs', side_effect=[OSError(errno.EEXIST)]) 308 def test_warning_exist_make_dir(self, mock_make): 146 309 with pytest.raises(OSError) as pytest_wrapped_e: 147 make_dir(testdir) 148 assert pytest_wrapped_e.type == OSError 310 make_dir('existing_dir') 311 assert pytest_wrapped_e.e.errno == errno.EEXIST 312 313 @patch('os.makedirs', side_effect=OSError) 314 def test_fail_any_make_dir(self, mock_makedir): 315 with pytest.raises(OSError): 316 make_dir('any_dir') 317 318 def test_fail_empty_make_dir(self): 319 with pytest.raises(OSError): 320 make_dir('') 149 321 150 322 def test_success_make_dir(self): 151 testdir = 'testing_mkdir' 152 make_dir(testdir) 153 assert os.path.exists(testdir) == True 154 os.rmdir(testdir) 155 156 def test_fail_put_file_to_ecserver(self): 157 ecuid=os.environ['ECUID'] 158 ecgid=os.environ['ECGID'] 159 with pytest.raises(SystemExit) as pytest_wrapped_e: 160 put_file_to_ecserver('TestData/', 'testfil.txt', 161 'ecgate', ecuid, ecgid) 162 assert pytest_wrapped_e.type == SystemExit 163 assert pytest_wrapped_e.value.code == '... ECACCESS-FILE-PUT FAILED!' 164 165 def test_success_put_file_to_ecserver(self): 166 ecuid=os.environ['ECUID'] 167 ecgid=os.environ['ECGID'] 168 result = put_file_to_ecserver('TestData/', 'testfile.txt', 169 'ecgate', ecuid, ecgid) 170 assert result == '' 323 testdir = '/testing_mkdir' 324 make_dir(self.testdir + testdir) 325 assert os.path.exists(self.testdir + testdir) == True 326 os.rmdir(self.testdir + testdir) 327 328 329 @patch('subprocess.check_output', side_effect=[subprocess.CalledProcessError(1,'test')]) 330 def test_fail_put_file_to_ecserver(self, mock_co): 331 with pytest.raises(SystemExit): 332 put_file_to_ecserver(self.testfilesdir, 'test_put_to_ecserver.txt', 333 'ecgate', 'ex_ECUID', 'ex_ECGID') 334 335 @patch('subprocess.check_output', return_value=0) 336 def test_general_success_put_file_to_ecserver(self, mock_co): 337 result = put_file_to_ecserver(self.testfilesdir, 'test_put_to_ecserver.txt', 338 'ecgate', 'ex_ECUID', 'ex_ECGID') 339 assert result == None 171 340 172 341 @pytest.mark.msuser_pw 342 @pytest.mark.gateway 173 343 @pytest.mark.skip(reason="easier to ignore for now - implement in final version") 174 def test_fullsuccess_put_file_to_ecserver(self): 175 ecuid=os.environ['ECUID'] 176 ecgid=os.environ['ECGID'] 177 put_file_to_ecserver('TestData/', 'testfile.txt', 'ecgate', ecuid, ecgid) 344 def test_full_success_put_file_to_ecserver(self): 345 ecuid = os.environ['ECUID'] 346 ecgid = os.environ['ECGID'] 347 put_file_to_ecserver(self.testfilesdir, 'test_put_to_ecserver.txt', 348 'ecgate', ecuid, ecgid) 178 349 assert subprocess.call(['ssh', ecuid+'@ecaccess.ecmwf.int' , 179 350 'test -e ' + 180 351 pipes.quote('/home/ms/'+ecgid+'/'+ecuid)]) == 0 181 352 182 def test_fail_submit_job_to_ecserver(self): 183 with pytest.raises(SystemExit) as pytest_wrapped_e: 184 submit_job_to_ecserver('ecgate', 'job.ksh') 185 assert pytest_wrapped_e.type == SystemExit 186 assert pytest_wrapped_e.value.code == '... ECACCESS-JOB-SUBMIT FAILED!' 187 353 @patch('subprocess.check_output', side_effect=[subprocess.CalledProcessError(1,'test'), 354 OSError]) 355 def test_fail_submit_job_to_ecserver(self, mock_co): 356 with pytest.raises(SystemExit): 357 job_id = submit_job_to_ecserver('ecgate', 'job.ksh') 358 359 @pytest.mark.msuser_pw 360 @pytest.mark.gateway 361 @pytest.mark.skip(reason="easier to ignore for now - implement in final version") 188 362 def test_success_submit_job_to_ecserver(self): 189 result = submit_job_to_ecserver('ecgate', 'TestData/testfile.txt') 190 assert result.strip().isdigit() == True 191 363 job_id = submit_job_to_ecserver('ecgate', 364 os.path.join(self.testfilesdir, 365 'test_put_to_ecserver.txt')) 366 assert job_id.strip().isdigit() == True 367 368 -
source/pythontest/pytest.ini
r25b14be r96e1533 3 3 markers = 4 4 msuser_pw: Test that can be executed only as a member-state user. Password required. 5 gateway: Test that can be executed only in the gateway mode.
Note: See TracChangeset
for help on using the changeset viewer.