1 | #!/opt/anaconda/bin/python |
---|
2 | # -*- coding: utf-8 -*- |
---|
3 | |
---|
4 | # for the gateway tests, the env vars of ECUID and ECGID have to be set upfront |
---|
5 | |
---|
6 | import os |
---|
7 | import sys |
---|
8 | import errno |
---|
9 | from exceptions import OSError |
---|
10 | import subprocess |
---|
11 | import pipes |
---|
12 | import pytest |
---|
13 | from mock import patch, call |
---|
14 | #from mock import PropertyMock |
---|
15 | |
---|
16 | |
---|
17 | import _config |
---|
18 | import _config_test |
---|
19 | from classes.ControlFile import ControlFile |
---|
20 | from mods.tools import (none_or_str, none_or_int, get_cmdline_arguments, |
---|
21 | read_ecenv, clean_up, my_error, send_mail, |
---|
22 | normal_exit, product, silent_remove, |
---|
23 | init128, to_param_id, get_list_as_string, make_dir, |
---|
24 | put_file_to_ecserver, submit_job_to_ecserver) |
---|
25 | |
---|
26 | class TestTools(object): |
---|
27 | ''' |
---|
28 | ''' |
---|
29 | |
---|
30 | def setup_method(self): |
---|
31 | self.testdir = _config_test.PATH_TEST_DIR |
---|
32 | self.testfilesdir = _config_test.PATH_TESTFILES_DIR |
---|
33 | self.c = ControlFile(self.testdir+'/Controls/CONTROL.test') |
---|
34 | |
---|
35 | def test_nonestr_none_or_int(self): |
---|
36 | assert None == none_or_int('None') |
---|
37 | |
---|
38 | def test_intstr_none_or_int(self): |
---|
39 | assert 42 == none_or_int('42') |
---|
40 | |
---|
41 | def test_nonestr_none_or_str(self): |
---|
42 | assert None == none_or_str('None') |
---|
43 | |
---|
44 | def test_anystr_none_or_str(self): |
---|
45 | assert 'test' == none_or_str('test') |
---|
46 | |
---|
47 | def test_fail_get_cmdline_arguments(self): |
---|
48 | sys.argv = ['dummy.py', '--wrong=1'] |
---|
49 | with pytest.raises(SystemExit): |
---|
50 | results = get_cmdline_arguments() |
---|
51 | |
---|
52 | def test_default_get_cmdline_arguments(self): |
---|
53 | cmd_dict_control = {'start_date':None, |
---|
54 | 'end_date':None, |
---|
55 | 'date_chunk':None, |
---|
56 | 'basetime':None, |
---|
57 | 'step':None, |
---|
58 | 'levelist':None, |
---|
59 | 'area':None, |
---|
60 | 'inputdir':None, |
---|
61 | 'outputdir':None, |
---|
62 | 'flexpart_root_scripts':None, |
---|
63 | 'ppid':None, |
---|
64 | 'job_template':'job.temp', |
---|
65 | 'queue':None, |
---|
66 | 'controlfile':'CONTROL.temp', |
---|
67 | 'debug':None, |
---|
68 | 'public':None, |
---|
69 | 'request':None} |
---|
70 | |
---|
71 | sys.argv = ['dummy.py'] |
---|
72 | |
---|
73 | results = get_cmdline_arguments() |
---|
74 | |
---|
75 | assert cmd_dict_control == vars(results) |
---|
76 | |
---|
77 | def test_input_get_cmdline_arguments(self): |
---|
78 | cmd_dict_control = {'start_date':'20180101', |
---|
79 | 'end_date':'20180101', |
---|
80 | 'date_chunk':3, |
---|
81 | 'basetime':12, |
---|
82 | 'step':'1', |
---|
83 | 'levelist':'1/to/10', |
---|
84 | 'area':'50/10/60/20', |
---|
85 | 'inputdir':'../work', |
---|
86 | 'outputdir':None, |
---|
87 | 'flexpart_root_scripts':'../', |
---|
88 | 'ppid':1234, |
---|
89 | 'job_template':'job.sh', |
---|
90 | 'queue':'ecgate', |
---|
91 | 'controlfile':'CONTROL.WORK', |
---|
92 | 'debug':1, |
---|
93 | 'public':None, |
---|
94 | 'request':0} |
---|
95 | |
---|
96 | sys.argv = ['dummy.py', |
---|
97 | '--start_date=20180101', |
---|
98 | '--end_date=20180101', |
---|
99 | '--date_chunk=3', |
---|
100 | '--basetime=12', |
---|
101 | '--step=1', |
---|
102 | '--levelist=1/to/10', |
---|
103 | '--area=50/10/60/20', |
---|
104 | '--inputdir=../work', |
---|
105 | '--outputdir=None', |
---|
106 | '--flexpart_root_scripts=../', |
---|
107 | '--ppid=1234', |
---|
108 | '--job_template=job.sh', |
---|
109 | '--queue=ecgate', |
---|
110 | '--controlfile=CONTROL.WORK', |
---|
111 | '--debug=1', |
---|
112 | '--public=None', |
---|
113 | '--request=0'] |
---|
114 | |
---|
115 | results = get_cmdline_arguments() |
---|
116 | |
---|
117 | assert cmd_dict_control == vars(results) |
---|
118 | |
---|
119 | def test_success_init128(self): |
---|
120 | table128 = init128(_config.PATH_GRIBTABLE) |
---|
121 | expected_sample = {'078': 'TCLW', '130': 'T', '034': 'SST'} |
---|
122 | # check a sample of parameters which must have been read in |
---|
123 | assert all((k in table128 and table128[k]==v) |
---|
124 | for k,v in expected_sample.iteritems()) |
---|
125 | |
---|
126 | @patch('__builtin__.open', side_effect=[OSError(errno.EEXIST)]) |
---|
127 | def test_fail_open_init128(self, mock_openfile): |
---|
128 | with pytest.raises(SystemExit): |
---|
129 | table128 = init128(_config.PATH_GRIBTABLE) |
---|
130 | |
---|
131 | @pytest.mark.parametrize( |
---|
132 | 'ipar_str, opar_listint', |
---|
133 | [('SP/LSP/SSHF', [134, 142, 146]), |
---|
134 | ('T', [130]), |
---|
135 | ('', []), |
---|
136 | (None, []), |
---|
137 | ('testtest', []), |
---|
138 | ('130/142', [130, 142]), |
---|
139 | (130, [130]), |
---|
140 | (50.56, [])]) |
---|
141 | def test_to_param_id(self, ipar_str, opar_listint): |
---|
142 | table128 = init128(_config.PATH_GRIBTABLE) |
---|
143 | ipars = to_param_id(ipar_str, table128) |
---|
144 | assert sorted(ipars) == sorted(opar_listint) |
---|
145 | |
---|
146 | @patch('traceback.format_stack', return_value='empty trace') |
---|
147 | @patch('mods.tools.send_mail', return_value=0) |
---|
148 | def test_success_my_error(self, mock_mail, mock_trace, capfd): |
---|
149 | with pytest.raises(SystemExit): |
---|
150 | my_error(['any_user'], 'Failed!') |
---|
151 | out, err = capfd.readouterr() |
---|
152 | assert out == "Failed!\n\nempty_trace\n" |
---|
153 | |
---|
154 | @patch('subprocess.Popen') |
---|
155 | @patch('os.getenv', return_value='user') |
---|
156 | @patch('os.path.expandvars', return_value='user') |
---|
157 | def test_success_userenv_twouser_send_mail(self, mock_os, mock_env, mock_popen, capfd): |
---|
158 | mock_popen.return_value = subprocess.Popen(["echo", "Hello Test!"], |
---|
159 | stdout=subprocess.PIPE) |
---|
160 | send_mail(['${USER}', 'any_user'], 'ERROR', message='error mail') |
---|
161 | out, err = capfd.readouterr() |
---|
162 | assert out == b'Email sent to user\nEmail sent to user\n' |
---|
163 | |
---|
164 | @patch('subprocess.Popen') |
---|
165 | @patch('os.path.expandvars', return_value='any_user') |
---|
166 | def test_success_send_mail(self, mock_os, mock_popen, capfd): |
---|
167 | mock_popen.return_value = subprocess.Popen(["echo", "Hello Test!"], |
---|
168 | stdout=subprocess.PIPE) |
---|
169 | send_mail(['any-user'], 'ERROR', message='error mail') |
---|
170 | out, err = capfd.readouterr() |
---|
171 | assert out == b'Email sent to any_user\n' |
---|
172 | |
---|
173 | @patch('subprocess.Popen', side_effect=[ValueError, OSError]) |
---|
174 | @patch('os.path.expandvars', return_value='any_user') |
---|
175 | def test_fail_valueerror_send_mail(self, mock_osvar, mock_popen): |
---|
176 | with pytest.raises(SystemExit): # ValueError |
---|
177 | send_mail(['any-user'], 'ERROR', message='error mail') |
---|
178 | with pytest.raises(SystemExit): # OSError |
---|
179 | send_mail(['any-user'], 'ERROR', message='error mail') |
---|
180 | |
---|
181 | def test_success_read_ecenv(self): |
---|
182 | envs_ref = {'ECUID': 'testuser', |
---|
183 | 'ECGID': 'testgroup', |
---|
184 | 'GATEWAY': 'gateway.test.ac.at', |
---|
185 | 'DESTINATION': 'user@destination' |
---|
186 | } |
---|
187 | envs = read_ecenv(self.testfilesdir + '/ECMWF_ENV.test') |
---|
188 | |
---|
189 | assert envs_ref == envs |
---|
190 | |
---|
191 | @patch('__builtin__.open', side_effect=[OSError(errno.EPERM)]) |
---|
192 | def test_fail_read_ecenv(self, mock_open): |
---|
193 | with pytest.raises(SystemExit): |
---|
194 | read_ecenv('any_file') |
---|
195 | |
---|
196 | @patch('glob.glob', return_value=[]) |
---|
197 | @patch('mods.tools.silent_remove') |
---|
198 | def test_empty_clean_up(self, mock_rm, mock_clean): |
---|
199 | clean_up(self.c) |
---|
200 | mock_rm.assert_not_called() |
---|
201 | |
---|
202 | @patch('glob.glob', return_value=['any_file','EIfile']) |
---|
203 | @patch('os.remove', return_value=0) |
---|
204 | def test_success_clean_up(self, mock_rm, mock_glob): |
---|
205 | # ectrans=0; ecstorage=0; ecapi=None; prefix not in filename |
---|
206 | clean_up(self.c) |
---|
207 | mock_rm.assert_has_calls([call('any_file'), call('EIfile')]) |
---|
208 | mock_rm.reset_mock() |
---|
209 | |
---|
210 | # ectrans=0; ecstorage=0; ecapi=False; prefix in filename |
---|
211 | self.c.prefix = 'EI' |
---|
212 | self.c.ecapi = False |
---|
213 | clean_up(self.c) |
---|
214 | mock_rm.assert_has_calls([call('any_file')]) |
---|
215 | mock_rm.reset_mock() |
---|
216 | |
---|
217 | # ectrans=0; ecstorage=0; ecapi=True; prefix in filename |
---|
218 | self.c.prefix = 'EI' |
---|
219 | self.c.ecapi = True |
---|
220 | clean_up(self.c) |
---|
221 | mock_rm.assert_has_calls([call('any_file')]) |
---|
222 | mock_rm.reset_mock() |
---|
223 | |
---|
224 | # ectrans=1; ecstorage=0; ecapi=True; prefix in filename |
---|
225 | self.c.prefix = 'EI' |
---|
226 | self.c.ecapi = True |
---|
227 | self.c.ectrans = 1 |
---|
228 | clean_up(self.c) |
---|
229 | mock_rm.assert_has_calls([call('any_file')]) |
---|
230 | mock_rm.reset_mock() |
---|
231 | |
---|
232 | # ectrans=1; ecstorage=0; ecapi=False; prefix in filename |
---|
233 | self.c.prefix = 'EI' |
---|
234 | self.c.ecapi = False |
---|
235 | self.c.ectrans = 1 |
---|
236 | clean_up(self.c) |
---|
237 | mock_rm.assert_has_calls([call('any_file'), call('EIfile')]) |
---|
238 | mock_rm.reset_mock() |
---|
239 | |
---|
240 | # ectrans=1; ecstorage=1; ecapi=False; prefix in filename |
---|
241 | self.c.prefix = 'EI' |
---|
242 | self.c.ecapi = False |
---|
243 | self.c.ectrans = 1 |
---|
244 | self.c.ecstorage = 1 |
---|
245 | clean_up(self.c) |
---|
246 | mock_rm.assert_has_calls([call('any_file'), call('EIfile')]) |
---|
247 | mock_rm.reset_mock() |
---|
248 | |
---|
249 | def test_default_normal_exit(self, capfd): |
---|
250 | normal_exit() |
---|
251 | out, err = capfd.readouterr() |
---|
252 | assert out == 'Done!\n' |
---|
253 | |
---|
254 | def test_message_normal_exit(self, capfd): |
---|
255 | normal_exit('Hi there!') |
---|
256 | out, err = capfd.readouterr() |
---|
257 | assert out == 'Hi there!\n' |
---|
258 | |
---|
259 | def test_int_normal_exit(self, capfd): |
---|
260 | normal_exit(42) |
---|
261 | out, err = capfd.readouterr() |
---|
262 | assert out == '42\n' |
---|
263 | |
---|
264 | @pytest.mark.parametrize( |
---|
265 | 'input1, input2, output_list', |
---|
266 | [('ABC','xy',[('A','x'),('A','y'),('B','x'),('B','y'),('C','x'),('C','y')]), |
---|
267 | (range(1), range(1), [(0,0),(0,1),(1,0),(1,1)])]) |
---|
268 | def test_success_product(self, input1, input2, output_list): |
---|
269 | index = 0 |
---|
270 | for prod in product(input1, input2): |
---|
271 | assert isinstance(prod, tuple) |
---|
272 | assert prod == output_list[index] |
---|
273 | index += 1 |
---|
274 | |
---|
275 | @pytest.mark.parametrize( |
---|
276 | 'input1, input2, output_list', |
---|
277 | [(1,1,(1,1))]) |
---|
278 | def test_fail_product(self, input1, input2, output_list): |
---|
279 | index = 0 |
---|
280 | with pytest.raises(SystemExit): |
---|
281 | for prod in product(input1, input2): |
---|
282 | assert isinstance(prod, tuple) |
---|
283 | assert prod == output_list[index] |
---|
284 | index += 1 |
---|
285 | |
---|
286 | def test_success_silent_remove(self): |
---|
287 | testfile = self.testfilesdir + 'test.txt' |
---|
288 | open(testfile, 'w').close() |
---|
289 | silent_remove(testfile) |
---|
290 | assert os.path.isfile(testfile) == False |
---|
291 | |
---|
292 | @patch('os.remove', side_effect=OSError(errno.ENOENT)) |
---|
293 | def test_fail_notexist_silent_remove(self, mock_rm): |
---|
294 | with pytest.raises(OSError) as pytest_wrapped_e: |
---|
295 | silent_remove('any_dir') |
---|
296 | assert pytest_wrapped_e.e.errno == errno.ENOENT |
---|
297 | |
---|
298 | @patch('os.remove', side_effect=OSError(errno.EEXIST)) |
---|
299 | def test_fail_any_silent_remove(self, mock_rm): |
---|
300 | with pytest.raises(OSError): |
---|
301 | silent_remove('any_dir') |
---|
302 | |
---|
303 | @pytest.mark.parametrize( |
---|
304 | 'input_list, output_list', |
---|
305 | [([],''), |
---|
306 | ([1, 2, 3.5, '...', 'testlist'], '1, 2, 3.5, ..., testlist'), |
---|
307 | ('2', '2')]) |
---|
308 | def test_success_get_list_as_string(self, input_list, output_list): |
---|
309 | assert output_list == get_list_as_string(input_list) |
---|
310 | |
---|
311 | @patch('os.makedirs', side_effect=[OSError(errno.EEXIST)]) |
---|
312 | def test_warning_exist_make_dir(self, mock_make): |
---|
313 | with pytest.raises(OSError) as pytest_wrapped_e: |
---|
314 | make_dir('existing_dir') |
---|
315 | assert pytest_wrapped_e.e.errno == errno.EEXIST |
---|
316 | |
---|
317 | @patch('os.makedirs', side_effect=OSError) |
---|
318 | def test_fail_any_make_dir(self, mock_makedir): |
---|
319 | with pytest.raises(OSError): |
---|
320 | make_dir('any_dir') |
---|
321 | |
---|
322 | def test_fail_empty_make_dir(self): |
---|
323 | with pytest.raises(OSError): |
---|
324 | make_dir('') |
---|
325 | |
---|
326 | def test_success_make_dir(self): |
---|
327 | testdir = '/testing_mkdir' |
---|
328 | make_dir(self.testdir + testdir) |
---|
329 | assert os.path.exists(self.testdir + testdir) == True |
---|
330 | os.rmdir(self.testdir + testdir) |
---|
331 | |
---|
332 | |
---|
333 | @patch('subprocess.check_output', side_effect=[subprocess.CalledProcessError(1,'test')]) |
---|
334 | def test_fail_put_file_to_ecserver(self, mock_co): |
---|
335 | with pytest.raises(SystemExit): |
---|
336 | put_file_to_ecserver(self.testfilesdir, 'test_put_to_ecserver.txt', |
---|
337 | 'ecgate', 'ex_ECUID', 'ex_ECGID') |
---|
338 | |
---|
339 | @patch('subprocess.check_output', return_value=0) |
---|
340 | def test_general_success_put_file_to_ecserver(self, mock_co): |
---|
341 | result = put_file_to_ecserver(self.testfilesdir, 'test_put_to_ecserver.txt', |
---|
342 | 'ecgate', 'ex_ECUID', 'ex_ECGID') |
---|
343 | assert result == None |
---|
344 | |
---|
345 | @pytest.mark.msuser_pw |
---|
346 | @pytest.mark.gateway |
---|
347 | @pytest.mark.skip(reason="easier to ignore for now - implement in final version") |
---|
348 | def test_full_success_put_file_to_ecserver(self): |
---|
349 | ecuid = os.environ['ECUID'] |
---|
350 | ecgid = os.environ['ECGID'] |
---|
351 | put_file_to_ecserver(self.testfilesdir, 'test_put_to_ecserver.txt', |
---|
352 | 'ecgate', ecuid, ecgid) |
---|
353 | assert subprocess.call(['ssh', ecuid+'@ecaccess.ecmwf.int' , |
---|
354 | 'test -e ' + |
---|
355 | pipes.quote('/home/ms/'+ecgid+'/'+ecuid)]) == 0 |
---|
356 | |
---|
357 | @patch('subprocess.check_output', side_effect=[subprocess.CalledProcessError(1,'test'), |
---|
358 | OSError]) |
---|
359 | def test_fail_submit_job_to_ecserver(self, mock_co): |
---|
360 | with pytest.raises(SystemExit): |
---|
361 | job_id = submit_job_to_ecserver('ecgate', 'job.ksh') |
---|
362 | |
---|
363 | @pytest.mark.msuser_pw |
---|
364 | @pytest.mark.gateway |
---|
365 | @pytest.mark.skip(reason="easier to ignore for now - implement in final version") |
---|
366 | def test_success_submit_job_to_ecserver(self): |
---|
367 | job_id = submit_job_to_ecserver('ecgate', |
---|
368 | os.path.join(self.testfilesdir, |
---|
369 | 'test_put_to_ecserver.txt')) |
---|
370 | assert job_id.strip().isdigit() == True |
---|
371 | |
---|
372 | |
---|