Context Navigation

source: flex_extract.git/source/python/install.py @ 6cda7c1

ctbtodev

Last change on this file since 6cda7c1 was 6cda7c1, checked in by Anne Philipp <anne.philipp@…>, 5 years ago
moved check install coditions to install module
Property mode set to `100755`
File size: 23.2 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - moved install_args_and_control in here
14	# - splitted code in smaller functions
15	# - delete convert build files in here instead of compile job script
16	# - changed static path names to Variables from config file
17	#
18	# @License:
19	# (C) Copyright 2015-2018.
20	#
21	# This software is licensed under the terms of the Apache Licence Version 2.0
22	# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
23	#
24	# @Program Functionality:
25	# Depending on the selected installation environment (locally or on the
26	# ECMWF server ecgate or cca) the program extracts the commandline
27	# arguments and the CONTROL file parameter and prepares the corresponding
28	# environment. The necessary files are collected in a tar-ball and placed
29	# at the target location. There its untared, the environment variables will
30	# be set and the Fortran code will be compiled. If the ECMWF environment is
31	# selected a job script is prepared and submitted for the remaining
32	# configurations after putting the tar-ball to the target ECMWF server.
33	#
34	# @Program Content:
35	# - main
36	# - get_install_cmdline_arguments
37	# - install_via_gateway
38	# - mk_tarball
39	# - un_tarball
40	# - mk_env_vars
41	# - mk_compilejob
42	# - mk_job_template
43	# - delete_convert_build
44	# - make_convert_build
45	#
46	#*******************************************************************************
47
48	# ------------------------------------------------------------------------------
49	# MODULES
50	# ------------------------------------------------------------------------------
51	import os
52	import sys
53	import glob
54	import subprocess
55	import inspect
56	import tarfile
57	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
58
59	# software specific classes and modules from flex_extract
60	import _config
61	from classes.ControlFile import ControlFile
62	from classes.UioFiles import UioFiles
63	from mods.tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
64
65	# ------------------------------------------------------------------------------
66	# FUNCTIONS
67	# ------------------------------------------------------------------------------
68	def main():
69	'''Controls the installation process. Calls the installation function
70	if target is specified.
71
72	Parameters
73	----------
74
75	Return
76	------
77	'''
78
79	args = get_install_cmdline_arguments()
80	c = ControlFile(args.controlfile)
81	c.assign_args_to_control(args)
82	check_install_conditions(c)
83
84	install_via_gateway(c)
85
86	return
87
88	def get_install_cmdline_arguments():
89	'''Decomposes the command line arguments and assigns them to variables.
90	Apply default values for non mentioned arguments.
91
92	Parameters
93	----------
94
95	Return
96	------
97	args : :obj:`Namespace`
98	Contains the commandline arguments from script/program call.
99	'''
100	parser = ArgumentParser(description='Install flex_extract software locally or \
101	on ECMWF machines',
102	formatter_class=ArgumentDefaultsHelpFormatter)
103
104	parser.add_argument('--target', dest='install_target', default=None,
105	help="Valid targets: local \| ecgate \| cca , \
106	the latter two are at ECMWF")
107	parser.add_argument("--makefile", dest="makefile", default=None,
108	help='Name of Makefile to use for compiling CONVERT2')
109	parser.add_argument("--ecuid", dest="ecuid", default=None,
110	help='user id at ECMWF')
111	parser.add_argument("--ecgid", dest="ecgid", default=None,
112	help='group id at ECMWF')
113	parser.add_argument("--gateway", dest="gateway", default=None,
114	help='name of local gateway server')
115	parser.add_argument("--destination", dest="destination", default=None,
116	help='ecaccess destination, e.g. leo@genericSftp')
117
118	parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
119	default=None, help="FLEXPART root directory on ECMWF \
120	servers (to find grib2flexpart and COMMAND file)\n\
121	Normally flex_extract resides in the scripts directory \
122	of the FLEXPART distribution.")
123
124	# arguments for job submission to ECMWF, only needed by submit.py
125	parser.add_argument("--job_template", dest='job_template',
126	default="job.temp.o",
127	help="job template file for submission to ECMWF")
128
129	parser.add_argument("--controlfile", dest="controlfile",
130	default='CONTROL.temp',
131	help="file with CONTROL parameters")
132
133	args = parser.parse_args()
134
135	return args
136
137
138	def install_via_gateway(c):
139	'''Perform the actual installation on local machine or prepare data
140	transfer to remote gate and submit a job script which will
141	install everything on the remote gate.
142
143	Parameters
144	----------
145	c : :obj:`ControlFile`
146	Contains all the parameters of CONTROL file and
147	command line.
148
149	Return
150	------
151
152	'''
153	import tarfile
154
155	ecd = _config.PATH_FLEXEXTRACT_DIR
156	tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
157	tar_file = os.path.join(ecd, tarball_name)
158
159	target_dirname = _config.FLEXEXTRACT_DIRNAME
160	fortran_executable = _config.FORTRAN_EXECUTABLE
161
162	if c.install_target.lower() != 'local': # ecgate or cca
163
164	mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
165	c.flexpart_root_scripts)
166
167	mk_job_template(c.ecuid, c.ecgid, c.gateway,
168	c.destination, c.flexpart_root_scripts)
169
170	mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
171
172	mk_tarball(tar_file, c.install_target)
173
174	put_file_to_ecserver(ecd, tarball_name, c.install_target,
175	c.ecuid, c.ecgid)
176
177	submit_job_to_ecserver(c.install_target,
178	os.path.join(_config.PATH_REL_JOBSCRIPTS,
179	_config.FILE_INSTALL_COMPILEJOB))
180
181	print('job compilation script has been submitted to ecgate for ' +
182	'installation in ' + c.flexpart_root_scripts +
183	'/' + target_dirname)
184	print('You should get an email with subject "flexcompile" within ' +
185	'the next few minutes!')
186
187	else: #local
188	if c.flexpart_root_scripts == _config.PATH_FLEXEXTRACT_DIR :
189	print('WARNING: FLEXPART_ROOT_SCRIPTS has not been specified')
190	print('flex_extract will be installed in here by compiling the ' +
191	'Fortran source in ' + _config.PATH_FORTRAN_SRC)
192	os.chdir(_config.PATH_FORTRAN_SRC)
193	else: # creates the target working directory for flex_extract
194	c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
195	c.flexpart_root_scripts))
196	if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
197	mk_tarball(tar_file, c.install_target)
198	make_dir(os.path.join(c.flexpart_root_scripts,
199	target_dirname))
200	os.chdir(os.path.join(c.flexpart_root_scripts,
201	target_dirname))
202	un_tarball(tar_file)
203	os.chdir(os.path.join(c.flexpart_root_scripts,
204	target_dirname,
205	_config.PATH_REL_FORTRAN_SRC))
206
207	# Create Fortran executable - CONVERT2
208	print('Install ' + target_dirname + ' software at ' +
209	c.install_target + ' in directory ' +
210	os.path.abspath(c.flexpart_root_scripts) + '\n')
211
212	delete_convert_build('.')
213	make_convert_build('.', c.makefile)
214
215	os.chdir(ecd)
216	if os.path.isfile(tar_file):
217	os.remove(tar_file)
218
219	return
220
221	def check_install_conditions(c):
222	'''Checks a couple of necessary attributes and conditions
223	for the installation such as if they exist and contain values.
224	Otherwise set default values.
225
226	Parameters
227	----------
228	c : :obj:`ControlFile`
229	Contains all the parameters of CONTROL file and
230	command line.
231
232
233	Return
234	------
235
236	'''
237
238	if c.install_target and \
239	c.install_target not in _config.INSTALL_TARGETS:
240	print('ERROR: unknown or missing installation target ')
241	print('target: ', c.install_target)
242	print('please specify correct installation target ' +
243	str(INSTALL_TARGETS))
244	print('use -h or --help for help')
245	sys.exit(1)
246
247	if c.install_target and c.install_target != 'local':
248	if not c.ecgid or not c.ecuid or \
249	not c.gateway or not c.destination:
250	print('Please enter your ECMWF user id and group id as well ' +
251	'as the \nname of the local gateway and the ectrans ' +
252	'destination ')
253	print('with command line options --ecuid --ecgid \
254	--gateway --destination')
255	print('Try "' + sys.argv[0].split('/')[-1] + \
256	' -h" to print usage information')
257	print('Please consult ecaccess documentation or ECMWF user \
258	support for further details')
259	sys.exit(1)
260
261	if not c.flexpart_root_scripts:
262	c.flexpart_root_scripts = '${HOME}'
263	else:
264	c.flexpart_root_scripts = c.flexpart_root_scripts
265	else: # local
266	if not c.flexpart_root_scripts:
267	c.flexpart_root_scripts = _config.PATH_FLEXEXTRACT_DIR
268
269	return
270
271
272	def mk_tarball(tarball_path, target):
273	'''Creates a tarball with all necessary files which need to be sent to the
274	installation directory.
275	It does not matter if this is local or remote.
276	Collects all python files, the Fortran source and makefiles,
277	the ECMWF_ENV file, the CONTROL files as well as the
278	template files.
279
280	Parameters
281	----------
282	tarball_path : :obj:`string`
283	The complete path to the tar file which will contain all
284	relevant data for flex_extract.
285
286	target : :obj:`string`
287	The queue where the job is submitted to.
288
289	Return
290	------
291
292	'''
293	from glob import glob
294
295	print('Create tarball ...')
296
297	# change to FLEXEXTRACT directory so that the tar can contain
298	# relative pathes to the files and directories
299	ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
300	os.chdir(ecd)
301
302	# get lists of the files to be added to the tar file
303	if target == 'local':
304	ECMWF_ENV_FILE = []
305	runfile = [os.path.relpath(x, ecd)
306	for x in UioFiles(_config.PATH_REL_RUN_DIR,
307	'run_local.sh').files]
308	else:
309	ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
310	runfile = [os.path.relpath(x, ecd)
311	for x in UioFiles(_config.PATH_REL_RUN_DIR,
312	'run.sh').files]
313
314	pyfiles = [os.path.relpath(x, ecd)
315	for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
316	pytestfiles = [os.path.relpath(x, ecd)
317	for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
318	controlfiles = [os.path.relpath(x, ecd)
319	for x in UioFiles(_config.PATH_REL_CONTROLFILES,
320	'CONTROL*').files]
321	tempfiles = [os.path.relpath(x, ecd)
322	for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.temp').files]
323	nlfiles = [os.path.relpath(x, ecd)
324	for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.nl').files]
325	gribtable = [os.path.relpath(x, ecd)
326	for x in UioFiles(_config.PATH_REL_TEMPLATES , 'grib').files]
327	ffiles = [os.path.relpath(x, ecd)
328	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '.f').files]
329	hfiles = [os.path.relpath(x, ecd)
330	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
331	makefiles = [os.path.relpath(x, ecd)
332	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'Makefile*').files]
333	jobdir = [_config.PATH_REL_JOBSCRIPTS]
334
335	# concatenate single lists to one for a better looping
336	filelist = pyfiles + pytestfiles + controlfiles + tempfiles + nlfiles + \
337	ffiles + gribtable + hfiles + makefiles + ECMWF_ENV_FILE + \
338	runfile + jobdir + \
339	['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
340
341	# create installation tar-file
342	exclude_files = [".ksh"]
343	try:
344	with tarfile.open(tarball_path, "w:gz") as tar_handle:
345	for file in filelist:
346	tar_handle.add(file, recursive=False,
347	filter=lambda tarinfo: None
348	if os.path.splitext(tarinfo.name)[1]
349	in exclude_files
350	else tarinfo)
351	except subprocess.CalledProcessError as e:
352	print('... ERROR CODE:\n ... ' + str(e.returncode))
353	print('... ERROR MESSAGE:\n ... ' + str(e))
354
355	sys.exit('\n... could not make installation tar ball!')
356	except OSError as e:
357	print('... ERROR CODE: ' + str(e.errno))
358	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
359
360	sys.exit('\n... error occured while trying to read tar-file ' +
361	str(tarball_path))
362
363	return
364
365
366	def un_tarball(tarball_path):
367	'''Extracts the given tarball into current directory.
368
369	Parameters
370	----------
371	tarball_path : :obj:`string`
372	The complete path to the tar file which will contain all
373	relevant data for flex_extract.
374
375	Return
376	------
377
378	'''
379
380	print('Untar ...')
381
382	try:
383	with tarfile.open(tarball_path) as tar_handle:
384	tar_handle.extractall()
385	except tarfile.TarError as e:
386	sys.exit('\n... error occured while trying to read tar-file ' +
387	str(tarball_path))
388	except OSError as e:
389	print('... ERROR CODE: ' + str(e.errno))
390	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
391
392	sys.exit('\n... error occured while trying to read tar-file ' +
393	str(tarball_path))
394
395	return
396
397	def mk_env_vars(ecuid, ecgid, gateway, destination):
398	'''Creates a file named ECMWF_ENV which contains the
399	necessary environmental variables at ECMWF servers.
400	It is based on the template ECMWF_ENV.template.
401
402	Parameters
403	----------
404	ecuid : :obj:`string`
405	The user id on ECMWF server.
406
407	ecgid : :obj:`string`
408	The group id on ECMWF server.
409
410	gateway : :obj:`string`
411	The gateway server the user is using.
412
413	destination : :obj:`string`
414	The remote destination which is used to transfer files
415	from ECMWF server to local gateway server.
416
417	Return
418	------
419
420	'''
421	from genshi.template.text import NewTextTemplate
422	from genshi.template import TemplateLoader
423	from genshi.template.eval import UndefinedError
424
425	try:
426	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
427	ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
428	cls=NewTextTemplate)
429
430	stream = ecmwfvars_template.generate(user_name = ecuid,
431	user_group = ecgid,
432	gateway_name = gateway,
433	destination_name = destination
434	)
435	except UndefinedError as e:
436	print('... ERROR ' + str(e))
437
438	sys.exit('\n... error occured while trying to generate template ' +
439	_config.PATH_ECMWF_ENV)
440	except OSError as e:
441	print('... ERROR CODE: ' + str(e.errno))
442	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
443
444	sys.exit('\n... error occured while trying to generate template ' +
445	_config.PATH_ECMWF_ENV)
446
447	try:
448	with open(_config.PATH_ECMWF_ENV, 'w') as f:
449	f.write(stream.render('text'))
450	except OSError as e:
451	print('... ERROR CODE: ' + str(e.errno))
452	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
453
454	sys.exit('\n... error occured while trying to write ' +
455	_config.PATH_ECMWF_ENV)
456
457	return
458
459	def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
460	'''Modifies the original job template file so that it is specified
461	for the user and the environment were it will be applied. Result
462	is stored in a new file "job.temp" in the python directory.
463
464	Parameters
465	----------
466	makefile : :obj:`string`
467	Name of the makefile which should be used to compile FORTRAN
468	CONVERT2 program.
469
470	target : :obj:`string`
471	The target where the installation should be done, e.g. the queue.
472
473	ecuid : :obj:`string`
474	The user id on ECMWF server.
475
476	ecgid : :obj:`string`
477	The group id on ECMWF server.
478
479	fp_root : :obj:`string`
480	Path to the root directory of FLEXPART environment or flex_extract
481	environment.
482
483	Return
484	------
485
486	'''
487	from genshi.template.text import NewTextTemplate
488	from genshi.template import TemplateLoader
489	from genshi.template.eval import UndefinedError
490
491	if fp_root == '../':
492	fp_root = '$HOME'
493
494	try:
495	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
496	compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
497	cls=NewTextTemplate)
498
499	stream = compile_template.generate(
500	usergroup = ecgid,
501	username = ecuid,
502	version_number = _config._VERSION_STR,
503	fp_root_scripts = fp_root,
504	makefile = makefile,
505	fortran_program = _config.FORTRAN_EXECUTABLE
506	)
507	except UndefinedError as e:
508	print('... ERROR ' + str(e))
509
510	sys.exit('\n... error occured while trying to generate template ' +
511	_config.TEMPFILE_INSTALL_COMPILEJOB)
512	except OSError as e:
513	print('... ERROR CODE: ' + str(e.errno))
514	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
515
516	sys.exit('\n... error occured while trying to generate template ' +
517	_config.TEMPFILE_INSTALL_COMPILEJOB)
518
519	try:
520	compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
521	_config.FILE_INSTALL_COMPILEJOB)
522
523	with open(compilejob, 'w') as f:
524	f.write(stream.render('text'))
525	except OSError as e:
526	print('... ERROR CODE: ' + str(e.errno))
527	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
528
529	sys.exit('\n... error occured while trying to write ' +
530	compilejob)
531
532	return
533
534	def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
535	'''Modifies the original job template file so that it is specified
536	for the user and the environment were it will be applied. Result
537	is stored in a new file.
538
539	Parameters
540	----------
541	ecuid : :obj:`string`
542	The user id on ECMWF server.
543
544	ecgid : :obj:`string`
545	The group id on ECMWF server.
546
547	gateway : :obj:`string`
548	The gateway server the user is using.
549
550	destination : :obj:`string`
551	The remote destination which is used to transfer files
552	from ECMWF server to local gateway server.
553
554	fp_root : :obj:`string`
555	Path to the root directory of FLEXPART environment or flex_extract
556	environment.
557
558	Return
559	------
560
561	'''
562	from genshi.template.text import NewTextTemplate
563	from genshi.template import TemplateLoader
564	from genshi.template.eval import UndefinedError
565
566	fp_root_path_to_python = os.path.join(fp_root,
567	_config.FLEXEXTRACT_DIRNAME,
568	_config.PATH_REL_PYTHON_SRC)
569
570	try:
571	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
572	compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
573	cls=NewTextTemplate)
574
575	stream = compile_template.generate(
576	usergroup = ecgid,
577	username = ecuid,
578	version_number = _config._VERSION_STR,
579	fp_root_path = fp_root_path_to_python,
580	)
581	except UndefinedError as e:
582	print('... ERROR ' + str(e))
583
584	sys.exit('\n... error occured while trying to generate template ' +
585	_config.TEMPFILE_INSTALL_JOB)
586	except OSError as e:
587	print('... ERROR CODE: ' + str(e.errno))
588	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
589
590	sys.exit('\n... error occured while trying to generate template ' +
591	_config.TEMPFILE_INSTALL_JOB)
592
593
594	try:
595	tempjobfile = os.path.join(_config.PATH_TEMPLATES,
596	_config.TEMPFILE_JOB)
597
598	with open(tempjobfile, 'w') as f:
599	f.write(stream.render('text'))
600	except OSError as e:
601	print('... ERROR CODE: ' + str(e.errno))
602	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
603
604	sys.exit('\n... error occured while trying to write ' +
605	tempjobfile)
606
607	return
608
609	def delete_convert_build(src_path):
610	'''Clean up the Fortran source directory and remove all
611	build files (e.g. \.o, \.mod and CONVERT2)
612
613	Parameters
614	----------
615	src_path : :obj:`string`
616	Path to the fortran source directory.
617
618	Return
619	------
620
621	'''
622
623	modfiles = UioFiles(src_path, '*.mod')
624	objfiles = UioFiles(src_path, '*.o')
625	exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
626
627	modfiles.delete_files()
628	objfiles.delete_files()
629	exefile.delete_files()
630
631	return
632
633	def make_convert_build(src_path, makefile):
634	'''Compiles the Fortran code and generates the executable.
635
636	Parameters
637	----------
638	src_path : :obj:`string`
639	Path to the fortran source directory.
640
641	makefile : :obj:`string`
642	The name of the makefile which should be used.
643
644	Return
645	------
646
647	'''
648
649	try:
650	print('Using makefile: ' + makefile)
651	p = subprocess.Popen(['make', '-f',
652	os.path.join(src_path, makefile)],
653	stdin=subprocess.PIPE,
654	stdout=subprocess.PIPE,
655	stderr=subprocess.PIPE,
656	bufsize=1)
657	pout, perr = p.communicate()
658	print(pout)
659	if p.returncode != 0:
660	print(perr)
661	print('Please edit ' + makefile +
662	' or try another Makefile in the src directory.')
663	print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
664	'and EMOSLIB must be adapted.')
665	print('Available Makefiles:')
666	print(UioFiles(src_path, 'Makefile*'))
667	sys.exit('Compilation failed!')
668	except ValueError as e:
669	print('ERROR: Makefile call failed:')
670	print(e)
671	else:
672	subprocess.check_call(['ls', '-l',
673	os.path.join(src_path,
674	_config.FORTRAN_EXECUTABLE)])
675
676	return
677
678
679	if __name__ == "__main__":
680	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: