Context Navigation

source: flex_extract.git/source/python/install.py @ 8ce3be6

ctbtodev

Last change on this file since 8ce3be6 was 8ce3be6, checked in by Anne Philipp <anne.philipp@…>, 5 years ago
python2 downgrade/ changed flexpartdir to installdir/ adaptations for description of command line parameters/ minor corrections
Property mode set to `100755`
File size: 23.8 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - moved install_args_and_control in here
14	# - splitted code in smaller functions
15	# - delete convert build files in here instead of compile job script
16	# - changed static path names to variables from config file
17	# - splitted install function into several smaller pieces
18	# - use of tarfile package in python
19	#
20	# @License:
21	# (C) Copyright 2014-2019.
22	# Anne Philipp, Leopold Haimberger
23	#
24	# This work is licensed under the Creative Commons Attribution 4.0
25	# International License. To view a copy of this license, visit
26	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
27	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
28	#
29	# @Methods:
30	# main
31	# get_install_cmdline_args
32	# install_via_gateway
33	# check_install_conditions
34	# mk_tarball
35	# un_tarball
36	# mk_env_vars
37	# mk_compilejob
38	# mk_job_template
39	# del_convert_build
40	# mk_convert_build
41	#
42	#*******************************************************************************
43	'''This script installs the flex_extract program.
44
45	Depending on the selected installation environment (locally or on the
46	ECMWF server ecgate or cca) the program extracts the commandline
47	arguments and the CONTROL file parameter and prepares the corresponding
48	environment.
49	The necessary files are collected in a tar-ball and placed
50	at the target location. There its untared, the environment variables will
51	be set and the Fortran code will be compiled.
52	If the ECMWF environment is selected a job script is prepared and submitted
53	for the remaining configurations after putting the tar-ball to the
54	target ECMWF server.
55
56	Type: install.py --help
57	to get information about command line parameters.
58	Read the documentation for usage instructions.
59	'''
60
61	# ------------------------------------------------------------------------------
62	# MODULES
63	# ------------------------------------------------------------------------------
64	from __future__ import print_function
65
66	import os
67	import sys
68	import glob
69	import subprocess
70	import inspect
71	import tarfile
72	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
73
74	# software specific classes and modules from flex_extract
75	import _config
76	from classes.ControlFile import ControlFile
77	from classes.UioFiles import UioFiles
78	from mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
79	silent_remove, execute_subprocess, none_or_str)
80
81	# ------------------------------------------------------------------------------
82	# FUNCTIONS
83	# ------------------------------------------------------------------------------
84	def main():
85	'''Controls the installation process. Calls the installation function
86	if target is specified.
87
88	Parameters
89	----------
90
91	Return
92	------
93	'''
94
95	args = get_install_cmdline_args()
96	c = ControlFile(args.controlfile)
97	c.assign_args_to_control(args)
98	check_install_conditions(c)
99
100	if c.install_target.lower() != 'local': # ecgate or cca
101	install_via_gateway(c)
102	else: # local
103	install_local(c)
104
105	return
106
107	def get_install_cmdline_args():
108	'''Decomposes the command line arguments and assigns them to variables.
109	Apply default values for non mentioned arguments.
110
111	Parameters
112	----------
113
114	Return
115	------
116	args : Namespace
117	Contains the commandline arguments from script/program call.
118	'''
119	parser = ArgumentParser(description='Install flex_extract software '
120	'locally or on ECMWF machines',
121	formatter_class=ArgumentDefaultsHelpFormatter)
122
123	parser.add_argument('--target', dest='install_target',
124	type=none_or_str, default=None,
125	help="Valid targets: local \| ecgate \| cca , \
126	the latter two are at ECMWF")
127	parser.add_argument("--makefile", dest="makefile",
128	type=none_or_str, default=None,
129	help='Name of Makefile to use for compiling the '
130	'Fortran program')
131	parser.add_argument("--ecuid", dest="ecuid",
132	type=none_or_str, default=None,
133	help='The user id at ECMWF.')
134	parser.add_argument("--ecgid", dest="ecgid",
135	type=none_or_str, default=None,
136	help='The group id at ECMWF.')
137	parser.add_argument("--gateway", dest="gateway",
138	type=none_or_str, default=None,
139	help='The name of the local gateway server.')
140	parser.add_argument("--destination", dest="destination",
141	type=none_or_str, default=None,
142	help='The ecaccess association, e.g. '
143	'myUser@genericSftp')
144
145	parser.add_argument("--installdir", dest="installdir",
146	type=none_or_str, default=None,
147	help='Root directory where '
148	'flex_extract will be installed to.')
149
150	# arguments for job submission to ECMWF, only needed by submit.py
151	parser.add_argument("--job_template", dest='job_template',
152	type=none_or_str, default="job.template",
153	help='The rudimentary template file to create a batch '
154	'job template for submission to ECMWF servers.')
155
156	parser.add_argument("--controlfile", dest="controlfile",
157	type=none_or_str, default='CONTROL_EA5',
158	help="The file with all CONTROL parameters.")
159
160	args = parser.parse_args()
161
162	return args
163
164
165	def install_via_gateway(c):
166	'''Prepare data transfer to remote gate and submit a job script which will
167	install everything on the remote gate.
168
169	Parameters
170	----------
171	c : ControlFile
172	Contains all the parameters of CONTROL file and
173	command line.
174
175	Return
176	------
177
178	'''
179	import tarfile
180
181	tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
182	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
183
184	mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
185	c.installdir)
186
187	mk_job_template(c.ecuid, c.ecgid, c.gateway,
188	c.destination, c.installdir)
189
190	mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
191
192	mk_tarball(tar_file, c.install_target)
193
194	put_file_to_ecserver(_config.PATH_FLEXEXTRACT_DIR, tarball_name,
195	c.install_target, c.ecuid, c.ecgid)
196
197	submit_job_to_ecserver(c.install_target,
198	os.path.join(_config.PATH_REL_JOBSCRIPTS,
199	_config.FILE_INSTALL_COMPILEJOB))
200
201	silent_remove(tar_file)
202
203	print('Job compilation script has been submitted to ecgate for ' +
204	'installation in ' + c.installdir +
205	'/' + _config.FLEXEXTRACT_DIRNAME)
206	print('You should get an email with subject "flexcompile" within ' +
207	'the next few minutes!')
208
209	return
210
211	def install_local(c):
212	'''Perform the actual installation on a local machine.
213
214	Parameters
215	----------
216	c : ControlFile
217	Contains all the parameters of CONTROL file and
218	command line.
219
220	Return
221	------
222
223	'''
224	import tarfile
225
226	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR,
227	_config.FLEXEXTRACT_DIRNAME + '.tar')
228
229	if c.installdir == _config.PATH_FLEXEXTRACT_DIR :
230	print('WARNING: installdir has not been specified')
231	print('flex_extract will be installed in here by compiling the ' +
232	'Fortran source in ' + _config.PATH_FORTRAN_SRC)
233	os.chdir(_config.PATH_FORTRAN_SRC)
234	else: # creates the target working directory for flex_extract
235	c.installdir = os.path.expandvars(os.path.expanduser(
236	c.installdir))
237	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) != \
238	os.path.abspath(c.installdir):
239	mk_tarball(tar_file, c.install_target)
240	make_dir(os.path.join(c.installdir,
241	_config.FLEXEXTRACT_DIRNAME))
242	os.chdir(os.path.join(c.installdir,
243	_config.FLEXEXTRACT_DIRNAME))
244	un_tarball(tar_file)
245	os.chdir(os.path.join(c.installdir,
246	_config.FLEXEXTRACT_DIRNAME,
247	_config.PATH_REL_FORTRAN_SRC))
248
249	# Create Fortran executable - CONVERT2
250	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software at ' +
251	c.install_target + ' in directory ' +
252	os.path.abspath(c.installdir) + '\n')
253
254	del_convert_build('.')
255	mk_convert_build('.', c.makefile)
256
257	os.chdir(_config.PATH_FLEXEXTRACT_DIR)
258	if os.path.isfile(tar_file):
259	os.remove(tar_file)
260
261	return
262
263
264	def check_install_conditions(c):
265	'''Checks a couple of necessary attributes and conditions
266	for the installation such as if they exist and contain values.
267	Otherwise set default values.
268
269	Parameters
270	----------
271	c : ControlFile
272	Contains all the parameters of CONTROL file and
273	command line.
274
275
276	Return
277	------
278
279	'''
280
281	if c.install_target and \
282	c.install_target not in _config.INSTALL_TARGETS:
283	print('ERROR: unknown or missing installation target ')
284	print('target: ', c.install_target)
285	print('please specify correct installation target ' +
286	str(INSTALL_TARGETS))
287	print('use -h or --help for help')
288	sys.exit(1)
289
290	if c.install_target and c.install_target != 'local':
291	if not c.ecgid or not c.ecuid or \
292	not c.gateway or not c.destination:
293	print('Please enter your ECMWF user id and group id as well ' +
294	'as the \nname of the local gateway and the ectrans ' +
295	'destination ')
296	print('with command line options --ecuid --ecgid \
297	--gateway --destination')
298	print('Try "' + sys.argv[0].split('/')[-1] + \
299	' -h" to print usage information')
300	print('Please consult ecaccess documentation or ECMWF user \
301	support for further details')
302	sys.exit(1)
303
304	if not c.installdir:
305	c.installdir = '${HOME}'
306	else: # local
307	if not c.installdir:
308	c.installdir = _config.PATH_FLEXEXTRACT_DIR
309
310	return
311
312
313	def mk_tarball(tarball_path, target):
314	'''Creates a tarball with all necessary files which need to be sent to the
315	installation directory.
316	It does not matter if this is local or remote.
317	Collects all python files, the Fortran source and makefiles,
318	the ECMWF_ENV file, the CONTROL files as well as the
319	template files.
320
321	Parameters
322	----------
323	tarball_path : str
324	The complete path to the tar file which will contain all
325	relevant data for flex_extract.
326
327	target : str
328	The queue where the job is submitted to.
329
330	Return
331	------
332
333	'''
334	from glob import glob
335
336	print('Create tarball ...')
337
338	# change to FLEXEXTRACT directory so that the tar can contain
339	# relative pathes to the files and directories
340	ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
341	os.chdir(ecd)
342
343	# get lists of the files to be added to the tar file
344	if target == 'local':
345	ECMWF_ENV_FILE = []
346	runfile = [os.path.relpath(x, ecd)
347	for x in UioFiles(_config.PATH_REL_RUN_DIR,
348	'run_local.sh').files]
349	else:
350	ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
351	runfile = [os.path.relpath(x, ecd)
352	for x in UioFiles(_config.PATH_REL_RUN_DIR,
353	'run.sh').files]
354
355	pyfiles = [os.path.relpath(x, ecd)
356	for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
357	pytestfiles = [os.path.relpath(x, ecd)
358	for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
359	controlfiles = [os.path.relpath(x, ecd)
360	for x in UioFiles(_config.PATH_REL_CONTROLFILES,
361	'CONTROL*').files]
362	testfiles = [os.path.relpath(x, ecd)
363	for x in UioFiles(_config.PATH_REL_TEST , '*').files]
364	tempfiles = [os.path.relpath(x, ecd)
365	for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.temp').files]
366	nlfiles = [os.path.relpath(x, ecd)
367	for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.nl').files]
368	gribtable = [os.path.relpath(x, ecd)
369	for x in UioFiles(_config.PATH_REL_TEMPLATES , 'grib').files]
370	ffiles = [os.path.relpath(x, ecd)
371	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '.f').files]
372	hfiles = [os.path.relpath(x, ecd)
373	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
374	makefiles = [os.path.relpath(x, ecd)
375	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'Makefile*').files]
376	jobdir = [_config.PATH_REL_JOBSCRIPTS]
377
378	# concatenate single lists to one for a better looping
379	filelist = pyfiles + pytestfiles + controlfiles + tempfiles + nlfiles + \
380	ffiles + gribtable + hfiles + makefiles + ECMWF_ENV_FILE + \
381	runfile + jobdir + testfiles +\
382	['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
383
384	# create installation tar-file
385	exclude_files = [".ksh"]
386	try:
387	with tarfile.open(tarball_path, "w:gz") as tar_handle:
388	for file in filelist:
389	tar_handle.add(file, recursive=False,
390	filter=lambda tarinfo: None
391	if os.path.splitext(tarinfo.name)[1]
392	in exclude_files
393	else tarinfo)
394	except tarfile.TarError as e:
395	sys.exit('\n... error occured while trying to create the tar-file ' +
396	str(tarball_path))
397
398	return
399
400
401	def un_tarball(tarball_path):
402	'''Extracts the given tarball into current directory.
403
404	Parameters
405	----------
406	tarball_path : str
407	The complete path to the tar file which will contain all
408	relevant data for flex_extract.
409
410	Return
411	------
412
413	'''
414
415	print('Untar ...')
416
417	try:
418	with tarfile.open(tarball_path) as tar_handle:
419	tar_handle.extractall()
420	except tarfile.TarError as e:
421	sys.exit('\n... error occured while trying to read tar-file ' +
422	str(tarball_path))
423	except OSError as e:
424	print('... ERROR CODE: ' + str(e.errno))
425	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
426
427	sys.exit('\n... error occured while trying to read tar-file ' +
428	str(tarball_path))
429
430	return
431
432	def mk_env_vars(ecuid, ecgid, gateway, destination):
433	'''Creates a file named ECMWF_ENV which contains the
434	necessary environmental variables at ECMWF servers.
435	It is based on the template ECMWF_ENV.template.
436
437	Parameters
438	----------
439	ecuid : str
440	The user id on ECMWF server.
441
442	ecgid : str
443	The group id on ECMWF server.
444
445	gateway : str
446	The gateway server the user is using.
447
448	destination : str
449	The remote destination which is used to transfer files
450	from ECMWF server to local gateway server.
451
452	Return
453	------
454
455	'''
456	from genshi.template.text import NewTextTemplate
457	from genshi.template import TemplateLoader
458	from genshi.template.eval import UndefinedError
459
460	try:
461	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
462	ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
463	cls=NewTextTemplate)
464
465	stream = ecmwfvars_template.generate(user_name = ecuid,
466	user_group = ecgid,
467	gateway_name = gateway,
468	destination_name = destination
469	)
470	except UndefinedError as e:
471	print('... ERROR ' + str(e))
472
473	sys.exit('\n... error occured while trying to generate template ' +
474	_config.PATH_ECMWF_ENV)
475	except OSError as e:
476	print('... ERROR CODE: ' + str(e.errno))
477	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
478
479	sys.exit('\n... error occured while trying to generate template ' +
480	_config.PATH_ECMWF_ENV)
481
482	try:
483	with open(_config.PATH_ECMWF_ENV, 'w') as f:
484	f.write(stream.render('text'))
485	except OSError as e:
486	print('... ERROR CODE: ' + str(e.errno))
487	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
488
489	sys.exit('\n... error occured while trying to write ' +
490	_config.PATH_ECMWF_ENV)
491
492	return
493
494	def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
495	'''Modifies the original job template file so that it is specified
496	for the user and the environment were it will be applied. Result
497	is stored in a new file "job.temp" in the python directory.
498
499	Parameters
500	----------
501	makefile : str
502	Name of the makefile which should be used to compile FORTRAN
503	CONVERT2 program.
504
505	target : str
506	The target where the installation should be done, e.g. the queue.
507
508	ecuid : str
509	The user id on ECMWF server.
510
511	ecgid : str
512	The group id on ECMWF server.
513
514	fp_root : str
515	Path to the root directory of FLEXPART environment or flex_extract
516	environment.
517
518	Return
519	------
520
521	'''
522	from genshi.template.text import NewTextTemplate
523	from genshi.template import TemplateLoader
524	from genshi.template.eval import UndefinedError
525
526	if fp_root == '../':
527	fp_root = '$HOME'
528
529	try:
530	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
531	compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
532	cls=NewTextTemplate)
533
534	stream = compile_template.generate(
535	usergroup = ecgid,
536	username = ecuid,
537	version_number = _config._VERSION_STR,
538	fp_root_scripts = fp_root,
539	makefile = makefile,
540	fortran_program = _config.FORTRAN_EXECUTABLE
541	)
542	except UndefinedError as e:
543	print('... ERROR ' + str(e))
544
545	sys.exit('\n... error occured while trying to generate template ' +
546	_config.TEMPFILE_INSTALL_COMPILEJOB)
547	except OSError as e:
548	print('... ERROR CODE: ' + str(e.errno))
549	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
550
551	sys.exit('\n... error occured while trying to generate template ' +
552	_config.TEMPFILE_INSTALL_COMPILEJOB)
553
554	try:
555	compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
556	_config.FILE_INSTALL_COMPILEJOB)
557
558	with open(compilejob, 'w') as f:
559	f.write(stream.render('text'))
560	except OSError as e:
561	print('... ERROR CODE: ' + str(e.errno))
562	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
563
564	sys.exit('\n... error occured while trying to write ' +
565	compilejob)
566
567	return
568
569	def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
570	'''Modifies the original job template file so that it is specified
571	for the user and the environment were it will be applied. Result
572	is stored in a new file.
573
574	Parameters
575	----------
576	ecuid : str
577	The user id on ECMWF server.
578
579	ecgid : str
580	The group id on ECMWF server.
581
582	gateway : str
583	The gateway server the user is using.
584
585	destination : str
586	The remote destination which is used to transfer files
587	from ECMWF server to local gateway server.
588
589	fp_root : str
590	Path to the root directory of FLEXPART environment or flex_extract
591	environment.
592
593	Return
594	------
595
596	'''
597	from genshi.template.text import NewTextTemplate
598	from genshi.template import TemplateLoader
599	from genshi.template.eval import UndefinedError
600
601	fp_root_path_to_python = os.path.join(fp_root,
602	_config.FLEXEXTRACT_DIRNAME,
603	_config.PATH_REL_PYTHON_SRC)
604	if '$' in fp_root_path_to_python:
605	ind = fp_root_path_to_python.index('$')
606	fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
607	fp_root_path_to_python[ind:]
608
609	try:
610	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
611	compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
612	cls=NewTextTemplate)
613
614	stream = compile_template.generate(
615	usergroup = ecgid,
616	username = ecuid,
617	version_number = _config._VERSION_STR,
618	fp_root_path = fp_root_path_to_python,
619	)
620	except UndefinedError as e:
621	print('... ERROR ' + str(e))
622
623	sys.exit('\n... error occured while trying to generate template ' +
624	_config.TEMPFILE_INSTALL_JOB)
625	except OSError as e:
626	print('... ERROR CODE: ' + str(e.errno))
627	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
628
629	sys.exit('\n... error occured while trying to generate template ' +
630	_config.TEMPFILE_INSTALL_JOB)
631
632
633	try:
634	tempjobfile = os.path.join(_config.PATH_TEMPLATES,
635	_config.TEMPFILE_JOB)
636
637	with open(tempjobfile, 'w') as f:
638	f.write(stream.render('text'))
639	except OSError as e:
640	print('... ERROR CODE: ' + str(e.errno))
641	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
642
643	sys.exit('\n... error occured while trying to write ' +
644	tempjobfile)
645
646	return
647
648	def del_convert_build(src_path):
649	'''Clean up the Fortran source directory and remove all
650	build files (e.g. \.o, \.mod and CONVERT2)
651
652	Parameters
653	----------
654	src_path : str
655	Path to the fortran source directory.
656
657	Return
658	------
659
660	'''
661
662	modfiles = UioFiles(src_path, '*.mod')
663	objfiles = UioFiles(src_path, '*.o')
664	exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
665
666	modfiles.delete_files()
667	objfiles.delete_files()
668	exefile.delete_files()
669
670	return
671
672	def mk_convert_build(src_path, makefile):
673	'''Compiles the Fortran code and generates the executable.
674
675	Parameters
676	----------
677	src_path : str
678	Path to the fortran source directory.
679
680	makefile : str
681	The name of the makefile which should be used.
682
683	Return
684	------
685
686	'''
687
688	try:
689	print('Using makefile: ' + makefile)
690	p = subprocess.Popen(['make', '-f',
691	os.path.join(src_path, makefile)],
692	stdin=subprocess.PIPE,
693	stdout=subprocess.PIPE,
694	stderr=subprocess.PIPE,
695	bufsize=1)
696	pout, perr = p.communicate()
697	print(pout.decode())
698	if p.returncode != 0:
699	print(perr.decode())
700	print('Please edit ' + makefile +
701	' or try another Makefile in the src directory.')
702	print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
703	'and EMOSLIB must be adapted.')
704	print('Available Makefiles:')
705	print(UioFiles(src_path, 'Makefile*'))
706	sys.exit('Compilation failed!')
707	except ValueError as e:
708	print('ERROR: Makefile call failed:')
709	print(e)
710	else:
711	execute_subprocess(['ls', '-l', os.path.join(src_path,
712	_config.FORTRAN_EXECUTABLE)], error_msg=
713	'FORTRAN EXECUTABLE COULD NOT BE FOUND!')
714
715	return
716
717
718	if __name__ == "__main__":
719	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: