Context Navigation

source: flex_extract.git/source/python/install.py @ 6f951ca

ctbtodev

Last change on this file since 6f951ca was 6f951ca, checked in by Anne Philipp <anne.philipp@…>, 5 years ago
new style of docstring params and updates in docstrings
Property mode set to `100755`
File size: 23.5 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - moved install_args_and_control in here
14	# - splitted code in smaller functions
15	# - delete convert build files in here instead of compile job script
16	# - changed static path names to variables from config file
17	# - splitted install function into several smaller pieces
18	# - use of tarfile package in python
19	#
20	# @License:
21	# (C) Copyright 2014-2019.
22	# Anne Philipp, Leopold Haimberger
23	#
24	# This work is licensed under the Creative Commons Attribution 4.0
25	# International License. To view a copy of this license, visit
26	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
27	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
28	#
29	# @Methods:
30	# main
31	# get_install_cmdline_args
32	# install_via_gateway
33	# check_install_conditions
34	# mk_tarball
35	# un_tarball
36	# mk_env_vars
37	# mk_compilejob
38	# mk_job_template
39	# del_convert_build
40	# mk_convert_build
41	#
42	#*******************************************************************************
43	'''This script installs the flex_extract program.
44
45	Depending on the selected installation environment (locally or on the
46	ECMWF server ecgate or cca) the program extracts the commandline
47	arguments and the CONTROL file parameter and prepares the corresponding
48	environment.
49	The necessary files are collected in a tar-ball and placed
50	at the target location. There its untared, the environment variables will
51	be set and the Fortran code will be compiled.
52	If the ECMWF environment is selected a job script is prepared and submitted
53	for the remaining configurations after putting the tar-ball to the
54	target ECMWF server.
55
56	Type: install.py --help
57	to get information about command line parameters.
58	Read the documentation for usage instructions.
59	'''
60
61	# ------------------------------------------------------------------------------
62	# MODULES
63	# ------------------------------------------------------------------------------
64	import os
65	import sys
66	import glob
67	import subprocess
68	import inspect
69	import tarfile
70	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
71
72	# software specific classes and modules from flex_extract
73	import _config
74	from classes.ControlFile import ControlFile
75	from classes.UioFiles import UioFiles
76	from mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
77	silent_remove, execute_subprocess)
78
79	# ------------------------------------------------------------------------------
80	# FUNCTIONS
81	# ------------------------------------------------------------------------------
82	def main():
83	'''Controls the installation process. Calls the installation function
84	if target is specified.
85
86	Parameters
87	----------
88
89	Return
90	------
91	'''
92
93	args = get_install_cmdline_args()
94	c = ControlFile(args.controlfile)
95	c.assign_args_to_control(args)
96	check_install_conditions(c)
97
98	install_via_gateway(c)
99
100	return
101
102	def get_install_cmdline_args():
103	'''Decomposes the command line arguments and assigns them to variables.
104	Apply default values for non mentioned arguments.
105
106	Parameters
107	----------
108
109	Return
110	------
111	args : Namespace
112	Contains the commandline arguments from script/program call.
113	'''
114	parser = ArgumentParser(description='Install flex_extract software locally or \
115	on ECMWF machines',
116	formatter_class=ArgumentDefaultsHelpFormatter)
117
118	parser.add_argument('--target', dest='install_target', default=None,
119	help="Valid targets: local \| ecgate \| cca , \
120	the latter two are at ECMWF")
121	parser.add_argument("--makefile", dest="makefile", default=None,
122	help='Name of Makefile to use for compiling CONVERT2')
123	parser.add_argument("--ecuid", dest="ecuid", default=None,
124	help='user id at ECMWF')
125	parser.add_argument("--ecgid", dest="ecgid", default=None,
126	help='group id at ECMWF')
127	parser.add_argument("--gateway", dest="gateway", default=None,
128	help='name of local gateway server')
129	parser.add_argument("--destination", dest="destination", default=None,
130	help='ecaccess destination, e.g. leo@genericSftp')
131
132	parser.add_argument("--flexpartdir", dest="flexpartdir",
133	default=None, help="FLEXPART root directory on ECMWF \
134	servers (to find grib2flexpart and COMMAND file)\n\
135	Normally flex_extract resides in the scripts directory \
136	of the FLEXPART distribution.")
137
138	# arguments for job submission to ECMWF, only needed by submit.py
139	parser.add_argument("--job_template", dest='job_template',
140	default="job.temp.o",
141	help="job template file for submission to ECMWF")
142
143	parser.add_argument("--controlfile", dest="controlfile",
144	default='CONTROL.temp',
145	help="file with CONTROL parameters")
146
147	args = parser.parse_args()
148
149	return args
150
151
152	def install_via_gateway(c):
153	'''Perform the actual installation on local machine or prepare data
154	transfer to remote gate and submit a job script which will
155	install everything on the remote gate.
156
157	Parameters
158	----------
159	c : ControlFile
160	Contains all the parameters of CONTROL file and
161	command line.
162
163	Return
164	------
165
166	'''
167	import tarfile
168
169	ecd = _config.PATH_FLEXEXTRACT_DIR
170	tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
171	tar_file = os.path.join(ecd, tarball_name)
172
173	target_dirname = _config.FLEXEXTRACT_DIRNAME
174	fortran_executable = _config.FORTRAN_EXECUTABLE
175
176	if c.install_target.lower() != 'local': # ecgate or cca
177
178	mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
179	c.flexpartdir)
180
181	mk_job_template(c.ecuid, c.ecgid, c.gateway,
182	c.destination, c.flexpartdir)
183
184	mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
185
186	mk_tarball(tar_file, c.install_target)
187
188	put_file_to_ecserver(ecd, tarball_name, c.install_target,
189	c.ecuid, c.ecgid)
190
191	submit_job_to_ecserver(c.install_target,
192	os.path.join(_config.PATH_REL_JOBSCRIPTS,
193	_config.FILE_INSTALL_COMPILEJOB))
194
195	silent_remove(tar_file)
196
197	print('job compilation script has been submitted to ecgate for ' +
198	'installation in ' + c.flexpartdir +
199	'/' + target_dirname)
200	print('You should get an email with subject "flexcompile" within ' +
201	'the next few minutes!')
202
203	else: #local
204	if c.flexpartdir == _config.PATH_FLEXEXTRACT_DIR :
205	print('WARNING: FLEXPARTDIR has not been specified')
206	print('flex_extract will be installed in here by compiling the ' +
207	'Fortran source in ' + _config.PATH_FORTRAN_SRC)
208	os.chdir(_config.PATH_FORTRAN_SRC)
209	else: # creates the target working directory for flex_extract
210	c.flexpartdir = os.path.expandvars(os.path.expanduser(
211	c.flexpartdir))
212	if os.path.abspath(ecd) != os.path.abspath(c.flexpartdir):
213	mk_tarball(tar_file, c.install_target)
214	make_dir(os.path.join(c.flexpartdir,
215	target_dirname))
216	os.chdir(os.path.join(c.flexpartdir,
217	target_dirname))
218	un_tarball(tar_file)
219	os.chdir(os.path.join(c.flexpartdir,
220	target_dirname,
221	_config.PATH_REL_FORTRAN_SRC))
222
223	# Create Fortran executable - CONVERT2
224	print('Install ' + target_dirname + ' software at ' +
225	c.install_target + ' in directory ' +
226	os.path.abspath(c.flexpartdir) + '\n')
227
228	del_convert_build('.')
229	mk_convert_build('.', c.makefile)
230
231	os.chdir(ecd)
232	if os.path.isfile(tar_file):
233	os.remove(tar_file)
234
235	return
236
237	def check_install_conditions(c):
238	'''Checks a couple of necessary attributes and conditions
239	for the installation such as if they exist and contain values.
240	Otherwise set default values.
241
242	Parameters
243	----------
244	c : ControlFile
245	Contains all the parameters of CONTROL file and
246	command line.
247
248
249	Return
250	------
251
252	'''
253
254	if c.install_target and \
255	c.install_target not in _config.INSTALL_TARGETS:
256	print('ERROR: unknown or missing installation target ')
257	print('target: ', c.install_target)
258	print('please specify correct installation target ' +
259	str(INSTALL_TARGETS))
260	print('use -h or --help for help')
261	sys.exit(1)
262
263	if c.install_target and c.install_target != 'local':
264	if not c.ecgid or not c.ecuid or \
265	not c.gateway or not c.destination:
266	print('Please enter your ECMWF user id and group id as well ' +
267	'as the \nname of the local gateway and the ectrans ' +
268	'destination ')
269	print('with command line options --ecuid --ecgid \
270	--gateway --destination')
271	print('Try "' + sys.argv[0].split('/')[-1] + \
272	' -h" to print usage information')
273	print('Please consult ecaccess documentation or ECMWF user \
274	support for further details')
275	sys.exit(1)
276
277	if not c.flexpartdir:
278	c.flexpartdir = '${HOME}'
279	else:
280	c.flexpartdir = c.flexpartdir
281	else: # local
282	if not c.flexpartdir:
283	c.flexpartdir = _config.PATH_FLEXEXTRACT_DIR
284
285	return
286
287
288	def mk_tarball(tarball_path, target):
289	'''Creates a tarball with all necessary files which need to be sent to the
290	installation directory.
291	It does not matter if this is local or remote.
292	Collects all python files, the Fortran source and makefiles,
293	the ECMWF_ENV file, the CONTROL files as well as the
294	template files.
295
296	Parameters
297	----------
298	tarball_path : str
299	The complete path to the tar file which will contain all
300	relevant data for flex_extract.
301
302	target : str
303	The queue where the job is submitted to.
304
305	Return
306	------
307
308	'''
309	from glob import glob
310
311	print('Create tarball ...')
312
313	# change to FLEXEXTRACT directory so that the tar can contain
314	# relative pathes to the files and directories
315	ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
316	os.chdir(ecd)
317
318	# get lists of the files to be added to the tar file
319	if target == 'local':
320	ECMWF_ENV_FILE = []
321	runfile = [os.path.relpath(x, ecd)
322	for x in UioFiles(_config.PATH_REL_RUN_DIR,
323	'run_local.sh').files]
324	else:
325	ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
326	runfile = [os.path.relpath(x, ecd)
327	for x in UioFiles(_config.PATH_REL_RUN_DIR,
328	'run.sh').files]
329
330	pyfiles = [os.path.relpath(x, ecd)
331	for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
332	pytestfiles = [os.path.relpath(x, ecd)
333	for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
334	controlfiles = [os.path.relpath(x, ecd)
335	for x in UioFiles(_config.PATH_REL_CONTROLFILES,
336	'CONTROL*').files]
337	testfiles = [os.path.relpath(x, ecd)
338	for x in UioFiles(_config.PATH_REL_TEST , '*').files]
339	tempfiles = [os.path.relpath(x, ecd)
340	for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.temp').files]
341	nlfiles = [os.path.relpath(x, ecd)
342	for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.nl').files]
343	gribtable = [os.path.relpath(x, ecd)
344	for x in UioFiles(_config.PATH_REL_TEMPLATES , 'grib').files]
345	ffiles = [os.path.relpath(x, ecd)
346	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '.f').files]
347	hfiles = [os.path.relpath(x, ecd)
348	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
349	makefiles = [os.path.relpath(x, ecd)
350	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'Makefile*').files]
351	jobdir = [_config.PATH_REL_JOBSCRIPTS]
352
353	# concatenate single lists to one for a better looping
354	filelist = pyfiles + pytestfiles + controlfiles + tempfiles + nlfiles + \
355	ffiles + gribtable + hfiles + makefiles + ECMWF_ENV_FILE + \
356	runfile + jobdir + testfiles +\
357	['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
358
359	# create installation tar-file
360	exclude_files = [".ksh"]
361	try:
362	with tarfile.open(tarball_path, "w:gz") as tar_handle:
363	for file in filelist:
364	tar_handle.add(file, recursive=False,
365	filter=lambda tarinfo: None
366	if os.path.splitext(tarinfo.name)[1]
367	in exclude_files
368	else tarinfo)
369	except subprocess.CalledProcessError as e:
370	print('... ERROR CODE:\n ... ' + str(e.returncode))
371	print('... ERROR MESSAGE:\n ... ' + str(e))
372
373	sys.exit('\n... could not make installation tar ball!')
374	except OSError as e:
375	print('... ERROR CODE: ' + str(e.errno))
376	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
377
378	sys.exit('\n... error occured while trying to read tar-file ' +
379	str(tarball_path))
380
381	return
382
383
384	def un_tarball(tarball_path):
385	'''Extracts the given tarball into current directory.
386
387	Parameters
388	----------
389	tarball_path : str
390	The complete path to the tar file which will contain all
391	relevant data for flex_extract.
392
393	Return
394	------
395
396	'''
397
398	print('Untar ...')
399
400	try:
401	with tarfile.open(tarball_path) as tar_handle:
402	tar_handle.extractall()
403	except tarfile.TarError as e:
404	sys.exit('\n... error occured while trying to read tar-file ' +
405	str(tarball_path))
406	except OSError as e:
407	print('... ERROR CODE: ' + str(e.errno))
408	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
409
410	sys.exit('\n... error occured while trying to read tar-file ' +
411	str(tarball_path))
412
413	return
414
415	def mk_env_vars(ecuid, ecgid, gateway, destination):
416	'''Creates a file named ECMWF_ENV which contains the
417	necessary environmental variables at ECMWF servers.
418	It is based on the template ECMWF_ENV.template.
419
420	Parameters
421	----------
422	ecuid : str
423	The user id on ECMWF server.
424
425	ecgid : str
426	The group id on ECMWF server.
427
428	gateway : str
429	The gateway server the user is using.
430
431	destination : str
432	The remote destination which is used to transfer files
433	from ECMWF server to local gateway server.
434
435	Return
436	------
437
438	'''
439	from genshi.template.text import NewTextTemplate
440	from genshi.template import TemplateLoader
441	from genshi.template.eval import UndefinedError
442
443	try:
444	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
445	ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
446	cls=NewTextTemplate)
447
448	stream = ecmwfvars_template.generate(user_name = ecuid,
449	user_group = ecgid,
450	gateway_name = gateway,
451	destination_name = destination
452	)
453	except UndefinedError as e:
454	print('... ERROR ' + str(e))
455
456	sys.exit('\n... error occured while trying to generate template ' +
457	_config.PATH_ECMWF_ENV)
458	except OSError as e:
459	print('... ERROR CODE: ' + str(e.errno))
460	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
461
462	sys.exit('\n... error occured while trying to generate template ' +
463	_config.PATH_ECMWF_ENV)
464
465	try:
466	with open(_config.PATH_ECMWF_ENV, 'w') as f:
467	f.write(stream.render('text'))
468	except OSError as e:
469	print('... ERROR CODE: ' + str(e.errno))
470	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
471
472	sys.exit('\n... error occured while trying to write ' +
473	_config.PATH_ECMWF_ENV)
474
475	return
476
477	def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
478	'''Modifies the original job template file so that it is specified
479	for the user and the environment were it will be applied. Result
480	is stored in a new file "job.temp" in the python directory.
481
482	Parameters
483	----------
484	makefile : str
485	Name of the makefile which should be used to compile FORTRAN
486	CONVERT2 program.
487
488	target : str
489	The target where the installation should be done, e.g. the queue.
490
491	ecuid : str
492	The user id on ECMWF server.
493
494	ecgid : str
495	The group id on ECMWF server.
496
497	fp_root : str
498	Path to the root directory of FLEXPART environment or flex_extract
499	environment.
500
501	Return
502	------
503
504	'''
505	from genshi.template.text import NewTextTemplate
506	from genshi.template import TemplateLoader
507	from genshi.template.eval import UndefinedError
508
509	if fp_root == '../':
510	fp_root = '$HOME'
511
512	try:
513	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
514	compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
515	cls=NewTextTemplate)
516
517	stream = compile_template.generate(
518	usergroup = ecgid,
519	username = ecuid,
520	version_number = _config._VERSION_STR,
521	fp_root_scripts = fp_root,
522	makefile = makefile,
523	fortran_program = _config.FORTRAN_EXECUTABLE
524	)
525	except UndefinedError as e:
526	print('... ERROR ' + str(e))
527
528	sys.exit('\n... error occured while trying to generate template ' +
529	_config.TEMPFILE_INSTALL_COMPILEJOB)
530	except OSError as e:
531	print('... ERROR CODE: ' + str(e.errno))
532	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
533
534	sys.exit('\n... error occured while trying to generate template ' +
535	_config.TEMPFILE_INSTALL_COMPILEJOB)
536
537	try:
538	compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
539	_config.FILE_INSTALL_COMPILEJOB)
540
541	with open(compilejob, 'w') as f:
542	f.write(stream.render('text'))
543	except OSError as e:
544	print('... ERROR CODE: ' + str(e.errno))
545	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
546
547	sys.exit('\n... error occured while trying to write ' +
548	compilejob)
549
550	return
551
552	def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
553	'''Modifies the original job template file so that it is specified
554	for the user and the environment were it will be applied. Result
555	is stored in a new file.
556
557	Parameters
558	----------
559	ecuid : str
560	The user id on ECMWF server.
561
562	ecgid : str
563	The group id on ECMWF server.
564
565	gateway : str
566	The gateway server the user is using.
567
568	destination : str
569	The remote destination which is used to transfer files
570	from ECMWF server to local gateway server.
571
572	fp_root : str
573	Path to the root directory of FLEXPART environment or flex_extract
574	environment.
575
576	Return
577	------
578
579	'''
580	from genshi.template.text import NewTextTemplate
581	from genshi.template import TemplateLoader
582	from genshi.template.eval import UndefinedError
583
584	fp_root_path_to_python = os.path.join(fp_root,
585	_config.FLEXEXTRACT_DIRNAME,
586	_config.PATH_REL_PYTHON_SRC)
587	if '$' in fp_root_path_to_python:
588	ind = fp_root_path_to_python.index('$')
589	fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
590	fp_root_path_to_python[ind:]
591
592	try:
593	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
594	compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
595	cls=NewTextTemplate)
596
597	stream = compile_template.generate(
598	usergroup = ecgid,
599	username = ecuid,
600	version_number = _config._VERSION_STR,
601	fp_root_path = fp_root_path_to_python,
602	)
603	except UndefinedError as e:
604	print('... ERROR ' + str(e))
605
606	sys.exit('\n... error occured while trying to generate template ' +
607	_config.TEMPFILE_INSTALL_JOB)
608	except OSError as e:
609	print('... ERROR CODE: ' + str(e.errno))
610	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
611
612	sys.exit('\n... error occured while trying to generate template ' +
613	_config.TEMPFILE_INSTALL_JOB)
614
615
616	try:
617	tempjobfile = os.path.join(_config.PATH_TEMPLATES,
618	_config.TEMPFILE_JOB)
619
620	with open(tempjobfile, 'w') as f:
621	f.write(stream.render('text'))
622	except OSError as e:
623	print('... ERROR CODE: ' + str(e.errno))
624	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
625
626	sys.exit('\n... error occured while trying to write ' +
627	tempjobfile)
628
629	return
630
631	def del_convert_build(src_path):
632	'''Clean up the Fortran source directory and remove all
633	build files (e.g. \.o, \.mod and CONVERT2)
634
635	Parameters
636	----------
637	src_path : str
638	Path to the fortran source directory.
639
640	Return
641	------
642
643	'''
644
645	modfiles = UioFiles(src_path, '*.mod')
646	objfiles = UioFiles(src_path, '*.o')
647	exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
648
649	modfiles.delete_files()
650	objfiles.delete_files()
651	exefile.delete_files()
652
653	return
654
655	def mk_convert_build(src_path, makefile):
656	'''Compiles the Fortran code and generates the executable.
657
658	Parameters
659	----------
660	src_path : str
661	Path to the fortran source directory.
662
663	makefile : str
664	The name of the makefile which should be used.
665
666	Return
667	------
668
669	'''
670
671	try:
672	print('Using makefile: ' + makefile)
673	p = subprocess.Popen(['make', '-f',
674	os.path.join(src_path, makefile)],
675	stdin=subprocess.PIPE,
676	stdout=subprocess.PIPE,
677	stderr=subprocess.PIPE,
678	bufsize=1)
679	pout, perr = p.communicate()
680	print(pout)
681	if p.returncode != 0:
682	print(perr)
683	print('Please edit ' + makefile +
684	' or try another Makefile in the src directory.')
685	print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
686	'and EMOSLIB must be adapted.')
687	print('Available Makefiles:')
688	print(UioFiles(src_path, 'Makefile*'))
689	sys.exit('Compilation failed!')
690	except ValueError as e:
691	print('ERROR: Makefile call failed:')
692	print(e)
693	else:
694	execute_subprocess(['ls', '-l', os.path.join(src_path,
695	_config.FORTRAN_EXECUTABLE)], error_msg=
696	'FORTRAN EXECUTABLE COULD NOT BE FOUND!')
697
698	return
699
700
701	if __name__ == "__main__":
702	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: