Context Navigation

source: flex_extract.git/source/python/install.py @ 79729d5

ctbtodev

Last change on this file since 79729d5 was 79729d5, checked in by Anne Philipp <anne.philipp@…>, 5 years ago
switched from python2 to python3
Property mode set to `100755`
File size: 23.2 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - moved install_args_and_control in here
14	# - splitted code in smaller functions
15	# - delete convert build files in here instead of compile job script
16	# - changed static path names to variables from config file
17	# - splitted install function into several smaller pieces
18	# - use of tarfile package in python
19	#
20	# @License:
21	# (C) Copyright 2014-2019.
22	# Anne Philipp, Leopold Haimberger
23	#
24	# This work is licensed under the Creative Commons Attribution 4.0
25	# International License. To view a copy of this license, visit
26	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
27	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
28	#
29	# @Methods:
30	# main
31	# get_install_cmdline_args
32	# install_via_gateway
33	# check_install_conditions
34	# mk_tarball
35	# un_tarball
36	# mk_env_vars
37	# mk_compilejob
38	# mk_job_template
39	# del_convert_build
40	# mk_convert_build
41	#
42	#*******************************************************************************
43	'''This script installs the flex_extract program.
44
45	Depending on the selected installation environment (locally or on the
46	ECMWF server ecgate or cca) the program extracts the commandline
47	arguments and the CONTROL file parameter and prepares the corresponding
48	environment.
49	The necessary files are collected in a tar-ball and placed
50	at the target location. There its untared, the environment variables will
51	be set and the Fortran code will be compiled.
52	If the ECMWF environment is selected a job script is prepared and submitted
53	for the remaining configurations after putting the tar-ball to the
54	target ECMWF server.
55
56	Type: install.py --help
57	to get information about command line parameters.
58	Read the documentation for usage instructions.
59	'''
60
61	# ------------------------------------------------------------------------------
62	# MODULES
63	# ------------------------------------------------------------------------------
64	import os
65	import sys
66	import glob
67	import subprocess
68	import inspect
69	import tarfile
70	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
71
72	# software specific classes and modules from flex_extract
73	import _config
74	from classes.ControlFile import ControlFile
75	from classes.UioFiles import UioFiles
76	from mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
77	silent_remove, execute_subprocess)
78
79	# ------------------------------------------------------------------------------
80	# FUNCTIONS
81	# ------------------------------------------------------------------------------
82	def main():
83	'''Controls the installation process. Calls the installation function
84	if target is specified.
85
86	Parameters
87	----------
88
89	Return
90	------
91	'''
92
93	args = get_install_cmdline_args()
94	c = ControlFile(args.controlfile)
95	c.assign_args_to_control(args)
96	check_install_conditions(c)
97
98	install_via_gateway(c)
99
100	return
101
102	def get_install_cmdline_args():
103	'''Decomposes the command line arguments and assigns them to variables.
104	Apply default values for non mentioned arguments.
105
106	Parameters
107	----------
108
109	Return
110	------
111	args : Namespace
112	Contains the commandline arguments from script/program call.
113	'''
114	parser = ArgumentParser(description='Install flex_extract software locally or \
115	on ECMWF machines',
116	formatter_class=ArgumentDefaultsHelpFormatter)
117
118	parser.add_argument('--target', dest='install_target', default=None,
119	help="Valid targets: local \| ecgate \| cca , \
120	the latter two are at ECMWF")
121	parser.add_argument("--makefile", dest="makefile", default=None,
122	help='Name of Makefile to use for compiling CONVERT2')
123	parser.add_argument("--ecuid", dest="ecuid", default=None,
124	help='user id at ECMWF')
125	parser.add_argument("--ecgid", dest="ecgid", default=None,
126	help='group id at ECMWF')
127	parser.add_argument("--gateway", dest="gateway", default=None,
128	help='name of local gateway server')
129	parser.add_argument("--destination", dest="destination", default=None,
130	help='ecaccess destination, e.g. leo@genericSftp')
131
132	parser.add_argument("--flexpartdir", dest="flexpartdir",
133	default=None, help="FLEXPART root directory on ECMWF \
134	servers (to find grib2flexpart and COMMAND file)\n\
135	Normally flex_extract resides in the scripts directory \
136	of the FLEXPART distribution.")
137
138	# arguments for job submission to ECMWF, only needed by submit.py
139	parser.add_argument("--job_template", dest='job_template',
140	default="job.temp.o",
141	help="job template file for submission to ECMWF")
142
143	parser.add_argument("--controlfile", dest="controlfile",
144	default='CONTROL.temp',
145	help="file with CONTROL parameters")
146
147	args = parser.parse_args()
148
149	return args
150
151
152	def install_via_gateway(c):
153	'''Perform the actual installation on local machine or prepare data
154	transfer to remote gate and submit a job script which will
155	install everything on the remote gate.
156
157	Parameters
158	----------
159	c : ControlFile
160	Contains all the parameters of CONTROL file and
161	command line.
162
163	Return
164	------
165
166	'''
167	import tarfile
168
169	ecd = _config.PATH_FLEXEXTRACT_DIR
170	tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
171	tar_file = os.path.join(ecd, tarball_name)
172
173	target_dirname = _config.FLEXEXTRACT_DIRNAME
174	fortran_executable = _config.FORTRAN_EXECUTABLE
175
176	if c.install_target.lower() != 'local': # ecgate or cca
177
178	mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
179	c.flexpartdir)
180
181	mk_job_template(c.ecuid, c.ecgid, c.gateway,
182	c.destination, c.flexpartdir)
183
184	mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
185
186	mk_tarball(tar_file, c.install_target)
187
188	put_file_to_ecserver(ecd, tarball_name, c.install_target,
189	c.ecuid, c.ecgid)
190
191	submit_job_to_ecserver(c.install_target,
192	os.path.join(_config.PATH_REL_JOBSCRIPTS,
193	_config.FILE_INSTALL_COMPILEJOB))
194
195	silent_remove(tar_file)
196
197	print('job compilation script has been submitted to ecgate for ' +
198	'installation in ' + c.flexpartdir +
199	'/' + target_dirname)
200	print('You should get an email with subject "flexcompile" within ' +
201	'the next few minutes!')
202
203	else: #local
204	if c.flexpartdir == _config.PATH_FLEXEXTRACT_DIR :
205	print('WARNING: FLEXPARTDIR has not been specified')
206	print('flex_extract will be installed in here by compiling the ' +
207	'Fortran source in ' + _config.PATH_FORTRAN_SRC)
208	os.chdir(_config.PATH_FORTRAN_SRC)
209	else: # creates the target working directory for flex_extract
210	c.flexpartdir = os.path.expandvars(os.path.expanduser(
211	c.flexpartdir))
212	if os.path.abspath(ecd) != os.path.abspath(c.flexpartdir):
213	mk_tarball(tar_file, c.install_target)
214	make_dir(os.path.join(c.flexpartdir,
215	target_dirname))
216	os.chdir(os.path.join(c.flexpartdir,
217	target_dirname))
218	un_tarball(tar_file)
219	os.chdir(os.path.join(c.flexpartdir,
220	target_dirname,
221	_config.PATH_REL_FORTRAN_SRC))
222
223	# Create Fortran executable - CONVERT2
224	print('Install ' + target_dirname + ' software at ' +
225	c.install_target + ' in directory ' +
226	os.path.abspath(c.flexpartdir) + '\n')
227
228	del_convert_build('.')
229	mk_convert_build('.', c.makefile)
230
231	os.chdir(ecd)
232	if os.path.isfile(tar_file):
233	os.remove(tar_file)
234
235	return
236
237	def check_install_conditions(c):
238	'''Checks a couple of necessary attributes and conditions
239	for the installation such as if they exist and contain values.
240	Otherwise set default values.
241
242	Parameters
243	----------
244	c : ControlFile
245	Contains all the parameters of CONTROL file and
246	command line.
247
248
249	Return
250	------
251
252	'''
253
254	if c.install_target and \
255	c.install_target not in _config.INSTALL_TARGETS:
256	print('ERROR: unknown or missing installation target ')
257	print('target: ', c.install_target)
258	print('please specify correct installation target ' +
259	str(INSTALL_TARGETS))
260	print('use -h or --help for help')
261	sys.exit(1)
262
263	if c.install_target and c.install_target != 'local':
264	if not c.ecgid or not c.ecuid or \
265	not c.gateway or not c.destination:
266	print('Please enter your ECMWF user id and group id as well ' +
267	'as the \nname of the local gateway and the ectrans ' +
268	'destination ')
269	print('with command line options --ecuid --ecgid \
270	--gateway --destination')
271	print('Try "' + sys.argv[0].split('/')[-1] + \
272	' -h" to print usage information')
273	print('Please consult ecaccess documentation or ECMWF user \
274	support for further details')
275	sys.exit(1)
276
277	if not c.flexpartdir:
278	c.flexpartdir = '${HOME}'
279	else:
280	c.flexpartdir = c.flexpartdir
281	else: # local
282	if not c.flexpartdir:
283	c.flexpartdir = _config.PATH_FLEXEXTRACT_DIR
284
285	return
286
287
288	def mk_tarball(tarball_path, target):
289	'''Creates a tarball with all necessary files which need to be sent to the
290	installation directory.
291	It does not matter if this is local or remote.
292	Collects all python files, the Fortran source and makefiles,
293	the ECMWF_ENV file, the CONTROL files as well as the
294	template files.
295
296	Parameters
297	----------
298	tarball_path : str
299	The complete path to the tar file which will contain all
300	relevant data for flex_extract.
301
302	target : str
303	The queue where the job is submitted to.
304
305	Return
306	------
307
308	'''
309	from glob import glob
310
311	print('Create tarball ...')
312
313	# change to FLEXEXTRACT directory so that the tar can contain
314	# relative pathes to the files and directories
315	ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
316	os.chdir(ecd)
317
318	# get lists of the files to be added to the tar file
319	if target == 'local':
320	ECMWF_ENV_FILE = []
321	runfile = [os.path.relpath(x, ecd)
322	for x in UioFiles(_config.PATH_REL_RUN_DIR,
323	'run_local.sh').files]
324	else:
325	ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
326	runfile = [os.path.relpath(x, ecd)
327	for x in UioFiles(_config.PATH_REL_RUN_DIR,
328	'run.sh').files]
329
330	pyfiles = [os.path.relpath(x, ecd)
331	for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
332	pytestfiles = [os.path.relpath(x, ecd)
333	for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
334	controlfiles = [os.path.relpath(x, ecd)
335	for x in UioFiles(_config.PATH_REL_CONTROLFILES,
336	'CONTROL*').files]
337	testfiles = [os.path.relpath(x, ecd)
338	for x in UioFiles(_config.PATH_REL_TEST , '*').files]
339	tempfiles = [os.path.relpath(x, ecd)
340	for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.temp').files]
341	nlfiles = [os.path.relpath(x, ecd)
342	for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.nl').files]
343	gribtable = [os.path.relpath(x, ecd)
344	for x in UioFiles(_config.PATH_REL_TEMPLATES , 'grib').files]
345	ffiles = [os.path.relpath(x, ecd)
346	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '.f').files]
347	hfiles = [os.path.relpath(x, ecd)
348	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
349	makefiles = [os.path.relpath(x, ecd)
350	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'Makefile*').files]
351	jobdir = [_config.PATH_REL_JOBSCRIPTS]
352
353	# concatenate single lists to one for a better looping
354	filelist = pyfiles + pytestfiles + controlfiles + tempfiles + nlfiles + \
355	ffiles + gribtable + hfiles + makefiles + ECMWF_ENV_FILE + \
356	runfile + jobdir + testfiles +\
357	['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
358
359	# create installation tar-file
360	exclude_files = [".ksh"]
361	try:
362	with tarfile.open(tarball_path, "w:gz") as tar_handle:
363	for file in filelist:
364	tar_handle.add(file, recursive=False,
365	filter=lambda tarinfo: None
366	if os.path.splitext(tarinfo.name)[1]
367	in exclude_files
368	else tarinfo)
369	except tarfile.TarError as e:
370	sys.exit('\n... error occured while trying to create the tar-file ' +
371	str(tarball_path))
372
373	return
374
375
376	def un_tarball(tarball_path):
377	'''Extracts the given tarball into current directory.
378
379	Parameters
380	----------
381	tarball_path : str
382	The complete path to the tar file which will contain all
383	relevant data for flex_extract.
384
385	Return
386	------
387
388	'''
389
390	print('Untar ...')
391
392	try:
393	with tarfile.open(tarball_path) as tar_handle:
394	tar_handle.extractall()
395	except tarfile.TarError as e:
396	sys.exit('\n... error occured while trying to read tar-file ' +
397	str(tarball_path))
398	except OSError as e:
399	print('... ERROR CODE: ' + str(e.errno))
400	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
401
402	sys.exit('\n... error occured while trying to read tar-file ' +
403	str(tarball_path))
404
405	return
406
407	def mk_env_vars(ecuid, ecgid, gateway, destination):
408	'''Creates a file named ECMWF_ENV which contains the
409	necessary environmental variables at ECMWF servers.
410	It is based on the template ECMWF_ENV.template.
411
412	Parameters
413	----------
414	ecuid : str
415	The user id on ECMWF server.
416
417	ecgid : str
418	The group id on ECMWF server.
419
420	gateway : str
421	The gateway server the user is using.
422
423	destination : str
424	The remote destination which is used to transfer files
425	from ECMWF server to local gateway server.
426
427	Return
428	------
429
430	'''
431	from genshi.template.text import NewTextTemplate
432	from genshi.template import TemplateLoader
433	from genshi.template.eval import UndefinedError
434
435	try:
436	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
437	ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
438	cls=NewTextTemplate)
439
440	stream = ecmwfvars_template.generate(user_name = ecuid,
441	user_group = ecgid,
442	gateway_name = gateway,
443	destination_name = destination
444	)
445	except UndefinedError as e:
446	print('... ERROR ' + str(e))
447
448	sys.exit('\n... error occured while trying to generate template ' +
449	_config.PATH_ECMWF_ENV)
450	except OSError as e:
451	print('... ERROR CODE: ' + str(e.errno))
452	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
453
454	sys.exit('\n... error occured while trying to generate template ' +
455	_config.PATH_ECMWF_ENV)
456
457	try:
458	with open(_config.PATH_ECMWF_ENV, 'w') as f:
459	f.write(stream.render('text'))
460	except OSError as e:
461	print('... ERROR CODE: ' + str(e.errno))
462	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
463
464	sys.exit('\n... error occured while trying to write ' +
465	_config.PATH_ECMWF_ENV)
466
467	return
468
469	def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
470	'''Modifies the original job template file so that it is specified
471	for the user and the environment were it will be applied. Result
472	is stored in a new file "job.temp" in the python directory.
473
474	Parameters
475	----------
476	makefile : str
477	Name of the makefile which should be used to compile FORTRAN
478	CONVERT2 program.
479
480	target : str
481	The target where the installation should be done, e.g. the queue.
482
483	ecuid : str
484	The user id on ECMWF server.
485
486	ecgid : str
487	The group id on ECMWF server.
488
489	fp_root : str
490	Path to the root directory of FLEXPART environment or flex_extract
491	environment.
492
493	Return
494	------
495
496	'''
497	from genshi.template.text import NewTextTemplate
498	from genshi.template import TemplateLoader
499	from genshi.template.eval import UndefinedError
500
501	if fp_root == '../':
502	fp_root = '$HOME'
503
504	try:
505	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
506	compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
507	cls=NewTextTemplate)
508
509	stream = compile_template.generate(
510	usergroup = ecgid,
511	username = ecuid,
512	version_number = _config._VERSION_STR,
513	fp_root_scripts = fp_root,
514	makefile = makefile,
515	fortran_program = _config.FORTRAN_EXECUTABLE
516	)
517	except UndefinedError as e:
518	print('... ERROR ' + str(e))
519
520	sys.exit('\n... error occured while trying to generate template ' +
521	_config.TEMPFILE_INSTALL_COMPILEJOB)
522	except OSError as e:
523	print('... ERROR CODE: ' + str(e.errno))
524	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
525
526	sys.exit('\n... error occured while trying to generate template ' +
527	_config.TEMPFILE_INSTALL_COMPILEJOB)
528
529	try:
530	compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
531	_config.FILE_INSTALL_COMPILEJOB)
532
533	with open(compilejob, 'w') as f:
534	f.write(stream.render('text'))
535	except OSError as e:
536	print('... ERROR CODE: ' + str(e.errno))
537	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
538
539	sys.exit('\n... error occured while trying to write ' +
540	compilejob)
541
542	return
543
544	def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
545	'''Modifies the original job template file so that it is specified
546	for the user and the environment were it will be applied. Result
547	is stored in a new file.
548
549	Parameters
550	----------
551	ecuid : str
552	The user id on ECMWF server.
553
554	ecgid : str
555	The group id on ECMWF server.
556
557	gateway : str
558	The gateway server the user is using.
559
560	destination : str
561	The remote destination which is used to transfer files
562	from ECMWF server to local gateway server.
563
564	fp_root : str
565	Path to the root directory of FLEXPART environment or flex_extract
566	environment.
567
568	Return
569	------
570
571	'''
572	from genshi.template.text import NewTextTemplate
573	from genshi.template import TemplateLoader
574	from genshi.template.eval import UndefinedError
575
576	fp_root_path_to_python = os.path.join(fp_root,
577	_config.FLEXEXTRACT_DIRNAME,
578	_config.PATH_REL_PYTHON_SRC)
579	if '$' in fp_root_path_to_python:
580	ind = fp_root_path_to_python.index('$')
581	fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
582	fp_root_path_to_python[ind:]
583
584	try:
585	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
586	compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
587	cls=NewTextTemplate)
588
589	stream = compile_template.generate(
590	usergroup = ecgid,
591	username = ecuid,
592	version_number = _config._VERSION_STR,
593	fp_root_path = fp_root_path_to_python,
594	)
595	except UndefinedError as e:
596	print('... ERROR ' + str(e))
597
598	sys.exit('\n... error occured while trying to generate template ' +
599	_config.TEMPFILE_INSTALL_JOB)
600	except OSError as e:
601	print('... ERROR CODE: ' + str(e.errno))
602	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
603
604	sys.exit('\n... error occured while trying to generate template ' +
605	_config.TEMPFILE_INSTALL_JOB)
606
607
608	try:
609	tempjobfile = os.path.join(_config.PATH_TEMPLATES,
610	_config.TEMPFILE_JOB)
611
612	with open(tempjobfile, 'w') as f:
613	f.write(stream.render('text'))
614	except OSError as e:
615	print('... ERROR CODE: ' + str(e.errno))
616	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
617
618	sys.exit('\n... error occured while trying to write ' +
619	tempjobfile)
620
621	return
622
623	def del_convert_build(src_path):
624	'''Clean up the Fortran source directory and remove all
625	build files (e.g. \.o, \.mod and CONVERT2)
626
627	Parameters
628	----------
629	src_path : str
630	Path to the fortran source directory.
631
632	Return
633	------
634
635	'''
636
637	modfiles = UioFiles(src_path, '*.mod')
638	objfiles = UioFiles(src_path, '*.o')
639	exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
640
641	modfiles.delete_files()
642	objfiles.delete_files()
643	exefile.delete_files()
644
645	return
646
647	def mk_convert_build(src_path, makefile):
648	'''Compiles the Fortran code and generates the executable.
649
650	Parameters
651	----------
652	src_path : str
653	Path to the fortran source directory.
654
655	makefile : str
656	The name of the makefile which should be used.
657
658	Return
659	------
660
661	'''
662
663	try:
664	print('Using makefile: ' + makefile)
665	p = subprocess.Popen(['make', '-f',
666	os.path.join(src_path, makefile)],
667	stdin=subprocess.PIPE,
668	stdout=subprocess.PIPE,
669	stderr=subprocess.PIPE,
670	bufsize=1)
671	pout, perr = p.communicate()
672	print(pout.decode())
673	if p.returncode != 0:
674	print(perr.decode())
675	print('Please edit ' + makefile +
676	' or try another Makefile in the src directory.')
677	print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
678	'and EMOSLIB must be adapted.')
679	print('Available Makefiles:')
680	print(UioFiles(src_path, 'Makefile*'))
681	sys.exit('Compilation failed!')
682	except ValueError as e:
683	print('ERROR: Makefile call failed:')
684	print(e)
685	else:
686	execute_subprocess(['ls', '-l', os.path.join(src_path,
687	_config.FORTRAN_EXECUTABLE)], error_msg=
688	'FORTRAN EXECUTABLE COULD NOT BE FOUND!')
689
690	return
691
692
693	if __name__ == "__main__":
694	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: