Context Navigation

source: flex_extract.git/Source/Python/install.py @ 8209738

ctbtodev

Last change on this file since 8209738 was 8209738, checked in by anphi <anne.philipp@…>, 4 years ago
language corrections in comment sections and print commands
Property mode set to `100755`
File size: 23.4 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - moved install_args_and_control in here
14	# - splitted code in smaller functions
15	# - delete convert build files in here instead of compile job script
16	# - changed static path names to variables from config file
17	# - splitted install function into several smaller pieces
18	# - use of tarfile package in python
19	#
20	# @License:
21	# (C) Copyright 2014-2019.
22	# Anne Philipp, Leopold Haimberger
23	#
24	# SPDX-License-Identifier: CC-BY-4.0
25	#
26	# This work is licensed under the Creative Commons Attribution 4.0
27	# International License. To view a copy of this license, visit
28	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
29	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
30	#
31	# @Methods:
32	# main
33	# get_install_cmdline_args
34	# install_via_gateway
35	# check_install_conditions
36	# mk_tarball
37	# un_tarball
38	# mk_env_vars
39	# mk_compilejob
40	# mk_job_template
41	# del_convert_build
42	# mk_convert_build
43	#
44	#*******************************************************************************
45	'''This script installs the flex_extract program.
46
47	Depending on the selected installation environment (locally or on the
48	ECMWF server ecgate or cca) the program extracts the command line
49	arguments and the CONTROL file parameter and prepares the corresponding
50	environment.
51	The necessary files are collected in a tar ball and placed
52	at the target location. There, is is untared, the environment variables are
53	set, and the Fortran code is compiled.
54	If the ECMWF environment is selected, a job script is prepared and submitted
55	for the remaining configurations after putting the tar ball on the
56	target ECMWF server.
57
58	Type: install.py --help
59	to get information about command line parameters.
60	Read the documentation for usage instructions.
61	'''
62
63	# ------------------------------------------------------------------------------
64	# MODULES
65	# ------------------------------------------------------------------------------
66	from __future__ import print_function
67
68	import os
69	import sys
70	import subprocess
71	import tarfile
72	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
73
74	# software specific classes and modules from flex_extract
75	import _config
76	from Classes.ControlFile import ControlFile
77	from Classes.UioFiles import UioFiles
78	from Mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
79	silent_remove, execute_subprocess, none_or_str)
80
81	# ------------------------------------------------------------------------------
82	# FUNCTIONS
83	# ------------------------------------------------------------------------------
84	def main():
85	'''Controls the installation process. Calls the installation function
86	if target is specified.
87
88	Parameters
89	----------
90
91	Return
92	------
93	'''
94
95	args = get_install_cmdline_args()
96	c = ControlFile(args.controlfile)
97	c.assign_args_to_control(args)
98	check_install_conditions(c)
99
100	if c.install_target.lower() != 'local': # ecgate or cca
101	install_via_gateway(c)
102	else: # local
103	install_local(c)
104
105	return
106
107	def get_install_cmdline_args():
108	'''Decomposes the command line arguments and assigns them to variables.
109	Apply default values for arguments not present.
110
111	Parameters
112	----------
113
114	Return
115	------
116	args : Namespace
117	Contains the commandline arguments from script/program call.
118	'''
119	parser = ArgumentParser(description='Install flex_extract software '
120	'locally or on ECMWF machines',
121	formatter_class=ArgumentDefaultsHelpFormatter)
122
123	parser.add_argument('--target', dest='install_target',
124	type=none_or_str, default=None,
125	help="Valid targets: local \| ecgate \| cca , \
126	the latter two are at ECMWF")
127	parser.add_argument("--makefile", dest="makefile",
128	type=none_or_str, default=None,
129	help='Name of makefile for compiling the '
130	'Fortran program')
131	parser.add_argument("--ecuid", dest="ecuid",
132	type=none_or_str, default=None,
133	help='User id at ECMWF')
134	parser.add_argument("--ecgid", dest="ecgid",
135	type=none_or_str, default=None,
136	help='Group id at ECMWF')
137	parser.add_argument("--gateway", dest="gateway",
138	type=none_or_str, default=None,
139	help='Name of the local gateway server')
140	parser.add_argument("--destination", dest="destination",
141	type=none_or_str, default=None,
142	help='ecaccess association, e.g. '
143	'myUser@genericSftp')
144
145	parser.add_argument("--installdir", dest="installdir",
146	type=none_or_str, default=None,
147	help='Root directory of the '
148	'flex_extract installation')
149
150	# arguments for job submission to ECMWF, only needed by submit.py
151	parser.add_argument("--job_template", dest='job_template',
152	type=none_or_str, default="job.template",
153	help='Rudimentary template file to create a batch '
154	'job template for submission to ECMWF servers')
155
156	parser.add_argument("--controlfile", dest="controlfile",
157	type=none_or_str, default='CONTROL_EA5',
158	help="A file that contains all CONTROL parameters.")
159
160	args = parser.parse_args()
161
162	return args
163
164
165	def install_via_gateway(c):
166	'''Prepare data transfer to remote gateway and submit a job script which will
167	install everything on the remote gateway.
168
169	Parameters
170	----------
171	c : ControlFile
172	Contains all the parameters of CONTROL file and
173	command line.
174
175	Return
176	------
177
178	'''
179
180	tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
181	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
182
183	mk_compilejob(c.makefile, c.ecuid, c.ecgid, c.installdir)
184
185	mk_job_template(c.ecuid, c.ecgid, c.installdir)
186
187	mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
188
189	mk_tarball(tar_file, c.install_target)
190
191	put_file_to_ecserver(_config.PATH_FLEXEXTRACT_DIR, tarball_name,
192	c.install_target, c.ecuid, c.ecgid)
193
194	submit_job_to_ecserver(c.install_target,
195	os.path.join(_config.PATH_REL_JOBSCRIPTS,
196	_config.FILE_INSTALL_COMPILEJOB))
197
198	silent_remove(tar_file)
199
200	print('Job compilation script has been submitted to ecgate for ' +
201	'installation in ' + c.installdir +
202	'/' + _config.FLEXEXTRACT_DIRNAME)
203	print('You should get an email with subject "flexcompile" within ' +
204	'the next few minutes!')
205
206	return
207
208	def install_local(c):
209	'''Perform the actual installation on a local machine.
210
211	Parameters
212	----------
213	c : ControlFile
214	Contains all the parameters of CONTROL file and
215	command line.
216
217	Return
218	------
219
220	'''
221
222	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR,
223	_config.FLEXEXTRACT_DIRNAME + '.tar')
224
225	if c.installdir == _config.PATH_FLEXEXTRACT_DIR:
226	print('WARNING: installdir has not been specified')
227	print('flex_extract will be installed in here by compiling the ' +
228	'Fortran source in ' + _config.PATH_FORTRAN_SRC)
229	os.chdir(_config.PATH_FORTRAN_SRC)
230	else: # creates the target working directory for flex_extract
231	c.installdir = os.path.expandvars(os.path.expanduser(
232	c.installdir))
233	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) != \
234	os.path.abspath(c.installdir):
235	mk_tarball(tar_file, c.install_target)
236	make_dir(os.path.join(c.installdir,
237	_config.FLEXEXTRACT_DIRNAME))
238	os.chdir(os.path.join(c.installdir,
239	_config.FLEXEXTRACT_DIRNAME))
240	un_tarball(tar_file)
241	os.chdir(os.path.join(c.installdir,
242	_config.FLEXEXTRACT_DIRNAME,
243	_config.PATH_REL_FORTRAN_SRC))
244
245	# Create Fortran executable - CONVERT2
246	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software at ' +
247	c.install_target + ' in directory ' +
248	os.path.abspath(c.installdir) + '\n')
249
250	del_convert_build('.')
251	mk_convert_build('.', c.makefile)
252
253	os.chdir(_config.PATH_FLEXEXTRACT_DIR)
254	if os.path.isfile(tar_file):
255	os.remove(tar_file)
256
257	return
258
259
260	def check_install_conditions(c):
261	'''Checks necessary attributes and conditions
262	for the installation, e.g. whether they exist and contain values.
263	Otherwise set default values.
264
265	Parameters
266	----------
267	c : ControlFile
268	Contains all the parameters of CONTROL file and
269	command line.
270
271
272	Return
273	------
274
275	'''
276
277	if c.install_target and \
278	c.install_target not in _config.INSTALL_TARGETS:
279	print('ERROR: unknown or missing installation target ')
280	print('target: ', c.install_target)
281	print('please specify correct installation target ' +
282	str(_config.INSTALL_TARGETS))
283	print('use -h or --help for help')
284	sys.exit(1)
285
286	if c.install_target and c.install_target != 'local':
287	if not c.ecgid or not c.ecuid:
288	print('Please enter your ECMWF user id and group id '
289	' with command line options --ecuid --ecgid')
290	print('Try "' + sys.argv[0].split('/')[-1] + \
291	' -h" to print usage information')
292	print('Please consult ecaccess documentation or ECMWF user '
293	'support for further details.\n')
294	sys.exit(1)
295	if not c.gateway or not c.destination:
296	print('WARNING: Parameters GATEWAY and DESTINATION were '
297	'not properly set for working on ECMWF server. \n'
298	'There will be no transfer of output files to the '
299	'local gateway server possible!')
300	if not c.installdir:
301	c.installdir = '${HOME}'
302	else: # local
303	if not c.installdir:
304	c.installdir = _config.PATH_FLEXEXTRACT_DIR
305
306	return
307
308
309	def mk_tarball(tarball_path, target):
310	'''Creates a tarball with all necessary files which need to be sent to the
311	installation directory.
312	It does not matter whether this is local or remote.
313	Collects all Python files, the Fortran source and makefiles,
314	the ECMWF_ENV file, the CONTROL files as well as the
315	template files.
316
317	Parameters
318	----------
319	tarball_path : str
320	The complete path to the tar file which will contain all
321	relevant data for flex_extract.
322
323	target : str
324	The queue where the job is submitted to.
325
326	Return
327	------
328
329	'''
330
331	print('Create tarball ...')
332
333	# change to FLEXEXTRACT directory so that the tar can contain
334	# relative pathes to the files and directories
335	ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
336	os.chdir(ecd)
337
338	# get lists of the files to be added to the tar file
339	if target == 'local':
340	ecmwf_env_file = []
341	runfile = [os.path.relpath(x, ecd)
342	for x in UioFiles(_config.PATH_REL_RUN_DIR,
343	'run_local.sh').files]
344	else:
345	ecmwf_env_file = [_config.PATH_REL_ECMWF_ENV]
346	runfile = [os.path.relpath(x, ecd)
347	for x in UioFiles(_config.PATH_REL_RUN_DIR,
348	'run.sh').files]
349
350	pyfiles = [os.path.relpath(x, ecd)
351	for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
352	pytestfiles = [os.path.relpath(x, ecd)
353	for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
354	controlfiles = [os.path.relpath(x, ecd)
355	for x in UioFiles(_config.PATH_REL_CONTROLFILES,
356	'CONTROL*').files]
357	testfiles = [os.path.relpath(x, ecd)
358	for x in UioFiles(_config.PATH_REL_TEST, '*').files]
359	tempfiles = [os.path.relpath(x, ecd)
360	for x in UioFiles(_config.PATH_REL_TEMPLATES, '*.temp').files]
361	nlfiles = [os.path.relpath(x, ecd)
362	for x in UioFiles(_config.PATH_REL_TEMPLATES, '*.nl').files]
363	gribtable = [os.path.relpath(x, ecd)
364	for x in UioFiles(_config.PATH_REL_TEMPLATES, 'grib').files]
365	ffiles = [os.path.relpath(x, ecd)
366	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f90').files]
367	hfiles = [os.path.relpath(x, ecd)
368	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
369	makefiles = [os.path.relpath(x, ecd)
370	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'makefile*').files]
371	jobdir = [_config.PATH_REL_JOBSCRIPTS]
372
373	# concatenate single lists to one for a better looping
374	filelist = pyfiles + pytestfiles + controlfiles + tempfiles + nlfiles + \
375	ffiles + gribtable + hfiles + makefiles + ecmwf_env_file + \
376	runfile + jobdir + testfiles +\
377	['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
378
379	# create installation tar-file
380	exclude_files = [".ksh", ".tar"]
381	try:
382	with tarfile.open(tarball_path, "w:gz") as tar_handle:
383	for filename in filelist:
384	tar_handle.add(filename, recursive=False,
385	filter=lambda tarinfo: None
386	if os.path.splitext(tarinfo.name)[1]
387	in exclude_files
388	else tarinfo)
389	except tarfile.TarError as e:
390	print('... ERROR: ' + str(e))
391
392	sys.exit('\n... error occured while trying to create the tar-file ' +
393	str(tarball_path))
394
395	return
396
397
398	def un_tarball(tarball_path):
399	'''Extracts the given tarball into current directory.
400
401	Parameters
402	----------
403	tarball_path : str
404	The complete path to the tar file which will contain all
405	relevant data for flex_extract.
406
407	Return
408	------
409
410	'''
411
412	print('Untar ...')
413
414	try:
415	with tarfile.open(tarball_path) as tar_handle:
416	tar_handle.extractall()
417	except tarfile.TarError as e:
418	sys.exit('\n... error occured while trying to read tar-file ' +
419	str(tarball_path))
420	except OSError as e:
421	print('... ERROR CODE: ' + str(e.errno))
422	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
423
424	sys.exit('\n... error occured while trying to read tar-file ' +
425	str(tarball_path))
426
427	return
428
429	def mk_env_vars(ecuid, ecgid, gateway, destination):
430	'''Creates a file named ECMWF_ENV which contains the
431	necessary environmental variables at ECMWF servers.
432	It is based on the template ECMWF_ENV.template.
433
434	Parameters
435	----------
436	ecuid : str
437	The user id on ECMWF server.
438
439	ecgid : str
440	The group id on ECMWF server.
441
442	gateway : str
443	The gateway server the user is using.
444
445	destination : str
446	The remote destination which is used to transfer files
447	from ECMWF server to local gateway server.
448
449	Return
450	------
451
452	'''
453	from genshi.template.text import NewTextTemplate
454	from genshi.template import TemplateLoader
455	from genshi.template.eval import UndefinedError
456
457	try:
458	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
459	ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
460	cls=NewTextTemplate)
461
462	stream = ecmwfvars_template.generate(user_name=ecuid,
463	user_group=ecgid,
464	gateway_name=gateway,
465	destination_name=destination
466	)
467	except UndefinedError as e:
468	print('... ERROR ' + str(e))
469
470	sys.exit('\n... error occured while trying to generate template ' +
471	_config.PATH_ECMWF_ENV)
472	except OSError as e:
473	print('... ERROR CODE: ' + str(e.errno))
474	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
475
476	sys.exit('\n... error occured while trying to generate template ' +
477	_config.PATH_ECMWF_ENV)
478
479	try:
480	with open(_config.PATH_ECMWF_ENV, 'w') as f:
481	f.write(stream.render('text'))
482	except OSError as e:
483	print('... ERROR CODE: ' + str(e.errno))
484	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
485
486	sys.exit('\n... error occured while trying to write ' +
487	_config.PATH_ECMWF_ENV)
488
489	return
490
491	def mk_compilejob(makefile, ecuid, ecgid, fp_root):
492	'''Modifies the original job template file so that it is specified
493	for the user and the environment were it will be applied. Result
494	is stored in a new file "job.temp" in the python directory.
495
496	Parameters
497	----------
498	makefile : str
499	Name of the makefile which should be used to compile the Fortran
500	program.
501
502	ecuid : str
503	The user id on ECMWF server.
504
505	ecgid : str
506	The group id on ECMWF server.
507
508	fp_root : str
509	Path to the root directory of FLEXPART environment or flex_extract
510	environment.
511
512	Return
513	------
514
515	'''
516	from genshi.template.text import NewTextTemplate
517	from genshi.template import TemplateLoader
518	from genshi.template.eval import UndefinedError
519
520	if fp_root == '../':
521	fp_root = '$HOME'
522
523	try:
524	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
525	compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
526	cls=NewTextTemplate)
527
528	stream = compile_template.generate(
529	usergroup=ecgid,
530	username=ecuid,
531	version_number=_config._VERSION_STR,
532	fp_root_scripts=fp_root,
533	makefile=makefile,
534	fortran_program=_config.FORTRAN_EXECUTABLE
535	)
536	except UndefinedError as e:
537	print('... ERROR ' + str(e))
538
539	sys.exit('\n... error occured while trying to generate template ' +
540	_config.TEMPFILE_INSTALL_COMPILEJOB)
541	except OSError as e:
542	print('... ERROR CODE: ' + str(e.errno))
543	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
544
545	sys.exit('\n... error occured while trying to generate template ' +
546	_config.TEMPFILE_INSTALL_COMPILEJOB)
547
548	try:
549	compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
550	_config.FILE_INSTALL_COMPILEJOB)
551
552	with open(compilejob, 'w') as f:
553	f.write(stream.render('text'))
554	except OSError as e:
555	print('... ERROR CODE: ' + str(e.errno))
556	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
557
558	sys.exit('\n... error occured while trying to write ' +
559	compilejob)
560
561	return
562
563	def mk_job_template(ecuid, ecgid, fp_root):
564	'''Modifies the original job template file so that it is specified
565	for the user and the environment were it will be applied. Result
566	is stored in a new file.
567
568	Parameters
569	----------
570	ecuid : str
571	The user id on ECMWF server.
572
573	ecgid : str
574	The group id on ECMWF server.
575
576	fp_root : str
577	Path to the root directory of FLEXPART environment or flex_extract
578	environment.
579
580	Return
581	------
582
583	'''
584	from genshi.template.text import NewTextTemplate
585	from genshi.template import TemplateLoader
586	from genshi.template.eval import UndefinedError
587
588	fp_root_path_to_python = os.path.join(fp_root,
589	_config.FLEXEXTRACT_DIRNAME,
590	_config.PATH_REL_PYTHON_SRC)
591	if '$' in fp_root_path_to_python:
592	ind = fp_root_path_to_python.index('$')
593	fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
594	fp_root_path_to_python[ind:]
595
596	try:
597	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
598	compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
599	cls=NewTextTemplate)
600
601	stream = compile_template.generate(
602	usergroup=ecgid,
603	username=ecuid,
604	version_number=_config._VERSION_STR,
605	fp_root_path=fp_root_path_to_python,
606	)
607	except UndefinedError as e:
608	print('... ERROR ' + str(e))
609
610	sys.exit('\n... error occured while trying to generate template ' +
611	_config.TEMPFILE_INSTALL_JOB)
612	except OSError as e:
613	print('... ERROR CODE: ' + str(e.errno))
614	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
615
616	sys.exit('\n... error occured while trying to generate template ' +
617	_config.TEMPFILE_INSTALL_JOB)
618
619
620	try:
621	tempjobfile = os.path.join(_config.PATH_TEMPLATES,
622	_config.TEMPFILE_JOB)
623
624	with open(tempjobfile, 'w') as f:
625	f.write(stream.render('text'))
626	except OSError as e:
627	print('... ERROR CODE: ' + str(e.errno))
628	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
629
630	sys.exit('\n... error occured while trying to write ' +
631	tempjobfile)
632
633	return
634
635	def del_convert_build(src_path):
636	'''Clean up the Fortran source directory and remove all
637	build files (e.g. \.o, \.mod and CONVERT2)
638
639	Parameters
640	----------
641	src_path : str
642	Path to the fortran source directory.
643
644	Return
645	------
646
647	'''
648
649	modfiles = UioFiles(src_path, '*.mod')
650	objfiles = UioFiles(src_path, '*.o')
651	exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
652
653	modfiles.delete_files()
654	objfiles.delete_files()
655	exefile.delete_files()
656
657	return
658
659	def mk_convert_build(src_path, makefile):
660	'''Compiles the Fortran code and generates the executable.
661
662	Parameters
663	----------
664	src_path : str
665	Path to the fortran source directory.
666
667	makefile : str
668	The name of the makefile which should be used.
669
670	Return
671	------
672
673	'''
674
675	try:
676	print('Using makefile: ' + makefile)
677	p = subprocess.Popen(['make', '-f',
678	os.path.join(src_path, makefile)],
679	stdin=subprocess.PIPE,
680	stdout=subprocess.PIPE,
681	stderr=subprocess.PIPE,
682	bufsize=1)
683	pout, perr = p.communicate()
684	print(pout.decode())
685	if p.returncode != 0:
686	print(perr.decode())
687	print('Please edit ' + makefile +
688	' or try another makefile in the src directory.')
689	print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
690	'and EMOSLIB must be adapted.')
691	print('Available makefiles:')
692	print(UioFiles(src_path, 'makefile*'))
693	sys.exit('Compilation failed!')
694	except ValueError as e:
695	print('ERROR: make of Fortran code failed:')
696	print(e)
697	else:
698	execute_subprocess(['ls', '-l',
699	os.path.join(src_path, _config.FORTRAN_EXECUTABLE)],
700	error_msg='FORTRAN EXECUTABLE COULD NOT BE FOUND!')
701
702	return
703
704
705	if __name__ == "__main__":
706	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: