Context Navigation

source: flex_extract.git/Source/Python/install.py @ 8028176

ctbtodev

Last change on this file since 8028176 was 8028176, checked in by Anne Philipp <anne.philipp@…>, 4 years ago
updated unit tests
Property mode set to `100755`
File size: 24.2 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - moved install_args_and_control in here
14	# - splitted code in smaller functions
15	# - delete fortran build files in here instead of compile job script
16	# - changed static path names to variables from config file
17	# - splitted install function into several smaller pieces
18	# - use of tarfile package in python
19	# June 2020 - Anne Philipp
20	# - renamed "convert" functions to "fortran" functions
21	# - reconfigured mk_tarball to select *.template files instead
22	# of .nl and .temp
23	# - added check for makefile settings
24	#
25	# @License:
26	# (C) Copyright 2014-2020.
27	# Anne Philipp, Leopold Haimberger
28	#
29	# SPDX-License-Identifier: CC-BY-4.0
30	#
31	# This work is licensed under the Creative Commons Attribution 4.0
32	# International License. To view a copy of this license, visit
33	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
34	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
35	#
36	# @Methods:
37	# main
38	# get_install_cmdline_args
39	# install_via_gateway
40	# check_install_conditions
41	# mk_tarball
42	# un_tarball
43	# mk_env_vars
44	# mk_compilejob
45	# mk_job_template
46	# del_fortran_build
47	# mk_fortran_build
48	#
49	#*******************************************************************************
50	'''This script installs the flex_extract program.
51
52	Depending on the selected installation environment (locally or on the
53	ECMWF server ecgate or cca) the program extracts the command line
54	arguments and the CONTROL file parameter and prepares the corresponding
55	environment.
56	The necessary files are collected in a tar ball and placed
57	at the target location. There, is is untared, the environment variables are
58	set, and the Fortran code is compiled.
59	If the ECMWF environment is selected, a job script is prepared and submitted
60	for the remaining configurations after putting the tar ball on the
61	target ECMWF server.
62
63	Type: install.py --help
64	to get information about command line parameters.
65	Read the documentation for usage instructions.
66	'''
67
68	# ------------------------------------------------------------------------------
69	# MODULES
70	# ------------------------------------------------------------------------------
71	from __future__ import print_function
72
73	import os
74	import sys
75	import subprocess
76	import tarfile
77	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
78
79	# software specific classes and modules from flex_extract
80	import _config
81	from Classes.ControlFile import ControlFile
82	from Classes.UioFiles import UioFiles
83	from Mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
84	silent_remove, execute_subprocess, none_or_str)
85
86	# ------------------------------------------------------------------------------
87	# FUNCTIONS
88	# ------------------------------------------------------------------------------
89	def main():
90	'''Controls the installation process. Calls the installation function
91	if target is specified.
92
93	Parameters
94	----------
95
96	Return
97	------
98	'''
99
100	args = get_install_cmdline_args()
101	c = ControlFile(args.controlfile)
102	c.assign_args_to_control(args)
103	check_install_conditions(c)
104
105	if c.install_target.lower() != 'local': # ecgate or cca
106	install_via_gateway(c)
107	else: # local
108	install_local(c)
109
110	return
111
112	def get_install_cmdline_args():
113	'''Decomposes the command line arguments and assigns them to variables.
114	Apply default values for arguments not present.
115
116	Parameters
117	----------
118
119	Return
120	------
121	args : Namespace
122	Contains the commandline arguments from script/program call.
123	'''
124	parser = ArgumentParser(description='Install flex_extract software '
125	'locally or on ECMWF machines',
126	formatter_class=ArgumentDefaultsHelpFormatter)
127
128	parser.add_argument('--target', dest='install_target',
129	type=none_or_str, default=None,
130	help="Valid targets: local \| ecgate \| cca , \
131	the latter two are at ECMWF")
132	parser.add_argument("--makefile", dest="makefile",
133	type=none_or_str, default=None,
134	help='Name of makefile for compiling the '
135	'Fortran program')
136	parser.add_argument("--ecuid", dest="ecuid",
137	type=none_or_str, default=None,
138	help='User id at ECMWF')
139	parser.add_argument("--ecgid", dest="ecgid",
140	type=none_or_str, default=None,
141	help='Group id at ECMWF')
142	parser.add_argument("--gateway", dest="gateway",
143	type=none_or_str, default=None,
144	help='Name of the local gateway server')
145	parser.add_argument("--destination", dest="destination",
146	type=none_or_str, default=None,
147	help='ecaccess association, e.g. '
148	'myUser@genericSftp')
149
150	parser.add_argument("--installdir", dest="installdir",
151	type=none_or_str, default=None,
152	help='Root directory of the '
153	'flex_extract installation')
154
155	# arguments for job submission to ECMWF, only needed by submit.py
156	parser.add_argument("--job_template", dest='job_template',
157	type=none_or_str, default="job.template",
158	help='Rudimentary template file to create a batch '
159	'job template for submission to ECMWF servers')
160
161	parser.add_argument("--controlfile", dest="controlfile",
162	type=none_or_str, default='CONTROL_EA5',
163	help="A file that contains all CONTROL parameters.")
164
165	args = parser.parse_args()
166
167	return args
168
169
170	def install_via_gateway(c):
171	'''Prepare data transfer to remote gateway and submit a job script which will
172	install everything on the remote gateway.
173
174	Parameters
175	----------
176	c : ControlFile
177	Contains all the parameters of CONTROL file and
178	command line.
179
180	Return
181	------
182
183	'''
184
185	tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
186	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
187
188	mk_compilejob(c.makefile, c.ecuid, c.ecgid, c.installdir)
189
190	mk_job_template(c.ecuid, c.ecgid, c.installdir)
191
192	mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
193
194	mk_tarball(tar_file, c.install_target)
195
196	put_file_to_ecserver(_config.PATH_FLEXEXTRACT_DIR, tarball_name,
197	c.install_target, c.ecuid, c.ecgid)
198
199	submit_job_to_ecserver(c.install_target,
200	os.path.join(_config.PATH_REL_JOBSCRIPTS,
201	_config.FILE_INSTALL_COMPILEJOB))
202
203	silent_remove(tar_file)
204
205	print('Job compilation script has been submitted to ecgate for ' +
206	'installation in ' + c.installdir +
207	'/' + _config.FLEXEXTRACT_DIRNAME)
208	print('You should get an email with subject "flexcompile" within ' +
209	'the next few minutes!')
210
211	return
212
213	def install_local(c):
214	'''Perform the actual installation on a local machine.
215
216	Parameters
217	----------
218	c : ControlFile
219	Contains all the parameters of CONTROL file and
220	command line.
221
222	Return
223	------
224
225	'''
226
227	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR,
228	_config.FLEXEXTRACT_DIRNAME + '.tar')
229
230	if c.installdir == _config.PATH_FLEXEXTRACT_DIR:
231	print('WARNING: installdir has not been specified')
232	print('flex_extract will be installed in here by compiling the ' +
233	'Fortran source in ' + _config.PATH_FORTRAN_SRC)
234	os.chdir(_config.PATH_FORTRAN_SRC)
235	else: # creates the target working directory for flex_extract
236	c.installdir = os.path.expandvars(os.path.expanduser(
237	c.installdir))
238	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) != \
239	os.path.abspath(c.installdir):
240	mk_tarball(tar_file, c.install_target)
241	make_dir(os.path.join(c.installdir,
242	_config.FLEXEXTRACT_DIRNAME))
243	os.chdir(os.path.join(c.installdir,
244	_config.FLEXEXTRACT_DIRNAME))
245	un_tarball(tar_file)
246	os.chdir(os.path.join(c.installdir,
247	_config.FLEXEXTRACT_DIRNAME,
248	_config.PATH_REL_FORTRAN_SRC))
249
250	# Create Fortran executable
251	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software at ' +
252	c.install_target + ' in directory ' +
253	os.path.abspath(c.installdir) + '\n')
254
255	del_fortran_build('.')
256	mk_fortran_build('.', c.makefile)
257
258	os.chdir(_config.PATH_FLEXEXTRACT_DIR)
259	if os.path.isfile(tar_file):
260	os.remove(tar_file)
261
262	return
263
264
265	def check_install_conditions(c):
266	'''Checks necessary attributes and conditions
267	for the installation, e.g. whether they exist and contain values.
268	Otherwise set default values.
269
270	Parameters
271	----------
272	c : ControlFile
273	Contains all the parameters of CONTROL file and
274	command line.
275
276
277	Return
278	------
279
280	'''
281
282	if c.install_target and \
283	c.install_target not in _config.INSTALL_TARGETS:
284	print('ERROR: unknown or missing installation target ')
285	print('target: ', c.install_target)
286	print('please specify correct installation target ' +
287	str(_config.INSTALL_TARGETS))
288	print('use -h or --help for help')
289	sys.exit(1)
290
291	if c.install_target and c.install_target != 'local':
292	if not c.ecgid or not c.ecuid:
293	print('Please enter your ECMWF user id and group id '
294	' with command line options --ecuid --ecgid')
295	print('Try "' + sys.argv[0].split('/')[-1] + \
296	' -h" to print usage information')
297	print('Please consult ecaccess documentation or ECMWF user '
298	'support for further details.\n')
299	sys.exit(1)
300	if not c.gateway or not c.destination:
301	print('WARNING: Parameters GATEWAY and DESTINATION were '
302	'not properly set for working on ECMWF server. \n'
303	'There will be no transfer of output files to the '
304	'local gateway server possible!')
305	if not c.installdir:
306	c.installdir = '${HOME}'
307	else: # local
308	if not c.installdir:
309	c.installdir = _config.PATH_FLEXEXTRACT_DIR
310
311	if not c.makefile:
312	print('WARNING: no makefile was specified.')
313	if c.install_target == 'local':
314	c.makefile = 'makefile_local_gfortran'
315	print('WARNING: default makefile selected: makefile_local_gfortan')
316	elif c.install_target == 'ecgate':
317	c.makefile = 'makefile_ecgate'
318	print('WARNING: default makefile selected: makefile_ecgate')
319	elif c.install_target == 'cca' or \
320	c.install_target == 'ccb':
321	c.makefile = 'makefile_cray'
322	print('WARNING: default makefile selected: makefile_cray')
323	else:
324	pass
325
326	return
327
328
329	def mk_tarball(tarball_path, target):
330	'''Creates a tarball with all necessary files which need to be sent to the
331	installation directory.
332	It does not matter whether this is local or remote.
333	Collects all Python files, the Fortran source and makefiles,
334	the ECMWF_ENV file, the CONTROL files as well as the
335	template files.
336
337	Parameters
338	----------
339	tarball_path : str
340	The complete path to the tar file which will contain all
341	relevant data for flex_extract.
342
343	target : str
344	The queue where the job is submitted to.
345
346	Return
347	------
348
349	'''
350
351	print('Create tarball ...')
352
353	# change to FLEXEXTRACT directory so that the tar can contain
354	# relative pathes to the files and directories
355	ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
356	os.chdir(ecd)
357
358	# get lists of the files to be added to the tar file
359	if target == 'local':
360	ecmwf_env_file = []
361	runfile = [os.path.relpath(x, ecd)
362	for x in UioFiles(_config.PATH_REL_RUN_DIR,
363	'run_local.sh').files]
364	else:
365	ecmwf_env_file = [_config.PATH_REL_ECMWF_ENV]
366	runfile = [os.path.relpath(x, ecd)
367	for x in UioFiles(_config.PATH_REL_RUN_DIR,
368	'run.sh').files]
369
370	pyfiles = [os.path.relpath(x, ecd)
371	for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
372	pytestfiles = [os.path.relpath(x, ecd)
373	for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
374	controlfiles = [os.path.relpath(x, ecd)
375	for x in UioFiles(_config.PATH_REL_CONTROLFILES,
376	'CONTROL*').files]
377	testfiles = [os.path.relpath(x, ecd)
378	for x in UioFiles(_config.PATH_REL_TEST+"/Installation", '*').files]
379	tempfiles = [os.path.relpath(x, ecd)
380	for x in UioFiles(_config.PATH_REL_TEMPLATES, '*.template').files]
381	gribtable = [os.path.relpath(x, ecd)
382	for x in UioFiles(_config.PATH_REL_TEMPLATES, 'grib').files]
383	ffiles = [os.path.relpath(x, ecd)
384	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f90').files]
385	hfiles = [os.path.relpath(x, ecd)
386	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
387	makefiles = [os.path.relpath(x, ecd)
388	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'makefile*').files]
389	jobdir = [os.path.relpath(x, ecd)
390	for x in UioFiles(_config.PATH_REL_JOBSCRIPTS, '*.md').files]
391
392	# concatenate single lists to one for a better looping
393	filelist = pyfiles + pytestfiles + controlfiles + tempfiles + \
394	ffiles + gribtable + hfiles + makefiles + ecmwf_env_file + \
395	runfile + jobdir + testfiles +\
396	['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
397
398	# create installation tar-file
399	exclude_files = [".ksh", ".tar"]
400	try:
401	with tarfile.open(tarball_path, "w:gz") as tar_handle:
402	for filename in filelist:
403	tar_handle.add(filename, recursive=False,
404	filter=lambda tarinfo: None
405	if os.path.splitext(tarinfo.name)[1]
406	in exclude_files
407	else tarinfo)
408	except tarfile.TarError as e:
409	print('... ERROR: ' + str(e))
410
411	sys.exit('\n... error occured while trying to create the tar-file ' +
412	str(tarball_path))
413
414	return
415
416
417	def un_tarball(tarball_path):
418	'''Extracts the given tarball into current directory.
419
420	Parameters
421	----------
422	tarball_path : str
423	The complete path to the tar file which will contain all
424	relevant data for flex_extract.
425
426	Return
427	------
428
429	'''
430
431	print('Untar ...')
432
433	try:
434	with tarfile.open(tarball_path) as tar_handle:
435	tar_handle.extractall()
436	except tarfile.TarError as e:
437	sys.exit('\n... error occured while trying to read tar-file ' +
438	str(tarball_path))
439	except OSError as e:
440	print('... ERROR CODE: ' + str(e.errno))
441	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
442
443	sys.exit('\n... error occured while trying to read tar-file ' +
444	str(tarball_path))
445
446	return
447
448	def mk_env_vars(ecuid, ecgid, gateway, destination):
449	'''Creates a file named ECMWF_ENV which contains the
450	necessary environmental variables at ECMWF servers.
451	It is based on the template ECMWF_ENV.template.
452
453	Parameters
454	----------
455	ecuid : str
456	The user id on ECMWF server.
457
458	ecgid : str
459	The group id on ECMWF server.
460
461	gateway : str
462	The gateway server the user is using.
463
464	destination : str
465	The remote destination which is used to transfer files
466	from ECMWF server to local gateway server.
467
468	Return
469	------
470
471	'''
472	from genshi.template.text import NewTextTemplate
473	from genshi.template import TemplateLoader
474	from genshi.template.eval import UndefinedError
475
476	try:
477	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
478	ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
479	cls=NewTextTemplate)
480
481	stream = ecmwfvars_template.generate(user_name=ecuid,
482	user_group=ecgid,
483	gateway_name=gateway,
484	destination_name=destination
485	)
486	except UndefinedError as e:
487	print('... ERROR ' + str(e))
488
489	sys.exit('\n... error occured while trying to generate template ' +
490	_config.PATH_ECMWF_ENV)
491	except OSError as e:
492	print('... ERROR CODE: ' + str(e.errno))
493	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
494
495	sys.exit('\n... error occured while trying to generate template ' +
496	_config.PATH_ECMWF_ENV)
497
498	try:
499	with open(_config.PATH_ECMWF_ENV, 'w') as f:
500	f.write(stream.render('text'))
501	except OSError as e:
502	print('... ERROR CODE: ' + str(e.errno))
503	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
504
505	sys.exit('\n... error occured while trying to write ' +
506	_config.PATH_ECMWF_ENV)
507
508	return
509
510	def mk_compilejob(makefile, ecuid, ecgid, fp_root):
511	'''Modifies the original job template file so that it is specified
512	for the user and the environment were it will be applied. Result
513	is stored in a new file "job.temp" in the python directory.
514
515	Parameters
516	----------
517	makefile : str
518	Name of the makefile which should be used to compile the Fortran
519	program.
520
521	ecuid : str
522	The user id on ECMWF server.
523
524	ecgid : str
525	The group id on ECMWF server.
526
527	fp_root : str
528	Path to the root directory of FLEXPART environment or flex_extract
529	environment.
530
531	Return
532	------
533
534	'''
535	from genshi.template.text import NewTextTemplate
536	from genshi.template import TemplateLoader
537	from genshi.template.eval import UndefinedError
538
539	if fp_root == '../':
540	fp_root = '$HOME'
541
542	try:
543	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
544	compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
545	cls=NewTextTemplate)
546
547	stream = compile_template.generate(
548	usergroup=ecgid,
549	username=ecuid,
550	version_number=_config._VERSION_STR,
551	fp_root_scripts=fp_root,
552	makefile=makefile,
553	fortran_program=_config.FORTRAN_EXECUTABLE
554	)
555	except UndefinedError as e:
556	print('... ERROR ' + str(e))
557
558	sys.exit('\n... error occured while trying to generate template ' +
559	_config.TEMPFILE_INSTALL_COMPILEJOB)
560	except OSError as e:
561	print('... ERROR CODE: ' + str(e.errno))
562	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
563
564	sys.exit('\n... error occured while trying to generate template ' +
565	_config.TEMPFILE_INSTALL_COMPILEJOB)
566
567	try:
568	compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
569	_config.FILE_INSTALL_COMPILEJOB)
570
571	with open(compilejob, 'w') as f:
572	f.write(stream.render('text'))
573	except OSError as e:
574	print('... ERROR CODE: ' + str(e.errno))
575	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
576
577	sys.exit('\n... error occured while trying to write ' +
578	compilejob)
579
580	return
581
582	def mk_job_template(ecuid, ecgid, fp_root):
583	'''Modifies the original job template file so that it is specified
584	for the user and the environment were it will be applied. Result
585	is stored in a new file.
586
587	Parameters
588	----------
589	ecuid : str
590	The user id on ECMWF server.
591
592	ecgid : str
593	The group id on ECMWF server.
594
595	fp_root : str
596	Path to the root directory of FLEXPART environment or flex_extract
597	environment.
598
599	Return
600	------
601
602	'''
603	from genshi.template.text import NewTextTemplate
604	from genshi.template import TemplateLoader
605	from genshi.template.eval import UndefinedError
606
607	fp_root_path_to_python = os.path.join(fp_root,
608	_config.FLEXEXTRACT_DIRNAME,
609	_config.PATH_REL_PYTHON_SRC)
610	if '$' in fp_root_path_to_python:
611	ind = fp_root_path_to_python.index('$')
612	fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
613	fp_root_path_to_python[ind:]
614
615	try:
616	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
617	compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
618	cls=NewTextTemplate)
619
620	stream = compile_template.generate(
621	usergroup=ecgid,
622	username=ecuid,
623	version_number=_config._VERSION_STR,
624	fp_root_path=fp_root_path_to_python,
625	)
626	except UndefinedError as e:
627	print('... ERROR ' + str(e))
628
629	sys.exit('\n... error occured while trying to generate template ' +
630	_config.TEMPFILE_INSTALL_JOB)
631	except OSError as e:
632	print('... ERROR CODE: ' + str(e.errno))
633	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
634
635	sys.exit('\n... error occured while trying to generate template ' +
636	_config.TEMPFILE_INSTALL_JOB)
637
638
639	try:
640	tempjobfile = os.path.join(_config.PATH_TEMPLATES,
641	_config.TEMPFILE_JOB)
642
643	with open(tempjobfile, 'w') as f:
644	f.write(stream.render('text'))
645	except OSError as e:
646	print('... ERROR CODE: ' + str(e.errno))
647	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
648
649	sys.exit('\n... error occured while trying to write ' +
650	tempjobfile)
651
652	return
653
654	def del_fortran_build(src_path):
655	'''Clean up the Fortran source directory and remove all
656	build files (e.g. \.o, \.mod and FORTRAN EXECUTABLE)
657
658	Parameters
659	----------
660	src_path : str
661	Path to the fortran source directory.
662
663	Return
664	------
665
666	'''
667
668	modfiles = UioFiles(src_path, '*.mod')
669	objfiles = UioFiles(src_path, '*.o')
670	exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
671
672	modfiles.delete_files()
673	objfiles.delete_files()
674	exefile.delete_files()
675
676	return
677
678	def mk_fortran_build(src_path, makefile):
679	'''Compiles the Fortran code and generates the executable.
680
681	Parameters
682	----------
683	src_path : str
684	Path to the fortran source directory.
685
686	makefile : str
687	The name of the makefile which should be used.
688
689	Return
690	------
691
692	'''
693
694	try:
695	print('Using makefile: ' + makefile)
696	p = subprocess.Popen(['make', '-f',
697	os.path.join(src_path, makefile)],
698	stdin=subprocess.PIPE,
699	stdout=subprocess.PIPE,
700	stderr=subprocess.PIPE,
701	bufsize=1)
702	pout, perr = p.communicate()
703	print(pout.decode())
704	if p.returncode != 0:
705	print(perr.decode())
706	print('Please edit ' + makefile +
707	' or try another makefile in the src directory.')
708	print('Most likely ECCODES_INCLUDE_DIR, ECCODES_LIB '
709	'and EMOSLIB must be adapted.')
710	print('Available makefiles:')
711	print(UioFiles(src_path, 'makefile*'))
712	sys.exit('Compilation failed!')
713	except ValueError as e:
714	print('ERROR: makefile call failed:')
715	print(e)
716	else:
717	execute_subprocess(['ls', '-l',
718	os.path.join(src_path, _config.FORTRAN_EXECUTABLE)],
719	error_msg='FORTRAN EXECUTABLE COULD NOT BE FOUND!')
720
721	return
722
723
724	if __name__ == "__main__":
725	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: