Context Navigation

source: flex_extract.git/Source/Python/install.py @ bbbe1e1

ctbtodev

Last change on this file since bbbe1e1 was bbbe1e1, checked in by anphi <anne.philipp@…>, 4 years ago
solved a failed merge
Property mode set to `100755`
File size: 24.1 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - moved install_args_and_control in here
14	# - splitted code in smaller functions
15	# - delete fortran build files in here instead of compile job script
16	# - changed static path names to variables from config file
17	# - splitted install function into several smaller pieces
18	# - use of tarfile package in python
19	# June 2020 - Anne Philipp
20	# - renamed "convert" functions to "fortran" functions
21	# - reconfigured mk_tarball to select *.template files instead
22	# of .nl and .temp
23	# - added check for makefile settings
24	#
25	# @License:
26	# (C) Copyright 2014-2020.
27	# Anne Philipp, Leopold Haimberger
28	#
29	# SPDX-License-Identifier: CC-BY-4.0
30	#
31	# This work is licensed under the Creative Commons Attribution 4.0
32	# International License. To view a copy of this license, visit
33	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
34	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
35	#
36	# @Methods:
37	# main
38	# get_install_cmdline_args
39	# install_via_gateway
40	# check_install_conditions
41	# mk_tarball
42	# un_tarball
43	# mk_env_vars
44	# mk_compilejob
45	# mk_job_template
46	# del_fortran_build
47	# mk_fortran_build
48	#
49	#*******************************************************************************
50	'''This script installs the flex_extract program.
51
52	Depending on the selected installation environment (locally or on the
53	ECMWF server ecgate or cca) the program extracts the command line
54	arguments and the CONTROL file parameter and prepares the corresponding
55	environment.
56	The necessary files are collected in a tar ball and placed
57	at the target location. There, is is untared, the environment variables are
58	set, and the Fortran code is compiled.
59	If the ECMWF environment is selected, a job script is prepared and submitted
60	for the remaining configurations after putting the tar ball on the
61	target ECMWF server.
62
63	Type: install.py --help
64	to get information about command line parameters.
65	Read the documentation for usage instructions.
66	'''
67
68	# ------------------------------------------------------------------------------
69	# MODULES
70	# ------------------------------------------------------------------------------
71	from __future__ import print_function
72
73	import os
74	import sys
75	import subprocess
76	import tarfile
77	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
78
79	# software specific classes and modules from flex_extract
80	import _config
81	from Classes.ControlFile import ControlFile
82	from Classes.UioFiles import UioFiles
83	from Mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
84	silent_remove, execute_subprocess, none_or_str)
85
86	# ------------------------------------------------------------------------------
87	# FUNCTIONS
88	# ------------------------------------------------------------------------------
89	def main():
90	'''Controls the installation process. Calls the installation function
91	if target is specified.
92
93	Parameters
94	----------
95
96	Return
97	------
98	'''
99
100	args = get_install_cmdline_args()
101	c = ControlFile(args.controlfile)
102	c.assign_args_to_control(args)
103	check_install_conditions(c)
104
105	if c.install_target.lower() != 'local': # ecgate or cca
106	install_via_gateway(c)
107	else: # local
108	install_local(c)
109
110	return
111
112	def get_install_cmdline_args():
113	'''Decomposes the command line arguments and assigns them to variables.
114	Apply default values for arguments not present.
115
116	Parameters
117	----------
118
119	Return
120	------
121	args : Namespace
122	Contains the commandline arguments from script/program call.
123	'''
124	parser = ArgumentParser(description='Install flex_extract software '
125	'locally or on ECMWF machines',
126	formatter_class=ArgumentDefaultsHelpFormatter)
127
128	parser.add_argument('--target', dest='install_target',
129	type=none_or_str, default=None,
130	help="Valid targets: local \| ecgate \| cca , \
131	the latter two are at ECMWF")
132	parser.add_argument("--makefile", dest="makefile",
133	type=none_or_str, default=None,
134	help='Name of makefile for compiling the '
135	'Fortran program')
136	parser.add_argument("--ecuid", dest="ecuid",
137	type=none_or_str, default=None,
138	help='User id at ECMWF')
139	parser.add_argument("--ecgid", dest="ecgid",
140	type=none_or_str, default=None,
141	help='Group id at ECMWF')
142	parser.add_argument("--gateway", dest="gateway",
143	type=none_or_str, default=None,
144	help='Name of the local gateway server')
145	parser.add_argument("--destination", dest="destination",
146	type=none_or_str, default=None,
147	help='ecaccess association, e.g. '
148	'myUser@genericSftp')
149
150	parser.add_argument("--installdir", dest="installdir",
151	type=none_or_str, default=None,
152	help='Root directory of the '
153	'flex_extract installation')
154
155	# arguments for job submission to ECMWF, only needed by submit.py
156	parser.add_argument("--job_template", dest='job_template',
157	type=none_or_str, default="job.template",
158	help='Rudimentary template file to create a batch '
159	'job template for submission to ECMWF servers')
160
161	parser.add_argument("--controlfile", dest="controlfile",
162	type=none_or_str, default='CONTROL_EA5',
163	help="A file that contains all CONTROL parameters.")
164
165	args = parser.parse_args()
166
167	return args
168
169
170	def install_via_gateway(c):
171	'''Prepare data transfer to remote gateway and submit a job script which will
172	install everything on the remote gateway.
173
174	Parameters
175	----------
176	c : ControlFile
177	Contains all the parameters of CONTROL file and
178	command line.
179
180	Return
181	------
182
183	'''
184
185	tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
186	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
187
188	mk_compilejob(c.makefile, c.ecuid, c.ecgid, c.installdir)
189
190	mk_job_template(c.ecuid, c.ecgid, c.installdir)
191
192	mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
193
194	mk_tarball(tar_file, c.install_target)
195
196	put_file_to_ecserver(_config.PATH_FLEXEXTRACT_DIR, tarball_name,
197	c.install_target, c.ecuid, c.ecgid)
198
199	submit_job_to_ecserver(c.install_target,
200	os.path.join(_config.PATH_REL_JOBSCRIPTS,
201	_config.FILE_INSTALL_COMPILEJOB))
202
203	silent_remove(tar_file)
204
205	print('Job compilation script has been submitted to ecgate for ' +
206	'installation in ' + c.installdir +
207	'/' + _config.FLEXEXTRACT_DIRNAME)
208	print('You should get an email with subject "flexcompile" within ' +
209	'the next few minutes!')
210
211	return
212
213	def install_local(c):
214	'''Perform the actual installation on a local machine.
215
216	Parameters
217	----------
218	c : ControlFile
219	Contains all the parameters of CONTROL file and
220	command line.
221
222	Return
223	------
224
225	'''
226
227	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR,
228	_config.FLEXEXTRACT_DIRNAME + '.tar')
229
230	if c.installdir == _config.PATH_FLEXEXTRACT_DIR:
231	print('WARNING: installdir has not been specified')
232	print('flex_extract will be installed in here by compiling the ' +
233	'Fortran source in ' + _config.PATH_FORTRAN_SRC)
234	os.chdir(_config.PATH_FORTRAN_SRC)
235	else: # creates the target working directory for flex_extract
236	c.installdir = os.path.expandvars(os.path.expanduser(
237	c.installdir))
238	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) != \
239	os.path.abspath(c.installdir):
240	mk_tarball(tar_file, c.install_target)
241	make_dir(os.path.join(c.installdir,
242	_config.FLEXEXTRACT_DIRNAME))
243	os.chdir(os.path.join(c.installdir,
244	_config.FLEXEXTRACT_DIRNAME))
245	un_tarball(tar_file)
246	os.chdir(os.path.join(c.installdir,
247	_config.FLEXEXTRACT_DIRNAME,
248	_config.PATH_REL_FORTRAN_SRC))
249
250	# Create Fortran executable
251	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software at ' +
252	c.install_target + ' in directory ' +
253	os.path.abspath(c.installdir) + '\n')
254
255	del_fortran_build('.')
256	mk_fortran_build('.', c.makefile)
257
258	os.chdir(_config.PATH_FLEXEXTRACT_DIR)
259	if os.path.isfile(tar_file):
260	os.remove(tar_file)
261
262	return
263
264
265	def check_install_conditions(c):
266	'''Checks necessary attributes and conditions
267	for the installation, e.g. whether they exist and contain values.
268	Otherwise set default values.
269
270	Parameters
271	----------
272	c : ControlFile
273	Contains all the parameters of CONTROL file and
274	command line.
275
276
277	Return
278	------
279
280	'''
281
282	if c.install_target and \
283	c.install_target not in _config.INSTALL_TARGETS:
284	print('ERROR: unknown or missing installation target ')
285	print('target: ', c.install_target)
286	print('please specify correct installation target ' +
287	str(_config.INSTALL_TARGETS))
288	print('use -h or --help for help')
289	sys.exit(1)
290
291	if c.install_target and c.install_target != 'local':
292	if not c.ecgid or not c.ecuid:
293	print('Please enter your ECMWF user id and group id '
294	' with command line options --ecuid --ecgid')
295	print('Try "' + sys.argv[0].split('/')[-1] + \
296	' -h" to print usage information')
297	print('Please consult ecaccess documentation or ECMWF user '
298	'support for further details.\n')
299	sys.exit(1)
300	if not c.gateway or not c.destination:
301	print('WARNING: Parameters GATEWAY and DESTINATION were '
302	'not properly set for working on ECMWF server. \n'
303	'There will be no transfer of output files to the '
304	'local gateway server possible!')
305	if not c.installdir:
306	c.installdir = '${HOME}'
307	else: # local
308	if not c.installdir:
309	c.installdir = _config.PATH_FLEXEXTRACT_DIR
310
311	if not c.makefile:
312	print('WARNING: no makefile was specified.')
313	if c.install_target == 'local':
314	c.makefile = 'makefile_local_gfortran'
315	print('WARNING: default makefile selected: makefile_local_gfortan')
316	elif c.install_target == 'ecgate':
317	c.makefile = 'makefile_ecgate'
318	print('WARNING: default makefile selected: makefile_ecgate')
319	elif c.install_target == 'cca' or \
320	c.install_target == 'ccb':
321	c.makefile = 'makefile_cray'
322	print('WARNING: default makefile selected: makefile_cray')
323	else:
324	pass
325
326	return
327
328
329	def mk_tarball(tarball_path, target):
330	'''Creates a tarball with all necessary files which need to be sent to the
331	installation directory.
332	It does not matter whether this is local or remote.
333	Collects all Python files, the Fortran source and makefiles,
334	the ECMWF_ENV file, the CONTROL files as well as the
335	template files.
336
337	Parameters
338	----------
339	tarball_path : str
340	The complete path to the tar file which will contain all
341	relevant data for flex_extract.
342
343	target : str
344	The queue where the job is submitted to.
345
346	Return
347	------
348
349	'''
350
351	print('Create tarball ...')
352
353	# change to FLEXEXTRACT directory so that the tar can contain
354	# relative pathes to the files and directories
355	ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
356	os.chdir(ecd)
357
358	# get lists of the files to be added to the tar file
359	if target == 'local':
360	ecmwf_env_file = []
361	runfile = [os.path.relpath(x, ecd)
362	for x in UioFiles(_config.PATH_REL_RUN_DIR,
363	'run_local.sh').files]
364	else:
365	ecmwf_env_file = [_config.PATH_REL_ECMWF_ENV]
366	runfile = [os.path.relpath(x, ecd)
367	for x in UioFiles(_config.PATH_REL_RUN_DIR,
368	'run.sh').files]
369
370	pyfiles = [os.path.relpath(x, ecd)
371	for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
372	pytestfiles = [os.path.relpath(x, ecd)
373	for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
374	controlfiles = [os.path.relpath(x, ecd)
375	for x in UioFiles(_config.PATH_REL_CONTROLFILES,
376	'CONTROL*').files]
377	testfiles = [os.path.relpath(x, ecd)
378	for x in UioFiles(_config.PATH_REL_TEST, '*').files]
379	tempfiles = [os.path.relpath(x, ecd)
380	for x in UioFiles(_config.PATH_REL_TEMPLATES, '*.template').files]
381	gribtable = [os.path.relpath(x, ecd)
382	for x in UioFiles(_config.PATH_REL_TEMPLATES, 'grib').files]
383	ffiles = [os.path.relpath(x, ecd)
384	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f90').files]
385	hfiles = [os.path.relpath(x, ecd)
386	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
387	makefiles = [os.path.relpath(x, ecd)
388	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'makefile*').files]
389	jobdir = [_config.PATH_REL_JOBSCRIPTS]
390
391	# concatenate single lists to one for a better looping
392	filelist = pyfiles + pytestfiles + controlfiles + tempfiles + \
393	ffiles + gribtable + hfiles + makefiles + ecmwf_env_file + \
394	runfile + jobdir + testfiles +\
395	['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
396
397	# create installation tar-file
398	exclude_files = [".ksh", ".tar"]
399	try:
400	with tarfile.open(tarball_path, "w:gz") as tar_handle:
401	for filename in filelist:
402	tar_handle.add(filename, recursive=False,
403	filter=lambda tarinfo: None
404	if os.path.splitext(tarinfo.name)[1]
405	in exclude_files
406	else tarinfo)
407	except tarfile.TarError as e:
408	print('... ERROR: ' + str(e))
409
410	sys.exit('\n... error occured while trying to create the tar-file ' +
411	str(tarball_path))
412
413	return
414
415
416	def un_tarball(tarball_path):
417	'''Extracts the given tarball into current directory.
418
419	Parameters
420	----------
421	tarball_path : str
422	The complete path to the tar file which will contain all
423	relevant data for flex_extract.
424
425	Return
426	------
427
428	'''
429
430	print('Untar ...')
431
432	try:
433	with tarfile.open(tarball_path) as tar_handle:
434	tar_handle.extractall()
435	except tarfile.TarError as e:
436	sys.exit('\n... error occured while trying to read tar-file ' +
437	str(tarball_path))
438	except OSError as e:
439	print('... ERROR CODE: ' + str(e.errno))
440	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
441
442	sys.exit('\n... error occured while trying to read tar-file ' +
443	str(tarball_path))
444
445	return
446
447	def mk_env_vars(ecuid, ecgid, gateway, destination):
448	'''Creates a file named ECMWF_ENV which contains the
449	necessary environmental variables at ECMWF servers.
450	It is based on the template ECMWF_ENV.template.
451
452	Parameters
453	----------
454	ecuid : str
455	The user id on ECMWF server.
456
457	ecgid : str
458	The group id on ECMWF server.
459
460	gateway : str
461	The gateway server the user is using.
462
463	destination : str
464	The remote destination which is used to transfer files
465	from ECMWF server to local gateway server.
466
467	Return
468	------
469
470	'''
471	from genshi.template.text import NewTextTemplate
472	from genshi.template import TemplateLoader
473	from genshi.template.eval import UndefinedError
474
475	try:
476	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
477	ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
478	cls=NewTextTemplate)
479
480	stream = ecmwfvars_template.generate(user_name=ecuid,
481	user_group=ecgid,
482	gateway_name=gateway,
483	destination_name=destination
484	)
485	except UndefinedError as e:
486	print('... ERROR ' + str(e))
487
488	sys.exit('\n... error occured while trying to generate template ' +
489	_config.PATH_ECMWF_ENV)
490	except OSError as e:
491	print('... ERROR CODE: ' + str(e.errno))
492	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
493
494	sys.exit('\n... error occured while trying to generate template ' +
495	_config.PATH_ECMWF_ENV)
496
497	try:
498	with open(_config.PATH_ECMWF_ENV, 'w') as f:
499	f.write(stream.render('text'))
500	except OSError as e:
501	print('... ERROR CODE: ' + str(e.errno))
502	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
503
504	sys.exit('\n... error occured while trying to write ' +
505	_config.PATH_ECMWF_ENV)
506
507	return
508
509	def mk_compilejob(makefile, ecuid, ecgid, fp_root):
510	'''Modifies the original job template file so that it is specified
511	for the user and the environment were it will be applied. Result
512	is stored in a new file "job.temp" in the python directory.
513
514	Parameters
515	----------
516	makefile : str
517	Name of the makefile which should be used to compile the Fortran
518	program.
519
520	ecuid : str
521	The user id on ECMWF server.
522
523	ecgid : str
524	The group id on ECMWF server.
525
526	fp_root : str
527	Path to the root directory of FLEXPART environment or flex_extract
528	environment.
529
530	Return
531	------
532
533	'''
534	from genshi.template.text import NewTextTemplate
535	from genshi.template import TemplateLoader
536	from genshi.template.eval import UndefinedError
537
538	if fp_root == '../':
539	fp_root = '$HOME'
540
541	try:
542	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
543	compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
544	cls=NewTextTemplate)
545
546	stream = compile_template.generate(
547	usergroup=ecgid,
548	username=ecuid,
549	version_number=_config._VERSION_STR,
550	fp_root_scripts=fp_root,
551	makefile=makefile,
552	fortran_program=_config.FORTRAN_EXECUTABLE
553	)
554	except UndefinedError as e:
555	print('... ERROR ' + str(e))
556
557	sys.exit('\n... error occured while trying to generate template ' +
558	_config.TEMPFILE_INSTALL_COMPILEJOB)
559	except OSError as e:
560	print('... ERROR CODE: ' + str(e.errno))
561	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
562
563	sys.exit('\n... error occured while trying to generate template ' +
564	_config.TEMPFILE_INSTALL_COMPILEJOB)
565
566	try:
567	compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
568	_config.FILE_INSTALL_COMPILEJOB)
569
570	with open(compilejob, 'w') as f:
571	f.write(stream.render('text'))
572	except OSError as e:
573	print('... ERROR CODE: ' + str(e.errno))
574	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
575
576	sys.exit('\n... error occured while trying to write ' +
577	compilejob)
578
579	return
580
581	def mk_job_template(ecuid, ecgid, fp_root):
582	'''Modifies the original job template file so that it is specified
583	for the user and the environment were it will be applied. Result
584	is stored in a new file.
585
586	Parameters
587	----------
588	ecuid : str
589	The user id on ECMWF server.
590
591	ecgid : str
592	The group id on ECMWF server.
593
594	fp_root : str
595	Path to the root directory of FLEXPART environment or flex_extract
596	environment.
597
598	Return
599	------
600
601	'''
602	from genshi.template.text import NewTextTemplate
603	from genshi.template import TemplateLoader
604	from genshi.template.eval import UndefinedError
605
606	fp_root_path_to_python = os.path.join(fp_root,
607	_config.FLEXEXTRACT_DIRNAME,
608	_config.PATH_REL_PYTHON_SRC)
609	if '$' in fp_root_path_to_python:
610	ind = fp_root_path_to_python.index('$')
611	fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
612	fp_root_path_to_python[ind:]
613
614	try:
615	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
616	compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
617	cls=NewTextTemplate)
618
619	stream = compile_template.generate(
620	usergroup=ecgid,
621	username=ecuid,
622	version_number=_config._VERSION_STR,
623	fp_root_path=fp_root_path_to_python,
624	)
625	except UndefinedError as e:
626	print('... ERROR ' + str(e))
627
628	sys.exit('\n... error occured while trying to generate template ' +
629	_config.TEMPFILE_INSTALL_JOB)
630	except OSError as e:
631	print('... ERROR CODE: ' + str(e.errno))
632	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
633
634	sys.exit('\n... error occured while trying to generate template ' +
635	_config.TEMPFILE_INSTALL_JOB)
636
637
638	try:
639	tempjobfile = os.path.join(_config.PATH_TEMPLATES,
640	_config.TEMPFILE_JOB)
641
642	with open(tempjobfile, 'w') as f:
643	f.write(stream.render('text'))
644	except OSError as e:
645	print('... ERROR CODE: ' + str(e.errno))
646	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
647
648	sys.exit('\n... error occured while trying to write ' +
649	tempjobfile)
650
651	return
652
653	def del_fortran_build(src_path):
654	'''Clean up the Fortran source directory and remove all
655	build files (e.g. \.o, \.mod and FORTRAN EXECUTABLE)
656
657	Parameters
658	----------
659	src_path : str
660	Path to the fortran source directory.
661
662	Return
663	------
664
665	'''
666
667	modfiles = UioFiles(src_path, '*.mod')
668	objfiles = UioFiles(src_path, '*.o')
669	exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
670
671	modfiles.delete_files()
672	objfiles.delete_files()
673	exefile.delete_files()
674
675	return
676
677	def mk_fortran_build(src_path, makefile):
678	'''Compiles the Fortran code and generates the executable.
679
680	Parameters
681	----------
682	src_path : str
683	Path to the fortran source directory.
684
685	makefile : str
686	The name of the makefile which should be used.
687
688	Return
689	------
690
691	'''
692
693	try:
694	print('Using makefile: ' + makefile)
695	p = subprocess.Popen(['make', '-f',
696	os.path.join(src_path, makefile)],
697	stdin=subprocess.PIPE,
698	stdout=subprocess.PIPE,
699	stderr=subprocess.PIPE,
700	bufsize=1)
701	pout, perr = p.communicate()
702	print(pout.decode())
703	if p.returncode != 0:
704	print(perr.decode())
705	print('Please edit ' + makefile +
706	' or try another makefile in the src directory.')
707	print('Most likely ECCODES_INCLUDE_DIR, ECCODES_LIB '
708	'and EMOSLIB must be adapted.')
709	print('Available makefiles:')
710	print(UioFiles(src_path, 'makefile*'))
711	sys.exit('Compilation failed!')
712	except ValueError as e:
713	print('ERROR: makefile call failed:')
714	print(e)
715	else:
716	execute_subprocess(['ls', '-l',
717	os.path.join(src_path, _config.FORTRAN_EXECUTABLE)],
718	error_msg='FORTRAN EXECUTABLE COULD NOT BE FOUND!')
719
720	return
721
722
723	if __name__ == "__main__":
724	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: