Context Navigation

source: flex_extract.git/Source/Python/install.py @ 5f67883

ctbtodev

Last change on this file since 5f67883 was 5f67883, checked in by Anne Philipp <anne.philipp@…>, 4 years ago
changed makefile defaults; changed template filenames
Property mode set to `100755`
File size: 24.0 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - moved install_args_and_control in here
14	# - splitted code in smaller functions
15	# - delete fortran build files in here instead of compile job script
16	# - changed static path names to variables from config file
17	# - splitted install function into several smaller pieces
18	# - use of tarfile package in python
19	#
20	# @License:
21	# (C) Copyright 2014-2020.
22	# Anne Philipp, Leopold Haimberger
23	#
24	# SPDX-License-Identifier: CC-BY-4.0
25	#
26	# This work is licensed under the Creative Commons Attribution 4.0
27	# International License. To view a copy of this license, visit
28	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
29	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
30	#
31	# @Methods:
32	# main
33	# get_install_cmdline_args
34	# install_via_gateway
35	# check_install_conditions
36	# mk_tarball
37	# un_tarball
38	# mk_env_vars
39	# mk_compilejob
40	# mk_job_template
41	# del_fortran_build
42	# mk_fortran_build
43	#
44	#*******************************************************************************
45	'''This script installs the flex_extract program.
46
47	Depending on the selected installation environment (locally or on the
48	ECMWF server ecgate or cca) the program extracts the commandline
49	arguments and the CONTROL file parameter and prepares the corresponding
50	environment.
51	The necessary files are collected in a tar-ball and placed
52	at the target location. There its untared, the environment variables will
53	be set and the Fortran code will be compiled.
54	If the ECMWF environment is selected a job script is prepared and submitted
55	for the remaining configurations after putting the tar-ball to the
56	target ECMWF server.
57
58	Type: install.py --help
59	to get information about command line parameters.
60	Read the documentation for usage instructions.
61	'''
62
63	# ------------------------------------------------------------------------------
64	# MODULES
65	# ------------------------------------------------------------------------------
66	from __future__ import print_function
67
68	import os
69	import sys
70	import subprocess
71	import tarfile
72	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
73
74	# software specific classes and modules from flex_extract
75	import _config
76	from Classes.ControlFile import ControlFile
77	from Classes.UioFiles import UioFiles
78	from Mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
79	silent_remove, execute_subprocess, none_or_str)
80
81	# ------------------------------------------------------------------------------
82	# FUNCTIONS
83	# ------------------------------------------------------------------------------
84	def main():
85	'''Controls the installation process. Calls the installation function
86	if target is specified.
87
88	Parameters
89	----------
90
91	Return
92	------
93	'''
94
95	args = get_install_cmdline_args()
96	c = ControlFile(args.controlfile)
97	c.assign_args_to_control(args)
98	check_install_conditions(c)
99
100	if c.install_target.lower() != 'local': # ecgate or cca
101	install_via_gateway(c)
102	else: # local
103	install_local(c)
104
105	return
106
107	def get_install_cmdline_args():
108	'''Decomposes the command line arguments and assigns them to variables.
109	Apply default values for non mentioned arguments.
110
111	Parameters
112	----------
113
114	Return
115	------
116	args : Namespace
117	Contains the commandline arguments from script/program call.
118	'''
119	parser = ArgumentParser(description='Install flex_extract software '
120	'locally or on ECMWF machines',
121	formatter_class=ArgumentDefaultsHelpFormatter)
122
123	parser.add_argument('--target', dest='install_target',
124	type=none_or_str, default=None,
125	help="Valid targets: local \| ecgate \| cca , \
126	the latter two are at ECMWF")
127	parser.add_argument("--makefile", dest="makefile",
128	type=none_or_str, default=None,
129	help='Name of Makefile to use for compiling the '
130	'Fortran program')
131	parser.add_argument("--ecuid", dest="ecuid",
132	type=none_or_str, default=None,
133	help='The user id at ECMWF.')
134	parser.add_argument("--ecgid", dest="ecgid",
135	type=none_or_str, default=None,
136	help='The group id at ECMWF.')
137	parser.add_argument("--gateway", dest="gateway",
138	type=none_or_str, default=None,
139	help='The name of the local gateway server.')
140	parser.add_argument("--destination", dest="destination",
141	type=none_or_str, default=None,
142	help='The ecaccess association, e.g. '
143	'myUser@genericSftp')
144
145	parser.add_argument("--installdir", dest="installdir",
146	type=none_or_str, default=None,
147	help='Root directory where '
148	'flex_extract will be installed to.')
149
150	# arguments for job submission to ECMWF, only needed by submit.py
151	parser.add_argument("--job_template", dest='job_template',
152	type=none_or_str, default="job.template",
153	help='The rudimentary template file to create a batch '
154	'job template for submission to ECMWF servers.')
155
156	parser.add_argument("--controlfile", dest="controlfile",
157	type=none_or_str, default='CONTROL_EA5',
158	help="The file with all CONTROL parameters.")
159
160	args = parser.parse_args()
161
162	return args
163
164
165	def install_via_gateway(c):
166	'''Prepare data transfer to remote gate and submit a job script which will
167	install everything on the remote gate.
168
169	Parameters
170	----------
171	c : ControlFile
172	Contains all the parameters of CONTROL file and
173	command line.
174
175	Return
176	------
177
178	'''
179
180	tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
181	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
182
183	mk_compilejob(c.makefile, c.ecuid, c.ecgid, c.installdir)
184
185	mk_job_template(c.ecuid, c.ecgid, c.installdir)
186
187	mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
188
189	mk_tarball(tar_file, c.install_target)
190
191	put_file_to_ecserver(_config.PATH_FLEXEXTRACT_DIR, tarball_name,
192	c.install_target, c.ecuid, c.ecgid)
193
194	submit_job_to_ecserver(c.install_target,
195	os.path.join(_config.PATH_REL_JOBSCRIPTS,
196	_config.FILE_INSTALL_COMPILEJOB))
197
198	silent_remove(tar_file)
199
200	print('Job compilation script has been submitted to ecgate for ' +
201	'installation in ' + c.installdir +
202	'/' + _config.FLEXEXTRACT_DIRNAME)
203	print('You should get an email with subject "flexcompile" within ' +
204	'the next few minutes!')
205
206	return
207
208	def install_local(c):
209	'''Perform the actual installation on a local machine.
210
211	Parameters
212	----------
213	c : ControlFile
214	Contains all the parameters of CONTROL file and
215	command line.
216
217	Return
218	------
219
220	'''
221
222	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR,
223	_config.FLEXEXTRACT_DIRNAME + '.tar')
224
225	if c.installdir == _config.PATH_FLEXEXTRACT_DIR:
226	print('WARNING: installdir has not been specified')
227	print('flex_extract will be installed in here by compiling the ' +
228	'Fortran source in ' + _config.PATH_FORTRAN_SRC)
229	os.chdir(_config.PATH_FORTRAN_SRC)
230	else: # creates the target working directory for flex_extract
231	c.installdir = os.path.expandvars(os.path.expanduser(
232	c.installdir))
233	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) != \
234	os.path.abspath(c.installdir):
235	mk_tarball(tar_file, c.install_target)
236	make_dir(os.path.join(c.installdir,
237	_config.FLEXEXTRACT_DIRNAME))
238	os.chdir(os.path.join(c.installdir,
239	_config.FLEXEXTRACT_DIRNAME))
240	un_tarball(tar_file)
241	os.chdir(os.path.join(c.installdir,
242	_config.FLEXEXTRACT_DIRNAME,
243	_config.PATH_REL_FORTRAN_SRC))
244
245	# Create Fortran executable
246	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software at ' +
247	c.install_target + ' in directory ' +
248	os.path.abspath(c.installdir) + '\n')
249
250	del_fortran_build('.')
251	mk_fortran_build('.', c.makefile)
252
253	os.chdir(_config.PATH_FLEXEXTRACT_DIR)
254	if os.path.isfile(tar_file):
255	os.remove(tar_file)
256
257	return
258
259
260	def check_install_conditions(c):
261	'''Checks a couple of necessary attributes and conditions
262	for the installation such as if they exist and contain values.
263	Otherwise set default values.
264
265	Parameters
266	----------
267	c : ControlFile
268	Contains all the parameters of CONTROL file and
269	command line.
270
271
272	Return
273	------
274
275	'''
276
277	if c.install_target and \
278	c.install_target not in _config.INSTALL_TARGETS:
279	print('ERROR: unknown or missing installation target ')
280	print('target: ', c.install_target)
281	print('please specify correct installation target ' +
282	str(_config.INSTALL_TARGETS))
283	print('use -h or --help for help')
284	sys.exit(1)
285
286	if c.install_target and c.install_target != 'local':
287	if not c.ecgid or not c.ecuid:
288	print('Please enter your ECMWF user id and group id '
289	' with command line options --ecuid --ecgid')
290	print('Try "' + sys.argv[0].split('/')[-1] + \
291	' -h" to print usage information')
292	print('Please consult ecaccess documentation or ECMWF user '
293	'support for further details.\n')
294	sys.exit(1)
295	if not c.gateway or not c.destination:
296	print('WARNING: Parameters GATEWAY and DESTINATION were '
297	'not properly set for working on ECMWF server. \n'
298	'There will be no transfer of output files to the '
299	'local gateway server possible!')
300	if not c.installdir:
301	c.installdir = '${HOME}'
302	else: # local
303	if not c.installdir:
304	c.installdir = _config.PATH_FLEXEXTRACT_DIR
305
306	if not c.makefile:
307	print('WARNING: no makefile was specified.')
308	if c.install_target == 'local':
309	c.makefile = 'makefile_local_gfortran'
310	print('WARNING: default makefile selected: makefile_local_gfortan')
311	elif c.install_target == 'ecgate':
312	c.makefile = 'makefile_ecgate'
313	print('WARNING: default makefile selected: makefile_ecgate')
314	elif c.install_target == 'cca' or \
315	c.install_target == 'ccb':
316	c.makefile = 'makefile_cray'
317	print('WARNING: default makefile selected: makefile_cray')
318	else:
319	pass
320
321	return
322
323
324	def mk_tarball(tarball_path, target):
325	'''Creates a tarball with all necessary files which need to be sent to the
326	installation directory.
327	It does not matter if this is local or remote.
328	Collects all python files, the Fortran source and makefiles,
329	the ECMWF_ENV file, the CONTROL files as well as the
330	template files.
331
332	Parameters
333	----------
334	tarball_path : str
335	The complete path to the tar file which will contain all
336	relevant data for flex_extract.
337
338	target : str
339	The queue where the job is submitted to.
340
341	Return
342	------
343
344	'''
345
346	print('Create tarball ...')
347
348	# change to FLEXEXTRACT directory so that the tar can contain
349	# relative pathes to the files and directories
350	ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
351	os.chdir(ecd)
352
353	# get lists of the files to be added to the tar file
354	if target == 'local':
355	ecmwf_env_file = []
356	runfile = [os.path.relpath(x, ecd)
357	for x in UioFiles(_config.PATH_REL_RUN_DIR,
358	'run_local.sh').files]
359	else:
360	ecmwf_env_file = [_config.PATH_REL_ECMWF_ENV]
361	runfile = [os.path.relpath(x, ecd)
362	for x in UioFiles(_config.PATH_REL_RUN_DIR,
363	'run.sh').files]
364
365	pyfiles = [os.path.relpath(x, ecd)
366	for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
367	pytestfiles = [os.path.relpath(x, ecd)
368	for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
369	controlfiles = [os.path.relpath(x, ecd)
370	for x in UioFiles(_config.PATH_REL_CONTROLFILES,
371	'CONTROL*').files]
372	testfiles = [os.path.relpath(x, ecd)
373	for x in UioFiles(_config.PATH_REL_TEST, '*').files]
374	tempfiles = [os.path.relpath(x, ecd)
375	for x in UioFiles(_config.PATH_REL_TEMPLATES, '*.temp').files]
376	nlfiles = [os.path.relpath(x, ecd)
377	for x in UioFiles(_config.PATH_REL_TEMPLATES, '*.nl').files]
378	gribtable = [os.path.relpath(x, ecd)
379	for x in UioFiles(_config.PATH_REL_TEMPLATES, 'grib').files]
380	ffiles = [os.path.relpath(x, ecd)
381	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f90').files]
382	hfiles = [os.path.relpath(x, ecd)
383	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
384	makefiles = [os.path.relpath(x, ecd)
385	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'makefile*').files]
386	jobdir = [_config.PATH_REL_JOBSCRIPTS]
387
388	# concatenate single lists to one for a better looping
389	filelist = pyfiles + pytestfiles + controlfiles + tempfiles + nlfiles + \
390	ffiles + gribtable + hfiles + makefiles + ecmwf_env_file + \
391	runfile + jobdir + testfiles +\
392	['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
393
394	# create installation tar-file
395	exclude_files = [".ksh", ".tar"]
396	try:
397	with tarfile.open(tarball_path, "w:gz") as tar_handle:
398	for filename in filelist:
399	tar_handle.add(filename, recursive=False,
400	filter=lambda tarinfo: None
401	if os.path.splitext(tarinfo.name)[1]
402	in exclude_files
403	else tarinfo)
404	except tarfile.TarError as e:
405	print('... ERROR: ' + str(e))
406
407	sys.exit('\n... error occured while trying to create the tar-file ' +
408	str(tarball_path))
409
410	return
411
412
413	def un_tarball(tarball_path):
414	'''Extracts the given tarball into current directory.
415
416	Parameters
417	----------
418	tarball_path : str
419	The complete path to the tar file which will contain all
420	relevant data for flex_extract.
421
422	Return
423	------
424
425	'''
426
427	print('Untar ...')
428
429	try:
430	with tarfile.open(tarball_path) as tar_handle:
431	tar_handle.extractall()
432	except tarfile.TarError as e:
433	sys.exit('\n... error occured while trying to read tar-file ' +
434	str(tarball_path))
435	except OSError as e:
436	print('... ERROR CODE: ' + str(e.errno))
437	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
438
439	sys.exit('\n... error occured while trying to read tar-file ' +
440	str(tarball_path))
441
442	return
443
444	def mk_env_vars(ecuid, ecgid, gateway, destination):
445	'''Creates a file named ECMWF_ENV which contains the
446	necessary environmental variables at ECMWF servers.
447	It is based on the template ECMWF_ENV.template.
448
449	Parameters
450	----------
451	ecuid : str
452	The user id on ECMWF server.
453
454	ecgid : str
455	The group id on ECMWF server.
456
457	gateway : str
458	The gateway server the user is using.
459
460	destination : str
461	The remote destination which is used to transfer files
462	from ECMWF server to local gateway server.
463
464	Return
465	------
466
467	'''
468	from genshi.template.text import NewTextTemplate
469	from genshi.template import TemplateLoader
470	from genshi.template.eval import UndefinedError
471
472	try:
473	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
474	ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
475	cls=NewTextTemplate)
476
477	stream = ecmwfvars_template.generate(user_name=ecuid,
478	user_group=ecgid,
479	gateway_name=gateway,
480	destination_name=destination
481	)
482	except UndefinedError as e:
483	print('... ERROR ' + str(e))
484
485	sys.exit('\n... error occured while trying to generate template ' +
486	_config.PATH_ECMWF_ENV)
487	except OSError as e:
488	print('... ERROR CODE: ' + str(e.errno))
489	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
490
491	sys.exit('\n... error occured while trying to generate template ' +
492	_config.PATH_ECMWF_ENV)
493
494	try:
495	with open(_config.PATH_ECMWF_ENV, 'w') as f:
496	f.write(stream.render('text'))
497	except OSError as e:
498	print('... ERROR CODE: ' + str(e.errno))
499	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
500
501	sys.exit('\n... error occured while trying to write ' +
502	_config.PATH_ECMWF_ENV)
503
504	return
505
506	def mk_compilejob(makefile, ecuid, ecgid, fp_root):
507	'''Modifies the original job template file so that it is specified
508	for the user and the environment were it will be applied. Result
509	is stored in a new file "job.temp" in the python directory.
510
511	Parameters
512	----------
513	makefile : str
514	Name of the makefile which should be used to compile FORTRAN
515	program.
516
517	ecuid : str
518	The user id on ECMWF server.
519
520	ecgid : str
521	The group id on ECMWF server.
522
523	fp_root : str
524	Path to the root directory of FLEXPART environment or flex_extract
525	environment.
526
527	Return
528	------
529
530	'''
531	from genshi.template.text import NewTextTemplate
532	from genshi.template import TemplateLoader
533	from genshi.template.eval import UndefinedError
534
535	if fp_root == '../':
536	fp_root = '$HOME'
537
538	try:
539	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
540	compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
541	cls=NewTextTemplate)
542
543	stream = compile_template.generate(
544	usergroup=ecgid,
545	username=ecuid,
546	version_number=_config._VERSION_STR,
547	fp_root_scripts=fp_root,
548	makefile=makefile,
549	fortran_program=_config.FORTRAN_EXECUTABLE
550	)
551	except UndefinedError as e:
552	print('... ERROR ' + str(e))
553
554	sys.exit('\n... error occured while trying to generate template ' +
555	_config.TEMPFILE_INSTALL_COMPILEJOB)
556	except OSError as e:
557	print('... ERROR CODE: ' + str(e.errno))
558	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
559
560	sys.exit('\n... error occured while trying to generate template ' +
561	_config.TEMPFILE_INSTALL_COMPILEJOB)
562
563	try:
564	compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
565	_config.FILE_INSTALL_COMPILEJOB)
566
567	with open(compilejob, 'w') as f:
568	f.write(stream.render('text'))
569	except OSError as e:
570	print('... ERROR CODE: ' + str(e.errno))
571	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
572
573	sys.exit('\n... error occured while trying to write ' +
574	compilejob)
575
576	return
577
578	def mk_job_template(ecuid, ecgid, fp_root):
579	'''Modifies the original job template file so that it is specified
580	for the user and the environment were it will be applied. Result
581	is stored in a new file.
582
583	Parameters
584	----------
585	ecuid : str
586	The user id on ECMWF server.
587
588	ecgid : str
589	The group id on ECMWF server.
590
591	fp_root : str
592	Path to the root directory of FLEXPART environment or flex_extract
593	environment.
594
595	Return
596	------
597
598	'''
599	from genshi.template.text import NewTextTemplate
600	from genshi.template import TemplateLoader
601	from genshi.template.eval import UndefinedError
602
603	fp_root_path_to_python = os.path.join(fp_root,
604	_config.FLEXEXTRACT_DIRNAME,
605	_config.PATH_REL_PYTHON_SRC)
606	if '$' in fp_root_path_to_python:
607	ind = fp_root_path_to_python.index('$')
608	fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
609	fp_root_path_to_python[ind:]
610
611	try:
612	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
613	compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
614	cls=NewTextTemplate)
615
616	stream = compile_template.generate(
617	usergroup=ecgid,
618	username=ecuid,
619	version_number=_config._VERSION_STR,
620	fp_root_path=fp_root_path_to_python,
621	)
622	except UndefinedError as e:
623	print('... ERROR ' + str(e))
624
625	sys.exit('\n... error occured while trying to generate template ' +
626	_config.TEMPFILE_INSTALL_JOB)
627	except OSError as e:
628	print('... ERROR CODE: ' + str(e.errno))
629	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
630
631	sys.exit('\n... error occured while trying to generate template ' +
632	_config.TEMPFILE_INSTALL_JOB)
633
634
635	try:
636	tempjobfile = os.path.join(_config.PATH_TEMPLATES,
637	_config.TEMPFILE_JOB)
638
639	with open(tempjobfile, 'w') as f:
640	f.write(stream.render('text'))
641	except OSError as e:
642	print('... ERROR CODE: ' + str(e.errno))
643	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
644
645	sys.exit('\n... error occured while trying to write ' +
646	tempjobfile)
647
648	return
649
650	def del_fortran_build(src_path):
651	'''Clean up the Fortran source directory and remove all
652	build files (e.g. \.o, \.mod and FORTRAN EXECUTABLE)
653
654	Parameters
655	----------
656	src_path : str
657	Path to the fortran source directory.
658
659	Return
660	------
661
662	'''
663
664	modfiles = UioFiles(src_path, '*.mod')
665	objfiles = UioFiles(src_path, '*.o')
666	exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
667
668	modfiles.delete_files()
669	objfiles.delete_files()
670	exefile.delete_files()
671
672	return
673
674	def mk_fortran_build(src_path, makefile):
675	'''Compiles the Fortran code and generates the executable.
676
677	Parameters
678	----------
679	src_path : str
680	Path to the fortran source directory.
681
682	makefile : str
683	The name of the makefile which should be used.
684
685	Return
686	------
687
688	'''
689
690	try:
691	print('Using makefile: ' + makefile)
692	p = subprocess.Popen(['make', '-f',
693	os.path.join(src_path, makefile)],
694	stdin=subprocess.PIPE,
695	stdout=subprocess.PIPE,
696	stderr=subprocess.PIPE,
697	bufsize=1)
698	pout, perr = p.communicate()
699	print(pout.decode())
700	if p.returncode != 0:
701	print(perr.decode())
702	print('Please edit ' + makefile +
703	' or try another makefile in the src directory.')
704	print('Most likely ECCODES_INCLUDE_DIR, ECCODES_LIB '
705	'and EMOSLIB must be adapted.')
706	print('Available makefiles:')
707	print(UioFiles(src_path, 'makefile*'))
708	sys.exit('Compilation failed!')
709	except ValueError as e:
710	print('ERROR: makefile call failed:')
711	print(e)
712	else:
713	execute_subprocess(['ls', '-l',
714	os.path.join(src_path, _config.FORTRAN_EXECUTABLE)],
715	error_msg='FORTRAN EXECUTABLE COULD NOT BE FOUND!')
716
717	return
718
719
720	if __name__ == "__main__":
721	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: