Context Navigation

source: flex_extract.git/Source/Python/install.py @ 44174de

ctbtodev

Last change on this file since 44174de was 44174de, checked in by Anne Philipp <anne.philipp@…>, 5 years ago
commented out the WRF parts since they are still under construction and added License SPDX tags
Property mode set to `100755`
File size: 23.8 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - moved install_args_and_control in here
14	# - splitted code in smaller functions
15	# - delete convert build files in here instead of compile job script
16	# - changed static path names to variables from config file
17	# - splitted install function into several smaller pieces
18	# - use of tarfile package in python
19	#
20	# @License:
21	# (C) Copyright 2014-2019.
22	# Anne Philipp, Leopold Haimberger
23	#
24	# SPDX-License-Identifier: CC-BY-4.0
25	#
26	# This work is licensed under the Creative Commons Attribution 4.0
27	# International License. To view a copy of this license, visit
28	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
29	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
30	#
31	# @Methods:
32	# main
33	# get_install_cmdline_args
34	# install_via_gateway
35	# check_install_conditions
36	# mk_tarball
37	# un_tarball
38	# mk_env_vars
39	# mk_compilejob
40	# mk_job_template
41	# del_convert_build
42	# mk_convert_build
43	#
44	#*******************************************************************************
45	'''This script installs the flex_extract program.
46
47	Depending on the selected installation environment (locally or on the
48	ECMWF server ecgate or cca) the program extracts the commandline
49	arguments and the CONTROL file parameter and prepares the corresponding
50	environment.
51	The necessary files are collected in a tar-ball and placed
52	at the target location. There its untared, the environment variables will
53	be set and the Fortran code will be compiled.
54	If the ECMWF environment is selected a job script is prepared and submitted
55	for the remaining configurations after putting the tar-ball to the
56	target ECMWF server.
57
58	Type: install.py --help
59	to get information about command line parameters.
60	Read the documentation for usage instructions.
61	'''
62
63	# ------------------------------------------------------------------------------
64	# MODULES
65	# ------------------------------------------------------------------------------
66	from __future__ import print_function
67
68	import os
69	import sys
70	import glob
71	import subprocess
72	import inspect
73	import tarfile
74	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
75
76	# software specific classes and modules from flex_extract
77	import _config
78	from Classes.ControlFile import ControlFile
79	from Classes.UioFiles import UioFiles
80	from Mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
81	silent_remove, execute_subprocess, none_or_str)
82
83	# ------------------------------------------------------------------------------
84	# FUNCTIONS
85	# ------------------------------------------------------------------------------
86	def main():
87	'''Controls the installation process. Calls the installation function
88	if target is specified.
89
90	Parameters
91	----------
92
93	Return
94	------
95	'''
96
97	args = get_install_cmdline_args()
98	c = ControlFile(args.controlfile)
99	c.assign_args_to_control(args)
100	check_install_conditions(c)
101
102	if c.install_target.lower() != 'local': # ecgate or cca
103	install_via_gateway(c)
104	else: # local
105	install_local(c)
106
107	return
108
109	def get_install_cmdline_args():
110	'''Decomposes the command line arguments and assigns them to variables.
111	Apply default values for non mentioned arguments.
112
113	Parameters
114	----------
115
116	Return
117	------
118	args : Namespace
119	Contains the commandline arguments from script/program call.
120	'''
121	parser = ArgumentParser(description='Install flex_extract software '
122	'locally or on ECMWF machines',
123	formatter_class=ArgumentDefaultsHelpFormatter)
124
125	parser.add_argument('--target', dest='install_target',
126	type=none_or_str, default=None,
127	help="Valid targets: local \| ecgate \| cca , \
128	the latter two are at ECMWF")
129	parser.add_argument("--makefile", dest="makefile",
130	type=none_or_str, default=None,
131	help='Name of Makefile to use for compiling the '
132	'Fortran program')
133	parser.add_argument("--ecuid", dest="ecuid",
134	type=none_or_str, default=None,
135	help='The user id at ECMWF.')
136	parser.add_argument("--ecgid", dest="ecgid",
137	type=none_or_str, default=None,
138	help='The group id at ECMWF.')
139	parser.add_argument("--gateway", dest="gateway",
140	type=none_or_str, default=None,
141	help='The name of the local gateway server.')
142	parser.add_argument("--destination", dest="destination",
143	type=none_or_str, default=None,
144	help='The ecaccess association, e.g. '
145	'myUser@genericSftp')
146
147	parser.add_argument("--installdir", dest="installdir",
148	type=none_or_str, default=None,
149	help='Root directory where '
150	'flex_extract will be installed to.')
151
152	# arguments for job submission to ECMWF, only needed by submit.py
153	parser.add_argument("--job_template", dest='job_template',
154	type=none_or_str, default="job.template",
155	help='The rudimentary template file to create a batch '
156	'job template for submission to ECMWF servers.')
157
158	parser.add_argument("--controlfile", dest="controlfile",
159	type=none_or_str, default='CONTROL_EA5',
160	help="The file with all CONTROL parameters.")
161
162	args = parser.parse_args()
163
164	return args
165
166
167	def install_via_gateway(c):
168	'''Prepare data transfer to remote gate and submit a job script which will
169	install everything on the remote gate.
170
171	Parameters
172	----------
173	c : ControlFile
174	Contains all the parameters of CONTROL file and
175	command line.
176
177	Return
178	------
179
180	'''
181	import tarfile
182
183	tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
184	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
185
186	mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
187	c.installdir)
188
189	mk_job_template(c.ecuid, c.ecgid, c.gateway,
190	c.destination, c.installdir)
191
192	mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
193
194	mk_tarball(tar_file, c.install_target)
195
196	put_file_to_ecserver(_config.PATH_FLEXEXTRACT_DIR, tarball_name,
197	c.install_target, c.ecuid, c.ecgid)
198
199	submit_job_to_ecserver(c.install_target,
200	os.path.join(_config.PATH_REL_JOBSCRIPTS,
201	_config.FILE_INSTALL_COMPILEJOB))
202
203	silent_remove(tar_file)
204
205	print('Job compilation script has been submitted to ecgate for ' +
206	'installation in ' + c.installdir +
207	'/' + _config.FLEXEXTRACT_DIRNAME)
208	print('You should get an email with subject "flexcompile" within ' +
209	'the next few minutes!')
210
211	return
212
213	def install_local(c):
214	'''Perform the actual installation on a local machine.
215
216	Parameters
217	----------
218	c : ControlFile
219	Contains all the parameters of CONTROL file and
220	command line.
221
222	Return
223	------
224
225	'''
226	import tarfile
227
228	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR,
229	_config.FLEXEXTRACT_DIRNAME + '.tar')
230
231	if c.installdir == _config.PATH_FLEXEXTRACT_DIR :
232	print('WARNING: installdir has not been specified')
233	print('flex_extract will be installed in here by compiling the ' +
234	'Fortran source in ' + _config.PATH_FORTRAN_SRC)
235	os.chdir(_config.PATH_FORTRAN_SRC)
236	else: # creates the target working directory for flex_extract
237	c.installdir = os.path.expandvars(os.path.expanduser(
238	c.installdir))
239	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) != \
240	os.path.abspath(c.installdir):
241	mk_tarball(tar_file, c.install_target)
242	make_dir(os.path.join(c.installdir,
243	_config.FLEXEXTRACT_DIRNAME))
244	os.chdir(os.path.join(c.installdir,
245	_config.FLEXEXTRACT_DIRNAME))
246	un_tarball(tar_file)
247	os.chdir(os.path.join(c.installdir,
248	_config.FLEXEXTRACT_DIRNAME,
249	_config.PATH_REL_FORTRAN_SRC))
250
251	# Create Fortran executable - CONVERT2
252	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software at ' +
253	c.install_target + ' in directory ' +
254	os.path.abspath(c.installdir) + '\n')
255
256	del_convert_build('.')
257	mk_convert_build('.', c.makefile)
258
259	os.chdir(_config.PATH_FLEXEXTRACT_DIR)
260	if os.path.isfile(tar_file):
261	os.remove(tar_file)
262
263	return
264
265
266	def check_install_conditions(c):
267	'''Checks a couple of necessary attributes and conditions
268	for the installation such as if they exist and contain values.
269	Otherwise set default values.
270
271	Parameters
272	----------
273	c : ControlFile
274	Contains all the parameters of CONTROL file and
275	command line.
276
277
278	Return
279	------
280
281	'''
282
283	if c.install_target and \
284	c.install_target not in _config.INSTALL_TARGETS:
285	print('ERROR: unknown or missing installation target ')
286	print('target: ', c.install_target)
287	print('please specify correct installation target ' +
288	str(INSTALL_TARGETS))
289	print('use -h or --help for help')
290	sys.exit(1)
291
292	if c.install_target and c.install_target != 'local':
293	if not c.ecgid or not c.ecuid or \
294	not c.gateway or not c.destination:
295	print('Please enter your ECMWF user id and group id as well ' +
296	'as the \nname of the local gateway and the ectrans ' +
297	'destination ')
298	print('with command line options --ecuid --ecgid \
299	--gateway --destination')
300	print('Try "' + sys.argv[0].split('/')[-1] + \
301	' -h" to print usage information')
302	print('Please consult ecaccess documentation or ECMWF user \
303	support for further details')
304	sys.exit(1)
305
306	if not c.installdir:
307	c.installdir = '${HOME}'
308	else: # local
309	if not c.installdir:
310	c.installdir = _config.PATH_FLEXEXTRACT_DIR
311
312	return
313
314
315	def mk_tarball(tarball_path, target):
316	'''Creates a tarball with all necessary files which need to be sent to the
317	installation directory.
318	It does not matter if this is local or remote.
319	Collects all python files, the Fortran source and makefiles,
320	the ECMWF_ENV file, the CONTROL files as well as the
321	template files.
322
323	Parameters
324	----------
325	tarball_path : str
326	The complete path to the tar file which will contain all
327	relevant data for flex_extract.
328
329	target : str
330	The queue where the job is submitted to.
331
332	Return
333	------
334
335	'''
336	from glob import glob
337
338	print('Create tarball ...')
339
340	# change to FLEXEXTRACT directory so that the tar can contain
341	# relative pathes to the files and directories
342	ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
343	os.chdir(ecd)
344
345	# get lists of the files to be added to the tar file
346	if target == 'local':
347	ECMWF_ENV_FILE = []
348	runfile = [os.path.relpath(x, ecd)
349	for x in UioFiles(_config.PATH_REL_RUN_DIR,
350	'run_local.sh').files]
351	else:
352	ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
353	runfile = [os.path.relpath(x, ecd)
354	for x in UioFiles(_config.PATH_REL_RUN_DIR,
355	'run.sh').files]
356
357	pyfiles = [os.path.relpath(x, ecd)
358	for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
359	pytestfiles = [os.path.relpath(x, ecd)
360	for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
361	controlfiles = [os.path.relpath(x, ecd)
362	for x in UioFiles(_config.PATH_REL_CONTROLFILES,
363	'CONTROL*').files]
364	testfiles = [os.path.relpath(x, ecd)
365	for x in UioFiles(_config.PATH_REL_TEST , '*').files]
366	tempfiles = [os.path.relpath(x, ecd)
367	for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.temp').files]
368	nlfiles = [os.path.relpath(x, ecd)
369	for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.nl').files]
370	gribtable = [os.path.relpath(x, ecd)
371	for x in UioFiles(_config.PATH_REL_TEMPLATES , 'grib').files]
372	ffiles = [os.path.relpath(x, ecd)
373	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '.f').files]
374	hfiles = [os.path.relpath(x, ecd)
375	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
376	makefiles = [os.path.relpath(x, ecd)
377	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'Makefile*').files]
378	jobdir = [_config.PATH_REL_JOBSCRIPTS]
379
380	# concatenate single lists to one for a better looping
381	filelist = pyfiles + pytestfiles + controlfiles + tempfiles + nlfiles + \
382	ffiles + gribtable + hfiles + makefiles + ECMWF_ENV_FILE + \
383	runfile + jobdir + testfiles +\
384	['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
385
386	# create installation tar-file
387	exclude_files = [".ksh"]
388	try:
389	with tarfile.open(tarball_path, "w:gz") as tar_handle:
390	for file in filelist:
391	tar_handle.add(file, recursive=False,
392	filter=lambda tarinfo: None
393	if os.path.splitext(tarinfo.name)[1]
394	in exclude_files
395	else tarinfo)
396	except tarfile.TarError as e:
397	sys.exit('\n... error occured while trying to create the tar-file ' +
398	str(tarball_path))
399
400	return
401
402
403	def un_tarball(tarball_path):
404	'''Extracts the given tarball into current directory.
405
406	Parameters
407	----------
408	tarball_path : str
409	The complete path to the tar file which will contain all
410	relevant data for flex_extract.
411
412	Return
413	------
414
415	'''
416
417	print('Untar ...')
418
419	try:
420	with tarfile.open(tarball_path) as tar_handle:
421	tar_handle.extractall()
422	except tarfile.TarError as e:
423	sys.exit('\n... error occured while trying to read tar-file ' +
424	str(tarball_path))
425	except OSError as e:
426	print('... ERROR CODE: ' + str(e.errno))
427	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
428
429	sys.exit('\n... error occured while trying to read tar-file ' +
430	str(tarball_path))
431
432	return
433
434	def mk_env_vars(ecuid, ecgid, gateway, destination):
435	'''Creates a file named ECMWF_ENV which contains the
436	necessary environmental variables at ECMWF servers.
437	It is based on the template ECMWF_ENV.template.
438
439	Parameters
440	----------
441	ecuid : str
442	The user id on ECMWF server.
443
444	ecgid : str
445	The group id on ECMWF server.
446
447	gateway : str
448	The gateway server the user is using.
449
450	destination : str
451	The remote destination which is used to transfer files
452	from ECMWF server to local gateway server.
453
454	Return
455	------
456
457	'''
458	from genshi.template.text import NewTextTemplate
459	from genshi.template import TemplateLoader
460	from genshi.template.eval import UndefinedError
461
462	try:
463	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
464	ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
465	cls=NewTextTemplate)
466
467	stream = ecmwfvars_template.generate(user_name = ecuid,
468	user_group = ecgid,
469	gateway_name = gateway,
470	destination_name = destination
471	)
472	except UndefinedError as e:
473	print('... ERROR ' + str(e))
474
475	sys.exit('\n... error occured while trying to generate template ' +
476	_config.PATH_ECMWF_ENV)
477	except OSError as e:
478	print('... ERROR CODE: ' + str(e.errno))
479	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
480
481	sys.exit('\n... error occured while trying to generate template ' +
482	_config.PATH_ECMWF_ENV)
483
484	try:
485	with open(_config.PATH_ECMWF_ENV, 'w') as f:
486	f.write(stream.render('text'))
487	except OSError as e:
488	print('... ERROR CODE: ' + str(e.errno))
489	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
490
491	sys.exit('\n... error occured while trying to write ' +
492	_config.PATH_ECMWF_ENV)
493
494	return
495
496	def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
497	'''Modifies the original job template file so that it is specified
498	for the user and the environment were it will be applied. Result
499	is stored in a new file "job.temp" in the python directory.
500
501	Parameters
502	----------
503	makefile : str
504	Name of the makefile which should be used to compile FORTRAN
505	CONVERT2 program.
506
507	target : str
508	The target where the installation should be done, e.g. the queue.
509
510	ecuid : str
511	The user id on ECMWF server.
512
513	ecgid : str
514	The group id on ECMWF server.
515
516	fp_root : str
517	Path to the root directory of FLEXPART environment or flex_extract
518	environment.
519
520	Return
521	------
522
523	'''
524	from genshi.template.text import NewTextTemplate
525	from genshi.template import TemplateLoader
526	from genshi.template.eval import UndefinedError
527
528	if fp_root == '../':
529	fp_root = '$HOME'
530
531	try:
532	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
533	compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
534	cls=NewTextTemplate)
535
536	stream = compile_template.generate(
537	usergroup = ecgid,
538	username = ecuid,
539	version_number = _config._VERSION_STR,
540	fp_root_scripts = fp_root,
541	makefile = makefile,
542	fortran_program = _config.FORTRAN_EXECUTABLE
543	)
544	except UndefinedError as e:
545	print('... ERROR ' + str(e))
546
547	sys.exit('\n... error occured while trying to generate template ' +
548	_config.TEMPFILE_INSTALL_COMPILEJOB)
549	except OSError as e:
550	print('... ERROR CODE: ' + str(e.errno))
551	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
552
553	sys.exit('\n... error occured while trying to generate template ' +
554	_config.TEMPFILE_INSTALL_COMPILEJOB)
555
556	try:
557	compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
558	_config.FILE_INSTALL_COMPILEJOB)
559
560	with open(compilejob, 'w') as f:
561	f.write(stream.render('text'))
562	except OSError as e:
563	print('... ERROR CODE: ' + str(e.errno))
564	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
565
566	sys.exit('\n... error occured while trying to write ' +
567	compilejob)
568
569	return
570
571	def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
572	'''Modifies the original job template file so that it is specified
573	for the user and the environment were it will be applied. Result
574	is stored in a new file.
575
576	Parameters
577	----------
578	ecuid : str
579	The user id on ECMWF server.
580
581	ecgid : str
582	The group id on ECMWF server.
583
584	gateway : str
585	The gateway server the user is using.
586
587	destination : str
588	The remote destination which is used to transfer files
589	from ECMWF server to local gateway server.
590
591	fp_root : str
592	Path to the root directory of FLEXPART environment or flex_extract
593	environment.
594
595	Return
596	------
597
598	'''
599	from genshi.template.text import NewTextTemplate
600	from genshi.template import TemplateLoader
601	from genshi.template.eval import UndefinedError
602
603	fp_root_path_to_python = os.path.join(fp_root,
604	_config.FLEXEXTRACT_DIRNAME,
605	_config.PATH_REL_PYTHON_SRC)
606	if '$' in fp_root_path_to_python:
607	ind = fp_root_path_to_python.index('$')
608	fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
609	fp_root_path_to_python[ind:]
610
611	try:
612	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
613	compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
614	cls=NewTextTemplate)
615
616	stream = compile_template.generate(
617	usergroup = ecgid,
618	username = ecuid,
619	version_number = _config._VERSION_STR,
620	fp_root_path = fp_root_path_to_python,
621	)
622	except UndefinedError as e:
623	print('... ERROR ' + str(e))
624
625	sys.exit('\n... error occured while trying to generate template ' +
626	_config.TEMPFILE_INSTALL_JOB)
627	except OSError as e:
628	print('... ERROR CODE: ' + str(e.errno))
629	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
630
631	sys.exit('\n... error occured while trying to generate template ' +
632	_config.TEMPFILE_INSTALL_JOB)
633
634
635	try:
636	tempjobfile = os.path.join(_config.PATH_TEMPLATES,
637	_config.TEMPFILE_JOB)
638
639	with open(tempjobfile, 'w') as f:
640	f.write(stream.render('text'))
641	except OSError as e:
642	print('... ERROR CODE: ' + str(e.errno))
643	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
644
645	sys.exit('\n... error occured while trying to write ' +
646	tempjobfile)
647
648	return
649
650	def del_convert_build(src_path):
651	'''Clean up the Fortran source directory and remove all
652	build files (e.g. \.o, \.mod and CONVERT2)
653
654	Parameters
655	----------
656	src_path : str
657	Path to the fortran source directory.
658
659	Return
660	------
661
662	'''
663
664	modfiles = UioFiles(src_path, '*.mod')
665	objfiles = UioFiles(src_path, '*.o')
666	exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
667
668	modfiles.delete_files()
669	objfiles.delete_files()
670	exefile.delete_files()
671
672	return
673
674	def mk_convert_build(src_path, makefile):
675	'''Compiles the Fortran code and generates the executable.
676
677	Parameters
678	----------
679	src_path : str
680	Path to the fortran source directory.
681
682	makefile : str
683	The name of the makefile which should be used.
684
685	Return
686	------
687
688	'''
689
690	try:
691	print('Using makefile: ' + makefile)
692	p = subprocess.Popen(['make', '-f',
693	os.path.join(src_path, makefile)],
694	stdin=subprocess.PIPE,
695	stdout=subprocess.PIPE,
696	stderr=subprocess.PIPE,
697	bufsize=1)
698	pout, perr = p.communicate()
699	print(pout.decode())
700	if p.returncode != 0:
701	print(perr.decode())
702	print('Please edit ' + makefile +
703	' or try another Makefile in the src directory.')
704	print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
705	'and EMOSLIB must be adapted.')
706	print('Available Makefiles:')
707	print(UioFiles(src_path, 'Makefile*'))
708	sys.exit('Compilation failed!')
709	except ValueError as e:
710	print('ERROR: Makefile call failed:')
711	print(e)
712	else:
713	execute_subprocess(['ls', '-l', os.path.join(src_path,
714	_config.FORTRAN_EXECUTABLE)], error_msg=
715	'FORTRAN EXECUTABLE COULD NOT BE FOUND!')
716
717	return
718
719
720	if __name__ == "__main__":
721	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: