Context Navigation

source: flex_extract.git/source/python/install.py @ 1eca806

ctbtodev

Last change on this file since 1eca806 was 1eca806, checked in by Anne Philipp <anne.philipp@…>, 5 years ago
added test directory to installation tar file
Property mode set to `100755`
File size: 23.4 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - moved install_args_and_control in here
14	# - splitted code in smaller functions
15	# - delete convert build files in here instead of compile job script
16	# - changed static path names to Variables from config file
17	#
18	# @License:
19	# (C) Copyright 2015-2018.
20	#
21	# This software is licensed under the terms of the Apache Licence Version 2.0
22	# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
23	#
24	# @Program Functionality:
25	# Depending on the selected installation environment (locally or on the
26	# ECMWF server ecgate or cca) the program extracts the commandline
27	# arguments and the CONTROL file parameter and prepares the corresponding
28	# environment. The necessary files are collected in a tar-ball and placed
29	# at the target location. There its untared, the environment variables will
30	# be set and the Fortran code will be compiled. If the ECMWF environment is
31	# selected a job script is prepared and submitted for the remaining
32	# configurations after putting the tar-ball to the target ECMWF server.
33	#
34	# @Program Content:
35	# - main
36	# - get_install_cmdline_args
37	# - install_via_gateway
38	# - mk_tarball
39	# - un_tarball
40	# - mk_env_vars
41	# - mk_compilejob
42	# - mk_job_template
43	# - del_convert_build
44	# - mk_convert_build
45	#
46	#*******************************************************************************
47
48	# ------------------------------------------------------------------------------
49	# MODULES
50	# ------------------------------------------------------------------------------
51	import os
52	import sys
53	import glob
54	import subprocess
55	import inspect
56	import tarfile
57	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
58
59	# software specific classes and modules from flex_extract
60	import _config
61	from classes.ControlFile import ControlFile
62	from classes.UioFiles import UioFiles
63	from mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
64	silent_remove, execute_subprocess)
65
66	# ------------------------------------------------------------------------------
67	# FUNCTIONS
68	# ------------------------------------------------------------------------------
69	def main():
70	'''Controls the installation process. Calls the installation function
71	if target is specified.
72
73	Parameters
74	----------
75
76	Return
77	------
78	'''
79
80	args = get_install_cmdline_args()
81	c = ControlFile(args.controlfile)
82	c.assign_args_to_control(args)
83	check_install_conditions(c)
84
85	install_via_gateway(c)
86
87	return
88
89	def get_install_cmdline_args():
90	'''Decomposes the command line arguments and assigns them to variables.
91	Apply default values for non mentioned arguments.
92
93	Parameters
94	----------
95
96	Return
97	------
98	args : :obj:`Namespace`
99	Contains the commandline arguments from script/program call.
100	'''
101	parser = ArgumentParser(description='Install flex_extract software locally or \
102	on ECMWF machines',
103	formatter_class=ArgumentDefaultsHelpFormatter)
104
105	parser.add_argument('--target', dest='install_target', default=None,
106	help="Valid targets: local \| ecgate \| cca , \
107	the latter two are at ECMWF")
108	parser.add_argument("--makefile", dest="makefile", default=None,
109	help='Name of Makefile to use for compiling CONVERT2')
110	parser.add_argument("--ecuid", dest="ecuid", default=None,
111	help='user id at ECMWF')
112	parser.add_argument("--ecgid", dest="ecgid", default=None,
113	help='group id at ECMWF')
114	parser.add_argument("--gateway", dest="gateway", default=None,
115	help='name of local gateway server')
116	parser.add_argument("--destination", dest="destination", default=None,
117	help='ecaccess destination, e.g. leo@genericSftp')
118
119	parser.add_argument("--flexpartdir", dest="flexpartdir",
120	default=None, help="FLEXPART root directory on ECMWF \
121	servers (to find grib2flexpart and COMMAND file)\n\
122	Normally flex_extract resides in the scripts directory \
123	of the FLEXPART distribution.")
124
125	# arguments for job submission to ECMWF, only needed by submit.py
126	parser.add_argument("--job_template", dest='job_template',
127	default="job.temp.o",
128	help="job template file for submission to ECMWF")
129
130	parser.add_argument("--controlfile", dest="controlfile",
131	default='CONTROL.temp',
132	help="file with CONTROL parameters")
133
134	args = parser.parse_args()
135
136	return args
137
138
139	def install_via_gateway(c):
140	'''Perform the actual installation on local machine or prepare data
141	transfer to remote gate and submit a job script which will
142	install everything on the remote gate.
143
144	Parameters
145	----------
146	c : :obj:`ControlFile`
147	Contains all the parameters of CONTROL file and
148	command line.
149
150	Return
151	------
152
153	'''
154	import tarfile
155
156	ecd = _config.PATH_FLEXEXTRACT_DIR
157	tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
158	tar_file = os.path.join(ecd, tarball_name)
159
160	target_dirname = _config.FLEXEXTRACT_DIRNAME
161	fortran_executable = _config.FORTRAN_EXECUTABLE
162
163	if c.install_target.lower() != 'local': # ecgate or cca
164
165	mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
166	c.flexpartdir)
167
168	mk_job_template(c.ecuid, c.ecgid, c.gateway,
169	c.destination, c.flexpartdir)
170
171	mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
172
173	mk_tarball(tar_file, c.install_target)
174
175	put_file_to_ecserver(ecd, tarball_name, c.install_target,
176	c.ecuid, c.ecgid)
177
178	submit_job_to_ecserver(c.install_target,
179	os.path.join(_config.PATH_REL_JOBSCRIPTS,
180	_config.FILE_INSTALL_COMPILEJOB))
181
182	silent_remove(tar_file)
183
184	print('job compilation script has been submitted to ecgate for ' +
185	'installation in ' + c.flexpartdir +
186	'/' + target_dirname)
187	print('You should get an email with subject "flexcompile" within ' +
188	'the next few minutes!')
189
190	else: #local
191	if c.flexpartdir == _config.PATH_FLEXEXTRACT_DIR :
192	print('WARNING: FLEXPARTDIR has not been specified')
193	print('flex_extract will be installed in here by compiling the ' +
194	'Fortran source in ' + _config.PATH_FORTRAN_SRC)
195	os.chdir(_config.PATH_FORTRAN_SRC)
196	else: # creates the target working directory for flex_extract
197	c.flexpartdir = os.path.expandvars(os.path.expanduser(
198	c.flexpartdir))
199	if os.path.abspath(ecd) != os.path.abspath(c.flexpartdir):
200	mk_tarball(tar_file, c.install_target)
201	make_dir(os.path.join(c.flexpartdir,
202	target_dirname))
203	os.chdir(os.path.join(c.flexpartdir,
204	target_dirname))
205	un_tarball(tar_file)
206	os.chdir(os.path.join(c.flexpartdir,
207	target_dirname,
208	_config.PATH_REL_FORTRAN_SRC))
209
210	# Create Fortran executable - CONVERT2
211	print('Install ' + target_dirname + ' software at ' +
212	c.install_target + ' in directory ' +
213	os.path.abspath(c.flexpartdir) + '\n')
214
215	del_convert_build('.')
216	mk_convert_build('.', c.makefile)
217
218	os.chdir(ecd)
219	if os.path.isfile(tar_file):
220	os.remove(tar_file)
221
222	return
223
224	def check_install_conditions(c):
225	'''Checks a couple of necessary attributes and conditions
226	for the installation such as if they exist and contain values.
227	Otherwise set default values.
228
229	Parameters
230	----------
231	c : :obj:`ControlFile`
232	Contains all the parameters of CONTROL file and
233	command line.
234
235
236	Return
237	------
238
239	'''
240
241	if c.install_target and \
242	c.install_target not in _config.INSTALL_TARGETS:
243	print('ERROR: unknown or missing installation target ')
244	print('target: ', c.install_target)
245	print('please specify correct installation target ' +
246	str(INSTALL_TARGETS))
247	print('use -h or --help for help')
248	sys.exit(1)
249
250	if c.install_target and c.install_target != 'local':
251	if not c.ecgid or not c.ecuid or \
252	not c.gateway or not c.destination:
253	print('Please enter your ECMWF user id and group id as well ' +
254	'as the \nname of the local gateway and the ectrans ' +
255	'destination ')
256	print('with command line options --ecuid --ecgid \
257	--gateway --destination')
258	print('Try "' + sys.argv[0].split('/')[-1] + \
259	' -h" to print usage information')
260	print('Please consult ecaccess documentation or ECMWF user \
261	support for further details')
262	sys.exit(1)
263
264	if not c.flexpartdir:
265	c.flexpartdir = '${HOME}'
266	else:
267	c.flexpartdir = c.flexpartdir
268	else: # local
269	if not c.flexpartdir:
270	c.flexpartdir = _config.PATH_FLEXEXTRACT_DIR
271
272	return
273
274
275	def mk_tarball(tarball_path, target):
276	'''Creates a tarball with all necessary files which need to be sent to the
277	installation directory.
278	It does not matter if this is local or remote.
279	Collects all python files, the Fortran source and makefiles,
280	the ECMWF_ENV file, the CONTROL files as well as the
281	template files.
282
283	Parameters
284	----------
285	tarball_path : :obj:`string`
286	The complete path to the tar file which will contain all
287	relevant data for flex_extract.
288
289	target : :obj:`string`
290	The queue where the job is submitted to.
291
292	Return
293	------
294
295	'''
296	from glob import glob
297
298	print('Create tarball ...')
299
300	# change to FLEXEXTRACT directory so that the tar can contain
301	# relative pathes to the files and directories
302	ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
303	os.chdir(ecd)
304
305	# get lists of the files to be added to the tar file
306	if target == 'local':
307	ECMWF_ENV_FILE = []
308	runfile = [os.path.relpath(x, ecd)
309	for x in UioFiles(_config.PATH_REL_RUN_DIR,
310	'run_local.sh').files]
311	else:
312	ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
313	runfile = [os.path.relpath(x, ecd)
314	for x in UioFiles(_config.PATH_REL_RUN_DIR,
315	'run.sh').files]
316
317	pyfiles = [os.path.relpath(x, ecd)
318	for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
319	pytestfiles = [os.path.relpath(x, ecd)
320	for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
321	controlfiles = [os.path.relpath(x, ecd)
322	for x in UioFiles(_config.PATH_REL_CONTROLFILES,
323	'CONTROL*').files]
324	testfiles = [os.path.relpath(x, ecd)
325	for x in UioFiles(_config.PATH_REL_TEST , '*').files]
326	tempfiles = [os.path.relpath(x, ecd)
327	for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.temp').files]
328	nlfiles = [os.path.relpath(x, ecd)
329	for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.nl').files]
330	gribtable = [os.path.relpath(x, ecd)
331	for x in UioFiles(_config.PATH_REL_TEMPLATES , 'grib').files]
332	ffiles = [os.path.relpath(x, ecd)
333	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '.f').files]
334	hfiles = [os.path.relpath(x, ecd)
335	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
336	makefiles = [os.path.relpath(x, ecd)
337	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'Makefile*').files]
338	jobdir = [_config.PATH_REL_JOBSCRIPTS]
339
340	# concatenate single lists to one for a better looping
341	filelist = pyfiles + pytestfiles + controlfiles + tempfiles + nlfiles + \
342	ffiles + gribtable + hfiles + makefiles + ECMWF_ENV_FILE + \
343	runfile + jobdir + testfiles +\
344	['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
345
346	# create installation tar-file
347	exclude_files = [".ksh"]
348	try:
349	with tarfile.open(tarball_path, "w:gz") as tar_handle:
350	for file in filelist:
351	tar_handle.add(file, recursive=False,
352	filter=lambda tarinfo: None
353	if os.path.splitext(tarinfo.name)[1]
354	in exclude_files
355	else tarinfo)
356	except subprocess.CalledProcessError as e:
357	print('... ERROR CODE:\n ... ' + str(e.returncode))
358	print('... ERROR MESSAGE:\n ... ' + str(e))
359
360	sys.exit('\n... could not make installation tar ball!')
361	except OSError as e:
362	print('... ERROR CODE: ' + str(e.errno))
363	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
364
365	sys.exit('\n... error occured while trying to read tar-file ' +
366	str(tarball_path))
367
368	return
369
370
371	def un_tarball(tarball_path):
372	'''Extracts the given tarball into current directory.
373
374	Parameters
375	----------
376	tarball_path : :obj:`string`
377	The complete path to the tar file which will contain all
378	relevant data for flex_extract.
379
380	Return
381	------
382
383	'''
384
385	print('Untar ...')
386
387	try:
388	with tarfile.open(tarball_path) as tar_handle:
389	tar_handle.extractall()
390	except tarfile.TarError as e:
391	sys.exit('\n... error occured while trying to read tar-file ' +
392	str(tarball_path))
393	except OSError as e:
394	print('... ERROR CODE: ' + str(e.errno))
395	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
396
397	sys.exit('\n... error occured while trying to read tar-file ' +
398	str(tarball_path))
399
400	return
401
402	def mk_env_vars(ecuid, ecgid, gateway, destination):
403	'''Creates a file named ECMWF_ENV which contains the
404	necessary environmental variables at ECMWF servers.
405	It is based on the template ECMWF_ENV.template.
406
407	Parameters
408	----------
409	ecuid : :obj:`string`
410	The user id on ECMWF server.
411
412	ecgid : :obj:`string`
413	The group id on ECMWF server.
414
415	gateway : :obj:`string`
416	The gateway server the user is using.
417
418	destination : :obj:`string`
419	The remote destination which is used to transfer files
420	from ECMWF server to local gateway server.
421
422	Return
423	------
424
425	'''
426	from genshi.template.text import NewTextTemplate
427	from genshi.template import TemplateLoader
428	from genshi.template.eval import UndefinedError
429
430	try:
431	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
432	ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
433	cls=NewTextTemplate)
434
435	stream = ecmwfvars_template.generate(user_name = ecuid,
436	user_group = ecgid,
437	gateway_name = gateway,
438	destination_name = destination
439	)
440	except UndefinedError as e:
441	print('... ERROR ' + str(e))
442
443	sys.exit('\n... error occured while trying to generate template ' +
444	_config.PATH_ECMWF_ENV)
445	except OSError as e:
446	print('... ERROR CODE: ' + str(e.errno))
447	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
448
449	sys.exit('\n... error occured while trying to generate template ' +
450	_config.PATH_ECMWF_ENV)
451
452	try:
453	with open(_config.PATH_ECMWF_ENV, 'w') as f:
454	f.write(stream.render('text'))
455	except OSError as e:
456	print('... ERROR CODE: ' + str(e.errno))
457	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
458
459	sys.exit('\n... error occured while trying to write ' +
460	_config.PATH_ECMWF_ENV)
461
462	return
463
464	def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
465	'''Modifies the original job template file so that it is specified
466	for the user and the environment were it will be applied. Result
467	is stored in a new file "job.temp" in the python directory.
468
469	Parameters
470	----------
471	makefile : :obj:`string`
472	Name of the makefile which should be used to compile FORTRAN
473	CONVERT2 program.
474
475	target : :obj:`string`
476	The target where the installation should be done, e.g. the queue.
477
478	ecuid : :obj:`string`
479	The user id on ECMWF server.
480
481	ecgid : :obj:`string`
482	The group id on ECMWF server.
483
484	fp_root : :obj:`string`
485	Path to the root directory of FLEXPART environment or flex_extract
486	environment.
487
488	Return
489	------
490
491	'''
492	from genshi.template.text import NewTextTemplate
493	from genshi.template import TemplateLoader
494	from genshi.template.eval import UndefinedError
495
496	if fp_root == '../':
497	fp_root = '$HOME'
498
499	try:
500	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
501	compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
502	cls=NewTextTemplate)
503
504	stream = compile_template.generate(
505	usergroup = ecgid,
506	username = ecuid,
507	version_number = _config._VERSION_STR,
508	fp_root_scripts = fp_root,
509	makefile = makefile,
510	fortran_program = _config.FORTRAN_EXECUTABLE
511	)
512	except UndefinedError as e:
513	print('... ERROR ' + str(e))
514
515	sys.exit('\n... error occured while trying to generate template ' +
516	_config.TEMPFILE_INSTALL_COMPILEJOB)
517	except OSError as e:
518	print('... ERROR CODE: ' + str(e.errno))
519	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
520
521	sys.exit('\n... error occured while trying to generate template ' +
522	_config.TEMPFILE_INSTALL_COMPILEJOB)
523
524	try:
525	compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
526	_config.FILE_INSTALL_COMPILEJOB)
527
528	with open(compilejob, 'w') as f:
529	f.write(stream.render('text'))
530	except OSError as e:
531	print('... ERROR CODE: ' + str(e.errno))
532	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
533
534	sys.exit('\n... error occured while trying to write ' +
535	compilejob)
536
537	return
538
539	def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
540	'''Modifies the original job template file so that it is specified
541	for the user and the environment were it will be applied. Result
542	is stored in a new file.
543
544	Parameters
545	----------
546	ecuid : :obj:`string`
547	The user id on ECMWF server.
548
549	ecgid : :obj:`string`
550	The group id on ECMWF server.
551
552	gateway : :obj:`string`
553	The gateway server the user is using.
554
555	destination : :obj:`string`
556	The remote destination which is used to transfer files
557	from ECMWF server to local gateway server.
558
559	fp_root : :obj:`string`
560	Path to the root directory of FLEXPART environment or flex_extract
561	environment.
562
563	Return
564	------
565
566	'''
567	from genshi.template.text import NewTextTemplate
568	from genshi.template import TemplateLoader
569	from genshi.template.eval import UndefinedError
570
571	fp_root_path_to_python = os.path.join(fp_root,
572	_config.FLEXEXTRACT_DIRNAME,
573	_config.PATH_REL_PYTHON_SRC)
574	if '$' in fp_root_path_to_python:
575	ind = fp_root_path_to_python.index('$')
576	fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
577	fp_root_path_to_python[ind:]
578
579	try:
580	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
581	compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
582	cls=NewTextTemplate)
583
584	stream = compile_template.generate(
585	usergroup = ecgid,
586	username = ecuid,
587	version_number = _config._VERSION_STR,
588	fp_root_path = fp_root_path_to_python,
589	)
590	except UndefinedError as e:
591	print('... ERROR ' + str(e))
592
593	sys.exit('\n... error occured while trying to generate template ' +
594	_config.TEMPFILE_INSTALL_JOB)
595	except OSError as e:
596	print('... ERROR CODE: ' + str(e.errno))
597	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
598
599	sys.exit('\n... error occured while trying to generate template ' +
600	_config.TEMPFILE_INSTALL_JOB)
601
602
603	try:
604	tempjobfile = os.path.join(_config.PATH_TEMPLATES,
605	_config.TEMPFILE_JOB)
606
607	with open(tempjobfile, 'w') as f:
608	f.write(stream.render('text'))
609	except OSError as e:
610	print('... ERROR CODE: ' + str(e.errno))
611	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
612
613	sys.exit('\n... error occured while trying to write ' +
614	tempjobfile)
615
616	return
617
618	def del_convert_build(src_path):
619	'''Clean up the Fortran source directory and remove all
620	build files (e.g. \.o, \.mod and CONVERT2)
621
622	Parameters
623	----------
624	src_path : :obj:`string`
625	Path to the fortran source directory.
626
627	Return
628	------
629
630	'''
631
632	modfiles = UioFiles(src_path, '*.mod')
633	objfiles = UioFiles(src_path, '*.o')
634	exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
635
636	modfiles.delete_files()
637	objfiles.delete_files()
638	exefile.delete_files()
639
640	return
641
642	def mk_convert_build(src_path, makefile):
643	'''Compiles the Fortran code and generates the executable.
644
645	Parameters
646	----------
647	src_path : :obj:`string`
648	Path to the fortran source directory.
649
650	makefile : :obj:`string`
651	The name of the makefile which should be used.
652
653	Return
654	------
655
656	'''
657
658	try:
659	print('Using makefile: ' + makefile)
660	p = subprocess.Popen(['make', '-f',
661	os.path.join(src_path, makefile)],
662	stdin=subprocess.PIPE,
663	stdout=subprocess.PIPE,
664	stderr=subprocess.PIPE,
665	bufsize=1)
666	pout, perr = p.communicate()
667	print(pout)
668	if p.returncode != 0:
669	print(perr)
670	print('Please edit ' + makefile +
671	' or try another Makefile in the src directory.')
672	print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
673	'and EMOSLIB must be adapted.')
674	print('Available Makefiles:')
675	print(UioFiles(src_path, 'Makefile*'))
676	sys.exit('Compilation failed!')
677	except ValueError as e:
678	print('ERROR: Makefile call failed:')
679	print(e)
680	else:
681	execute_subprocess(['ls', '-l', os.path.join(src_path,
682	_config.FORTRAN_EXECUTABLE)], error_msg=
683	'FORTRAN EXECUTABLE COULD NOT BE FOUND!')
684
685	return
686
687
688	if __name__ == "__main__":
689	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: