Context Navigation

source: flex_extract.git/Source/Python/install.py @ 47be2684

ctbtodev

Last change on this file since 47be2684 was 47be2684, checked in by Leopold Haimberger <leopold.haimberger@…>, 4 years ago
Adaptations to allow for a system installation with separate user and system path. Updated documentation
Property mode set to `100755`
File size: 29.4 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - moved install_args_and_control in here
14	# - splitted code in smaller functions
15	# - delete fortran build files in here instead of compile job script
16	# - changed static path names to variables from config file
17	# - splitted install function into several smaller pieces
18	# - use of tarfile package in python
19	# June 2020 - Anne Philipp
20	# - renamed "convert" functions to "fortran" functions
21	# - reconfigured mk_tarball to select *.template files instead
22	# of .nl and .temp
23	# - added check for makefile settings
24	#
25	# @License:
26	# (C) Copyright 2014-2020.
27	# Anne Philipp, Leopold Haimberger
28	#
29	# SPDX-License-Identifier: CC-BY-4.0
30	#
31	# This work is licensed under the Creative Commons Attribution 4.0
32	# International License. To view a copy of this license, visit
33	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
34	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
35	#
36	# @Methods:
37	# main
38	# get_install_cmdline_args
39	# install_via_gateway
40	# check_install_conditions
41	# mk_tarball
42	# un_tarball
43	# mk_env_vars
44	# mk_compilejob
45	# mk_job_template
46	# del_fortran_build
47	# mk_fortran_build
48	#
49	#*******************************************************************************
50	'''This script installs the flex_extract program.
51
52	Depending on the selected installation environment (locally or on the
53	ECMWF server ecgate or cca) the program extracts the command line
54	arguments and the CONTROL file parameter and prepares the corresponding
55	environment.
56	The necessary files are collected in a tar ball and placed
57	at the target location. There, is is untared, the environment variables are
58	set, and the Fortran code is compiled.
59	If the ECMWF environment is selected, a job script is prepared and submitted
60	for the remaining configurations after putting the tar ball on the
61	target ECMWF server.
62
63	Type: install.py --help
64	to get information about command line parameters.
65	Read the documentation for usage instructions.
66	'''
67
68	# ------------------------------------------------------------------------------
69	# MODULES
70	# ------------------------------------------------------------------------------
71	from __future__ import print_function
72
73	import os
74	import sys
75	import subprocess
76	import tarfile
77	import shutil
78	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
79
80	# software specific classes and modules from flex_extract
81	import _config
82	from Classes.ControlFile import ControlFile
83	from Classes.UioFiles import UioFiles
84	from Mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
85	silent_remove, execute_subprocess, none_or_str,
86	overwrite_lines_in_file, check_for_string_in_file)
87
88	# ------------------------------------------------------------------------------
89	# FUNCTIONS
90	# ------------------------------------------------------------------------------
91	def main():
92	'''Controls the installation process. Calls the installation function
93	if target is specified.
94
95	Parameters
96	----------
97
98	Return
99	------
100	'''
101
102	args = get_install_cmdline_args()
103	c = ControlFile(args.controlfile)
104	c.assign_args_to_control(args)
105	check_install_conditions(c)
106
107	if c.install_target.lower() not in ['local', 'syslocal']: # ecgate or cca
108	install_via_gateway(c)
109	else: # local
110	install_local(c)
111
112	print("SUCCESS: INSTALLATION FINISHED!")
113
114	return
115
116	def get_install_cmdline_args():
117	'''Decomposes the command line arguments and assigns them to variables.
118	Apply default values for arguments not present.
119
120	Parameters
121	----------
122
123	Return
124	------
125	args : Namespace
126	Contains the commandline arguments from script/program call.
127	'''
128	parser = ArgumentParser(description='Install flex_extract software '
129	'locally or on ECMWF machines',
130	formatter_class=ArgumentDefaultsHelpFormatter)
131
132	parser.add_argument('--target', dest='install_target',
133	type=none_or_str, default=None,
134	help="Valid targets: syslocal \| local \| ecgate \| cca , \
135	the latter two are at ECMWF")
136	parser.add_argument("--makefile", dest="makefile",
137	type=none_or_str, default=None,
138	help='Name of makefile for compiling the '
139	'Fortran program')
140	parser.add_argument("--ecuid", dest="ecuid",
141	type=none_or_str, default=None,
142	help='User id at ECMWF')
143	parser.add_argument("--ecgid", dest="ecgid",
144	type=none_or_str, default=None,
145	help='Group id at ECMWF')
146	parser.add_argument("--gateway", dest="gateway",
147	type=none_or_str, default=None,
148	help='Name of the local gateway server')
149	parser.add_argument("--destination", dest="destination",
150	type=none_or_str, default=None,
151	help='ecaccess association, e.g. '
152	'myUser@genericSftp')
153
154	parser.add_argument("--installdir", dest="installdir",
155	type=none_or_str, default=None,
156	help='Root (user) directory of the '
157	'flex_extract installation')
158	parser.add_argument("--sysinstalldir", dest="sysinstalldir",
159	type=none_or_str, default=None,
160	help='System installation path; where '
161	'executables are stored.')
162
163	# arguments for job submission to ECMWF, only needed by submit.py
164	parser.add_argument("--job_template", dest='job_template',
165	type=none_or_str, default="job.template",
166	help='Rudimentary template file to create a batch '
167	'job template for submission to ECMWF servers')
168
169	parser.add_argument("--controlfile", dest="controlfile",
170	type=none_or_str, default='CONTROL_EA5',
171	help="A file that contains all CONTROL parameters.")
172
173	args = parser.parse_args()
174
175	return args
176
177
178	def install_via_gateway(c):
179	'''Prepare data transfer to remote gateway and submit a job script which will
180	install everything on the remote gateway.
181
182	Parameters
183	----------
184	c : ControlFile
185	Contains all the parameters of CONTROL file and
186	command line.
187
188	Return
189	------
190
191	'''
192
193	tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
194	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
195
196	mk_compilejob(c.makefile, c.ecuid, c.ecgid, c.installdir)
197
198	mk_job_template(c.ecuid, c.ecgid, c.installdir)
199
200	mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
201
202	mk_tarball(tar_file, c.install_target)
203
204	put_file_to_ecserver(_config.PATH_FLEXEXTRACT_DIR, tarball_name,
205	c.install_target, c.ecuid, c.ecgid)
206
207	submit_job_to_ecserver(c.install_target,
208	os.path.join(_config.PATH_REL_JOBSCRIPTS,
209	_config.FILE_INSTALL_COMPILEJOB))
210
211	silent_remove(tar_file)
212
213	print('Job compilation script has been submitted to ecgate for ' +
214	'installation in ' + c.installdir +
215	'/' + _config.FLEXEXTRACT_DIRNAME)
216	print('You should get an email with subject "flexcompile" within ' +
217	'the next few minutes!')
218
219	return
220
221	def install_local(c):
222	'''Perform the actual installation on a local machine.
223
224	Parameters
225	----------
226	c : ControlFile
227	Contains all the parameters of CONTROL file and
228	command line.
229
230	Return
231	------
232
233	'''
234
235	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR,
236	_config.FLEXEXTRACT_DIRNAME + '.tar')
237
238	c.installdir = os.path.abspath(os.path.expandvars(os.path.expanduser(
239	c.installdir)))
240	c.sysinstalldir = os.path.abspath(os.path.expandvars(os.path.expanduser(
241	c.sysinstalldir)))
242
243	# this is standard installation into a single directory
244	if c.install_target == 'local':
245
246	# installation into the current directory
247	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.installdir:
248	print('WARNING: installdir has not been specified')
249	print('flex_extract will be installed in current dir by compiling the ' +
250	'Fortran source in ' + _config.PATH_FORTRAN_SRC)
251	os.chdir(_config.PATH_FORTRAN_SRC)
252	# installation into a different path
253	elif os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) != c.installdir :
254
255	# creates the target working directory for flex_extract
256	mk_tarball(tar_file, c.install_target)
257	make_dir(os.path.join(c.installdir,
258	_config.FLEXEXTRACT_DIRNAME))
259	os.chdir(os.path.join(c.installdir,
260	_config.FLEXEXTRACT_DIRNAME))
261	un_tarball(tar_file)
262	os.chdir(os.path.join(c.installdir,
263	_config.FLEXEXTRACT_DIRNAME,
264	_config.PATH_REL_FORTRAN_SRC))
265
266	# Create Fortran executable
267	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software at ' +
268	c.install_target + ' in directory ' + c.installdir + '\n')
269
270	del_fortran_build('.')
271	mk_fortran_build('.', c.makefile)
272	os.chdir('../../')
273	# make sure that the correct calling of submit.py script is in run_local.sh
274	overwrite_lines_in_file('Run/run_local.sh',
275	'pyscript=', 'pyscript=../Source/Python/submit.py\n')
276
277	# this is system installation were executables and user files are separated
278	elif c.install_target == 'syslocal':
279	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.sysinstalldir :
280	sys.exit('ERROR: System installation path is equal to user '
281	'installation path.\n Please change either the system '
282	'installation path or use installation target "local".')
283	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.installdir :
284	print('Flex_extract will be installed in current directory!')
285	else: # install user part in different dir
286	print('Flex_extract will be installed in ' + c.installdir )
287
288	c.installdir = os.path.join(c.installdir,_config.FLEXEXTRACT_DIRNAME)
289	if os.path.isdir(c.installdir):
290	shutil.rmtree(c.installdir)
291
292	# copy all files except Python and Fortranfiles to this dir
293	shutil.copytree(_config.PATH_FLEXEXTRACT_DIR,
294	c.installdir, symlinks=True)
295	shutil.rmtree(os.path.join(c.installdir,'Source'))
296	shutil.rmtree(os.path.join(c.installdir,'.git'))
297	for x in UioFiles(c.installdir, '*~').files:
298	silent_remove(x)
299
300	os.remove(os.path.join(c.installdir,'setup.sh'))
301	os.remove(os.path.join(c.installdir,'setup_local.sh'))
302
303	# configure run_local script correctly
304	# check if source of system config file is already in run_local.sh,
305	# if not, add it
306	if not check_for_string_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
307	'source .setup.rc'):
308	overwrite_lines_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
309	'# PATH TO SUBMISSION SCRIPT',
310	'# PATH TO SUBMISSION SCRIPT\nsource '+_config.FILE_SYS_CONFIG+'\n')
311	# make sure that the correct calling of submit.py script is in run_local.sh
312	overwrite_lines_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
313	'pyscript=', 'pyscript=submit.py\n')
314
315	# change permission for file to executable
316	execute_subprocess(['chmod', '0775',
317	os.path.join(os.path.abspath(c.installdir),'Run/run_local.sh')])
318
319
320	# create systemdir
321	c.sysinstalldir = os.path.join(c.sysinstalldir,_config.FLEXEXTRACT_DIRNAME)
322	if os.path.isdir(c.sysinstalldir):
323	shutil.rmtree(c.sysinstalldir)
324
325	# create setup file for running flex_extract with system installation
326	with open(os.path.join(os.path.abspath(c.installdir),'Run/.setup.rc'),'w') as fio:
327	fio.write('#!/bin/bash \n')
328	fio.write('export FLEXEXTRACT_USER_DIR='+os.path.abspath(c.installdir)+'\n')
329	fio.write('export PATH='+os.path.abspath(c.sysinstalldir)+'/Python:${PATH}\n')
330	fio.write('export PATH='+os.path.abspath(c.sysinstalldir)+':${PATH}\n')
331
332	# copy all Python and Fortranfiles to this dir
333	shutil.copytree(_config.PATH_SOURCES, c.sysinstalldir, symlinks=True)
334
335	os.chdir(os.path.join(c.sysinstalldir,'Fortran'))
336	# Create Fortran executable
337	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software as ' +
338	c.install_target + ' in directory ' +
339	os.path.abspath(c.sysinstalldir) + '\n')
340
341	del_fortran_build('.')
342	mk_fortran_build('.', c.makefile)
343
344	outfile = [x for x in UioFiles('.','*.out').files]
345	test=os.path.join(c.sysinstalldir,'calc_etadot')
346	if len(outfile) != 1:
347	print('WARNING: Multiple executables for Fortran code are available!')
348	# move executable one dir up and delete Fortran dir
349	os.chdir('..')
350	shutil.move(outfile[0], os.path.join(c.sysinstalldir,'calc_etadot'))
351	shutil.rmtree(os.path.join(os.path.abspath(c.sysinstalldir),'Fortran'))
352
353	os.chdir(_config.PATH_FLEXEXTRACT_DIR)
354	if os.path.isfile(tar_file):
355	os.remove(tar_file)
356
357	return
358
359
360	def check_install_conditions(c):
361	'''Checks necessary attributes and conditions
362	for the installation, e.g. whether they exist and contain values.
363	Otherwise set default values.
364
365	Parameters
366	----------
367	c : ControlFile
368	Contains all the parameters of CONTROL file and
369	command line.
370
371
372	Return
373	------
374
375	'''
376
377	if c.install_target and \
378	c.install_target not in _config.INSTALL_TARGETS:
379	print('ERROR: unknown or missing installation target ')
380	print('target: ', c.install_target)
381	print('please specify correct installation target ' +
382	str(_config.INSTALL_TARGETS))
383	print('use -h or --help for help')
384	sys.exit(1)
385
386	if c.install_target and c.install_target not in ['local', 'syslocal']:
387	if not c.ecgid or not c.ecuid:
388	print('Please enter your ECMWF user id and group id '
389	' with command line options --ecuid --ecgid')
390	print('Try "' + sys.argv[0].split('/')[-1] + \
391	' -h" to print usage information')
392	print('Please consult ecaccess documentation or ECMWF user '
393	'support for further details.\n')
394	sys.exit(1)
395	if not c.gateway or not c.destination:
396	print('WARNING: Parameters GATEWAY and DESTINATION were '
397	'not properly set for working on ECMWF server. \n'
398	'There will be no transfer of output files to the '
399	'local gateway server possible!')
400	if not c.installdir:
401	c.installdir = '${HOME}'
402	elif c.install_target == 'local':
403	if not c.installdir:
404	c.installdir = _config.PATH_FLEXEXTRACT_DIR
405	elif c.install_target == 'syslocal':
406	if not c.installdir:
407	c.installdir = _config.PATH_FLEXEXTRACT_DIR
408	if not c.sysinstalldir:
409	print('ERROR: System installation was selected but '
410	'no system installation path was defined.')
411	sys.exit()
412
413	if not c.makefile:
414	print('WARNING: no makefile was specified.')
415	if c.install_target == 'local':
416	c.makefile = 'makefile_local_gfortran'
417	print('WARNING: default makefile selected: makefile_local_gfortan')
418	elif c.install_target == 'ecgate':
419	c.makefile = 'makefile_ecgate'
420	print('WARNING: default makefile selected: makefile_ecgate')
421	elif c.install_target == 'cca' or \
422	c.install_target == 'ccb':
423	c.makefile = 'makefile_cray'
424	print('WARNING: default makefile selected: makefile_cray')
425	else:
426	pass
427
428	return
429
430
431	def mk_tarball(tarball_path, target):
432	'''Creates a tarball with all necessary files which need to be sent to the
433	installation directory.
434	It does not matter whether this is local or remote.
435	Collects all Python files, the Fortran source and makefiles,
436	the ECMWF_ENV file, the CONTROL files as well as the
437	template files.
438
439	Parameters
440	----------
441	tarball_path : str
442	The complete path to the tar file which will contain all
443	relevant data for flex_extract.
444
445	target : str
446	The queue where the job is submitted to.
447
448	Return
449	------
450
451	'''
452
453	print('Create tarball ...')
454
455	# change to FLEXEXTRACT directory so that the tar can contain
456	# relative pathes to the files and directories
457	ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
458	os.chdir(ecd)
459
460	# get lists of the files to be added to the tar file
461	if target == 'local':
462	ecmwf_env_file = []
463	runfile = [os.path.relpath(x, ecd)
464	for x in UioFiles(_config.PATH_REL_RUN_DIR,
465	'run_local.sh').files]
466	else:
467	ecmwf_env_file = [_config.PATH_REL_ECMWF_ENV]
468	runfile = [os.path.relpath(x, ecd)
469	for x in UioFiles(_config.PATH_REL_RUN_DIR,
470	'run.sh').files]
471
472	pyfiles = [os.path.relpath(x, ecd)
473	for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
474	pytestfiles = [os.path.relpath(x, ecd)
475	for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
476	controlfiles = [os.path.relpath(x, ecd)
477	for x in UioFiles(_config.PATH_REL_CONTROLFILES,
478	'CONTROL*').files]
479	testfiles = [os.path.relpath(x, ecd)
480	for x in UioFiles(_config.PATH_REL_TEST+"/Installation", '*').files]
481	tempfiles = [os.path.relpath(x, ecd)
482	for x in UioFiles(_config.PATH_REL_TEMPLATES, '*.template').files]
483	gribtable = [os.path.relpath(x, ecd)
484	for x in UioFiles(_config.PATH_REL_TEMPLATES, 'grib').files]
485	ffiles = [os.path.relpath(x, ecd)
486	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f90').files]
487	hfiles = [os.path.relpath(x, ecd)
488	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
489	makefiles = [os.path.relpath(x, ecd)
490	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'makefile*').files]
491	jobdir = [os.path.relpath(x, ecd)
492	for x in UioFiles(_config.PATH_REL_JOBSCRIPTS, '*.md').files]
493
494	# concatenate single lists to one for a better looping
495	filelist = pyfiles + pytestfiles + controlfiles + tempfiles + \
496	ffiles + gribtable + hfiles + makefiles + ecmwf_env_file + \
497	runfile + jobdir + testfiles +\
498	['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
499
500	# create installation tar-file
501	exclude_files = [".ksh", ".tar"]
502	try:
503	with tarfile.open(tarball_path, "w:gz") as tar_handle:
504	for filename in filelist:
505	tar_handle.add(filename, recursive=False,
506	filter=lambda tarinfo: None
507	if os.path.splitext(tarinfo.name)[1]
508	in exclude_files
509	else tarinfo)
510	except tarfile.TarError as e:
511	print('... ERROR: ' + str(e))
512
513	sys.exit('\n... error occured while trying to create the tar-file ' +
514	str(tarball_path))
515
516	return
517
518
519	def un_tarball(tarball_path):
520	'''Extracts the given tarball into current directory.
521
522	Parameters
523	----------
524	tarball_path : str
525	The complete path to the tar file which will contain all
526	relevant data for flex_extract.
527
528	Return
529	------
530
531	'''
532
533	print('Untar ...')
534
535	try:
536	with tarfile.open(tarball_path) as tar_handle:
537	tar_handle.extractall()
538	except tarfile.TarError as e:
539	sys.exit('\n... error occured while trying to read tar-file ' +
540	str(tarball_path))
541	except OSError as e:
542	print('... ERROR CODE: ' + str(e.errno))
543	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
544
545	sys.exit('\n... error occured while trying to read tar-file ' +
546	str(tarball_path))
547
548	return
549
550	def mk_env_vars(ecuid, ecgid, gateway, destination):
551	'''Creates a file named ECMWF_ENV which contains the
552	necessary environmental variables at ECMWF servers.
553	It is based on the template ECMWF_ENV.template.
554
555	Parameters
556	----------
557	ecuid : str
558	The user id on ECMWF server.
559
560	ecgid : str
561	The group id on ECMWF server.
562
563	gateway : str
564	The gateway server the user is using.
565
566	destination : str
567	The remote destination which is used to transfer files
568	from ECMWF server to local gateway server.
569
570	Return
571	------
572
573	'''
574	from genshi.template.text import NewTextTemplate
575	from genshi.template import TemplateLoader
576	from genshi.template.eval import UndefinedError
577
578	try:
579	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
580	ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
581	cls=NewTextTemplate)
582
583	stream = ecmwfvars_template.generate(user_name=ecuid,
584	user_group=ecgid,
585	gateway_name=gateway,
586	destination_name=destination
587	)
588	except UndefinedError as e:
589	print('... ERROR ' + str(e))
590
591	sys.exit('\n... error occured while trying to generate template ' +
592	_config.PATH_ECMWF_ENV)
593	except OSError as e:
594	print('... ERROR CODE: ' + str(e.errno))
595	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
596
597	sys.exit('\n... error occured while trying to generate template ' +
598	_config.PATH_ECMWF_ENV)
599
600	try:
601	with open(_config.PATH_ECMWF_ENV, 'w') as f:
602	f.write(stream.render('text'))
603	except OSError as e:
604	print('... ERROR CODE: ' + str(e.errno))
605	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
606
607	sys.exit('\n... error occured while trying to write ' +
608	_config.PATH_ECMWF_ENV)
609
610	return
611
612	def mk_compilejob(makefile, ecuid, ecgid, fp_root):
613	'''Modifies the original job template file so that it is specified
614	for the user and the environment were it will be applied. Result
615	is stored in a new file "job.temp" in the python directory.
616
617	Parameters
618	----------
619	makefile : str
620	Name of the makefile which should be used to compile the Fortran
621	program.
622
623	ecuid : str
624	The user id on ECMWF server.
625
626	ecgid : str
627	The group id on ECMWF server.
628
629	fp_root : str
630	Path to the root directory of FLEXPART environment or flex_extract
631	environment.
632
633	Return
634	------
635
636	'''
637	from genshi.template.text import NewTextTemplate
638	from genshi.template import TemplateLoader
639	from genshi.template.eval import UndefinedError
640
641	if fp_root == '../':
642	fp_root = '$HOME'
643
644	try:
645	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
646	compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
647	cls=NewTextTemplate)
648
649	stream = compile_template.generate(
650	usergroup=ecgid,
651	username=ecuid,
652	version_number=_config._VERSION_STR,
653	fp_root_scripts=fp_root,
654	makefile=makefile,
655	fortran_program=_config.FORTRAN_EXECUTABLE
656	)
657	except UndefinedError as e:
658	print('... ERROR ' + str(e))
659
660	sys.exit('\n... error occured while trying to generate template ' +
661	_config.TEMPFILE_INSTALL_COMPILEJOB)
662	except OSError as e:
663	print('... ERROR CODE: ' + str(e.errno))
664	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
665
666	sys.exit('\n... error occured while trying to generate template ' +
667	_config.TEMPFILE_INSTALL_COMPILEJOB)
668
669	try:
670	compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
671	_config.FILE_INSTALL_COMPILEJOB)
672
673	with open(compilejob, 'w') as f:
674	f.write(stream.render('text'))
675	except OSError as e:
676	print('... ERROR CODE: ' + str(e.errno))
677	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
678
679	sys.exit('\n... error occured while trying to write ' +
680	compilejob)
681
682	return
683
684	def mk_job_template(ecuid, ecgid, fp_root):
685	'''Modifies the original job template file so that it is specified
686	for the user and the environment were it will be applied. Result
687	is stored in a new file.
688
689	Parameters
690	----------
691	ecuid : str
692	The user id on ECMWF server.
693
694	ecgid : str
695	The group id on ECMWF server.
696
697	fp_root : str
698	Path to the root directory of FLEXPART environment or flex_extract
699	environment.
700
701	Return
702	------
703
704	'''
705	from genshi.template.text import NewTextTemplate
706	from genshi.template import TemplateLoader
707	from genshi.template.eval import UndefinedError
708
709	fp_root_path_to_python = os.path.join(fp_root,
710	_config.FLEXEXTRACT_DIRNAME,
711	_config.PATH_REL_PYTHON_SRC)
712	if '$' in fp_root_path_to_python:
713	ind = fp_root_path_to_python.index('$')
714	fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
715	fp_root_path_to_python[ind:]
716
717	try:
718	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
719	compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
720	cls=NewTextTemplate)
721
722	stream = compile_template.generate(
723	usergroup=ecgid,
724	username=ecuid,
725	version_number=_config._VERSION_STR,
726	fp_root_path=fp_root_path_to_python,
727	)
728	except UndefinedError as e:
729	print('... ERROR ' + str(e))
730
731	sys.exit('\n... error occured while trying to generate template ' +
732	_config.TEMPFILE_INSTALL_JOB)
733	except OSError as e:
734	print('... ERROR CODE: ' + str(e.errno))
735	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
736
737	sys.exit('\n... error occured while trying to generate template ' +
738	_config.TEMPFILE_INSTALL_JOB)
739
740
741	try:
742	tempjobfile = os.path.join(_config.PATH_TEMPLATES,
743	_config.TEMPFILE_JOB)
744
745	with open(tempjobfile, 'w') as f:
746	f.write(stream.render('text'))
747	except OSError as e:
748	print('... ERROR CODE: ' + str(e.errno))
749	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
750
751	sys.exit('\n... error occured while trying to write ' +
752	tempjobfile)
753
754	return
755
756	def del_fortran_build(src_path):
757	'''Clean up the Fortran source directory and remove all
758	build files (e.g. \.o, \.mod and FORTRAN EXECUTABLE)
759
760	Parameters
761	----------
762	src_path : str
763	Path to the fortran source directory.
764
765	Return
766	------
767
768	'''
769
770	modfiles = UioFiles(src_path, '*.mod')
771	objfiles = UioFiles(src_path, '*.o')
772	exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
773
774	modfiles.delete_files()
775	objfiles.delete_files()
776	exefile.delete_files()
777
778	return
779
780	def mk_fortran_build(src_path, makefile):
781	'''Compiles the Fortran code and generates the executable.
782
783	Parameters
784	----------
785	src_path : str
786	Path to the fortran source directory.
787
788	makefile : str
789	The name of the makefile which should be used.
790
791	Return
792	------
793
794	'''
795
796	try:
797	print('Using makefile: ' + makefile)
798	p = subprocess.Popen(['make', '-f',
799	os.path.join(src_path, makefile)],
800	stdin=subprocess.PIPE,
801	stdout=subprocess.PIPE,
802	stderr=subprocess.PIPE,
803	bufsize=1)
804	pout, perr = p.communicate()
805	print(pout.decode())
806	if p.returncode != 0:
807	print(perr.decode())
808	print('Please edit ' + makefile +
809	' or try another makefile in the src directory.')
810	print('Most likely ECCODES_INCLUDE_DIR, ECCODES_LIB '
811	'and EMOSLIB must be adapted.')
812	print('Available makefiles:')
813	print(UioFiles(src_path, 'makefile*'))
814	sys.exit('Compilation failed!')
815	except ValueError as e:
816	print('ERROR: makefile call failed:')
817	print(e)
818	else:
819	execute_subprocess(['ls', '-l',
820	os.path.join(src_path, _config.FORTRAN_EXECUTABLE)],
821	error_msg='FORTRAN EXECUTABLE COULD NOT BE FOUND!')
822
823	return
824
825
826	if __name__ == "__main__":
827	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: