Context Navigation

source: flex_extract.git/Source/Python/install.py @ b6ea29e

dev

Last change on this file since b6ea29e was b6ea29e, checked in by Anne Tipka <anne.tipka@…>, 22 months ago
Adaptations for Bologna servers in Python part
Property mode set to `100755`
File size: 29.6 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - moved install_args_and_control in here
14	# - splitted code in smaller functions
15	# - delete fortran build files in here instead of compile job script
16	# - changed static path names to variables from config file
17	# - splitted install function into several smaller pieces
18	# - use of tarfile package in python
19	# June 2020 - Anne Philipp
20	# - renamed "convert" functions to "fortran" functions
21	# - reconfigured mk_tarball to select *.template files instead
22	# of .nl and .temp
23	# - added check for makefile settings
24	# August 2020 - Leopold Haimberger
25	# - added a new installation section for system installation (if-else block)
26	# - read new argument from command line
27	# - write .setup.rc for a system installation into Run directory
28	# - copy executables to system path and user files to user path
29	#
30	# @License:
31	# (C) Copyright 2014-2020.
32	# Anne Philipp, Leopold Haimberger
33	#
34	# SPDX-License-Identifier: CC-BY-4.0
35	#
36	# This work is licensed under the Creative Commons Attribution 4.0
37	# International License. To view a copy of this license, visit
38	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
39	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
40	#
41	# @Methods:
42	# main
43	# get_install_cmdline_args
44	# install_via_gateway
45	# check_install_conditions
46	# mk_tarball
47	# un_tarball
48	# mk_env_vars
49	# mk_compilejob
50	# mk_job_template
51	# del_fortran_build
52	# mk_fortran_build
53	#
54	#*******************************************************************************
55	'''This script installs the flex_extract program.
56
57	Depending on the selected installation environment (locally or on the
58	ECMWF server ecgate or cca) the program extracts the command line
59	arguments and the CONTROL file parameter and prepares the corresponding
60	environment.
61	The necessary files are collected in a tar ball and placed
62	at the target location. There, is is untared, the environment variables are
63	set, and the Fortran code is compiled.
64	If the ECMWF environment is selected, a job script is prepared and submitted
65	for the remaining configurations after putting the tar ball on the
66	target ECMWF server.
67
68	Type: install.py --help
69	to get information about command line parameters.
70	Read the documentation for usage instructions.
71	'''
72
73	# ------------------------------------------------------------------------------
74	# MODULES
75	# ------------------------------------------------------------------------------
76	from __future__ import print_function
77
78	import os
79	import sys
80	import subprocess
81	import tarfile
82	import shutil
83	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
84
85	# software specific classes and modules from flex_extract
86	import _config
87	from Classes.ControlFile import ControlFile
88	from Classes.UioFiles import UioFiles
89	from Mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
90	silent_remove, execute_subprocess, none_or_str,
91	overwrite_lines_in_file, check_for_string_in_file)
92
93	# ------------------------------------------------------------------------------
94	# FUNCTIONS
95	# ------------------------------------------------------------------------------
96	def main():
97	'''Controls the installation process. Calls the installation function
98	if target is specified.
99
100	Parameters
101	----------
102
103	Return
104	------
105	'''
106
107	args = get_install_cmdline_args()
108	c = ControlFile(args.controlfile)
109	c.assign_args_to_control(args)
110	check_install_conditions(c)
111
112	if c.install_target.lower() not in ['local', 'syslocal']: # ecmwf servers e.g. ecgate (ecs) and hpc
113	install_via_gateway(c)
114	else: # local
115	install_local(c)
116
117	print("SUCCESS: INSTALLATION FINISHED!")
118
119	return
120
121	def get_install_cmdline_args():
122	'''Decomposes the command line arguments and assigns them to variables.
123	Apply default values for arguments not present.
124
125	Parameters
126	----------
127
128	Return
129	------
130	args : Namespace
131	Contains the commandline arguments from script/program call.
132	'''
133	parser = ArgumentParser(description='Install flex_extract software '
134	'locally or on ECMWF machines',
135	formatter_class=ArgumentDefaultsHelpFormatter)
136
137	parser.add_argument('--target', dest='install_target',
138	type=none_or_str, default=None,
139	help="Valid targets: syslocal \| local \| ecgate \| cca \| \
140	ccb \| ecs \| hpc , the latter 5 are at ECMWF servers")
141	parser.add_argument("--makefile", dest="makefile",
142	type=none_or_str, default=None,
143	help='Name of makefile for compiling the '
144	'Fortran program')
145	parser.add_argument("--ecuid", dest="ecuid",
146	type=none_or_str, default=None,
147	help='User id at ECMWF')
148	parser.add_argument("--ecgid", dest="ecgid",
149	type=none_or_str, default=None,
150	help='Group id at ECMWF')
151	parser.add_argument("--gateway", dest="gateway",
152	type=none_or_str, default=None,
153	help='Name of the local gateway server')
154	parser.add_argument("--destination", dest="destination",
155	type=none_or_str, default=None,
156	help='ecaccess association, e.g. '
157	'myUser@genericSftp')
158
159	parser.add_argument("--installdir", dest="installdir",
160	type=none_or_str, default=None,
161	help='Root (user) directory of the '
162	'flex_extract installation')
163	parser.add_argument("--sysinstalldir", dest="sysinstalldir",
164	type=none_or_str, default=None,
165	help='System installation path; where '
166	'executables are stored.')
167
168	# arguments for job submission to ECMWF, only needed by submit.py
169	parser.add_argument("--job_template", dest='job_template',
170	type=none_or_str, default="job.template",
171	help='Rudimentary template file to create a batch '
172	'job template for submission to ECMWF servers')
173
174	parser.add_argument("--controlfile", dest="controlfile",
175	type=none_or_str, default='CONTROL_EA5',
176	help="A file that contains all CONTROL parameters.")
177
178	args = parser.parse_args()
179
180	return args
181
182
183	def install_via_gateway(c):
184	'''Prepare data transfer to remote gateway and submit a job script which will
185	install everything on the remote gateway.
186
187	Parameters
188	----------
189	c : ControlFile
190	Contains all the parameters of CONTROL file and
191	command line.
192
193	Return
194	------
195
196	'''
197
198	tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
199	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
200
201	mk_compilejob(c.makefile, c.ecuid, c.installdir)
202
203	mk_job_template(c.ecuid, c.ecgid, c.installdir)
204
205	mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
206
207	mk_tarball(tar_file, c.install_target)
208
209	put_file_to_ecserver(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
210
211	submit_job_to_ecserver(c.install_target,
212	os.path.join(_config.PATH_REL_JOBSCRIPTS,
213	_config.FILE_INSTALL_COMPILEJOB))
214
215	silent_remove(tar_file)
216
217	print('Job compilation script has been submitted to ecgate for ' +
218	'installation in ' + c.installdir +
219	'/' + _config.FLEXEXTRACT_DIRNAME)
220	print('You should get an email with subject "flexcompile" within ' +
221	'the next few minutes!')
222
223	return
224
225	def install_local(c):
226	'''Perform the actual installation on a local machine.
227
228	Parameters
229	----------
230	c : ControlFile
231	Contains all the parameters of CONTROL file and
232	command line.
233
234	Return
235	------
236
237	'''
238
239	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR,
240	_config.FLEXEXTRACT_DIRNAME + '.tar')
241
242	# this is standard installation into a single directory
243	if c.install_target == 'local':
244	c.installdir = os.path.abspath(os.path.expandvars(os.path.expanduser(
245	c.installdir)))
246
247	# installation into the current directory
248	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.installdir:
249	print('WARNING: installdir has not been specified')
250	print('flex_extract will be installed in current dir by compiling the ' +
251	'Fortran source in ' + _config.PATH_FORTRAN_SRC)
252	os.chdir(_config.PATH_FORTRAN_SRC)
253	# installation into a different path
254	elif os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) != c.installdir :
255
256	# creates the target working directory for flex_extract
257	mk_tarball(tar_file, c.install_target)
258	make_dir(os.path.join(c.installdir,
259	_config.FLEXEXTRACT_DIRNAME))
260	os.chdir(os.path.join(c.installdir,
261	_config.FLEXEXTRACT_DIRNAME))
262	un_tarball(tar_file)
263	os.chdir(os.path.join(c.installdir,
264	_config.FLEXEXTRACT_DIRNAME,
265	_config.PATH_REL_FORTRAN_SRC))
266
267	# Create Fortran executable
268	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software at ' +
269	c.install_target + ' in directory ' + c.installdir + '\n')
270
271	del_fortran_build('.')
272	mk_fortran_build('.', c.makefile)
273	os.chdir('../../')
274	# make sure that the correct calling of submit.py script is in run_local.sh
275	overwrite_lines_in_file('Run/run_local.sh',
276	'pyscript=', 'pyscript=../Source/Python/submit.py\n')
277
278	# this is system installation were executables and user files are separated
279	elif c.install_target == 'syslocal':
280	c.sysinstalldir = os.path.abspath(os.path.expandvars(os.path.expanduser(
281	c.sysinstalldir)))
282	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.sysinstalldir :
283	sys.exit('ERROR: System installation path is equal to user '
284	'installation path.\n Please change either the system '
285	'installation path or use installation target "local".')
286	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.installdir :
287	print('Flex_extract will be installed in current directory!')
288	else: # install user part in different dir
289	print('Flex_extract will be installed in ' + c.installdir )
290
291	c.installdir = os.path.join(c.installdir,_config.FLEXEXTRACT_DIRNAME)
292	if os.path.isdir(c.installdir):
293	shutil.rmtree(c.installdir)
294
295	# copy all files except Python and Fortranfiles to this dir
296	shutil.copytree(_config.PATH_FLEXEXTRACT_DIR,
297	c.installdir, symlinks=True)
298	shutil.rmtree(os.path.join(c.installdir,'Source'))
299	shutil.rmtree(os.path.join(c.installdir,'.git'))
300	for x in UioFiles(c.installdir, '*~').files:
301	silent_remove(x)
302
303	os.remove(os.path.join(c.installdir,'setup.sh'))
304	os.remove(os.path.join(c.installdir,'setup_local.sh'))
305
306	# configure run_local script correctly
307	# check if source of system config file is already in run_local.sh,
308	# if not, add it
309	if not check_for_string_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
310	'source .setup.rc'):
311	overwrite_lines_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
312	'# PATH TO SUBMISSION SCRIPT',
313	'# PATH TO SUBMISSION SCRIPT\nsource '+_config.FILE_SYS_CONFIG+'\n')
314	# make sure that the correct calling of submit.py script is in run_local.sh
315	overwrite_lines_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
316	'pyscript=', 'pyscript=submit.py\n')
317
318	# change permission for file to executable
319	execute_subprocess(['chmod', '0775',
320	os.path.join(os.path.abspath(c.installdir),'Run/run_local.sh')])
321
322
323	# create systemdir
324	c.sysinstalldir = os.path.join(c.sysinstalldir,_config.FLEXEXTRACT_DIRNAME)
325	if os.path.isdir(c.sysinstalldir):
326	shutil.rmtree(c.sysinstalldir)
327
328	# create setup file for running flex_extract with system installation
329	with open(os.path.join(os.path.abspath(c.installdir),'Run/.setup.rc'),'w') as fio:
330	fio.write('#!/bin/bash \n')
331	fio.write('export FLEXEXTRACT_USER_DIR='+os.path.abspath(c.installdir)+'\n')
332	fio.write('export PATH='+os.path.abspath(c.sysinstalldir)+'/Python:${PATH}\n')
333	fio.write('export PATH='+os.path.abspath(c.sysinstalldir)+':${PATH}\n')
334
335	# copy all Python and Fortranfiles to this dir
336	shutil.copytree(_config.PATH_SOURCES, c.sysinstalldir, symlinks=True)
337
338	os.chdir(os.path.join(c.sysinstalldir,'Fortran'))
339	# Create Fortran executable
340	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software as ' +
341	c.install_target + ' in directory ' +
342	os.path.abspath(c.sysinstalldir) + '\n')
343
344	del_fortran_build('.')
345	mk_fortran_build('.', c.makefile)
346
347	outfile = [x for x in UioFiles('.','*.out').files]
348	test=os.path.join(c.sysinstalldir,'calc_etadot')
349	if len(outfile) != 1:
350	print('WARNING: Multiple executables for Fortran code are available!')
351	# move executable one dir up and delete Fortran dir
352	os.chdir('..')
353	shutil.move(outfile[0], os.path.join(c.sysinstalldir,'calc_etadot'))
354	shutil.rmtree(os.path.join(os.path.abspath(c.sysinstalldir),'Fortran'))
355
356	os.chdir(_config.PATH_FLEXEXTRACT_DIR)
357	if os.path.isfile(tar_file):
358	os.remove(tar_file)
359
360	return
361
362
363	def check_install_conditions(c):
364	'''Checks necessary attributes and conditions
365	for the installation, e.g. whether they exist and contain values.
366	Otherwise set default values.
367
368	Parameters
369	----------
370	c : ControlFile
371	Contains all the parameters of CONTROL file and
372	command line.
373
374
375	Return
376	------
377
378	'''
379
380	if c.install_target and \
381	c.install_target not in _config.INSTALL_TARGETS:
382	print('ERROR: unknown or missing installation target ')
383	print('target: ', c.install_target)
384	print('please specify correct installation target ' +
385	str(_config.INSTALL_TARGETS))
386	print('use -h or --help for help')
387	sys.exit(1)
388
389	if c.install_target and c.install_target not in ['local', 'syslocal']:
390	if not c.ecgid or not c.ecuid:
391	print('Please enter your ECMWF user id and group id '
392	' with command line options --ecuid --ecgid')
393	print('Try "' + sys.argv[0].split('/')[-1] + \
394	' -h" to print usage information')
395	print('Please consult ecaccess documentation or ECMWF user '
396	'support for further details.\n')
397	sys.exit(1)
398	if not c.gateway or not c.destination:
399	print('WARNING: Parameters GATEWAY and DESTINATION were '
400	'not properly set for working on ECMWF server. \n'
401	'There will be no transfer of output files to the '
402	'local gateway server possible!')
403	if not c.installdir:
404	c.installdir = '${HOME}'
405	elif c.install_target == 'local':
406	if not c.installdir:
407	c.installdir = _config.PATH_FLEXEXTRACT_DIR
408	elif c.install_target == 'syslocal':
409	if not c.installdir:
410	c.installdir = _config.PATH_FLEXEXTRACT_DIR
411	if not c.sysinstalldir:
412	print('ERROR: System installation was selected but '
413	'no system installation path was defined.')
414	sys.exit()
415
416	if not c.makefile:
417	print('WARNING: no makefile was specified.')
418	if c.install_target == 'local':
419	c.makefile = 'makefile_local_gfortran'
420	print('WARNING: default makefile selected: makefile_local_gfortan')
421	elif c.install_target == 'ecgate':
422	c.makefile = 'makefile_ecgate'
423	print('WARNING: default makefile selected: makefile_ecgate')
424	elif c.install_target == 'cca' or \
425	c.install_target == 'ccb':
426	c.makefile = 'makefile_cray'
427	print('WARNING: default makefile selected: makefile_cray')
428	else:
429	pass
430
431	return
432
433
434	def mk_tarball(tarball_path, target):
435	'''Creates a tarball with all necessary files which need to be sent to the
436	installation directory.
437	It does not matter whether this is local or remote.
438	Collects all Python files, the Fortran source and makefiles,
439	the ECMWF_ENV file, the CONTROL files as well as the
440	template files.
441
442	Parameters
443	----------
444	tarball_path : str
445	The complete path to the tar file which will contain all
446	relevant data for flex_extract.
447
448	target : str
449	The queue where the job is submitted to.
450
451	Return
452	------
453
454	'''
455
456	print('Create tarball ...')
457
458	# change to FLEXEXTRACT directory so that the tar can contain
459	# relative pathes to the files and directories
460	ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
461	os.chdir(ecd)
462
463	# get lists of the files to be added to the tar file
464	if target == 'local':
465	ecmwf_env_file = []
466	runfile = [os.path.relpath(x, ecd)
467	for x in UioFiles(_config.PATH_REL_RUN_DIR,
468	'run_local.sh').files]
469	else:
470	ecmwf_env_file = [_config.PATH_REL_ECMWF_ENV]
471	runfile = [os.path.relpath(x, ecd)
472	for x in UioFiles(_config.PATH_REL_RUN_DIR,
473	'run.sh').files]
474
475	pyfiles = [os.path.relpath(x, ecd)
476	for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
477	pytestfiles = [os.path.relpath(x, ecd)
478	for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
479	controlfiles = [os.path.relpath(x, ecd)
480	for x in UioFiles(_config.PATH_REL_CONTROLFILES,
481	'CONTROL*').files]
482	testfiles = [os.path.relpath(x, ecd)
483	for x in UioFiles(_config.PATH_REL_TEST+"/Installation", '*').files]
484	tempfiles = [os.path.relpath(x, ecd)
485	for x in UioFiles(_config.PATH_REL_TEMPLATES, '*.template').files]
486	gribtable = [os.path.relpath(x, ecd)
487	for x in UioFiles(_config.PATH_REL_TEMPLATES, 'grib').files]
488	ffiles = [os.path.relpath(x, ecd)
489	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f90').files]
490	hfiles = [os.path.relpath(x, ecd)
491	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
492	makefiles = [os.path.relpath(x, ecd)
493	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'makefile*').files]
494	jobdir = [os.path.relpath(x, ecd)
495	for x in UioFiles(_config.PATH_REL_JOBSCRIPTS, '*.md').files]
496
497	# concatenate single lists to one for a better looping
498	filelist = pyfiles + pytestfiles + controlfiles + tempfiles + \
499	ffiles + gribtable + hfiles + makefiles + ecmwf_env_file + \
500	runfile + jobdir + testfiles +\
501	['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
502
503	# create installation tar-file
504	exclude_files = [".ksh", ".tar"]
505	try:
506	with tarfile.open(tarball_path, "w:gz") as tar_handle:
507	for filename in filelist:
508	tar_handle.add(filename, recursive=False,
509	filter=lambda tarinfo: None
510	if os.path.splitext(tarinfo.name)[1]
511	in exclude_files
512	else tarinfo)
513	except tarfile.TarError as e:
514	print('... ERROR: ' + str(e))
515
516	sys.exit('\n... error occured while trying to create the tar-file ' +
517	str(tarball_path))
518
519	return
520
521
522	def un_tarball(tarball_path):
523	'''Extracts the given tarball into current directory.
524
525	Parameters
526	----------
527	tarball_path : str
528	The complete path to the tar file which will contain all
529	relevant data for flex_extract.
530
531	Return
532	------
533
534	'''
535
536	print('Untar ...')
537
538	try:
539	with tarfile.open(tarball_path) as tar_handle:
540	tar_handle.extractall()
541	except tarfile.TarError as e:
542	sys.exit('\n... error occured while trying to read tar-file ' +
543	str(tarball_path))
544	except OSError as e:
545	print('... ERROR CODE: ' + str(e.errno))
546	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
547
548	sys.exit('\n... error occured while trying to read tar-file ' +
549	str(tarball_path))
550
551	return
552
553	def mk_env_vars(ecuid, ecgid, gateway, destination):
554	'''Creates a file named ECMWF_ENV which contains the
555	necessary environmental variables at ECMWF servers.
556	It is based on the template ECMWF_ENV.template.
557
558	Parameters
559	----------
560	ecuid : str
561	The user id on ECMWF server.
562
563	ecgid : str
564	The group id on ECMWF server.
565
566	gateway : str
567	The gateway server the user is using.
568
569	destination : str
570	The remote destination which is used to transfer files
571	from ECMWF server to local gateway server.
572
573	Return
574	------
575
576	'''
577	from genshi.template.text import NewTextTemplate
578	from genshi.template import TemplateLoader
579	from genshi.template.eval import UndefinedError
580
581	try:
582	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
583	ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
584	cls=NewTextTemplate)
585
586	stream = ecmwfvars_template.generate(user_name=ecuid,
587	user_group=ecgid,
588	gateway_name=gateway,
589	destination_name=destination
590	)
591	except UndefinedError as e:
592	print('... ERROR ' + str(e))
593
594	sys.exit('\n... error occured while trying to generate template ' +
595	_config.PATH_ECMWF_ENV)
596	except OSError as e:
597	print('... ERROR CODE: ' + str(e.errno))
598	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
599
600	sys.exit('\n... error occured while trying to generate template ' +
601	_config.PATH_ECMWF_ENV)
602
603	try:
604	with open(_config.PATH_ECMWF_ENV, 'w') as f:
605	f.write(stream.render('text'))
606	except OSError as e:
607	print('... ERROR CODE: ' + str(e.errno))
608	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
609
610	sys.exit('\n... error occured while trying to write ' +
611	_config.PATH_ECMWF_ENV)
612
613	return
614
615	def mk_compilejob(makefile, ecuid, fp_root):
616	'''Modifies the original job template file so that it is specified
617	for the user and the environment were it will be applied. Result
618	is stored in a new file "compilejob.ksh" in the Jobscript directory.
619
620	Parameters
621	----------
622	makefile : str
623	Name of the makefile which should be used to compile the Fortran
624	program.
625
626	ecuid : str
627	The user id on ECMWF server.
628
629	fp_root : str
630	Path to the root directory of FLEXPART environment or flex_extract
631	environment.
632
633	Return
634	------
635
636	'''
637	from genshi.template.text import NewTextTemplate
638	from genshi.template import TemplateLoader
639	from genshi.template.eval import UndefinedError
640
641	if fp_root == '../':
642	fp_root = '$HOME'
643
644	try:
645	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
646	compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
647	cls=NewTextTemplate)
648
649	stream = compile_template.generate(
650	username=ecuid,
651	version_number=_config._VERSION_STR,
652	fp_root_scripts=fp_root,
653	makefile=makefile,
654	fortran_program=_config.FORTRAN_EXECUTABLE
655	)
656	except UndefinedError as e:
657	print('... ERROR ' + str(e))
658
659	sys.exit('\n... error occured while trying to generate template ' +
660	_config.TEMPFILE_INSTALL_COMPILEJOB)
661	except OSError as e:
662	print('... ERROR CODE: ' + str(e.errno))
663	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
664
665	sys.exit('\n... error occured while trying to generate template ' +
666	_config.TEMPFILE_INSTALL_COMPILEJOB)
667
668	try:
669	compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
670	_config.FILE_INSTALL_COMPILEJOB)
671
672	with open(compilejob, 'w') as f:
673	f.write(stream.render('text'))
674	except OSError as e:
675	print('... ERROR CODE: ' + str(e.errno))
676	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
677
678	sys.exit('\n... error occured while trying to write ' +
679	compilejob)
680
681	return
682
683	def mk_job_template(ecuid, ecgid, fp_root):
684	'''Modifies the original job template file so that it is specified
685	for the user and the environment were it will be applied. Result
686	is stored in a new file.
687
688	Parameters
689	----------
690	ecuid : str
691	The user id on ECMWF server.
692
693	ecgid : str
694	The group id on ECMWF server.
695
696	fp_root : str
697	Path to the root directory of FLEXPART environment or flex_extract
698	environment.
699
700	Return
701	------
702
703	'''
704	from genshi.template.text import NewTextTemplate
705	from genshi.template import TemplateLoader
706	from genshi.template.eval import UndefinedError
707
708	fp_root_path_to_python = os.path.join(fp_root,
709	_config.FLEXEXTRACT_DIRNAME,
710	_config.PATH_REL_PYTHON_SRC)
711	if '$' in fp_root_path_to_python:
712	ind = fp_root_path_to_python.index('$')
713	fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
714	fp_root_path_to_python[ind:]
715
716	try:
717	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
718	compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
719	cls=NewTextTemplate)
720
721	stream = compile_template.generate(
722	usergroup=ecgid,
723	username=ecuid,
724	version_number=_config._VERSION_STR,
725	fp_root_path=fp_root_path_to_python,
726	)
727	except UndefinedError as e:
728	print('... ERROR ' + str(e))
729
730	sys.exit('\n... error occured while trying to generate template ' +
731	_config.TEMPFILE_INSTALL_JOB)
732	except OSError as e:
733	print('... ERROR CODE: ' + str(e.errno))
734	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
735
736	sys.exit('\n... error occured while trying to generate template ' +
737	_config.TEMPFILE_INSTALL_JOB)
738
739
740	try:
741	tempjobfile = os.path.join(_config.PATH_TEMPLATES,
742	_config.TEMPFILE_JOB)
743
744	with open(tempjobfile, 'w') as f:
745	f.write(stream.render('text'))
746	except OSError as e:
747	print('... ERROR CODE: ' + str(e.errno))
748	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
749
750	sys.exit('\n... error occured while trying to write ' +
751	tempjobfile)
752
753	return
754
755	def del_fortran_build(src_path):
756	'''Clean up the Fortran source directory and remove all
757	build files (e.g. \.o, \.mod and FORTRAN EXECUTABLE)
758
759	Parameters
760	----------
761	src_path : str
762	Path to the fortran source directory.
763
764	Return
765	------
766
767	'''
768
769	modfiles = UioFiles(src_path, '*.mod')
770	objfiles = UioFiles(src_path, '*.o')
771	exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
772
773	modfiles.delete_files()
774	objfiles.delete_files()
775	exefile.delete_files()
776
777	return
778
779	def mk_fortran_build(src_path, makefile):
780	'''Compiles the Fortran code and generates the executable.
781
782	Parameters
783	----------
784	src_path : str
785	Path to the fortran source directory.
786
787	makefile : str
788	The name of the makefile which should be used.
789
790	Return
791	------
792
793	'''
794
795	try:
796	print('Using makefile: ' + makefile)
797	p = subprocess.Popen(['make', '-f',
798	os.path.join(src_path, makefile)],
799	stdin=subprocess.PIPE,
800	stdout=subprocess.PIPE,
801	stderr=subprocess.PIPE,
802	bufsize=1)
803	pout, perr = p.communicate()
804	print(pout.decode())
805	if p.returncode != 0:
806	print(perr.decode())
807	print('Please edit ' + makefile +
808	' or try another makefile in the src directory.')
809	print('Most likely ECCODES_INCLUDE_DIR, ECCODES_LIB '
810	'and EMOSLIB must be adapted.')
811	print('Available makefiles:')
812	print(UioFiles(src_path, 'makefile*'))
813	sys.exit('Compilation failed!')
814	except ValueError as e:
815	print('ERROR: makefile call failed:')
816	print(e)
817	else:
818	execute_subprocess(['ls', '-l',
819	os.path.join(src_path, _config.FORTRAN_EXECUTABLE)],
820	error_msg='FORTRAN EXECUTABLE COULD NOT BE FOUND!')
821
822	return
823
824
825	if __name__ == "__main__":
826	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: