Context Navigation

source: flex_extract.git/Source/Python/install.py @ 5adaf8a

dev

Last change on this file since 5adaf8a was 5adaf8a, checked in by Anne Tipka <anne.tipka@…>, 22 months ago
bug fix: sysinstalldir assignment
Property mode set to `100755`
File size: 29.7 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - moved install_args_and_control in here
14	# - splitted code in smaller functions
15	# - delete fortran build files in here instead of compile job script
16	# - changed static path names to variables from config file
17	# - splitted install function into several smaller pieces
18	# - use of tarfile package in python
19	# June 2020 - Anne Philipp
20	# - renamed "convert" functions to "fortran" functions
21	# - reconfigured mk_tarball to select *.template files instead
22	# of .nl and .temp
23	# - added check for makefile settings
24	# August 2020 - Leopold Haimberger
25	# - added a new installation section for system installation (if-else block)
26	# - read new argument from command line
27	# - write .setup.rc for a system installation into Run directory
28	# - copy executables to system path and user files to user path
29	#
30	# @License:
31	# (C) Copyright 2014-2020.
32	# Anne Philipp, Leopold Haimberger
33	#
34	# SPDX-License-Identifier: CC-BY-4.0
35	#
36	# This work is licensed under the Creative Commons Attribution 4.0
37	# International License. To view a copy of this license, visit
38	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
39	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
40	#
41	# @Methods:
42	# main
43	# get_install_cmdline_args
44	# install_via_gateway
45	# check_install_conditions
46	# mk_tarball
47	# un_tarball
48	# mk_env_vars
49	# mk_compilejob
50	# mk_job_template
51	# del_fortran_build
52	# mk_fortran_build
53	#
54	#*******************************************************************************
55	'''This script installs the flex_extract program.
56
57	Depending on the selected installation environment (locally or on the
58	ECMWF server ecgate or cca) the program extracts the command line
59	arguments and the CONTROL file parameter and prepares the corresponding
60	environment.
61	The necessary files are collected in a tar ball and placed
62	at the target location. There, is is untared, the environment variables are
63	set, and the Fortran code is compiled.
64	If the ECMWF environment is selected, a job script is prepared and submitted
65	for the remaining configurations after putting the tar ball on the
66	target ECMWF server.
67
68	Type: install.py --help
69	to get information about command line parameters.
70	Read the documentation for usage instructions.
71	'''
72
73	# ------------------------------------------------------------------------------
74	# MODULES
75	# ------------------------------------------------------------------------------
76	from __future__ import print_function
77
78	import os
79	import sys
80	import subprocess
81	import tarfile
82	import shutil
83	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
84
85	# software specific classes and modules from flex_extract
86	import _config
87	from Classes.ControlFile import ControlFile
88	from Classes.UioFiles import UioFiles
89	from Mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
90	silent_remove, execute_subprocess, none_or_str,
91	overwrite_lines_in_file, check_for_string_in_file)
92
93	# ------------------------------------------------------------------------------
94	# FUNCTIONS
95	# ------------------------------------------------------------------------------
96	def main():
97	'''Controls the installation process. Calls the installation function
98	if target is specified.
99
100	Parameters
101	----------
102
103	Return
104	------
105	'''
106
107	args = get_install_cmdline_args()
108	c = ControlFile(args.controlfile)
109	c.assign_args_to_control(args)
110	check_install_conditions(c)
111
112	if c.install_target.lower() not in ['local', 'syslocal']: # ecgate or cca
113	install_via_gateway(c)
114	else: # local
115	install_local(c)
116
117	print("SUCCESS: INSTALLATION FINISHED!")
118
119	return
120
121	def get_install_cmdline_args():
122	'''Decomposes the command line arguments and assigns them to variables.
123	Apply default values for arguments not present.
124
125	Parameters
126	----------
127
128	Return
129	------
130	args : Namespace
131	Contains the commandline arguments from script/program call.
132	'''
133	parser = ArgumentParser(description='Install flex_extract software '
134	'locally or on ECMWF machines',
135	formatter_class=ArgumentDefaultsHelpFormatter)
136
137	parser.add_argument('--target', dest='install_target',
138	type=none_or_str, default=None,
139	help="Valid targets: syslocal \| local \| ecgate \| cca , \
140	the latter two are at ECMWF")
141	parser.add_argument("--makefile", dest="makefile",
142	type=none_or_str, default=None,
143	help='Name of makefile for compiling the '
144	'Fortran program')
145	parser.add_argument("--ecuid", dest="ecuid",
146	type=none_or_str, default=None,
147	help='User id at ECMWF')
148	parser.add_argument("--ecgid", dest="ecgid",
149	type=none_or_str, default=None,
150	help='Group id at ECMWF')
151	parser.add_argument("--gateway", dest="gateway",
152	type=none_or_str, default=None,
153	help='Name of the local gateway server')
154	parser.add_argument("--destination", dest="destination",
155	type=none_or_str, default=None,
156	help='ecaccess association, e.g. '
157	'myUser@genericSftp')
158
159	parser.add_argument("--installdir", dest="installdir",
160	type=none_or_str, default=None,
161	help='Root (user) directory of the '
162	'flex_extract installation')
163	parser.add_argument("--sysinstalldir", dest="sysinstalldir",
164	type=none_or_str, default=None,
165	help='System installation path; where '
166	'executables are stored.')
167
168	# arguments for job submission to ECMWF, only needed by submit.py
169	parser.add_argument("--job_template", dest='job_template',
170	type=none_or_str, default="job.template",
171	help='Rudimentary template file to create a batch '
172	'job template for submission to ECMWF servers')
173
174	parser.add_argument("--controlfile", dest="controlfile",
175	type=none_or_str, default='CONTROL_EA5',
176	help="A file that contains all CONTROL parameters.")
177
178	args = parser.parse_args()
179
180	return args
181
182
183	def install_via_gateway(c):
184	'''Prepare data transfer to remote gateway and submit a job script which will
185	install everything on the remote gateway.
186
187	Parameters
188	----------
189	c : ControlFile
190	Contains all the parameters of CONTROL file and
191	command line.
192
193	Return
194	------
195
196	'''
197
198	tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
199	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
200
201	mk_compilejob(c.makefile, c.ecuid, c.ecgid, c.installdir)
202
203	mk_job_template(c.ecuid, c.ecgid, c.installdir)
204
205	mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
206
207	mk_tarball(tar_file, c.install_target)
208
209	put_file_to_ecserver(_config.PATH_FLEXEXTRACT_DIR, tarball_name,
210	c.install_target, c.ecuid, c.ecgid)
211
212	submit_job_to_ecserver(c.install_target,
213	os.path.join(_config.PATH_REL_JOBSCRIPTS,
214	_config.FILE_INSTALL_COMPILEJOB))
215
216	silent_remove(tar_file)
217
218	print('Job compilation script has been submitted to ecgate for ' +
219	'installation in ' + c.installdir +
220	'/' + _config.FLEXEXTRACT_DIRNAME)
221	print('You should get an email with subject "flexcompile" within ' +
222	'the next few minutes!')
223
224	return
225
226	def install_local(c):
227	'''Perform the actual installation on a local machine.
228
229	Parameters
230	----------
231	c : ControlFile
232	Contains all the parameters of CONTROL file and
233	command line.
234
235	Return
236	------
237
238	'''
239
240	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR,
241	_config.FLEXEXTRACT_DIRNAME + '.tar')
242
243	# this is standard installation into a single directory
244	if c.install_target == 'local':
245	c.installdir = os.path.abspath(os.path.expandvars(os.path.expanduser(
246	c.installdir)))
247
248	# installation into the current directory
249	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.installdir:
250	print('WARNING: installdir has not been specified')
251	print('flex_extract will be installed in current dir by compiling the ' +
252	'Fortran source in ' + _config.PATH_FORTRAN_SRC)
253	os.chdir(_config.PATH_FORTRAN_SRC)
254	# installation into a different path
255	elif os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) != c.installdir :
256
257	# creates the target working directory for flex_extract
258	mk_tarball(tar_file, c.install_target)
259	make_dir(os.path.join(c.installdir,
260	_config.FLEXEXTRACT_DIRNAME))
261	os.chdir(os.path.join(c.installdir,
262	_config.FLEXEXTRACT_DIRNAME))
263	un_tarball(tar_file)
264	os.chdir(os.path.join(c.installdir,
265	_config.FLEXEXTRACT_DIRNAME,
266	_config.PATH_REL_FORTRAN_SRC))
267
268	# Create Fortran executable
269	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software at ' +
270	c.install_target + ' in directory ' + c.installdir + '\n')
271
272	del_fortran_build('.')
273	mk_fortran_build('.', c.makefile)
274	os.chdir('../../')
275	# make sure that the correct calling of submit.py script is in run_local.sh
276	overwrite_lines_in_file('Run/run_local.sh',
277	'pyscript=', 'pyscript=../Source/Python/submit.py\n')
278
279	# this is system installation were executables and user files are separated
280	elif c.install_target == 'syslocal':
281	c.sysinstalldir = os.path.abspath(os.path.expandvars(os.path.expanduser(
282	c.sysinstalldir)))
283	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.sysinstalldir :
284	sys.exit('ERROR: System installation path is equal to user '
285	'installation path.\n Please change either the system '
286	'installation path or use installation target "local".')
287	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.installdir :
288	print('Flex_extract will be installed in current directory!')
289	else: # install user part in different dir
290	print('Flex_extract will be installed in ' + c.installdir )
291
292	c.installdir = os.path.join(c.installdir,_config.FLEXEXTRACT_DIRNAME)
293	if os.path.isdir(c.installdir):
294	shutil.rmtree(c.installdir)
295
296	# copy all files except Python and Fortranfiles to this dir
297	shutil.copytree(_config.PATH_FLEXEXTRACT_DIR,
298	c.installdir, symlinks=True)
299	shutil.rmtree(os.path.join(c.installdir,'Source'))
300	shutil.rmtree(os.path.join(c.installdir,'.git'))
301	for x in UioFiles(c.installdir, '*~').files:
302	silent_remove(x)
303
304	os.remove(os.path.join(c.installdir,'setup.sh'))
305	os.remove(os.path.join(c.installdir,'setup_local.sh'))
306
307	# configure run_local script correctly
308	# check if source of system config file is already in run_local.sh,
309	# if not, add it
310	if not check_for_string_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
311	'source .setup.rc'):
312	overwrite_lines_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
313	'# PATH TO SUBMISSION SCRIPT',
314	'# PATH TO SUBMISSION SCRIPT\nsource '+_config.FILE_SYS_CONFIG+'\n')
315	# make sure that the correct calling of submit.py script is in run_local.sh
316	overwrite_lines_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
317	'pyscript=', 'pyscript=submit.py\n')
318
319	# change permission for file to executable
320	execute_subprocess(['chmod', '0775',
321	os.path.join(os.path.abspath(c.installdir),'Run/run_local.sh')])
322
323
324	# create systemdir
325	c.sysinstalldir = os.path.join(c.sysinstalldir,_config.FLEXEXTRACT_DIRNAME)
326	if os.path.isdir(c.sysinstalldir):
327	shutil.rmtree(c.sysinstalldir)
328
329	# create setup file for running flex_extract with system installation
330	with open(os.path.join(os.path.abspath(c.installdir),'Run/.setup.rc'),'w') as fio:
331	fio.write('#!/bin/bash \n')
332	fio.write('export FLEXEXTRACT_USER_DIR='+os.path.abspath(c.installdir)+'\n')
333	fio.write('export PATH='+os.path.abspath(c.sysinstalldir)+'/Python:${PATH}\n')
334	fio.write('export PATH='+os.path.abspath(c.sysinstalldir)+':${PATH}\n')
335
336	# copy all Python and Fortranfiles to this dir
337	shutil.copytree(_config.PATH_SOURCES, c.sysinstalldir, symlinks=True)
338
339	os.chdir(os.path.join(c.sysinstalldir,'Fortran'))
340	# Create Fortran executable
341	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software as ' +
342	c.install_target + ' in directory ' +
343	os.path.abspath(c.sysinstalldir) + '\n')
344
345	del_fortran_build('.')
346	mk_fortran_build('.', c.makefile)
347
348	outfile = [x for x in UioFiles('.','*.out').files]
349	test=os.path.join(c.sysinstalldir,'calc_etadot')
350	if len(outfile) != 1:
351	print('WARNING: Multiple executables for Fortran code are available!')
352	# move executable one dir up and delete Fortran dir
353	os.chdir('..')
354	shutil.move(outfile[0], os.path.join(c.sysinstalldir,'calc_etadot'))
355	shutil.rmtree(os.path.join(os.path.abspath(c.sysinstalldir),'Fortran'))
356
357	os.chdir(_config.PATH_FLEXEXTRACT_DIR)
358	if os.path.isfile(tar_file):
359	os.remove(tar_file)
360
361	return
362
363
364	def check_install_conditions(c):
365	'''Checks necessary attributes and conditions
366	for the installation, e.g. whether they exist and contain values.
367	Otherwise set default values.
368
369	Parameters
370	----------
371	c : ControlFile
372	Contains all the parameters of CONTROL file and
373	command line.
374
375
376	Return
377	------
378
379	'''
380
381	if c.install_target and \
382	c.install_target not in _config.INSTALL_TARGETS:
383	print('ERROR: unknown or missing installation target ')
384	print('target: ', c.install_target)
385	print('please specify correct installation target ' +
386	str(_config.INSTALL_TARGETS))
387	print('use -h or --help for help')
388	sys.exit(1)
389
390	if c.install_target and c.install_target not in ['local', 'syslocal']:
391	if not c.ecgid or not c.ecuid:
392	print('Please enter your ECMWF user id and group id '
393	' with command line options --ecuid --ecgid')
394	print('Try "' + sys.argv[0].split('/')[-1] + \
395	' -h" to print usage information')
396	print('Please consult ecaccess documentation or ECMWF user '
397	'support for further details.\n')
398	sys.exit(1)
399	if not c.gateway or not c.destination:
400	print('WARNING: Parameters GATEWAY and DESTINATION were '
401	'not properly set for working on ECMWF server. \n'
402	'There will be no transfer of output files to the '
403	'local gateway server possible!')
404	if not c.installdir:
405	c.installdir = '${HOME}'
406	elif c.install_target == 'local':
407	if not c.installdir:
408	c.installdir = _config.PATH_FLEXEXTRACT_DIR
409	elif c.install_target == 'syslocal':
410	if not c.installdir:
411	c.installdir = _config.PATH_FLEXEXTRACT_DIR
412	if not c.sysinstalldir:
413	print('ERROR: System installation was selected but '
414	'no system installation path was defined.')
415	sys.exit()
416
417	if not c.makefile:
418	print('WARNING: no makefile was specified.')
419	if c.install_target == 'local':
420	c.makefile = 'makefile_local_gfortran'
421	print('WARNING: default makefile selected: makefile_local_gfortan')
422	elif c.install_target == 'ecgate':
423	c.makefile = 'makefile_ecgate'
424	print('WARNING: default makefile selected: makefile_ecgate')
425	elif c.install_target == 'cca' or \
426	c.install_target == 'ccb':
427	c.makefile = 'makefile_cray'
428	print('WARNING: default makefile selected: makefile_cray')
429	else:
430	pass
431
432	return
433
434
435	def mk_tarball(tarball_path, target):
436	'''Creates a tarball with all necessary files which need to be sent to the
437	installation directory.
438	It does not matter whether this is local or remote.
439	Collects all Python files, the Fortran source and makefiles,
440	the ECMWF_ENV file, the CONTROL files as well as the
441	template files.
442
443	Parameters
444	----------
445	tarball_path : str
446	The complete path to the tar file which will contain all
447	relevant data for flex_extract.
448
449	target : str
450	The queue where the job is submitted to.
451
452	Return
453	------
454
455	'''
456
457	print('Create tarball ...')
458
459	# change to FLEXEXTRACT directory so that the tar can contain
460	# relative pathes to the files and directories
461	ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
462	os.chdir(ecd)
463
464	# get lists of the files to be added to the tar file
465	if target == 'local':
466	ecmwf_env_file = []
467	runfile = [os.path.relpath(x, ecd)
468	for x in UioFiles(_config.PATH_REL_RUN_DIR,
469	'run_local.sh').files]
470	else:
471	ecmwf_env_file = [_config.PATH_REL_ECMWF_ENV]
472	runfile = [os.path.relpath(x, ecd)
473	for x in UioFiles(_config.PATH_REL_RUN_DIR,
474	'run.sh').files]
475
476	pyfiles = [os.path.relpath(x, ecd)
477	for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
478	pytestfiles = [os.path.relpath(x, ecd)
479	for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
480	controlfiles = [os.path.relpath(x, ecd)
481	for x in UioFiles(_config.PATH_REL_CONTROLFILES,
482	'CONTROL*').files]
483	testfiles = [os.path.relpath(x, ecd)
484	for x in UioFiles(_config.PATH_REL_TEST+"/Installation", '*').files]
485	tempfiles = [os.path.relpath(x, ecd)
486	for x in UioFiles(_config.PATH_REL_TEMPLATES, '*.template').files]
487	gribtable = [os.path.relpath(x, ecd)
488	for x in UioFiles(_config.PATH_REL_TEMPLATES, 'grib').files]
489	ffiles = [os.path.relpath(x, ecd)
490	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f90').files]
491	hfiles = [os.path.relpath(x, ecd)
492	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
493	makefiles = [os.path.relpath(x, ecd)
494	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'makefile*').files]
495	jobdir = [os.path.relpath(x, ecd)
496	for x in UioFiles(_config.PATH_REL_JOBSCRIPTS, '*.md').files]
497
498	# concatenate single lists to one for a better looping
499	filelist = pyfiles + pytestfiles + controlfiles + tempfiles + \
500	ffiles + gribtable + hfiles + makefiles + ecmwf_env_file + \
501	runfile + jobdir + testfiles +\
502	['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
503
504	# create installation tar-file
505	exclude_files = [".ksh", ".tar"]
506	try:
507	with tarfile.open(tarball_path, "w:gz") as tar_handle:
508	for filename in filelist:
509	tar_handle.add(filename, recursive=False,
510	filter=lambda tarinfo: None
511	if os.path.splitext(tarinfo.name)[1]
512	in exclude_files
513	else tarinfo)
514	except tarfile.TarError as e:
515	print('... ERROR: ' + str(e))
516
517	sys.exit('\n... error occured while trying to create the tar-file ' +
518	str(tarball_path))
519
520	return
521
522
523	def un_tarball(tarball_path):
524	'''Extracts the given tarball into current directory.
525
526	Parameters
527	----------
528	tarball_path : str
529	The complete path to the tar file which will contain all
530	relevant data for flex_extract.
531
532	Return
533	------
534
535	'''
536
537	print('Untar ...')
538
539	try:
540	with tarfile.open(tarball_path) as tar_handle:
541	tar_handle.extractall()
542	except tarfile.TarError as e:
543	sys.exit('\n... error occured while trying to read tar-file ' +
544	str(tarball_path))
545	except OSError as e:
546	print('... ERROR CODE: ' + str(e.errno))
547	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
548
549	sys.exit('\n... error occured while trying to read tar-file ' +
550	str(tarball_path))
551
552	return
553
554	def mk_env_vars(ecuid, ecgid, gateway, destination):
555	'''Creates a file named ECMWF_ENV which contains the
556	necessary environmental variables at ECMWF servers.
557	It is based on the template ECMWF_ENV.template.
558
559	Parameters
560	----------
561	ecuid : str
562	The user id on ECMWF server.
563
564	ecgid : str
565	The group id on ECMWF server.
566
567	gateway : str
568	The gateway server the user is using.
569
570	destination : str
571	The remote destination which is used to transfer files
572	from ECMWF server to local gateway server.
573
574	Return
575	------
576
577	'''
578	from genshi.template.text import NewTextTemplate
579	from genshi.template import TemplateLoader
580	from genshi.template.eval import UndefinedError
581
582	try:
583	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
584	ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
585	cls=NewTextTemplate)
586
587	stream = ecmwfvars_template.generate(user_name=ecuid,
588	user_group=ecgid,
589	gateway_name=gateway,
590	destination_name=destination
591	)
592	except UndefinedError as e:
593	print('... ERROR ' + str(e))
594
595	sys.exit('\n... error occured while trying to generate template ' +
596	_config.PATH_ECMWF_ENV)
597	except OSError as e:
598	print('... ERROR CODE: ' + str(e.errno))
599	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
600
601	sys.exit('\n... error occured while trying to generate template ' +
602	_config.PATH_ECMWF_ENV)
603
604	try:
605	with open(_config.PATH_ECMWF_ENV, 'w') as f:
606	f.write(stream.render('text'))
607	except OSError as e:
608	print('... ERROR CODE: ' + str(e.errno))
609	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
610
611	sys.exit('\n... error occured while trying to write ' +
612	_config.PATH_ECMWF_ENV)
613
614	return
615
616	def mk_compilejob(makefile, ecuid, ecgid, fp_root):
617	'''Modifies the original job template file so that it is specified
618	for the user and the environment were it will be applied. Result
619	is stored in a new file "job.temp" in the python directory.
620
621	Parameters
622	----------
623	makefile : str
624	Name of the makefile which should be used to compile the Fortran
625	program.
626
627	ecuid : str
628	The user id on ECMWF server.
629
630	ecgid : str
631	The group id on ECMWF server.
632
633	fp_root : str
634	Path to the root directory of FLEXPART environment or flex_extract
635	environment.
636
637	Return
638	------
639
640	'''
641	from genshi.template.text import NewTextTemplate
642	from genshi.template import TemplateLoader
643	from genshi.template.eval import UndefinedError
644
645	if fp_root == '../':
646	fp_root = '$HOME'
647
648	try:
649	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
650	compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
651	cls=NewTextTemplate)
652
653	stream = compile_template.generate(
654	usergroup=ecgid,
655	username=ecuid,
656	version_number=_config._VERSION_STR,
657	fp_root_scripts=fp_root,
658	makefile=makefile,
659	fortran_program=_config.FORTRAN_EXECUTABLE
660	)
661	except UndefinedError as e:
662	print('... ERROR ' + str(e))
663
664	sys.exit('\n... error occured while trying to generate template ' +
665	_config.TEMPFILE_INSTALL_COMPILEJOB)
666	except OSError as e:
667	print('... ERROR CODE: ' + str(e.errno))
668	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
669
670	sys.exit('\n... error occured while trying to generate template ' +
671	_config.TEMPFILE_INSTALL_COMPILEJOB)
672
673	try:
674	compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
675	_config.FILE_INSTALL_COMPILEJOB)
676
677	with open(compilejob, 'w') as f:
678	f.write(stream.render('text'))
679	except OSError as e:
680	print('... ERROR CODE: ' + str(e.errno))
681	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
682
683	sys.exit('\n... error occured while trying to write ' +
684	compilejob)
685
686	return
687
688	def mk_job_template(ecuid, ecgid, fp_root):
689	'''Modifies the original job template file so that it is specified
690	for the user and the environment were it will be applied. Result
691	is stored in a new file.
692
693	Parameters
694	----------
695	ecuid : str
696	The user id on ECMWF server.
697
698	ecgid : str
699	The group id on ECMWF server.
700
701	fp_root : str
702	Path to the root directory of FLEXPART environment or flex_extract
703	environment.
704
705	Return
706	------
707
708	'''
709	from genshi.template.text import NewTextTemplate
710	from genshi.template import TemplateLoader
711	from genshi.template.eval import UndefinedError
712
713	fp_root_path_to_python = os.path.join(fp_root,
714	_config.FLEXEXTRACT_DIRNAME,
715	_config.PATH_REL_PYTHON_SRC)
716	if '$' in fp_root_path_to_python:
717	ind = fp_root_path_to_python.index('$')
718	fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
719	fp_root_path_to_python[ind:]
720
721	try:
722	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
723	compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
724	cls=NewTextTemplate)
725
726	stream = compile_template.generate(
727	usergroup=ecgid,
728	username=ecuid,
729	version_number=_config._VERSION_STR,
730	fp_root_path=fp_root_path_to_python,
731	)
732	except UndefinedError as e:
733	print('... ERROR ' + str(e))
734
735	sys.exit('\n... error occured while trying to generate template ' +
736	_config.TEMPFILE_INSTALL_JOB)
737	except OSError as e:
738	print('... ERROR CODE: ' + str(e.errno))
739	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
740
741	sys.exit('\n... error occured while trying to generate template ' +
742	_config.TEMPFILE_INSTALL_JOB)
743
744
745	try:
746	tempjobfile = os.path.join(_config.PATH_TEMPLATES,
747	_config.TEMPFILE_JOB)
748
749	with open(tempjobfile, 'w') as f:
750	f.write(stream.render('text'))
751	except OSError as e:
752	print('... ERROR CODE: ' + str(e.errno))
753	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
754
755	sys.exit('\n... error occured while trying to write ' +
756	tempjobfile)
757
758	return
759
760	def del_fortran_build(src_path):
761	'''Clean up the Fortran source directory and remove all
762	build files (e.g. \.o, \.mod and FORTRAN EXECUTABLE)
763
764	Parameters
765	----------
766	src_path : str
767	Path to the fortran source directory.
768
769	Return
770	------
771
772	'''
773
774	modfiles = UioFiles(src_path, '*.mod')
775	objfiles = UioFiles(src_path, '*.o')
776	exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
777
778	modfiles.delete_files()
779	objfiles.delete_files()
780	exefile.delete_files()
781
782	return
783
784	def mk_fortran_build(src_path, makefile):
785	'''Compiles the Fortran code and generates the executable.
786
787	Parameters
788	----------
789	src_path : str
790	Path to the fortran source directory.
791
792	makefile : str
793	The name of the makefile which should be used.
794
795	Return
796	------
797
798	'''
799
800	try:
801	print('Using makefile: ' + makefile)
802	p = subprocess.Popen(['make', '-f',
803	os.path.join(src_path, makefile)],
804	stdin=subprocess.PIPE,
805	stdout=subprocess.PIPE,
806	stderr=subprocess.PIPE,
807	bufsize=1)
808	pout, perr = p.communicate()
809	print(pout.decode())
810	if p.returncode != 0:
811	print(perr.decode())
812	print('Please edit ' + makefile +
813	' or try another makefile in the src directory.')
814	print('Most likely ECCODES_INCLUDE_DIR, ECCODES_LIB '
815	'and EMOSLIB must be adapted.')
816	print('Available makefiles:')
817	print(UioFiles(src_path, 'makefile*'))
818	sys.exit('Compilation failed!')
819	except ValueError as e:
820	print('ERROR: makefile call failed:')
821	print(e)
822	else:
823	execute_subprocess(['ls', '-l',
824	os.path.join(src_path, _config.FORTRAN_EXECUTABLE)],
825	error_msg='FORTRAN EXECUTABLE COULD NOT BE FOUND!')
826
827	return
828
829
830	if __name__ == "__main__":
831	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: