Context Navigation

source: flex_extract.git/Source/Python/install.py @ 75db9b0

dev

Last change on this file since 75db9b0 was 75db9b0, checked in by anphi <anne.philipp@…>, 4 years ago
added history information of changes to v7.1.2
Property mode set to `100755`
File size: 29.7 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - moved install_args_and_control in here
14	# - splitted code in smaller functions
15	# - delete fortran build files in here instead of compile job script
16	# - changed static path names to variables from config file
17	# - splitted install function into several smaller pieces
18	# - use of tarfile package in python
19	# June 2020 - Anne Philipp
20	# - renamed "convert" functions to "fortran" functions
21	# - reconfigured mk_tarball to select *.template files instead
22	# of .nl and .temp
23	# - added check for makefile settings
24	# August 2020 - Leopold Haimberger
25	# - added a new installation section for system installation (if-else block)
26	# - read new argument from command line
27	# - write .setup.rc for a system installation into Run directory
28	# - copy executables to system path and user files to user path
29	#
30	# @License:
31	# (C) Copyright 2014-2020.
32	# Anne Philipp, Leopold Haimberger
33	#
34	# SPDX-License-Identifier: CC-BY-4.0
35	#
36	# This work is licensed under the Creative Commons Attribution 4.0
37	# International License. To view a copy of this license, visit
38	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
39	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
40	#
41	# @Methods:
42	# main
43	# get_install_cmdline_args
44	# install_via_gateway
45	# check_install_conditions
46	# mk_tarball
47	# un_tarball
48	# mk_env_vars
49	# mk_compilejob
50	# mk_job_template
51	# del_fortran_build
52	# mk_fortran_build
53	#
54	#*******************************************************************************
55	'''This script installs the flex_extract program.
56
57	Depending on the selected installation environment (locally or on the
58	ECMWF server ecgate or cca) the program extracts the command line
59	arguments and the CONTROL file parameter and prepares the corresponding
60	environment.
61	The necessary files are collected in a tar ball and placed
62	at the target location. There, is is untared, the environment variables are
63	set, and the Fortran code is compiled.
64	If the ECMWF environment is selected, a job script is prepared and submitted
65	for the remaining configurations after putting the tar ball on the
66	target ECMWF server.
67
68	Type: install.py --help
69	to get information about command line parameters.
70	Read the documentation for usage instructions.
71	'''
72
73	# ------------------------------------------------------------------------------
74	# MODULES
75	# ------------------------------------------------------------------------------
76	from __future__ import print_function
77
78	import os
79	import sys
80	import subprocess
81	import tarfile
82	import shutil
83	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
84
85	# software specific classes and modules from flex_extract
86	import _config
87	from Classes.ControlFile import ControlFile
88	from Classes.UioFiles import UioFiles
89	from Mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
90	silent_remove, execute_subprocess, none_or_str,
91	overwrite_lines_in_file, check_for_string_in_file)
92
93	# ------------------------------------------------------------------------------
94	# FUNCTIONS
95	# ------------------------------------------------------------------------------
96	def main():
97	'''Controls the installation process. Calls the installation function
98	if target is specified.
99
100	Parameters
101	----------
102
103	Return
104	------
105	'''
106
107	args = get_install_cmdline_args()
108	c = ControlFile(args.controlfile)
109	c.assign_args_to_control(args)
110	check_install_conditions(c)
111
112	if c.install_target.lower() not in ['local', 'syslocal']: # ecgate or cca
113	install_via_gateway(c)
114	else: # local
115	install_local(c)
116
117	print("SUCCESS: INSTALLATION FINISHED!")
118
119	return
120
121	def get_install_cmdline_args():
122	'''Decomposes the command line arguments and assigns them to variables.
123	Apply default values for arguments not present.
124
125	Parameters
126	----------
127
128	Return
129	------
130	args : Namespace
131	Contains the commandline arguments from script/program call.
132	'''
133	parser = ArgumentParser(description='Install flex_extract software '
134	'locally or on ECMWF machines',
135	formatter_class=ArgumentDefaultsHelpFormatter)
136
137	parser.add_argument('--target', dest='install_target',
138	type=none_or_str, default=None,
139	help="Valid targets: syslocal \| local \| ecgate \| cca , \
140	the latter two are at ECMWF")
141	parser.add_argument("--makefile", dest="makefile",
142	type=none_or_str, default=None,
143	help='Name of makefile for compiling the '
144	'Fortran program')
145	parser.add_argument("--ecuid", dest="ecuid",
146	type=none_or_str, default=None,
147	help='User id at ECMWF')
148	parser.add_argument("--ecgid", dest="ecgid",
149	type=none_or_str, default=None,
150	help='Group id at ECMWF')
151	parser.add_argument("--gateway", dest="gateway",
152	type=none_or_str, default=None,
153	help='Name of the local gateway server')
154	parser.add_argument("--destination", dest="destination",
155	type=none_or_str, default=None,
156	help='ecaccess association, e.g. '
157	'myUser@genericSftp')
158
159	parser.add_argument("--installdir", dest="installdir",
160	type=none_or_str, default=None,
161	help='Root (user) directory of the '
162	'flex_extract installation')
163	parser.add_argument("--sysinstalldir", dest="sysinstalldir",
164	type=none_or_str, default=None,
165	help='System installation path; where '
166	'executables are stored.')
167
168	# arguments for job submission to ECMWF, only needed by submit.py
169	parser.add_argument("--job_template", dest='job_template',
170	type=none_or_str, default="job.template",
171	help='Rudimentary template file to create a batch '
172	'job template for submission to ECMWF servers')
173
174	parser.add_argument("--controlfile", dest="controlfile",
175	type=none_or_str, default='CONTROL_EA5',
176	help="A file that contains all CONTROL parameters.")
177
178	args = parser.parse_args()
179
180	return args
181
182
183	def install_via_gateway(c):
184	'''Prepare data transfer to remote gateway and submit a job script which will
185	install everything on the remote gateway.
186
187	Parameters
188	----------
189	c : ControlFile
190	Contains all the parameters of CONTROL file and
191	command line.
192
193	Return
194	------
195
196	'''
197
198	tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
199	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
200
201	mk_compilejob(c.makefile, c.ecuid, c.ecgid, c.installdir)
202
203	mk_job_template(c.ecuid, c.ecgid, c.installdir)
204
205	mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
206
207	mk_tarball(tar_file, c.install_target)
208
209	put_file_to_ecserver(_config.PATH_FLEXEXTRACT_DIR, tarball_name,
210	c.install_target, c.ecuid, c.ecgid)
211
212	submit_job_to_ecserver(c.install_target,
213	os.path.join(_config.PATH_REL_JOBSCRIPTS,
214	_config.FILE_INSTALL_COMPILEJOB))
215
216	silent_remove(tar_file)
217
218	print('Job compilation script has been submitted to ecgate for ' +
219	'installation in ' + c.installdir +
220	'/' + _config.FLEXEXTRACT_DIRNAME)
221	print('You should get an email with subject "flexcompile" within ' +
222	'the next few minutes!')
223
224	return
225
226	def install_local(c):
227	'''Perform the actual installation on a local machine.
228
229	Parameters
230	----------
231	c : ControlFile
232	Contains all the parameters of CONTROL file and
233	command line.
234
235	Return
236	------
237
238	'''
239
240	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR,
241	_config.FLEXEXTRACT_DIRNAME + '.tar')
242
243	c.installdir = os.path.abspath(os.path.expandvars(os.path.expanduser(
244	c.installdir)))
245	c.sysinstalldir = os.path.abspath(os.path.expandvars(os.path.expanduser(
246	c.sysinstalldir)))
247
248	# this is standard installation into a single directory
249	if c.install_target == 'local':
250
251	# installation into the current directory
252	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.installdir:
253	print('WARNING: installdir has not been specified')
254	print('flex_extract will be installed in current dir by compiling the ' +
255	'Fortran source in ' + _config.PATH_FORTRAN_SRC)
256	os.chdir(_config.PATH_FORTRAN_SRC)
257	# installation into a different path
258	elif os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) != c.installdir :
259
260	# creates the target working directory for flex_extract
261	mk_tarball(tar_file, c.install_target)
262	make_dir(os.path.join(c.installdir,
263	_config.FLEXEXTRACT_DIRNAME))
264	os.chdir(os.path.join(c.installdir,
265	_config.FLEXEXTRACT_DIRNAME))
266	un_tarball(tar_file)
267	os.chdir(os.path.join(c.installdir,
268	_config.FLEXEXTRACT_DIRNAME,
269	_config.PATH_REL_FORTRAN_SRC))
270
271	# Create Fortran executable
272	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software at ' +
273	c.install_target + ' in directory ' + c.installdir + '\n')
274
275	del_fortran_build('.')
276	mk_fortran_build('.', c.makefile)
277	os.chdir('../../')
278	# make sure that the correct calling of submit.py script is in run_local.sh
279	overwrite_lines_in_file('Run/run_local.sh',
280	'pyscript=', 'pyscript=../Source/Python/submit.py\n')
281
282	# this is system installation were executables and user files are separated
283	elif c.install_target == 'syslocal':
284	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.sysinstalldir :
285	sys.exit('ERROR: System installation path is equal to user '
286	'installation path.\n Please change either the system '
287	'installation path or use installation target "local".')
288	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.installdir :
289	print('Flex_extract will be installed in current directory!')
290	else: # install user part in different dir
291	print('Flex_extract will be installed in ' + c.installdir )
292
293	c.installdir = os.path.join(c.installdir,_config.FLEXEXTRACT_DIRNAME)
294	if os.path.isdir(c.installdir):
295	shutil.rmtree(c.installdir)
296
297	# copy all files except Python and Fortranfiles to this dir
298	shutil.copytree(_config.PATH_FLEXEXTRACT_DIR,
299	c.installdir, symlinks=True)
300	shutil.rmtree(os.path.join(c.installdir,'Source'))
301	shutil.rmtree(os.path.join(c.installdir,'.git'))
302	for x in UioFiles(c.installdir, '*~').files:
303	silent_remove(x)
304
305	os.remove(os.path.join(c.installdir,'setup.sh'))
306	os.remove(os.path.join(c.installdir,'setup_local.sh'))
307
308	# configure run_local script correctly
309	# check if source of system config file is already in run_local.sh,
310	# if not, add it
311	if not check_for_string_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
312	'source .setup.rc'):
313	overwrite_lines_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
314	'# PATH TO SUBMISSION SCRIPT',
315	'# PATH TO SUBMISSION SCRIPT\nsource '+_config.FILE_SYS_CONFIG+'\n')
316	# make sure that the correct calling of submit.py script is in run_local.sh
317	overwrite_lines_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
318	'pyscript=', 'pyscript=submit.py\n')
319
320	# change permission for file to executable
321	execute_subprocess(['chmod', '0775',
322	os.path.join(os.path.abspath(c.installdir),'Run/run_local.sh')])
323
324
325	# create systemdir
326	c.sysinstalldir = os.path.join(c.sysinstalldir,_config.FLEXEXTRACT_DIRNAME)
327	if os.path.isdir(c.sysinstalldir):
328	shutil.rmtree(c.sysinstalldir)
329
330	# create setup file for running flex_extract with system installation
331	with open(os.path.join(os.path.abspath(c.installdir),'Run/.setup.rc'),'w') as fio:
332	fio.write('#!/bin/bash \n')
333	fio.write('export FLEXEXTRACT_USER_DIR='+os.path.abspath(c.installdir)+'\n')
334	fio.write('export PATH='+os.path.abspath(c.sysinstalldir)+'/Python:${PATH}\n')
335	fio.write('export PATH='+os.path.abspath(c.sysinstalldir)+':${PATH}\n')
336
337	# copy all Python and Fortranfiles to this dir
338	shutil.copytree(_config.PATH_SOURCES, c.sysinstalldir, symlinks=True)
339
340	os.chdir(os.path.join(c.sysinstalldir,'Fortran'))
341	# Create Fortran executable
342	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software as ' +
343	c.install_target + ' in directory ' +
344	os.path.abspath(c.sysinstalldir) + '\n')
345
346	del_fortran_build('.')
347	mk_fortran_build('.', c.makefile)
348
349	outfile = [x for x in UioFiles('.','*.out').files]
350	test=os.path.join(c.sysinstalldir,'calc_etadot')
351	if len(outfile) != 1:
352	print('WARNING: Multiple executables for Fortran code are available!')
353	# move executable one dir up and delete Fortran dir
354	os.chdir('..')
355	shutil.move(outfile[0], os.path.join(c.sysinstalldir,'calc_etadot'))
356	shutil.rmtree(os.path.join(os.path.abspath(c.sysinstalldir),'Fortran'))
357
358	os.chdir(_config.PATH_FLEXEXTRACT_DIR)
359	if os.path.isfile(tar_file):
360	os.remove(tar_file)
361
362	return
363
364
365	def check_install_conditions(c):
366	'''Checks necessary attributes and conditions
367	for the installation, e.g. whether they exist and contain values.
368	Otherwise set default values.
369
370	Parameters
371	----------
372	c : ControlFile
373	Contains all the parameters of CONTROL file and
374	command line.
375
376
377	Return
378	------
379
380	'''
381
382	if c.install_target and \
383	c.install_target not in _config.INSTALL_TARGETS:
384	print('ERROR: unknown or missing installation target ')
385	print('target: ', c.install_target)
386	print('please specify correct installation target ' +
387	str(_config.INSTALL_TARGETS))
388	print('use -h or --help for help')
389	sys.exit(1)
390
391	if c.install_target and c.install_target not in ['local', 'syslocal']:
392	if not c.ecgid or not c.ecuid:
393	print('Please enter your ECMWF user id and group id '
394	' with command line options --ecuid --ecgid')
395	print('Try "' + sys.argv[0].split('/')[-1] + \
396	' -h" to print usage information')
397	print('Please consult ecaccess documentation or ECMWF user '
398	'support for further details.\n')
399	sys.exit(1)
400	if not c.gateway or not c.destination:
401	print('WARNING: Parameters GATEWAY and DESTINATION were '
402	'not properly set for working on ECMWF server. \n'
403	'There will be no transfer of output files to the '
404	'local gateway server possible!')
405	if not c.installdir:
406	c.installdir = '${HOME}'
407	elif c.install_target == 'local':
408	if not c.installdir:
409	c.installdir = _config.PATH_FLEXEXTRACT_DIR
410	elif c.install_target == 'syslocal':
411	if not c.installdir:
412	c.installdir = _config.PATH_FLEXEXTRACT_DIR
413	if not c.sysinstalldir:
414	print('ERROR: System installation was selected but '
415	'no system installation path was defined.')
416	sys.exit()
417
418	if not c.makefile:
419	print('WARNING: no makefile was specified.')
420	if c.install_target == 'local':
421	c.makefile = 'makefile_local_gfortran'
422	print('WARNING: default makefile selected: makefile_local_gfortan')
423	elif c.install_target == 'ecgate':
424	c.makefile = 'makefile_ecgate'
425	print('WARNING: default makefile selected: makefile_ecgate')
426	elif c.install_target == 'cca' or \
427	c.install_target == 'ccb':
428	c.makefile = 'makefile_cray'
429	print('WARNING: default makefile selected: makefile_cray')
430	else:
431	pass
432
433	return
434
435
436	def mk_tarball(tarball_path, target):
437	'''Creates a tarball with all necessary files which need to be sent to the
438	installation directory.
439	It does not matter whether this is local or remote.
440	Collects all Python files, the Fortran source and makefiles,
441	the ECMWF_ENV file, the CONTROL files as well as the
442	template files.
443
444	Parameters
445	----------
446	tarball_path : str
447	The complete path to the tar file which will contain all
448	relevant data for flex_extract.
449
450	target : str
451	The queue where the job is submitted to.
452
453	Return
454	------
455
456	'''
457
458	print('Create tarball ...')
459
460	# change to FLEXEXTRACT directory so that the tar can contain
461	# relative pathes to the files and directories
462	ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
463	os.chdir(ecd)
464
465	# get lists of the files to be added to the tar file
466	if target == 'local':
467	ecmwf_env_file = []
468	runfile = [os.path.relpath(x, ecd)
469	for x in UioFiles(_config.PATH_REL_RUN_DIR,
470	'run_local.sh').files]
471	else:
472	ecmwf_env_file = [_config.PATH_REL_ECMWF_ENV]
473	runfile = [os.path.relpath(x, ecd)
474	for x in UioFiles(_config.PATH_REL_RUN_DIR,
475	'run.sh').files]
476
477	pyfiles = [os.path.relpath(x, ecd)
478	for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
479	pytestfiles = [os.path.relpath(x, ecd)
480	for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
481	controlfiles = [os.path.relpath(x, ecd)
482	for x in UioFiles(_config.PATH_REL_CONTROLFILES,
483	'CONTROL*').files]
484	testfiles = [os.path.relpath(x, ecd)
485	for x in UioFiles(_config.PATH_REL_TEST+"/Installation", '*').files]
486	tempfiles = [os.path.relpath(x, ecd)
487	for x in UioFiles(_config.PATH_REL_TEMPLATES, '*.template').files]
488	gribtable = [os.path.relpath(x, ecd)
489	for x in UioFiles(_config.PATH_REL_TEMPLATES, 'grib').files]
490	ffiles = [os.path.relpath(x, ecd)
491	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f90').files]
492	hfiles = [os.path.relpath(x, ecd)
493	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
494	makefiles = [os.path.relpath(x, ecd)
495	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'makefile*').files]
496	jobdir = [os.path.relpath(x, ecd)
497	for x in UioFiles(_config.PATH_REL_JOBSCRIPTS, '*.md').files]
498
499	# concatenate single lists to one for a better looping
500	filelist = pyfiles + pytestfiles + controlfiles + tempfiles + \
501	ffiles + gribtable + hfiles + makefiles + ecmwf_env_file + \
502	runfile + jobdir + testfiles +\
503	['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
504
505	# create installation tar-file
506	exclude_files = [".ksh", ".tar"]
507	try:
508	with tarfile.open(tarball_path, "w:gz") as tar_handle:
509	for filename in filelist:
510	tar_handle.add(filename, recursive=False,
511	filter=lambda tarinfo: None
512	if os.path.splitext(tarinfo.name)[1]
513	in exclude_files
514	else tarinfo)
515	except tarfile.TarError as e:
516	print('... ERROR: ' + str(e))
517
518	sys.exit('\n... error occured while trying to create the tar-file ' +
519	str(tarball_path))
520
521	return
522
523
524	def un_tarball(tarball_path):
525	'''Extracts the given tarball into current directory.
526
527	Parameters
528	----------
529	tarball_path : str
530	The complete path to the tar file which will contain all
531	relevant data for flex_extract.
532
533	Return
534	------
535
536	'''
537
538	print('Untar ...')
539
540	try:
541	with tarfile.open(tarball_path) as tar_handle:
542	tar_handle.extractall()
543	except tarfile.TarError as e:
544	sys.exit('\n... error occured while trying to read tar-file ' +
545	str(tarball_path))
546	except OSError as e:
547	print('... ERROR CODE: ' + str(e.errno))
548	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
549
550	sys.exit('\n... error occured while trying to read tar-file ' +
551	str(tarball_path))
552
553	return
554
555	def mk_env_vars(ecuid, ecgid, gateway, destination):
556	'''Creates a file named ECMWF_ENV which contains the
557	necessary environmental variables at ECMWF servers.
558	It is based on the template ECMWF_ENV.template.
559
560	Parameters
561	----------
562	ecuid : str
563	The user id on ECMWF server.
564
565	ecgid : str
566	The group id on ECMWF server.
567
568	gateway : str
569	The gateway server the user is using.
570
571	destination : str
572	The remote destination which is used to transfer files
573	from ECMWF server to local gateway server.
574
575	Return
576	------
577
578	'''
579	from genshi.template.text import NewTextTemplate
580	from genshi.template import TemplateLoader
581	from genshi.template.eval import UndefinedError
582
583	try:
584	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
585	ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
586	cls=NewTextTemplate)
587
588	stream = ecmwfvars_template.generate(user_name=ecuid,
589	user_group=ecgid,
590	gateway_name=gateway,
591	destination_name=destination
592	)
593	except UndefinedError as e:
594	print('... ERROR ' + str(e))
595
596	sys.exit('\n... error occured while trying to generate template ' +
597	_config.PATH_ECMWF_ENV)
598	except OSError as e:
599	print('... ERROR CODE: ' + str(e.errno))
600	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
601
602	sys.exit('\n... error occured while trying to generate template ' +
603	_config.PATH_ECMWF_ENV)
604
605	try:
606	with open(_config.PATH_ECMWF_ENV, 'w') as f:
607	f.write(stream.render('text'))
608	except OSError as e:
609	print('... ERROR CODE: ' + str(e.errno))
610	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
611
612	sys.exit('\n... error occured while trying to write ' +
613	_config.PATH_ECMWF_ENV)
614
615	return
616
617	def mk_compilejob(makefile, ecuid, ecgid, fp_root):
618	'''Modifies the original job template file so that it is specified
619	for the user and the environment were it will be applied. Result
620	is stored in a new file "job.temp" in the python directory.
621
622	Parameters
623	----------
624	makefile : str
625	Name of the makefile which should be used to compile the Fortran
626	program.
627
628	ecuid : str
629	The user id on ECMWF server.
630
631	ecgid : str
632	The group id on ECMWF server.
633
634	fp_root : str
635	Path to the root directory of FLEXPART environment or flex_extract
636	environment.
637
638	Return
639	------
640
641	'''
642	from genshi.template.text import NewTextTemplate
643	from genshi.template import TemplateLoader
644	from genshi.template.eval import UndefinedError
645
646	if fp_root == '../':
647	fp_root = '$HOME'
648
649	try:
650	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
651	compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
652	cls=NewTextTemplate)
653
654	stream = compile_template.generate(
655	usergroup=ecgid,
656	username=ecuid,
657	version_number=_config._VERSION_STR,
658	fp_root_scripts=fp_root,
659	makefile=makefile,
660	fortran_program=_config.FORTRAN_EXECUTABLE
661	)
662	except UndefinedError as e:
663	print('... ERROR ' + str(e))
664
665	sys.exit('\n... error occured while trying to generate template ' +
666	_config.TEMPFILE_INSTALL_COMPILEJOB)
667	except OSError as e:
668	print('... ERROR CODE: ' + str(e.errno))
669	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
670
671	sys.exit('\n... error occured while trying to generate template ' +
672	_config.TEMPFILE_INSTALL_COMPILEJOB)
673
674	try:
675	compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
676	_config.FILE_INSTALL_COMPILEJOB)
677
678	with open(compilejob, 'w') as f:
679	f.write(stream.render('text'))
680	except OSError as e:
681	print('... ERROR CODE: ' + str(e.errno))
682	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
683
684	sys.exit('\n... error occured while trying to write ' +
685	compilejob)
686
687	return
688
689	def mk_job_template(ecuid, ecgid, fp_root):
690	'''Modifies the original job template file so that it is specified
691	for the user and the environment were it will be applied. Result
692	is stored in a new file.
693
694	Parameters
695	----------
696	ecuid : str
697	The user id on ECMWF server.
698
699	ecgid : str
700	The group id on ECMWF server.
701
702	fp_root : str
703	Path to the root directory of FLEXPART environment or flex_extract
704	environment.
705
706	Return
707	------
708
709	'''
710	from genshi.template.text import NewTextTemplate
711	from genshi.template import TemplateLoader
712	from genshi.template.eval import UndefinedError
713
714	fp_root_path_to_python = os.path.join(fp_root,
715	_config.FLEXEXTRACT_DIRNAME,
716	_config.PATH_REL_PYTHON_SRC)
717	if '$' in fp_root_path_to_python:
718	ind = fp_root_path_to_python.index('$')
719	fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
720	fp_root_path_to_python[ind:]
721
722	try:
723	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
724	compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
725	cls=NewTextTemplate)
726
727	stream = compile_template.generate(
728	usergroup=ecgid,
729	username=ecuid,
730	version_number=_config._VERSION_STR,
731	fp_root_path=fp_root_path_to_python,
732	)
733	except UndefinedError as e:
734	print('... ERROR ' + str(e))
735
736	sys.exit('\n... error occured while trying to generate template ' +
737	_config.TEMPFILE_INSTALL_JOB)
738	except OSError as e:
739	print('... ERROR CODE: ' + str(e.errno))
740	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
741
742	sys.exit('\n... error occured while trying to generate template ' +
743	_config.TEMPFILE_INSTALL_JOB)
744
745
746	try:
747	tempjobfile = os.path.join(_config.PATH_TEMPLATES,
748	_config.TEMPFILE_JOB)
749
750	with open(tempjobfile, 'w') as f:
751	f.write(stream.render('text'))
752	except OSError as e:
753	print('... ERROR CODE: ' + str(e.errno))
754	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
755
756	sys.exit('\n... error occured while trying to write ' +
757	tempjobfile)
758
759	return
760
761	def del_fortran_build(src_path):
762	'''Clean up the Fortran source directory and remove all
763	build files (e.g. \.o, \.mod and FORTRAN EXECUTABLE)
764
765	Parameters
766	----------
767	src_path : str
768	Path to the fortran source directory.
769
770	Return
771	------
772
773	'''
774
775	modfiles = UioFiles(src_path, '*.mod')
776	objfiles = UioFiles(src_path, '*.o')
777	exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
778
779	modfiles.delete_files()
780	objfiles.delete_files()
781	exefile.delete_files()
782
783	return
784
785	def mk_fortran_build(src_path, makefile):
786	'''Compiles the Fortran code and generates the executable.
787
788	Parameters
789	----------
790	src_path : str
791	Path to the fortran source directory.
792
793	makefile : str
794	The name of the makefile which should be used.
795
796	Return
797	------
798
799	'''
800
801	try:
802	print('Using makefile: ' + makefile)
803	p = subprocess.Popen(['make', '-f',
804	os.path.join(src_path, makefile)],
805	stdin=subprocess.PIPE,
806	stdout=subprocess.PIPE,
807	stderr=subprocess.PIPE,
808	bufsize=1)
809	pout, perr = p.communicate()
810	print(pout.decode())
811	if p.returncode != 0:
812	print(perr.decode())
813	print('Please edit ' + makefile +
814	' or try another makefile in the src directory.')
815	print('Most likely ECCODES_INCLUDE_DIR, ECCODES_LIB '
816	'and EMOSLIB must be adapted.')
817	print('Available makefiles:')
818	print(UioFiles(src_path, 'makefile*'))
819	sys.exit('Compilation failed!')
820	except ValueError as e:
821	print('ERROR: makefile call failed:')
822	print(e)
823	else:
824	execute_subprocess(['ls', '-l',
825	os.path.join(src_path, _config.FORTRAN_EXECUTABLE)],
826	error_msg='FORTRAN EXECUTABLE COULD NOT BE FOUND!')
827
828	return
829
830
831	if __name__ == "__main__":
832	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: