Context Navigation

source: flex_extract.git/Source/Python/install.py @ 01bd8aa

dev

Last change on this file since 01bd8aa was 01bd8aa, checked in by Anne Tipka <anne.tipka@…>, 22 months ago
added history comments to previous commit changes and minor additional corrections
Property mode set to `100755`
File size: 29.8 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - moved install_args_and_control in here
14	# - splitted code in smaller functions
15	# - delete fortran build files in here instead of compile job script
16	# - changed static path names to variables from config file
17	# - splitted install function into several smaller pieces
18	# - use of tarfile package in python
19	# June 2020 - Anne Philipp
20	# - renamed "convert" functions to "fortran" functions
21	# - reconfigured mk_tarball to select *.template files instead
22	# of .nl and .temp
23	# - added check for makefile settings
24	# August 2020 - Leopold Haimberger
25	# - added a new installation section for system installation (if-else block)
26	# - read new argument from command line
27	# - write .setup.rc for a system installation into Run directory
28	# - copy executables to system path and user files to user path
29	# July 2022 - Anne Tipka (formerly Philipp)
30	# - BugFix: sysinstalldir must be defined in related else path
31	# - template files are now using $HOME and $SCRATCH and hence don't
32	# need ecmwf user group id. Eliminated from function mk_compile_job
33	# and mk_job_template
34	#
35	# @License:
36	# (C) Copyright 2014-2020.
37	# Anne Philipp, Leopold Haimberger
38	#
39	# SPDX-License-Identifier: CC-BY-4.0
40	#
41	# This work is licensed under the Creative Commons Attribution 4.0
42	# International License. To view a copy of this license, visit
43	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
44	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
45	#
46	# @Methods:
47	# main
48	# get_install_cmdline_args
49	# install_via_gateway
50	# check_install_conditions
51	# mk_tarball
52	# un_tarball
53	# mk_env_vars
54	# mk_compilejob
55	# mk_job_template
56	# del_fortran_build
57	# mk_fortran_build
58	#
59	#*******************************************************************************
60	'''This script installs the flex_extract program.
61
62	Depending on the selected installation environment (locally or on the
63	ECMWF server ecgate or cca) the program extracts the command line
64	arguments and the CONTROL file parameter and prepares the corresponding
65	environment.
66	The necessary files are collected in a tar ball and placed
67	at the target location. There, is is untared, the environment variables are
68	set, and the Fortran code is compiled.
69	If the ECMWF environment is selected, a job script is prepared and submitted
70	for the remaining configurations after putting the tar ball on the
71	target ECMWF server.
72
73	Type: install.py --help
74	to get information about command line parameters.
75	Read the documentation for usage instructions.
76	'''
77
78	# ------------------------------------------------------------------------------
79	# MODULES
80	# ------------------------------------------------------------------------------
81	from __future__ import print_function
82
83	import os
84	import sys
85	import subprocess
86	import tarfile
87	import shutil
88	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
89
90	# software specific classes and modules from flex_extract
91	import _config
92	from Classes.ControlFile import ControlFile
93	from Classes.UioFiles import UioFiles
94	from Mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
95	silent_remove, execute_subprocess, none_or_str,
96	overwrite_lines_in_file, check_for_string_in_file)
97
98	# ------------------------------------------------------------------------------
99	# FUNCTIONS
100	# ------------------------------------------------------------------------------
101	def main():
102	'''Controls the installation process. Calls the installation function
103	if target is specified.
104
105	Parameters
106	----------
107
108	Return
109	------
110	'''
111
112	args = get_install_cmdline_args()
113	c = ControlFile(args.controlfile)
114	c.assign_args_to_control(args)
115	check_install_conditions(c)
116
117	if c.install_target.lower() not in ['local', 'syslocal']: # ecmwf servers e.g. ecgate (ecs) and hpc
118	install_via_gateway(c)
119	else: # local
120	install_local(c)
121
122	print("SUCCESS: INSTALLATION FINISHED!")
123
124	return
125
126	def get_install_cmdline_args():
127	'''Decomposes the command line arguments and assigns them to variables.
128	Apply default values for arguments not present.
129
130	Parameters
131	----------
132
133	Return
134	------
135	args : Namespace
136	Contains the commandline arguments from script/program call.
137	'''
138	parser = ArgumentParser(description='Install flex_extract software '
139	'locally or on ECMWF machines',
140	formatter_class=ArgumentDefaultsHelpFormatter)
141
142	parser.add_argument('--target', dest='install_target',
143	type=none_or_str, default=None,
144	help="Valid targets: syslocal \| local \| ecgate \| cca \| \
145	ccb \| ecs \| hpc , the latter 5 are at ECMWF servers")
146	parser.add_argument("--makefile", dest="makefile",
147	type=none_or_str, default=None,
148	help='Name of makefile for compiling the '
149	'Fortran program')
150	parser.add_argument("--ecuid", dest="ecuid",
151	type=none_or_str, default=None,
152	help='User id at ECMWF')
153	parser.add_argument("--ecgid", dest="ecgid",
154	type=none_or_str, default=None,
155	help='Group id at ECMWF')
156	parser.add_argument("--gateway", dest="gateway",
157	type=none_or_str, default=None,
158	help='Name of the local gateway server')
159	parser.add_argument("--destination", dest="destination",
160	type=none_or_str, default=None,
161	help='ecaccess association, e.g. '
162	'myUser@genericSftp')
163
164	parser.add_argument("--installdir", dest="installdir",
165	type=none_or_str, default=None,
166	help='Root (user) directory of the '
167	'flex_extract installation')
168	parser.add_argument("--sysinstalldir", dest="sysinstalldir",
169	type=none_or_str, default=None,
170	help='System installation path; where '
171	'executables are stored.')
172
173	# arguments for job submission to ECMWF, only needed by submit.py
174	parser.add_argument("--job_template", dest='job_template',
175	type=none_or_str, default="job.template",
176	help='Rudimentary template file to create a batch '
177	'job template for submission to ECMWF servers')
178
179	parser.add_argument("--controlfile", dest="controlfile",
180	type=none_or_str, default='CONTROL_EA5',
181	help="A file that contains all CONTROL parameters.")
182
183	args = parser.parse_args()
184
185	return args
186
187
188	def install_via_gateway(c):
189	'''Prepare data transfer to remote gateway and submit a job script which will
190	install everything on the remote gateway.
191
192	Parameters
193	----------
194	c : ControlFile
195	Contains all the parameters of CONTROL file and
196	command line.
197
198	Return
199	------
200
201	'''
202
203	tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
204	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
205
206	mk_compilejob(c.makefile, c.ecuid, c.installdir)
207
208	mk_job_template(c.ecuid, c.ecgid, c.installdir)
209
210	mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
211
212	mk_tarball(tar_file, c.install_target)
213
214	put_file_to_ecserver(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
215
216	submit_job_to_ecserver(c.install_target,
217	os.path.join(_config.PATH_REL_JOBSCRIPTS,
218	_config.FILE_INSTALL_COMPILEJOB))
219
220	silent_remove(tar_file)
221
222	print('Job compilation script has been submitted to ecgate for ' +
223	'installation in ' + c.installdir +
224	'/' + _config.FLEXEXTRACT_DIRNAME)
225	print('You should get an email with subject "flexcompile" within ' +
226	'the next few minutes!')
227
228	return
229
230	def install_local(c):
231	'''Perform the actual installation on a local machine.
232
233	Parameters
234	----------
235	c : ControlFile
236	Contains all the parameters of CONTROL file and
237	command line.
238
239	Return
240	------
241
242	'''
243
244	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR,
245	_config.FLEXEXTRACT_DIRNAME + '.tar')
246
247	# this is standard installation into a single directory
248	if c.install_target == 'local':
249	c.installdir = os.path.abspath(os.path.expandvars(os.path.expanduser(
250	c.installdir)))
251
252	# installation into the current directory
253	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.installdir:
254	print('WARNING: installdir has not been specified')
255	print('flex_extract will be installed in current dir by compiling the ' +
256	'Fortran source in ' + _config.PATH_FORTRAN_SRC)
257	os.chdir(_config.PATH_FORTRAN_SRC)
258	# installation into a different path
259	elif os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) != c.installdir :
260
261	# creates the target working directory for flex_extract
262	mk_tarball(tar_file, c.install_target)
263	make_dir(os.path.join(c.installdir,
264	_config.FLEXEXTRACT_DIRNAME))
265	os.chdir(os.path.join(c.installdir,
266	_config.FLEXEXTRACT_DIRNAME))
267	un_tarball(tar_file)
268	os.chdir(os.path.join(c.installdir,
269	_config.FLEXEXTRACT_DIRNAME,
270	_config.PATH_REL_FORTRAN_SRC))
271
272	# Create Fortran executable
273	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software at ' +
274	c.install_target + ' in directory ' + c.installdir + '\n')
275
276	del_fortran_build('.')
277	mk_fortran_build('.', c.makefile)
278	os.chdir('../../')
279	# make sure that the correct calling of submit.py script is in run_local.sh
280	overwrite_lines_in_file('Run/run_local.sh',
281	'pyscript=', 'pyscript=../Source/Python/submit.py\n')
282
283	# this is system installation were executables and user files are separated
284	elif c.install_target == 'syslocal':
285	c.sysinstalldir = os.path.abspath(os.path.expandvars(os.path.expanduser(
286	c.sysinstalldir)))
287	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.sysinstalldir :
288	sys.exit('ERROR: System installation path is equal to user '
289	'installation path.\n Please change either the system '
290	'installation path or use installation target "local".')
291	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.installdir :
292	print('Flex_extract will be installed in current directory!')
293	else: # install user part in different dir
294	print('Flex_extract will be installed in ' + c.installdir )
295
296	c.installdir = os.path.join(c.installdir,_config.FLEXEXTRACT_DIRNAME)
297	if os.path.isdir(c.installdir):
298	shutil.rmtree(c.installdir)
299
300	# copy all files except Python and Fortranfiles to this dir
301	shutil.copytree(_config.PATH_FLEXEXTRACT_DIR,
302	c.installdir, symlinks=True)
303	shutil.rmtree(os.path.join(c.installdir,'Source'))
304	shutil.rmtree(os.path.join(c.installdir,'.git'))
305	for x in UioFiles(c.installdir, '*~').files:
306	silent_remove(x)
307
308	os.remove(os.path.join(c.installdir,'setup.sh'))
309	os.remove(os.path.join(c.installdir,'setup_local.sh'))
310
311	# configure run_local script correctly
312	# check if source of system config file is already in run_local.sh,
313	# if not, add it
314	if not check_for_string_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
315	'source .setup.rc'):
316	overwrite_lines_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
317	'# PATH TO SUBMISSION SCRIPT',
318	'# PATH TO SUBMISSION SCRIPT\nsource '+_config.FILE_SYS_CONFIG+'\n')
319	# make sure that the correct calling of submit.py script is in run_local.sh
320	overwrite_lines_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
321	'pyscript=', 'pyscript=submit.py\n')
322
323	# change permission for file to executable
324	execute_subprocess(['chmod', '0775',
325	os.path.join(os.path.abspath(c.installdir),'Run/run_local.sh')])
326
327
328	# create systemdir
329	c.sysinstalldir = os.path.join(c.sysinstalldir,_config.FLEXEXTRACT_DIRNAME)
330	if os.path.isdir(c.sysinstalldir):
331	shutil.rmtree(c.sysinstalldir)
332
333	# create setup file for running flex_extract with system installation
334	with open(os.path.join(os.path.abspath(c.installdir),'Run/.setup.rc'),'w') as fio:
335	fio.write('#!/bin/bash \n')
336	fio.write('export FLEXEXTRACT_USER_DIR='+os.path.abspath(c.installdir)+'\n')
337	fio.write('export PATH='+os.path.abspath(c.sysinstalldir)+'/Python:${PATH}\n')
338	fio.write('export PATH='+os.path.abspath(c.sysinstalldir)+':${PATH}\n')
339
340	# copy all Python and Fortranfiles to this dir
341	shutil.copytree(_config.PATH_SOURCES, c.sysinstalldir, symlinks=True)
342
343	os.chdir(os.path.join(c.sysinstalldir,'Fortran'))
344	# Create Fortran executable
345	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software as ' +
346	c.install_target + ' in directory ' +
347	os.path.abspath(c.sysinstalldir) + '\n')
348
349	del_fortran_build('.')
350	mk_fortran_build('.', c.makefile)
351
352	outfile = [x for x in UioFiles('.','*.out').files]
353	test=os.path.join(c.sysinstalldir,'calc_etadot')
354	if len(outfile) != 1:
355	print('WARNING: Multiple executables for Fortran code are available!')
356	# move executable one dir up and delete Fortran dir
357	os.chdir('..')
358	shutil.move(outfile[0], os.path.join(c.sysinstalldir,'calc_etadot'))
359	shutil.rmtree(os.path.join(os.path.abspath(c.sysinstalldir),'Fortran'))
360
361	os.chdir(_config.PATH_FLEXEXTRACT_DIR)
362	if os.path.isfile(tar_file):
363	os.remove(tar_file)
364
365	return
366
367
368	def check_install_conditions(c):
369	'''Checks necessary attributes and conditions
370	for the installation, e.g. whether they exist and contain values.
371	Otherwise set default values.
372
373	Parameters
374	----------
375	c : ControlFile
376	Contains all the parameters of CONTROL file and
377	command line.
378
379
380	Return
381	------
382
383	'''
384
385	if c.install_target and \
386	c.install_target not in _config.INSTALL_TARGETS:
387	print('ERROR: unknown or missing installation target ')
388	print('target: ', c.install_target)
389	print('please specify correct installation target ' +
390	str(_config.INSTALL_TARGETS))
391	print('use -h or --help for help')
392	sys.exit(1)
393
394	if c.install_target and c.install_target not in ['local', 'syslocal']:
395	if not c.ecgid or not c.ecuid:
396	print('Please enter your ECMWF user id and group id '
397	' with command line options --ecuid --ecgid')
398	print('Try "' + sys.argv[0].split('/')[-1] + \
399	' -h" to print usage information')
400	print('Please consult ecaccess documentation or ECMWF user '
401	'support for further details.\n')
402	sys.exit(1)
403	if not c.gateway or not c.destination:
404	print('WARNING: Parameters GATEWAY and DESTINATION were '
405	'not properly set for working on ECMWF server. \n'
406	'There will be no transfer of output files to the '
407	'local gateway server possible!')
408	if not c.installdir:
409	c.installdir = '${HOME}'
410	elif c.install_target == 'local':
411	if not c.installdir:
412	c.installdir = _config.PATH_FLEXEXTRACT_DIR
413	elif c.install_target == 'syslocal':
414	if not c.installdir:
415	c.installdir = _config.PATH_FLEXEXTRACT_DIR
416	if not c.sysinstalldir:
417	print('ERROR: System installation was selected but '
418	'no system installation path was defined.')
419	sys.exit()
420
421	if not c.makefile:
422	print('WARNING: no makefile was specified.')
423	if c.install_target == 'local':
424	c.makefile = 'makefile_local_gfortran'
425	print('WARNING: default makefile selected: makefile_local_gfortan')
426	elif c.install_target == 'ecgate':
427	c.makefile = 'makefile_ecgate'
428	print('WARNING: default makefile selected: makefile_ecgate')
429	elif c.install_target == 'cca' or \
430	c.install_target == 'ccb':
431	c.makefile = 'makefile_cray'
432	print('WARNING: default makefile selected: makefile_cray')
433	else:
434	pass
435
436	return
437
438
439	def mk_tarball(tarball_path, target):
440	'''Creates a tarball with all necessary files which need to be sent to the
441	installation directory.
442	It does not matter whether this is local or remote.
443	Collects all Python files, the Fortran source and makefiles,
444	the ECMWF_ENV file, the CONTROL files as well as the
445	template files.
446
447	Parameters
448	----------
449	tarball_path : str
450	The complete path to the tar file which will contain all
451	relevant data for flex_extract.
452
453	target : str
454	The queue where the job is submitted to.
455
456	Return
457	------
458
459	'''
460
461	print('Create tarball ...')
462
463	# change to FLEXEXTRACT directory so that the tar can contain
464	# relative pathes to the files and directories
465	ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
466	os.chdir(ecd)
467
468	# get lists of the files to be added to the tar file
469	if target == 'local':
470	ecmwf_env_file = []
471	runfile = [os.path.relpath(x, ecd)
472	for x in UioFiles(_config.PATH_REL_RUN_DIR,
473	'run_local.sh').files]
474	else:
475	ecmwf_env_file = [_config.PATH_REL_ECMWF_ENV]
476	runfile = [os.path.relpath(x, ecd)
477	for x in UioFiles(_config.PATH_REL_RUN_DIR,
478	'run.sh').files]
479
480	pyfiles = [os.path.relpath(x, ecd)
481	for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
482	pytestfiles = [os.path.relpath(x, ecd)
483	for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
484	controlfiles = [os.path.relpath(x, ecd)
485	for x in UioFiles(_config.PATH_REL_CONTROLFILES,
486	'CONTROL*').files]
487	testfiles = [os.path.relpath(x, ecd)
488	for x in UioFiles(_config.PATH_REL_TEST+"/Installation", '*').files]
489	tempfiles = [os.path.relpath(x, ecd)
490	for x in UioFiles(_config.PATH_REL_TEMPLATES, '*.template').files]
491	gribtable = [os.path.relpath(x, ecd)
492	for x in UioFiles(_config.PATH_REL_TEMPLATES, 'grib').files]
493	ffiles = [os.path.relpath(x, ecd)
494	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f90').files]
495	hfiles = [os.path.relpath(x, ecd)
496	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
497	makefiles = [os.path.relpath(x, ecd)
498	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'makefile*').files]
499	jobdir = [os.path.relpath(x, ecd)
500	for x in UioFiles(_config.PATH_REL_JOBSCRIPTS, '*.md').files]
501
502	# concatenate single lists to one for a better looping
503	filelist = pyfiles + pytestfiles + controlfiles + tempfiles + \
504	ffiles + gribtable + hfiles + makefiles + ecmwf_env_file + \
505	runfile + jobdir + testfiles +\
506	['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
507
508	# create installation tar-file
509	exclude_files = [".ksh", ".tar"]
510	try:
511	with tarfile.open(tarball_path, "w:gz") as tar_handle:
512	for filename in filelist:
513	tar_handle.add(filename, recursive=False,
514	filter=lambda tarinfo: None
515	if os.path.splitext(tarinfo.name)[1]
516	in exclude_files
517	else tarinfo)
518	except tarfile.TarError as e:
519	print('... ERROR: ' + str(e))
520
521	sys.exit('\n... error occured while trying to create the tar-file ' +
522	str(tarball_path))
523
524	return
525
526
527	def un_tarball(tarball_path):
528	'''Extracts the given tarball into current directory.
529
530	Parameters
531	----------
532	tarball_path : str
533	The complete path to the tar file which will contain all
534	relevant data for flex_extract.
535
536	Return
537	------
538
539	'''
540
541	print('Untar ...')
542
543	try:
544	with tarfile.open(tarball_path) as tar_handle:
545	tar_handle.extractall()
546	except tarfile.TarError as e:
547	sys.exit('\n... error occured while trying to read tar-file ' +
548	str(tarball_path))
549	except OSError as e:
550	print('... ERROR CODE: ' + str(e.errno))
551	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
552
553	sys.exit('\n... error occured while trying to read tar-file ' +
554	str(tarball_path))
555
556	return
557
558	def mk_env_vars(ecuid, ecgid, gateway, destination):
559	'''Creates a file named ECMWF_ENV which contains the
560	necessary environmental variables at ECMWF servers.
561	It is based on the template ECMWF_ENV.template.
562
563	Parameters
564	----------
565	ecuid : str
566	The user id on ECMWF server.
567
568	ecgid : str
569	The group id on ECMWF server.
570
571	gateway : str
572	The gateway server the user is using.
573
574	destination : str
575	The remote destination which is used to transfer files
576	from ECMWF server to local gateway server.
577
578	Return
579	------
580
581	'''
582	from genshi.template.text import NewTextTemplate
583	from genshi.template import TemplateLoader
584	from genshi.template.eval import UndefinedError
585
586	try:
587	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
588	ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
589	cls=NewTextTemplate)
590
591	stream = ecmwfvars_template.generate(user_name=ecuid,
592	user_group=ecgid,
593	gateway_name=gateway,
594	destination_name=destination
595	)
596	except UndefinedError as e:
597	print('... ERROR ' + str(e))
598
599	sys.exit('\n... error occured while trying to generate template ' +
600	_config.PATH_ECMWF_ENV)
601	except OSError as e:
602	print('... ERROR CODE: ' + str(e.errno))
603	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
604
605	sys.exit('\n... error occured while trying to generate template ' +
606	_config.PATH_ECMWF_ENV)
607
608	try:
609	with open(_config.PATH_ECMWF_ENV, 'w') as f:
610	f.write(stream.render('text'))
611	except OSError as e:
612	print('... ERROR CODE: ' + str(e.errno))
613	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
614
615	sys.exit('\n... error occured while trying to write ' +
616	_config.PATH_ECMWF_ENV)
617
618	return
619
620	def mk_compilejob(makefile, ecuid, fp_root):
621	'''Modifies the original job template file so that it is specified
622	for the user and the environment were it will be applied. Result
623	is stored in a new file "compilejob.ksh" in the Jobscript directory.
624
625	Parameters
626	----------
627	makefile : str
628	Name of the makefile which should be used to compile the Fortran
629	program.
630
631	ecuid : str
632	The user id on ECMWF server.
633
634	fp_root : str
635	Path to the root directory of FLEXPART environment or flex_extract
636	environment.
637
638	Return
639	------
640
641	'''
642	from genshi.template.text import NewTextTemplate
643	from genshi.template import TemplateLoader
644	from genshi.template.eval import UndefinedError
645
646	if fp_root == '../':
647	fp_root = '$HOME'
648
649	try:
650	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
651	compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
652	cls=NewTextTemplate)
653
654	stream = compile_template.generate(
655	username=ecuid,
656	version_number=_config._VERSION_STR,
657	fp_root_scripts=fp_root,
658	makefile=makefile,
659	fortran_program=_config.FORTRAN_EXECUTABLE
660	)
661	except UndefinedError as e:
662	print('... ERROR ' + str(e))
663
664	sys.exit('\n... error occured while trying to generate template ' +
665	_config.TEMPFILE_INSTALL_COMPILEJOB)
666	except OSError as e:
667	print('... ERROR CODE: ' + str(e.errno))
668	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
669
670	sys.exit('\n... error occured while trying to generate template ' +
671	_config.TEMPFILE_INSTALL_COMPILEJOB)
672
673	try:
674	compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
675	_config.FILE_INSTALL_COMPILEJOB)
676
677	with open(compilejob, 'w') as f:
678	f.write(stream.render('text'))
679	except OSError as e:
680	print('... ERROR CODE: ' + str(e.errno))
681	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
682
683	sys.exit('\n... error occured while trying to write ' +
684	compilejob)
685
686	return
687
688	def mk_job_template(ecuid, fp_root):
689	'''Modifies the original job template file so that it is specified
690	for the user and the environment were it will be applied. Result
691	is stored in a new file.
692
693	Parameters
694	----------
695	ecuid : str
696	The user id on ECMWF server.
697
698	fp_root : str
699	Path to the root directory of FLEXPART environment or flex_extract
700	environment.
701
702	Return
703	------
704
705	'''
706	from genshi.template.text import NewTextTemplate
707	from genshi.template import TemplateLoader
708	from genshi.template.eval import UndefinedError
709
710	fp_root_path_to_python = os.path.join(fp_root,
711	_config.FLEXEXTRACT_DIRNAME,
712	_config.PATH_REL_PYTHON_SRC)
713	if '$' in fp_root_path_to_python:
714	ind = fp_root_path_to_python.index('$')
715	fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
716	fp_root_path_to_python[ind:]
717
718	try:
719	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
720	compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
721	cls=NewTextTemplate)
722
723	stream = compile_template.generate(
724	username=ecuid,
725	version_number=_config._VERSION_STR,
726	fp_root_path=fp_root_path_to_python,
727	)
728	except UndefinedError as e:
729	print('... ERROR ' + str(e))
730
731	sys.exit('\n... error occured while trying to generate template ' +
732	_config.TEMPFILE_INSTALL_JOB)
733	except OSError as e:
734	print('... ERROR CODE: ' + str(e.errno))
735	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
736
737	sys.exit('\n... error occured while trying to generate template ' +
738	_config.TEMPFILE_INSTALL_JOB)
739
740
741	try:
742	tempjobfile = os.path.join(_config.PATH_TEMPLATES,
743	_config.TEMPFILE_JOB)
744
745	with open(tempjobfile, 'w') as f:
746	f.write(stream.render('text'))
747	except OSError as e:
748	print('... ERROR CODE: ' + str(e.errno))
749	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
750
751	sys.exit('\n... error occured while trying to write ' +
752	tempjobfile)
753
754	return
755
756	def del_fortran_build(src_path):
757	'''Clean up the Fortran source directory and remove all
758	build files (e.g. \.o, \.mod and FORTRAN EXECUTABLE)
759
760	Parameters
761	----------
762	src_path : str
763	Path to the fortran source directory.
764
765	Return
766	------
767
768	'''
769
770	modfiles = UioFiles(src_path, '*.mod')
771	objfiles = UioFiles(src_path, '*.o')
772	exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
773
774	modfiles.delete_files()
775	objfiles.delete_files()
776	exefile.delete_files()
777
778	return
779
780	def mk_fortran_build(src_path, makefile):
781	'''Compiles the Fortran code and generates the executable.
782
783	Parameters
784	----------
785	src_path : str
786	Path to the fortran source directory.
787
788	makefile : str
789	The name of the makefile which should be used.
790
791	Return
792	------
793
794	'''
795
796	try:
797	print('Using makefile: ' + makefile)
798	p = subprocess.Popen(['make', '-f',
799	os.path.join(src_path, makefile)],
800	stdin=subprocess.PIPE,
801	stdout=subprocess.PIPE,
802	stderr=subprocess.PIPE,
803	bufsize=1)
804	pout, perr = p.communicate()
805	print(pout.decode())
806	if p.returncode != 0:
807	print(perr.decode())
808	print('Please edit ' + makefile +
809	' or try another makefile in the src directory.')
810	print('Most likely ECCODES_INCLUDE_DIR, ECCODES_LIB '
811	'and EMOSLIB must be adapted.')
812	print('Available makefiles:')
813	print(UioFiles(src_path, 'makefile*'))
814	sys.exit('Compilation failed!')
815	except ValueError as e:
816	print('ERROR: makefile call failed:')
817	print(e)
818	else:
819	execute_subprocess(['ls', '-l',
820	os.path.join(src_path, _config.FORTRAN_EXECUTABLE)],
821	error_msg='FORTRAN EXECUTABLE COULD NOT BE FOUND!')
822
823	return
824
825
826	if __name__ == "__main__":
827	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: