Context Navigation

source: flex_extract.git/Source/Python/install.py @ 94d3227

dev

Last change on this file since 94d3227 was 94d3227, checked in by Anne Tipka <anne.tipka@…>, 19 months ago
switch back to using ecaccess-job-submit for installation script
Property mode set to `100755`
File size: 30.0 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - moved install_args_and_control in here
14	# - splitted code in smaller functions
15	# - delete fortran build files in here instead of compile job script
16	# - changed static path names to variables from config file
17	# - splitted install function into several smaller pieces
18	# - use of tarfile package in python
19	# June 2020 - Anne Philipp
20	# - renamed "convert" functions to "fortran" functions
21	# - reconfigured mk_tarball to select *.template files instead
22	# of .nl and .temp
23	# - added check for makefile settings
24	# August 2020 - Leopold Haimberger
25	# - added a new installation section for system installation (if-else block)
26	# - read new argument from command line
27	# - write .setup.rc for a system installation into Run directory
28	# - copy executables to system path and user files to user path
29	# July 2022 - Anne Tipka (formerly Philipp)
30	# - BugFix: sysinstalldir must be defined in related else path
31	# - template files are now using $HOME and $SCRATCH and hence don't
32	# need ecmwf user group id. Eliminated from function mk_compile_job
33	# and mk_job_template
34	#
35	# @License:
36	# (C) Copyright 2014-2020.
37	# Anne Philipp, Leopold Haimberger
38	#
39	# SPDX-License-Identifier: CC-BY-4.0
40	#
41	# This work is licensed under the Creative Commons Attribution 4.0
42	# International License. To view a copy of this license, visit
43	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
44	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
45	#
46	# @Methods:
47	# main
48	# get_install_cmdline_args
49	# install_via_gateway
50	# check_install_conditions
51	# mk_tarball
52	# un_tarball
53	# mk_env_vars
54	# mk_compilejob
55	# mk_job_template
56	# del_fortran_build
57	# mk_fortran_build
58	#
59	#*******************************************************************************
60	'''This script installs the flex_extract program.
61
62	Depending on the selected installation environment (locally or on the
63	ECMWF server ecgate or cca) the program extracts the command line
64	arguments and the CONTROL file parameter and prepares the corresponding
65	environment.
66	The necessary files are collected in a tar ball and placed
67	at the target location. There, is is untared, the environment variables are
68	set, and the Fortran code is compiled.
69	If the ECMWF environment is selected, a job script is prepared and submitted
70	for the remaining configurations after putting the tar ball on the
71	target ECMWF server.
72
73	Type: install.py --help
74	to get information about command line parameters.
75	Read the documentation for usage instructions.
76	'''
77
78	# ------------------------------------------------------------------------------
79	# MODULES
80	# ------------------------------------------------------------------------------
81	from __future__ import print_function
82
83	import os
84	import sys
85	import subprocess
86	import tarfile
87	import shutil
88	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
89
90	# software specific classes and modules from flex_extract
91	import _config
92	from Classes.ControlFile import ControlFile
93	from Classes.UioFiles import UioFiles
94	from Mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
95	silent_remove, execute_subprocess, none_or_str,
96	overwrite_lines_in_file, check_for_string_in_file,
97	submit_sbatch_job)
98
99	# ------------------------------------------------------------------------------
100	# FUNCTIONS
101	# ------------------------------------------------------------------------------
102	def main():
103	'''Controls the installation process. Calls the installation function
104	if target is specified.
105
106	Parameters
107	----------
108
109	Return
110	------
111	'''
112
113	args = get_install_cmdline_args()
114	c = ControlFile(args.controlfile)
115	c.assign_args_to_control(args)
116	check_install_conditions(c)
117
118	if c.install_target.lower() not in ['local', 'syslocal']: # ecmwf servers e.g. ecgate (ecs) and hpc
119	install_via_gateway(c)
120	else: # local
121	install_local(c)
122
123	print("SUCCESS: INSTALLATION FINISHED!")
124
125	return
126
127	def get_install_cmdline_args():
128	'''Decomposes the command line arguments and assigns them to variables.
129	Apply default values for arguments not present.
130
131	Parameters
132	----------
133
134	Return
135	------
136	args : Namespace
137	Contains the commandline arguments from script/program call.
138	'''
139	parser = ArgumentParser(description='Install flex_extract software '
140	'locally or on ECMWF machines',
141	formatter_class=ArgumentDefaultsHelpFormatter)
142
143	parser.add_argument('--target', dest='install_target',
144	type=none_or_str, default=None,
145	help="Valid targets: syslocal \| local \| ecgate \| cca \| \
146	ccb \| ecs \| hpc , the latter 5 are at ECMWF servers")
147	parser.add_argument("--makefile", dest="makefile",
148	type=none_or_str, default=None,
149	help='Name of makefile for compiling the '
150	'Fortran program')
151	parser.add_argument("--ecuid", dest="ecuid",
152	type=none_or_str, default=None,
153	help='User id at ECMWF')
154	parser.add_argument("--ecgid", dest="ecgid",
155	type=none_or_str, default=None,
156	help='Group id at ECMWF')
157	parser.add_argument("--gateway", dest="gateway",
158	type=none_or_str, default=None,
159	help='Name of the local gateway server')
160	parser.add_argument("--destination", dest="destination",
161	type=none_or_str, default=None,
162	help='ecaccess association, e.g. '
163	'myUser@genericSftp')
164
165	parser.add_argument("--installdir", dest="installdir",
166	type=none_or_str, default=None,
167	help='Root (user) directory of the '
168	'flex_extract installation')
169	parser.add_argument("--sysinstalldir", dest="sysinstalldir",
170	type=none_or_str, default=None,
171	help='System installation path; where '
172	'executables are stored.')
173
174	# arguments for job submission to ECMWF, only needed by submit.py
175	parser.add_argument("--job_template", dest='job_template',
176	type=none_or_str, default="job.template",
177	help='Rudimentary template file to create a batch '
178	'job template for submission to ECMWF servers')
179
180	parser.add_argument("--controlfile", dest="controlfile",
181	type=none_or_str, default='CONTROL_EA5',
182	help="A file that contains all CONTROL parameters.")
183
184	args = parser.parse_args()
185
186	return args
187
188
189	def install_via_gateway(c):
190	'''Prepare data transfer to remote gateway and submit a job script which will
191	install everything on the remote gateway.
192
193	Parameters
194	----------
195	c : ControlFile
196	Contains all the parameters of CONTROL file and
197	command line.
198
199	Return
200	------
201
202	'''
203
204	tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
205	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
206
207	mk_compilejob(c.makefile, c.ecuid, c.installdir)
208
209	mk_job_template(c.ecuid, c.installdir)
210
211	mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
212
213	mk_tarball(tar_file, c.install_target)
214
215	put_file_to_ecserver(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
216
217	# submit_sbatch_job(os.path.join(_config.PATH_REL_JOBSCRIPTS,
218	# _config.FILE_INSTALL_COMPILEJOB))
219
220	submit_job_to_ecserver(c.install_target,
221	os.path.join(_config.PATH_REL_JOBSCRIPTS,
222	_config.FILE_INSTALL_COMPILEJOB))
223
224	silent_remove(tar_file)
225
226	print('Job compilation script has been submitted to ecgate for ' +
227	'installation in ' + c.installdir +
228	'/' + _config.FLEXEXTRACT_DIRNAME)
229	print('You should get an email with subject "flexcompile" within ' +
230	'the next few minutes!')
231
232	return
233
234	def install_local(c):
235	'''Perform the actual installation on a local machine.
236
237	Parameters
238	----------
239	c : ControlFile
240	Contains all the parameters of CONTROL file and
241	command line.
242
243	Return
244	------
245
246	'''
247
248	tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR,
249	_config.FLEXEXTRACT_DIRNAME + '.tar')
250
251	# this is standard installation into a single directory
252	if c.install_target == 'local':
253	c.installdir = os.path.abspath(os.path.expandvars(os.path.expanduser(
254	c.installdir)))
255
256	# installation into the current directory
257	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.installdir:
258	print('WARNING: installdir has not been specified')
259	print('flex_extract will be installed in current dir by compiling the ' +
260	'Fortran source in ' + _config.PATH_FORTRAN_SRC)
261	os.chdir(_config.PATH_FORTRAN_SRC)
262	# installation into a different path
263	elif os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) != c.installdir :
264
265	# creates the target working directory for flex_extract
266	mk_tarball(tar_file, c.install_target)
267	make_dir(os.path.join(c.installdir,
268	_config.FLEXEXTRACT_DIRNAME))
269	os.chdir(os.path.join(c.installdir,
270	_config.FLEXEXTRACT_DIRNAME))
271	un_tarball(tar_file)
272	os.chdir(os.path.join(c.installdir,
273	_config.FLEXEXTRACT_DIRNAME,
274	_config.PATH_REL_FORTRAN_SRC))
275
276	# Create Fortran executable
277	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software at ' +
278	c.install_target + ' in directory ' + c.installdir + '\n')
279
280	del_fortran_build('.')
281	mk_fortran_build('.', c.makefile)
282	os.chdir('../../')
283	# make sure that the correct calling of submit.py script is in run_local.sh
284	overwrite_lines_in_file('Run/run_local.sh',
285	'pyscript=', 'pyscript=../Source/Python/submit.py\n')
286
287	# this is system installation were executables and user files are separated
288	elif c.install_target == 'syslocal':
289	c.sysinstalldir = os.path.abspath(os.path.expandvars(os.path.expanduser(
290	c.sysinstalldir)))
291	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.sysinstalldir :
292	sys.exit('ERROR: System installation path is equal to user '
293	'installation path.\n Please change either the system '
294	'installation path or use installation target "local".')
295	if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.installdir :
296	print('Flex_extract will be installed in current directory!')
297	else: # install user part in different dir
298	print('Flex_extract will be installed in ' + c.installdir )
299
300	c.installdir = os.path.join(c.installdir,_config.FLEXEXTRACT_DIRNAME)
301	if os.path.isdir(c.installdir):
302	shutil.rmtree(c.installdir)
303
304	# copy all files except Python and Fortranfiles to this dir
305	shutil.copytree(_config.PATH_FLEXEXTRACT_DIR,
306	c.installdir, symlinks=True)
307	shutil.rmtree(os.path.join(c.installdir,'Source'))
308	shutil.rmtree(os.path.join(c.installdir,'.git'))
309	for x in UioFiles(c.installdir, '*~').files:
310	silent_remove(x)
311
312	os.remove(os.path.join(c.installdir,'setup.sh'))
313	os.remove(os.path.join(c.installdir,'setup_local.sh'))
314
315	# configure run_local script correctly
316	# check if source of system config file is already in run_local.sh,
317	# if not, add it
318	if not check_for_string_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
319	'source .setup.rc'):
320	overwrite_lines_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
321	'# PATH TO SUBMISSION SCRIPT',
322	'# PATH TO SUBMISSION SCRIPT\nsource '+_config.FILE_SYS_CONFIG+'\n')
323	# make sure that the correct calling of submit.py script is in run_local.sh
324	overwrite_lines_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
325	'pyscript=', 'pyscript=submit.py\n')
326
327	# change permission for file to executable
328	execute_subprocess(['chmod', '0775',
329	os.path.join(os.path.abspath(c.installdir),'Run/run_local.sh')])
330
331
332	# create systemdir
333	c.sysinstalldir = os.path.join(c.sysinstalldir,_config.FLEXEXTRACT_DIRNAME)
334	if os.path.isdir(c.sysinstalldir):
335	shutil.rmtree(c.sysinstalldir)
336
337	# create setup file for running flex_extract with system installation
338	with open(os.path.join(os.path.abspath(c.installdir),'Run/.setup.rc'),'w') as fio:
339	fio.write('#!/bin/bash \n')
340	fio.write('export FLEXEXTRACT_USER_DIR='+os.path.abspath(c.installdir)+'\n')
341	fio.write('export PATH='+os.path.abspath(c.sysinstalldir)+'/Python:${PATH}\n')
342	fio.write('export PATH='+os.path.abspath(c.sysinstalldir)+':${PATH}\n')
343
344	# copy all Python and Fortranfiles to this dir
345	shutil.copytree(_config.PATH_SOURCES, c.sysinstalldir, symlinks=True)
346
347	os.chdir(os.path.join(c.sysinstalldir,'Fortran'))
348	# Create Fortran executable
349	print('Install ' + _config.FLEXEXTRACT_DIRNAME + ' software as ' +
350	c.install_target + ' in directory ' +
351	os.path.abspath(c.sysinstalldir) + '\n')
352
353	del_fortran_build('.')
354	mk_fortran_build('.', c.makefile)
355
356	outfile = [x for x in UioFiles('.','*.out').files]
357	test=os.path.join(c.sysinstalldir,'calc_etadot')
358	if len(outfile) != 1:
359	print('WARNING: Multiple executables for Fortran code are available!')
360	# move executable one dir up and delete Fortran dir
361	os.chdir('..')
362	shutil.move(outfile[0], os.path.join(c.sysinstalldir,'calc_etadot'))
363	shutil.rmtree(os.path.join(os.path.abspath(c.sysinstalldir),'Fortran'))
364
365	os.chdir(_config.PATH_FLEXEXTRACT_DIR)
366	if os.path.isfile(tar_file):
367	os.remove(tar_file)
368
369	return
370
371
372	def check_install_conditions(c):
373	'''Checks necessary attributes and conditions
374	for the installation, e.g. whether they exist and contain values.
375	Otherwise set default values.
376
377	Parameters
378	----------
379	c : ControlFile
380	Contains all the parameters of CONTROL file and
381	command line.
382
383
384	Return
385	------
386
387	'''
388
389	if c.install_target and \
390	c.install_target not in _config.INSTALL_TARGETS:
391	print('ERROR: unknown or missing installation target ')
392	print('target: ', c.install_target)
393	print('please specify correct installation target ' +
394	str(_config.INSTALL_TARGETS))
395	print('use -h or --help for help')
396	sys.exit(1)
397
398	if c.install_target and c.install_target not in ['local', 'syslocal']:
399	if not c.ecgid or not c.ecuid:
400	print('Please enter your ECMWF user id and group id '
401	' with command line options --ecuid --ecgid')
402	print('Try "' + sys.argv[0].split('/')[-1] + \
403	' -h" to print usage information')
404	print('Please consult ecaccess documentation or ECMWF user '
405	'support for further details.\n')
406	sys.exit(1)
407	if not c.gateway or not c.destination:
408	print('WARNING: Parameters GATEWAY and DESTINATION were '
409	'not properly set for working on ECMWF server. \n'
410	'There will be no transfer of output files to the '
411	'local gateway server possible!')
412	if not c.installdir:
413	c.installdir = '${HOME}'
414	elif c.install_target == 'local':
415	if not c.installdir:
416	c.installdir = _config.PATH_FLEXEXTRACT_DIR
417	elif c.install_target == 'syslocal':
418	if not c.installdir:
419	c.installdir = _config.PATH_FLEXEXTRACT_DIR
420	if not c.sysinstalldir:
421	print('ERROR: System installation was selected but '
422	'no system installation path was defined.')
423	sys.exit()
424
425	if not c.makefile:
426	print('WARNING: no makefile was specified.')
427	if c.install_target == 'local':
428	c.makefile = 'makefile_local_gfortran'
429	print('WARNING: default makefile selected: makefile_local_gfortan')
430	elif c.install_target == 'ecgate':
431	c.makefile = 'makefile_ecgate'
432	print('WARNING: default makefile selected: makefile_ecgate')
433	elif c.install_target == 'cca' or \
434	c.install_target == 'ccb':
435	c.makefile = 'makefile_cray'
436	print('WARNING: default makefile selected: makefile_cray')
437	else:
438	pass
439
440	return
441
442
443	def mk_tarball(tarball_path, target):
444	'''Creates a tarball with all necessary files which need to be sent to the
445	installation directory.
446	It does not matter whether this is local or remote.
447	Collects all Python files, the Fortran source and makefiles,
448	the ECMWF_ENV file, the CONTROL files as well as the
449	template files.
450
451	Parameters
452	----------
453	tarball_path : str
454	The complete path to the tar file which will contain all
455	relevant data for flex_extract.
456
457	target : str
458	The queue where the job is submitted to.
459
460	Return
461	------
462
463	'''
464
465	print('Create tarball ...')
466
467	# change to FLEXEXTRACT directory so that the tar can contain
468	# relative pathes to the files and directories
469	ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
470	os.chdir(ecd)
471
472	# get lists of the files to be added to the tar file
473	if target == 'local':
474	ecmwf_env_file = []
475	runfile = [os.path.relpath(x, ecd)
476	for x in UioFiles(_config.PATH_REL_RUN_DIR,
477	'run_local.sh').files]
478	else:
479	ecmwf_env_file = [_config.PATH_REL_ECMWF_ENV]
480	runfile = [os.path.relpath(x, ecd)
481	for x in UioFiles(_config.PATH_REL_RUN_DIR,
482	'run.sh').files]
483
484	pyfiles = [os.path.relpath(x, ecd)
485	for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
486	pytestfiles = [os.path.relpath(x, ecd)
487	for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
488	controlfiles = [os.path.relpath(x, ecd)
489	for x in UioFiles(_config.PATH_REL_CONTROLFILES,
490	'CONTROL*').files]
491	testfiles = [os.path.relpath(x, ecd)
492	for x in UioFiles(_config.PATH_REL_TEST+"/Installation", '*').files]
493	tempfiles = [os.path.relpath(x, ecd)
494	for x in UioFiles(_config.PATH_REL_TEMPLATES, '*.template').files]
495	gribtable = [os.path.relpath(x, ecd)
496	for x in UioFiles(_config.PATH_REL_TEMPLATES, 'grib').files]
497	ffiles = [os.path.relpath(x, ecd)
498	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f90').files]
499	hfiles = [os.path.relpath(x, ecd)
500	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
501	makefiles = [os.path.relpath(x, ecd)
502	for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'makefile*').files]
503	jobdir = [os.path.relpath(x, ecd)
504	for x in UioFiles(_config.PATH_REL_JOBSCRIPTS, '*.md').files]
505
506	# concatenate single lists to one for a better looping
507	filelist = pyfiles + pytestfiles + controlfiles + tempfiles + \
508	ffiles + gribtable + hfiles + makefiles + ecmwf_env_file + \
509	runfile + jobdir + testfiles +\
510	['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
511
512	# create installation tar-file
513	exclude_files = [".ksh", ".tar"]
514	try:
515	with tarfile.open(tarball_path, "w:gz") as tar_handle:
516	for filename in filelist:
517	tar_handle.add(filename, recursive=False,
518	filter=lambda tarinfo: None
519	if os.path.splitext(tarinfo.name)[1]
520	in exclude_files
521	else tarinfo)
522	except tarfile.TarError as e:
523	print('... ERROR: ' + str(e))
524
525	sys.exit('\n... error occured while trying to create the tar-file ' +
526	str(tarball_path))
527
528	return
529
530
531	def un_tarball(tarball_path):
532	'''Extracts the given tarball into current directory.
533
534	Parameters
535	----------
536	tarball_path : str
537	The complete path to the tar file which will contain all
538	relevant data for flex_extract.
539
540	Return
541	------
542
543	'''
544
545	print('Untar ...')
546
547	try:
548	with tarfile.open(tarball_path) as tar_handle:
549	tar_handle.extractall()
550	except tarfile.TarError as e:
551	sys.exit('\n... error occured while trying to read tar-file ' +
552	str(tarball_path))
553	except OSError as e:
554	print('... ERROR CODE: ' + str(e.errno))
555	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
556
557	sys.exit('\n... error occured while trying to read tar-file ' +
558	str(tarball_path))
559
560	return
561
562	def mk_env_vars(ecuid, ecgid, gateway, destination):
563	'''Creates a file named ECMWF_ENV which contains the
564	necessary environmental variables at ECMWF servers.
565	It is based on the template ECMWF_ENV.template.
566
567	Parameters
568	----------
569	ecuid : str
570	The user id on ECMWF server.
571
572	ecgid : str
573	The group id on ECMWF server.
574
575	gateway : str
576	The gateway server the user is using.
577
578	destination : str
579	The remote destination which is used to transfer files
580	from ECMWF server to local gateway server.
581
582	Return
583	------
584
585	'''
586	from genshi.template.text import NewTextTemplate
587	from genshi.template import TemplateLoader
588	from genshi.template.eval import UndefinedError
589
590	try:
591	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
592	ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
593	cls=NewTextTemplate)
594
595	stream = ecmwfvars_template.generate(user_name=ecuid,
596	user_group=ecgid,
597	gateway_name=gateway,
598	destination_name=destination
599	)
600	except UndefinedError as e:
601	print('... ERROR ' + str(e))
602
603	sys.exit('\n... error occured while trying to generate template ' +
604	_config.PATH_ECMWF_ENV)
605	except OSError as e:
606	print('... ERROR CODE: ' + str(e.errno))
607	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
608
609	sys.exit('\n... error occured while trying to generate template ' +
610	_config.PATH_ECMWF_ENV)
611
612	try:
613	with open(_config.PATH_ECMWF_ENV, 'w') as f:
614	f.write(stream.render('text'))
615	except OSError as e:
616	print('... ERROR CODE: ' + str(e.errno))
617	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
618
619	sys.exit('\n... error occured while trying to write ' +
620	_config.PATH_ECMWF_ENV)
621
622	return
623
624	def mk_compilejob(makefile, ecuid, fp_root):
625	'''Modifies the original job template file so that it is specified
626	for the user and the environment were it will be applied. Result
627	is stored in a new file "compilejob.ksh" in the Jobscript directory.
628
629	Parameters
630	----------
631	makefile : str
632	Name of the makefile which should be used to compile the Fortran
633	program.
634
635	ecuid : str
636	The user id on ECMWF server.
637
638	fp_root : str
639	Path to the root directory of FLEXPART environment or flex_extract
640	environment.
641
642	Return
643	------
644
645	'''
646	from genshi.template.text import NewTextTemplate
647	from genshi.template import TemplateLoader
648	from genshi.template.eval import UndefinedError
649
650	if fp_root == '../':
651	fp_root = '$HOME'
652
653	try:
654	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
655	compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
656	cls=NewTextTemplate)
657
658	stream = compile_template.generate(
659	username=ecuid,
660	version_number=_config._VERSION_STR,
661	fp_root_scripts=fp_root,
662	makefile=makefile,
663	fortran_program=_config.FORTRAN_EXECUTABLE
664	)
665	except UndefinedError as e:
666	print('... ERROR ' + str(e))
667
668	sys.exit('\n... error occured while trying to generate template ' +
669	_config.TEMPFILE_INSTALL_COMPILEJOB)
670	except OSError as e:
671	print('... ERROR CODE: ' + str(e.errno))
672	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
673
674	sys.exit('\n... error occured while trying to generate template ' +
675	_config.TEMPFILE_INSTALL_COMPILEJOB)
676
677	try:
678	compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
679	_config.FILE_INSTALL_COMPILEJOB)
680
681	with open(compilejob, 'w') as f:
682	f.write(stream.render('text'))
683	except OSError as e:
684	print('... ERROR CODE: ' + str(e.errno))
685	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
686
687	sys.exit('\n... error occured while trying to write ' +
688	compilejob)
689
690	return
691
692	def mk_job_template(ecuid, fp_root):
693	'''Modifies the original job template file so that it is specified
694	for the user and the environment were it will be applied. Result
695	is stored in a new file.
696
697	Parameters
698	----------
699	ecuid : str
700	The user id on ECMWF server.
701
702	fp_root : str
703	Path to the root directory of FLEXPART environment or flex_extract
704	environment.
705
706	Return
707	------
708
709	'''
710	from genshi.template.text import NewTextTemplate
711	from genshi.template import TemplateLoader
712	from genshi.template.eval import UndefinedError
713
714	fp_root_path_to_python = os.path.join(fp_root,
715	_config.FLEXEXTRACT_DIRNAME,
716	_config.PATH_REL_PYTHON_SRC)
717	if '$' in fp_root_path_to_python:
718	ind = fp_root_path_to_python.index('$')
719	fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
720	fp_root_path_to_python[ind:]
721
722	try:
723	loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
724	compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
725	cls=NewTextTemplate)
726
727	stream = compile_template.generate(
728	username=ecuid,
729	version_number=_config._VERSION_STR,
730	fp_root_path=fp_root_path_to_python,
731	)
732	except UndefinedError as e:
733	print('... ERROR ' + str(e))
734
735	sys.exit('\n... error occured while trying to generate template ' +
736	_config.TEMPFILE_INSTALL_JOB)
737	except OSError as e:
738	print('... ERROR CODE: ' + str(e.errno))
739	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
740
741	sys.exit('\n... error occured while trying to generate template ' +
742	_config.TEMPFILE_INSTALL_JOB)
743
744
745	try:
746	tempjobfile = os.path.join(_config.PATH_TEMPLATES,
747	_config.TEMPFILE_JOB)
748
749	with open(tempjobfile, 'w') as f:
750	f.write(stream.render('text'))
751	except OSError as e:
752	print('... ERROR CODE: ' + str(e.errno))
753	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
754
755	sys.exit('\n... error occured while trying to write ' +
756	tempjobfile)
757
758	return
759
760	def del_fortran_build(src_path):
761	'''Clean up the Fortran source directory and remove all
762	build files (e.g. \.o, \.mod and FORTRAN EXECUTABLE)
763
764	Parameters
765	----------
766	src_path : str
767	Path to the fortran source directory.
768
769	Return
770	------
771
772	'''
773
774	modfiles = UioFiles(src_path, '*.mod')
775	objfiles = UioFiles(src_path, '*.o')
776	exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
777
778	modfiles.delete_files()
779	objfiles.delete_files()
780	exefile.delete_files()
781
782	return
783
784	def mk_fortran_build(src_path, makefile):
785	'''Compiles the Fortran code and generates the executable.
786
787	Parameters
788	----------
789	src_path : str
790	Path to the fortran source directory.
791
792	makefile : str
793	The name of the makefile which should be used.
794
795	Return
796	------
797
798	'''
799
800	try:
801	print('Using makefile: ' + makefile)
802	p = subprocess.Popen(['make', '-f',
803	os.path.join(src_path, makefile)],
804	stdin=subprocess.PIPE,
805	stdout=subprocess.PIPE,
806	stderr=subprocess.PIPE,
807	bufsize=1)
808	pout, perr = p.communicate()
809	print(pout.decode())
810	if p.returncode != 0:
811	print(perr.decode())
812	print('Please edit ' + makefile +
813	' or try another makefile in the src directory.')
814	print('Most likely ECCODES_INCLUDE_DIR, ECCODES_LIB '
815	'and EMOSLIB must be adapted.')
816	print('Available makefiles:')
817	print(UioFiles(src_path, 'makefile*'))
818	sys.exit('Compilation failed!')
819	except ValueError as e:
820	print('ERROR: makefile call failed:')
821	print(e)
822	else:
823	execute_subprocess(['ls', '-l',
824	os.path.join(src_path, _config.FORTRAN_EXECUTABLE)],
825	error_msg='FORTRAN EXECUTABLE COULD NOT BE FOUND!')
826
827	return
828
829
830	if __name__ == "__main__":
831	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: