source: flex_extract.git/source/python/install.py @ 274f9ef

ctbtodev
Last change on this file since 274f9ef was 274f9ef, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

Converted docstrings to numpy style and build first structure for sphinxdocumentation (incl API)

  • Property mode set to 100755
File size: 17.3 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Leopold Haimberger (University of Vienna)
5#
6# @Date: November 2015
7#
8# @Change History:
9#
10#    February 2018 - Anne Philipp (University of Vienna):
11#        - applied PEP8 style guide
12#        - added documentation
13#        - moved install_args_and_control in here
14#        - splitted code in smaller functions
15#        - delete convert build files in here instead of compile job script
16#        - changed static path names to Variables from config file
17#
18# @License:
19#    (C) Copyright 2015-2018.
20#
21#    This software is licensed under the terms of the Apache Licence Version 2.0
22#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
23#
24# @Program Functionality:
25#    Depending on the selected installation environment (locally or on the
26#    ECMWF server ecgate or cca) the program extracts the commandline
27#    arguments and the CONTROL file parameter and prepares the corresponding
28#    environment. The necessary files are collected in a tar-ball and placed
29#    at the target location. There its untared, the environment variables will
30#    be set and the Fortran code will be compiled. If the ECMWF environment is
31#    selected a job script is prepared and submitted for the remaining
32#    configurations after putting the tar-ball to the target ECMWF server.
33#
34# @Program Content:
35#    - main
36#    - get_install_cmdline_arguments
37#    - install_via_gateway
38#    - mk_tarball
39#    - un_tarball
40#    - mk_env_vars
41#    - mk_compilejob
42#    - mk_job_template
43#    - delete_convert_build
44#    - make_convert_build
45#
46#*******************************************************************************
47
48# ------------------------------------------------------------------------------
49# MODULES
50# ------------------------------------------------------------------------------
51import os
52import sys
53import glob
54import subprocess
55import inspect
56from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
57
58# software specific classes and modules from flex_extract
59import _config
60from classes.ControlFile import ControlFile
61from classes.UioFiles import UioFiles
62from mods.tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
63
64# ------------------------------------------------------------------------------
65# FUNCTIONS
66# ------------------------------------------------------------------------------
67def main():
68    '''Controls the installation process. Calls the installation function
69    if target is specified.
70
71    Parameters
72    ----------
73
74    Return
75    ------
76    '''
77
78    args = get_install_cmdline_arguments()
79    c = ControlFile(args.controlfile)
80    c.assign_args_to_control(args)
81    c.check_install_conditions()
82
83    install_via_gateway(c)
84
85    return
86
87def get_install_cmdline_arguments():
88    '''Decomposes the command line arguments and assigns them to variables.
89    Apply default values for non mentioned arguments.
90
91    Parameters
92    ----------
93
94    Return
95    ------
96    args : :obj:`Namespace`
97        Contains the commandline arguments from script/program call.
98    '''
99    parser = ArgumentParser(description='Install flex_extract software locally or \
100                            on ECMWF machines',
101                            formatter_class=ArgumentDefaultsHelpFormatter)
102
103    parser.add_argument('--target', dest='install_target', default=None,
104                        help="Valid targets: local | ecgate | cca , \
105                        the latter two are at ECMWF")
106    parser.add_argument("--makefile", dest="makefile", default=None,
107                        help='Name of Makefile to use for compiling CONVERT2')
108    parser.add_argument("--ecuid", dest="ecuid", default=None,
109                        help='user id at ECMWF')
110    parser.add_argument("--ecgid", dest="ecgid", default=None,
111                        help='group id at ECMWF')
112    parser.add_argument("--gateway", dest="gateway", default=None,
113                        help='name of local gateway server')
114    parser.add_argument("--destination", dest="destination", default=None,
115                        help='ecaccess destination, e.g. leo@genericSftp')
116
117    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
118                        default=None, help="FLEXPART root directory on ECMWF \
119                        servers (to find grib2flexpart and COMMAND file)\n\
120                        Normally flex_extract resides in the scripts directory \
121                        of the FLEXPART distribution.")
122
123    # arguments for job submission to ECMWF, only needed by submit.py
124    parser.add_argument("--job_template", dest='job_template',
125                        default="job.temp.o",
126                        help="job template file for submission to ECMWF")
127
128    parser.add_argument("--controlfile", dest="controlfile",
129                        default='CONTROL.temp',
130                        help="file with CONTROL parameters")
131
132    args = parser.parse_args()
133
134    return args
135
136
137def install_via_gateway(c):
138    '''Perform the actual installation on local machine or prepare data
139    transfer to remote gate and submit a job script which will
140    install everything on the remote gate.
141
142    Parameters
143    ----------
144    c : :obj:`ControlFile`
145        Contains all the parameters of CONTROL file and
146        command line.
147
148    Return
149    ------
150
151    '''
152    import tarfile
153
154    ecd = _config.PATH_FLEXEXTRACT_DIR
155    tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
156    tar_file = os.path.join(ecd, tarball_name)
157
158    target_dirname = _config.FLEXEXTRACT_DIRNAME
159    fortran_executable = _config.FORTRAN_EXECUTABLE
160
161    if c.install_target.lower() != 'local': # ecgate or cca
162
163        mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
164                      c.flexpart_root_scripts)
165
166        mk_job_template(c.ecuid, c.ecgid, c.gateway,
167                        c.destination, c.flexpart_root_scripts)
168
169        mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
170
171        mk_tarball(tar_file, c.install_target)
172
173        put_file_to_ecserver(ecd, tarball_name, c.install_target,
174                             c.ecuid, c.ecgid)
175
176        submit_job_to_ecserver(c.install_target,
177                               os.path.join(_config.PATH_REL_JOBSCRIPTS,
178                                            _config.FILE_INSTALL_COMPILEJOB))
179
180        print('job compilation script has been submitted to ecgate for ' +
181              'installation in ' + c.flexpart_root_scripts +
182               '/' + target_dirname)
183        print('You should get an email with subject "flexcompile" within ' +
184              'the next few minutes!')
185
186    else: #local
187        if c.flexpart_root_scripts == _config.PATH_FLEXEXTRACT_DIR :
188            print('WARNING: FLEXPART_ROOT_SCRIPTS has not been specified')
189            print('flex_extract will be installed in here by compiling the ' +
190                  'Fortran source in ' + _config.PATH_FORTRAN_SRC)
191            os.chdir(_config.PATH_FORTRAN_SRC)
192        else: # creates the target working directory for flex_extract
193            c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
194                                        c.flexpart_root_scripts))
195            if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
196                mk_tarball(tar_file, c.install_target)
197                make_dir(os.path.join(c.flexpart_root_scripts,
198                                      target_dirname))
199                os.chdir(os.path.join(c.flexpart_root_scripts,
200                                      target_dirname))
201                un_tarball(tar_file)
202                os.chdir(os.path.join(c.flexpart_root_scripts,
203                                      target_dirname,
204                                      _config.PATH_REL_FORTRAN_SRC))
205
206        # Create Fortran executable - CONVERT2
207        print('Install ' + target_dirname + ' software at ' +
208              c.install_target + ' in directory ' +
209              os.path.abspath(c.flexpart_root_scripts) + '\n')
210
211        delete_convert_build('.')
212        make_convert_build('.', c.makefile)
213
214        os.chdir(ecd)
215        if os.path.isfile(tar_file):
216            os.remove(tar_file)
217
218    return
219
220def mk_tarball(tarball_path, target):
221    '''Creates a tarball with all necessary files which need to be sent to the
222    installation directory.
223    It does not matter if this is local or remote.
224    Collects all python files, the Fortran source and makefiles,
225    the ECMWF_ENV file, the CONTROL files as well as the
226    template files.
227
228    Parameters
229    ----------
230    tarball_path : :obj:`string`
231        The complete path to the tar file which will contain all
232        relevant data for flex_extract.
233
234    target : :obj:`string`
235        The queue where the job is submitted to.
236
237    Return
238    ------
239
240    '''
241    import tarfile
242    from glob import glob
243
244    print('Create tarball ...')
245
246    # change to FLEXEXTRACT directory so that the tar can contain
247    # relative pathes to the files and directories
248    ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
249    os.chdir(ecd)
250
251    # get lists of the files to be added to the tar file
252    if target == 'local':
253        ECMWF_ENV_FILE = []
254    else:
255        ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
256
257    pyfiles = [os.path.relpath(x, ecd)
258               for x in UioFiles(_config.PATH_LOCAL_PYTHON, '*py').files]
259    controlfiles = [os.path.relpath(x, ecd)
260                    for x in UioFiles(_config.PATH_CONTROLFILES,
261                                      'CONTROL*').files]
262    tempfiles = [os.path.relpath(x, ecd)
263                 for x in UioFiles(_config.PATH_TEMPLATES , '*').files]
264    ffiles = [os.path.relpath(x, ecd)
265              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.f*').files]
266    hfiles = [os.path.relpath(x, ecd)
267              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.h').files]
268    makefiles = [os.path.relpath(x, ecd)
269                 for x in UioFiles(_config.PATH_FORTRAN_SRC, 'Makefile*').files]
270
271    # concatenate single lists to one for a better looping
272    filelist = pyfiles + controlfiles + tempfiles + ffiles + hfiles + \
273               makefiles + ECMWF_ENV_FILE
274
275    # create installation tar-file
276    try:
277        with tarfile.open(tarball_path, "w:gz") as tar_handle:
278            for file in filelist:
279                tar_handle.add(file)
280
281    except subprocess.CalledProcessError as e:
282        print('... ERROR CODE:\n ... ' + str(e.returncode))
283        print('... ERROR MESSAGE:\n ... ' + str(e))
284
285        sys.exit('... could not make installation tar ball!')
286
287    return
288
289
290def un_tarball(tarball_path):
291    '''Extracts the given tarball into current directory.
292
293    Parameters
294    ----------
295    tarball_path : :obj:`string`
296        The complete path to the tar file which will contain all
297        relevant data for flex_extract.
298
299    Return
300    ------
301
302    '''
303    import tarfile
304
305    print('Untar ...')
306
307    with tarfile.open(tarball_path) as tar_handle:
308        tar_handle.extractall()
309
310    return
311
312def mk_env_vars(ecuid, ecgid, gateway, destination):
313    '''Creates a file named ECMWF_ENV which contains the
314    necessary environmental variables at ECMWF servers.
315    It is based on the template ECMWF_ENV.template.
316
317    Parameters
318    ----------
319    ecuid : :obj:`string`
320        The user id on ECMWF server.
321
322    ecgid : :obj:`string`
323        The group id on ECMWF server.
324
325    gateway : :obj:`string`
326        The gateway server the user is using.
327
328    destination : :obj:`string`
329        The remote destination which is used to transfer files
330        from ECMWF server to local gateway server.
331
332    Return
333    ------
334
335    '''
336    from genshi.template.text import NewTextTemplate
337    from genshi.template import  TemplateLoader
338
339    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
340    ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
341                                     cls=NewTextTemplate)
342
343    stream = ecmwfvars_template.generate(user_name = ecuid,
344                                       user_group = ecgid,
345                                       gateway_name = gateway,
346                                       destination_name = destination
347                                       )
348
349    with open(_config.PATH_ECMWF_ENV, 'w') as f:
350        f.write(stream.render('text'))
351
352    return
353
354def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
355    '''Modifies the original job template file so that it is specified
356    for the user and the environment were it will be applied. Result
357    is stored in a new file "job.temp" in the python directory.
358
359    Parameters
360    ----------
361    makefile : :obj:`string`
362        Name of the makefile which should be used to compile FORTRAN
363        CONVERT2 program.
364
365    target : :obj:`string`
366        The target where the installation should be done, e.g. the queue.
367
368    ecuid : :obj:`string`
369        The user id on ECMWF server.
370
371    ecgid : :obj:`string`
372        The group id on ECMWF server.
373
374    fp_root : :obj:`string`
375       Path to the root directory of FLEXPART environment or flex_extract
376       environment.
377
378    Return
379    ------
380
381    '''
382    from genshi.template.text import NewTextTemplate
383    from genshi.template import  TemplateLoader
384
385    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
386    compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
387                                   cls=NewTextTemplate)
388
389    if fp_root == '../':
390        fp_root = '$HOME'
391
392    stream = compile_template.generate(
393        usergroup = ecgid,
394        username = ecuid,
395        version_number = _config._VERSION_STR,
396        fp_root_scripts = fp_root,
397        makefile = makefile,
398        fortran_program = _config.FORTRAN_EXECUTABLE
399    )
400
401    compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
402                              _config.FILE_INSTALL_COMPILEJOB)
403
404    with open(compilejob, 'w') as f:
405        f.write(stream.render('text'))
406
407    return
408
409def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
410    '''Modifies the original job template file so that it is specified
411    for the user and the environment were it will be applied. Result
412    is stored in a new file.
413
414    Parameters
415    ----------
416    ecuid : :obj:`string`
417        The user id on ECMWF server.
418
419    ecgid : :obj:`string`
420        The group id on ECMWF server.
421
422    gateway : :obj:`string`
423        The gateway server the user is using.
424
425    destination : :obj:`string`
426        The remote destination which is used to transfer files
427        from ECMWF server to local gateway server.
428
429    fp_root : :obj:`string`
430       Path to the root directory of FLEXPART environment or flex_extract
431       environment.
432
433    Return
434    ------
435
436    '''
437    from genshi.template.text import NewTextTemplate
438    from genshi.template import  TemplateLoader
439
440    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
441    compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
442                                   cls=NewTextTemplate)
443
444    fp_root_path_to_python = os.path.join(fp_root,
445                                          _config.FLEXEXTRACT_DIRNAME,
446                                          _config.PATH_REL_PYTHON)
447
448    stream = compile_template.generate(
449        usergroup = ecgid,
450        username = ecuid,
451        version_number = _config._VERSION_STR,
452        fp_root_path = fp_root_path_to_python,
453    )
454
455    tempjobfile = os.path.join(_config.PATH_TEMPLATES,
456                               _config.TEMPFILE_JOB)
457
458    with open(tempjobfile, 'w') as f:
459        f.write(stream.render('text'))
460
461    return
462
463def delete_convert_build(src_path):
464    '''Clean up the Fortran source directory and remove all
465    build files (e.g. \*.o, \*.mod and CONVERT2)
466
467    Parameters
468    ----------
469    src_path : :obj:`string`
470        Path to the fortran source directory.
471
472    Return
473    ------
474
475    '''
476
477    modfiles = UioFiles(src_path, '*.mod')
478    objfiles = UioFiles(src_path, '*.o')
479    exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
480
481    modfiles.delete_files()
482    objfiles.delete_files()
483    exefile.delete_files()
484
485    return
486
487def make_convert_build(src_path, makefile):
488    '''Compiles the Fortran code and generates the executable.
489
490    Parameters
491    ----------
492    src_path : :obj:`string`
493        Path to the fortran source directory.
494
495    makefile : :obj:`string`
496        The name of the makefile which should be used.
497
498    Return
499    ------
500
501    '''
502
503    try:
504        print('Using makefile: ' + makefile)
505        p = subprocess.Popen(['make', '-f',
506                              os.path.join(src_path, makefile)],
507                             stdin=subprocess.PIPE,
508                             stdout=subprocess.PIPE,
509                             stderr=subprocess.PIPE,
510                             bufsize=1)
511        pout, perr = p.communicate()
512        print(pout)
513        if p.returncode != 0:
514            print(perr)
515            print('Please edit ' + makefile +
516                  ' or try another Makefile in the src directory.')
517            print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
518                  'and EMOSLIB must be adapted.')
519            print('Available Makefiles:')
520            print(UioFiles(src_path, 'Makefile*'))
521            sys.exit('Compilation failed!')
522    except ValueError as e:
523        print('ERROR: Makefile call failed:')
524        print(e)
525    else:
526        subprocess.check_call(['ls', '-l',
527                               os.path.join(src_path,
528                                            _config.FORTRAN_EXECUTABLE)])
529
530    return
531
532
533if __name__ == "__main__":
534    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG