source: flex_extract.git/source/python/install.py @ c5074d2

ctbtodev
Last change on this file since c5074d2 was c5074d2, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

added making of namelist file and jobscript via genshi templates

  • Property mode set to 100755
File size: 17.7 KB
RevLine 
[d69b677]1#!/usr/bin/env python
[64cf353]2# -*- coding: utf-8 -*-
[991df6a]3#*******************************************************************************
4# @Author: Leopold Haimberger (University of Vienna)
5#
6# @Date: November 2015
7#
8# @Change History:
9#
10#    February 2018 - Anne Philipp (University of Vienna):
11#        - applied PEP8 style guide
12#        - added documentation
[ff99eae]13#        - moved install_args_and_control in here
[2fb99de]14#        - splitted code in smaller functions
15#        - delete convert build files in here instead of compile job script
16#        - changed static path names to Variables from config file
[991df6a]17#
18# @License:
19#    (C) Copyright 2015-2018.
20#
21#    This software is licensed under the terms of the Apache Licence Version 2.0
22#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
23#
24# @Program Functionality:
25#    Depending on the selected installation environment (locally or on the
26#    ECMWF server ecgate or cca) the program extracts the commandline
27#    arguments and the CONTROL file parameter and prepares the corresponding
28#    environment. The necessary files are collected in a tar-ball and placed
29#    at the target location. There its untared, the environment variables will
30#    be set and the Fortran code will be compiled. If the ECMWF environment is
31#    selected a job script is prepared and submitted for the remaining
32#    configurations after putting the tar-ball to the target ECMWF server.
33#
34# @Program Content:
35#    - main
[54a8a01]36#    - get_install_cmdline_arguments
[991df6a]37#    - install_via_gateway
[54a8a01]38#    - mk_tarball
[2fb99de]39#    - un_tarball
[54a8a01]40#    - mk_env_vars
41#    - mk_compilejob
42#    - mk_job_template
43#    - delete_convert_build
44#    - make_convert_build
[991df6a]45#
46#*******************************************************************************
47
[64cf353]48# ------------------------------------------------------------------------------
49# MODULES
50# ------------------------------------------------------------------------------
[991df6a]51import os
52import sys
[2fb99de]53import glob
[d69b677]54import subprocess
55import inspect
[ff99eae]56from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
[991df6a]57
58# software specific classes and modules from flex_extract
[2fb99de]59import _config
[25b14be]60from classes.ControlFile import ControlFile
61from classes.UioFiles import UioFiles
62from mods.tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
[d69b677]63
[64cf353]64# ------------------------------------------------------------------------------
65# FUNCTIONS
66# ------------------------------------------------------------------------------
[991df6a]67def main():
68    '''
69    @Description:
70        Controls the installation process. Calls the installation function
71        if target is specified.
72
73    @Intput:
74        <nothing>
75
76    @Return:
77        <nothing>
78    '''
79
[54a8a01]80    args = get_install_cmdline_arguments()
[4971f63]81    c = ControlFile(args.controlfile)
[54a8a01]82    c.assign_args_to_control(args)
83    c.check_install_conditions()
[991df6a]84
[54a8a01]85    install_via_gateway(c)
[991df6a]86
[54a8a01]87    return
[991df6a]88
[54a8a01]89def get_install_cmdline_arguments():
[efdb01a]90    '''
91    @Description:
[54a8a01]92        Decomposes the command line arguments and assigns them to variables.
93        Apply default values for non mentioned arguments.
[efdb01a]94
95    @Input:
96        <nothing>
97
98    @Return:
99        args: instance of ArgumentParser
100            Contains the commandline arguments from script/program call.
101    '''
[54a8a01]102    parser = ArgumentParser(description='Install flex_extract software locally or \
[efdb01a]103                            on ECMWF machines',
104                            formatter_class=ArgumentDefaultsHelpFormatter)
105
[54a8a01]106    parser.add_argument('--target', dest='install_target', default=None,
[efdb01a]107                        help="Valid targets: local | ecgate | cca , \
108                        the latter two are at ECMWF")
[54a8a01]109    parser.add_argument("--makefile", dest="makefile", default=None,
[efdb01a]110                        help='Name of Makefile to use for compiling CONVERT2')
[54a8a01]111    parser.add_argument("--ecuid", dest="ecuid", default=None,
[efdb01a]112                        help='user id at ECMWF')
[54a8a01]113    parser.add_argument("--ecgid", dest="ecgid", default=None,
[efdb01a]114                        help='group id at ECMWF')
[54a8a01]115    parser.add_argument("--gateway", dest="gateway", default=None,
[efdb01a]116                        help='name of local gateway server')
[54a8a01]117    parser.add_argument("--destination", dest="destination", default=None,
[efdb01a]118                        help='ecaccess destination, e.g. leo@genericSftp')
119
120    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
[54a8a01]121                        default=None, help="FLEXPART root directory on ECMWF \
122                        servers (to find grib2flexpart and COMMAND file)\n\
123                        Normally flex_extract resides in the scripts directory \
[2fb99de]124                        of the FLEXPART distribution.")
[efdb01a]125
[54a8a01]126    # arguments for job submission to ECMWF, only needed by submit.py
[efdb01a]127    parser.add_argument("--job_template", dest='job_template',
128                        default="job.temp.o",
129                        help="job template file for submission to ECMWF")
130
131    parser.add_argument("--controlfile", dest="controlfile",
132                        default='CONTROL.temp',
[991df6a]133                        help="file with CONTROL parameters")
[efdb01a]134
135    args = parser.parse_args()
136
[54a8a01]137    return args
138
[efdb01a]139
[54a8a01]140def install_via_gateway(c):
[a4b6cef]141    '''
[991df6a]142    @Description:
143        Perform the actual installation on local machine or prepare data
144        transfer to remote gate and submit a job script which will
145        install everything on the remote gate.
[a4b6cef]146
[991df6a]147    @Input:
148        c: instance of class ControlFile
[27fe969]149            Contains all the parameters of CONTROL file and
150            command line.
151            For more information about format and content of the parameter
152            see documentation.
[991df6a]153
[54a8a01]154    @Return:
155        <nothing>
156    '''
[2fb99de]157    import tarfile
158
159    ecd = _config.PATH_FLEXEXTRACT_DIR
160    tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
161    tar_file = os.path.join(ecd, tarball_name)
[54a8a01]162
[2fb99de]163    target_dirname = _config.FLEXEXTRACT_DIRNAME
164    fortran_executable = _config.FORTRAN_EXECUTABLE
[54a8a01]165
[97e09f4]166    if c.install_target.lower() != 'local': # ecgate or cca
[54a8a01]167
[2fb99de]168        mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
[54a8a01]169                      c.flexpart_root_scripts)
170
[2fb99de]171        mk_job_template(c.ecuid, c.ecgid, c.gateway,
[54a8a01]172                        c.destination, c.flexpart_root_scripts)
173
[2fb99de]174        mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
[54a8a01]175
[25b14be]176        mk_tarball(tar_file, c.install_target)
[54a8a01]177
178        put_file_to_ecserver(ecd, tarball_name, c.install_target,
179                             c.ecuid, c.ecgid)
180
[2fb99de]181        submit_job_to_ecserver(c.install_target,
[25b14be]182                               os.path.join(_config.PATH_REL_JOBSCRIPTS,
[2fb99de]183                                            _config.FILE_INSTALL_COMPILEJOB))
[54a8a01]184
[2fb99de]185        print('job compilation script has been submitted to ecgate for ' +
186              'installation in ' + c.flexpart_root_scripts +
187               '/' + target_dirname)
188        print('You should get an email with subject "flexcompile" within ' +
189              'the next few minutes!')
[54a8a01]190
191    else: #local
[2fb99de]192        if c.flexpart_root_scripts == _config.PATH_FLEXEXTRACT_DIR :
193            print('WARNING: FLEXPART_ROOT_SCRIPTS has not been specified')
194            print('flex_extract will be installed in here by compiling the ' +
195                  'Fortran source in ' + _config.PATH_FORTRAN_SRC)
196            os.chdir(_config.PATH_FORTRAN_SRC)
[54a8a01]197        else: # creates the target working directory for flex_extract
198            c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
[2fb99de]199                                        c.flexpart_root_scripts))
[54a8a01]200            if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
[25b14be]201                mk_tarball(tar_file, c.install_target)
[2fb99de]202                make_dir(os.path.join(c.flexpart_root_scripts,
203                                      target_dirname))
204                os.chdir(os.path.join(c.flexpart_root_scripts,
205                                      target_dirname))
206                un_tarball(tar_file)
207                os.chdir(os.path.join(c.flexpart_root_scripts,
208                                      target_dirname,
[25b14be]209                                      _config.PATH_REL_FORTRAN_SRC))
[54a8a01]210
211        # Create Fortran executable - CONVERT2
[2fb99de]212        print('Install ' + target_dirname + ' software at ' +
213              c.install_target + ' in directory ' +
214              os.path.abspath(c.flexpart_root_scripts) + '\n')
215
216        delete_convert_build('.')
217        make_convert_build('.', c.makefile)
[54a8a01]218
[2fb99de]219        os.chdir(ecd)
220        if os.path.isfile(tar_file):
221            os.remove(tar_file)
[54a8a01]222
223    return
224
[25b14be]225def mk_tarball(tarball_path, target):
[54a8a01]226    '''
227    @Description:
[2fb99de]228        Creates a tarball with all necessary files which need to be sent to the
[54a8a01]229        installation directory.
230        It does not matter if this is local or remote.
231        Collects all python files, the Fortran source and makefiles,
[2fb99de]232        the ECMWF_ENV file, the CONTROL files as well as the
233        template files.
[54a8a01]234
235    @Input:
[2fb99de]236        tarball_path: string
237            The complete path to the tar file which will contain all
238            relevant data for flex_extract.
[54a8a01]239
[25b14be]240        target: string
241            The queue where the job is submitted to.
242
[54a8a01]243    @Return:
244        <nothing>
245    '''
[2fb99de]246    import tarfile
247    from glob import glob
248
249    print('Create tarball ...')
250
251    # change to FLEXEXTRACT directory so that the tar can contain
252    # relative pathes to the files and directories
253    ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
254    os.chdir(ecd)
255
256    # get lists of the files to be added to the tar file
[25b14be]257    if target == 'local':
258        ECMWF_ENV_FILE = []
259    else:
260        ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
261
[2fb99de]262    pyfiles = [os.path.relpath(x, ecd)
[25b14be]263               for x in UioFiles(_config.PATH_LOCAL_PYTHON, '*py').files]
[2fb99de]264    controlfiles = [os.path.relpath(x, ecd)
[25b14be]265                    for x in UioFiles(_config.PATH_CONTROLFILES,
266                                      'CONTROL*').files]
[2fb99de]267    tempfiles = [os.path.relpath(x, ecd)
[25b14be]268                 for x in UioFiles(_config.PATH_TEMPLATES , '*').files]
[2fb99de]269    ffiles = [os.path.relpath(x, ecd)
[25b14be]270              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.f*').files]
[2fb99de]271    hfiles = [os.path.relpath(x, ecd)
[25b14be]272              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.h').files]
[2fb99de]273    makefiles = [os.path.relpath(x, ecd)
[25b14be]274                 for x in UioFiles(_config.PATH_FORTRAN_SRC, 'Makefile*').files]
[2fb99de]275
276    # concatenate single lists to one for a better looping
277    filelist = pyfiles + controlfiles + tempfiles + ffiles + hfiles + \
278               makefiles + ECMWF_ENV_FILE
279
280    # create installation tar-file
[54a8a01]281    try:
[2fb99de]282        with tarfile.open(tarball_path, "w:gz") as tar_handle:
283            for file in filelist:
284                tar_handle.add(file)
285
[54a8a01]286    except subprocess.CalledProcessError as e:
[2fb99de]287        print('... ERROR CODE:\n ... ' + str(e.returncode))
288        print('... ERROR MESSAGE:\n ... ' + str(e))
289
290        sys.exit('... could not make installation tar ball!')
[54a8a01]291
292    return
293
[2fb99de]294
295def un_tarball(tarball_path):
296    '''
297    @Description:
298        Extracts the given tarball into current directory.
299
300    @Input:
301        tarball_path: string
302            The complete path to the tar file which will contain all
303            relevant data for flex_extract.
304
305    @Return:
306        <nothing>
307    '''
308    import tarfile
309
310    print('Untar ...')
311
312    with tarfile.open(tarball_path) as tar_handle:
313        tar_handle.extractall()
314
315    return
316
317def mk_env_vars(ecuid, ecgid, gateway, destination):
[54a8a01]318    '''
319    @Description:
320        Creates a file named ECMWF_ENV which contains the
321        necessary environmental variables at ECMWF servers.
[0aaeb04]322        It is based on the template ECMWF_ENV.template.
[54a8a01]323
324    @Input:
325        ecuid: string
326            The user id on ECMWF server.
327
328        ecgid: string
329            The group id on ECMWF server.
330
331        gateway: string
332            The gateway server the user is using.
333
334        destination: string
335            The remote destination which is used to transfer files
336            from ECMWF server to local gateway server.
337
338    @Return:
339        <nothing>
340    '''
[0aaeb04]341    from genshi.template.text import NewTextTemplate
342    from genshi.template import  TemplateLoader
[54a8a01]343
[0aaeb04]344    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
345    ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
346                                     cls=NewTextTemplate)
347
348    stream = ecmwfvars_template.generate(user_name = ecuid,
349                                       user_group = ecgid,
350                                       gateway_name = gateway,
351                                       destination_name = destination
352                                       )
353
354    with open(_config.PATH_ECMWF_ENV, 'w') as f:
355        f.write(stream.render('text'))
[54a8a01]356
357    return
358
[2fb99de]359def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
[54a8a01]360    '''
361    @Description:
362        Modifies the original job template file so that it is specified
363        for the user and the environment were it will be applied. Result
364        is stored in a new file "job.temp" in the python directory.
365
366    @Input:
367        makefile: string
368            Name of the makefile which should be used to compile FORTRAN
369            CONVERT2 program.
370
[991df6a]371        target: string
[54a8a01]372            The target where the installation should be done, e.g. the queue.
373
374        ecuid: string
375            The user id on ECMWF server.
376
377        ecgid: string
378            The group id on ECMWF server.
379
380        fp_root: string
381           Path to the root directory of FLEXPART environment or flex_extract
382           environment.
[a4b6cef]383
[991df6a]384    @Return:
385        <nothing>
386    '''
[0aaeb04]387    from genshi.template.text import NewTextTemplate
388    from genshi.template import  TemplateLoader
389
390    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
391    compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
392                                   cls=NewTextTemplate)
393
394    if fp_root == '../':
395        fp_root = '$HOME'
396
397    stream = compile_template.generate(
398        usergroup = ecgid,
[c5074d2]399        username = ecuid,
[0aaeb04]400        version_number = _config._VERSION_STR,
401        fp_root_scripts = fp_root,
402        makefile = makefile,
403        fortran_program = _config.FORTRAN_EXECUTABLE
404    )
405
406    compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
[2fb99de]407                              _config.FILE_INSTALL_COMPILEJOB)
[0aaeb04]408
409    with open(compilejob, 'w') as f:
410        f.write(stream.render('text'))
411
[54a8a01]412    return
413
[2fb99de]414def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
[54a8a01]415    '''
416    @Description:
417        Modifies the original job template file so that it is specified
418        for the user and the environment were it will be applied. Result
[2fb99de]419        is stored in a new file.
[54a8a01]420
421    @Input:
422        ecuid: string
423            The user id on ECMWF server.
424
425        ecgid: string
426            The group id on ECMWF server.
427
428        gateway: string
429            The gateway server the user is using.
430
431        destination: string
432            The remote destination which is used to transfer files
433            from ECMWF server to local gateway server.
434
435        fp_root: string
436           Path to the root directory of FLEXPART environment or flex_extract
437           environment.
438
439    @Return:
440        <nothing>
441    '''
[c5074d2]442    from genshi.template.text import NewTextTemplate
443    from genshi.template import  TemplateLoader
444
445    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
446    compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
447                                   cls=NewTextTemplate)
448
449    fp_root_path_to_python = os.path.join(fp_root,
450                                          _config.FLEXEXTRACT_DIRNAME,
451                                          _config.PATH_REL_PYTHON)
452
453    stream = compile_template.generate(
454        usergroup = ecgid,
455        username = ecuid,
456        version_number = _config._VERSION_STR,
457        fp_root_path = fp_root_path_to_python,
458    )
459
460    tempjobfile = os.path.join(_config.PATH_TEMPLATES,
461                               _config.TEMPFILE_JOB)
462
463    with open(tempjobfile, 'w') as f:
464        f.write(stream.render('text'))
465
[54a8a01]466    return
467
[2fb99de]468def delete_convert_build(src_path):
[54a8a01]469    '''
470    @Description:
471        Clean up the Fortran source directory and remove all
472        build files (e.g. *.o, *.mod and CONVERT2)
473
474    @Input:
[2fb99de]475        src_path: string
476            Path to the fortran source directory.
[54a8a01]477
478    @Return:
479        <nothing>
480    '''
481
[2fb99de]482    modfiles = UioFiles(src_path, '*.mod')
483    objfiles = UioFiles(src_path, '*.o')
484    exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
[54a8a01]485
486    modfiles.delete_files()
487    objfiles.delete_files()
488    exefile.delete_files()
489
490    return
491
[2fb99de]492def make_convert_build(src_path, makefile):
[54a8a01]493    '''
494    @Description:
495        Compiles the Fortran code and generates the executable.
496
497    @Input:
[2fb99de]498        src_path: string
499            Path to the fortran source directory.
[54a8a01]500
501        makefile: string
502            The name of the makefile which should be used.
503
504    @Return:
505        <nothing>
506    '''
507
508    try:
[2fb99de]509        print('Using makefile: ' + makefile)
510        p = subprocess.Popen(['make', '-f',
511                              os.path.join(src_path, makefile)],
[54a8a01]512                             stdin=subprocess.PIPE,
513                             stdout=subprocess.PIPE,
514                             stderr=subprocess.PIPE,
515                             bufsize=1)
516        pout, perr = p.communicate()
[2fb99de]517        print(pout)
[54a8a01]518        if p.returncode != 0:
[2fb99de]519            print(perr)
520            print('Please edit ' + makefile +
521                  ' or try another Makefile in the src directory.')
522            print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
523                  'and EMOSLIB must be adapted.')
524            print('Available Makefiles:')
525            print(UioFiles(src_path, 'Makefile*'))
[54a8a01]526            sys.exit('Compilation failed!')
527    except ValueError as e:
[2fb99de]528        print('ERROR: Makefile call failed:')
529        print(e)
[d69b677]530    else:
[2fb99de]531        subprocess.check_call(['ls', '-l',
532                               os.path.join(src_path,
533                                            _config.FORTRAN_EXECUTABLE)])
[a4b6cef]534
[d69b677]535    return
536
[a4b6cef]537
[d69b677]538if __name__ == "__main__":
539    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG