source: flex_extract.git/source/python/install.py @ c5074d2

ctbtodev
Last change on this file since c5074d2 was c5074d2, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

added making of namelist file and jobscript via genshi templates

  • Property mode set to 100755
File size: 17.7 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Leopold Haimberger (University of Vienna)
5#
6# @Date: November 2015
7#
8# @Change History:
9#
10#    February 2018 - Anne Philipp (University of Vienna):
11#        - applied PEP8 style guide
12#        - added documentation
13#        - moved install_args_and_control in here
14#        - splitted code in smaller functions
15#        - delete convert build files in here instead of compile job script
16#        - changed static path names to Variables from config file
17#
18# @License:
19#    (C) Copyright 2015-2018.
20#
21#    This software is licensed under the terms of the Apache Licence Version 2.0
22#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
23#
24# @Program Functionality:
25#    Depending on the selected installation environment (locally or on the
26#    ECMWF server ecgate or cca) the program extracts the commandline
27#    arguments and the CONTROL file parameter and prepares the corresponding
28#    environment. The necessary files are collected in a tar-ball and placed
29#    at the target location. There its untared, the environment variables will
30#    be set and the Fortran code will be compiled. If the ECMWF environment is
31#    selected a job script is prepared and submitted for the remaining
32#    configurations after putting the tar-ball to the target ECMWF server.
33#
34# @Program Content:
35#    - main
36#    - get_install_cmdline_arguments
37#    - install_via_gateway
38#    - mk_tarball
39#    - un_tarball
40#    - mk_env_vars
41#    - mk_compilejob
42#    - mk_job_template
43#    - delete_convert_build
44#    - make_convert_build
45#
46#*******************************************************************************
47
48# ------------------------------------------------------------------------------
49# MODULES
50# ------------------------------------------------------------------------------
51import os
52import sys
53import glob
54import subprocess
55import inspect
56from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
57
58# software specific classes and modules from flex_extract
59import _config
60from classes.ControlFile import ControlFile
61from classes.UioFiles import UioFiles
62from mods.tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
63
64# ------------------------------------------------------------------------------
65# FUNCTIONS
66# ------------------------------------------------------------------------------
67def main():
68    '''
69    @Description:
70        Controls the installation process. Calls the installation function
71        if target is specified.
72
73    @Intput:
74        <nothing>
75
76    @Return:
77        <nothing>
78    '''
79
80    args = get_install_cmdline_arguments()
81    c = ControlFile(args.controlfile)
82    c.assign_args_to_control(args)
83    c.check_install_conditions()
84
85    install_via_gateway(c)
86
87    return
88
89def get_install_cmdline_arguments():
90    '''
91    @Description:
92        Decomposes the command line arguments and assigns them to variables.
93        Apply default values for non mentioned arguments.
94
95    @Input:
96        <nothing>
97
98    @Return:
99        args: instance of ArgumentParser
100            Contains the commandline arguments from script/program call.
101    '''
102    parser = ArgumentParser(description='Install flex_extract software locally or \
103                            on ECMWF machines',
104                            formatter_class=ArgumentDefaultsHelpFormatter)
105
106    parser.add_argument('--target', dest='install_target', default=None,
107                        help="Valid targets: local | ecgate | cca , \
108                        the latter two are at ECMWF")
109    parser.add_argument("--makefile", dest="makefile", default=None,
110                        help='Name of Makefile to use for compiling CONVERT2')
111    parser.add_argument("--ecuid", dest="ecuid", default=None,
112                        help='user id at ECMWF')
113    parser.add_argument("--ecgid", dest="ecgid", default=None,
114                        help='group id at ECMWF')
115    parser.add_argument("--gateway", dest="gateway", default=None,
116                        help='name of local gateway server')
117    parser.add_argument("--destination", dest="destination", default=None,
118                        help='ecaccess destination, e.g. leo@genericSftp')
119
120    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
121                        default=None, help="FLEXPART root directory on ECMWF \
122                        servers (to find grib2flexpart and COMMAND file)\n\
123                        Normally flex_extract resides in the scripts directory \
124                        of the FLEXPART distribution.")
125
126    # arguments for job submission to ECMWF, only needed by submit.py
127    parser.add_argument("--job_template", dest='job_template',
128                        default="job.temp.o",
129                        help="job template file for submission to ECMWF")
130
131    parser.add_argument("--controlfile", dest="controlfile",
132                        default='CONTROL.temp',
133                        help="file with CONTROL parameters")
134
135    args = parser.parse_args()
136
137    return args
138
139
140def install_via_gateway(c):
141    '''
142    @Description:
143        Perform the actual installation on local machine or prepare data
144        transfer to remote gate and submit a job script which will
145        install everything on the remote gate.
146
147    @Input:
148        c: instance of class ControlFile
149            Contains all the parameters of CONTROL file and
150            command line.
151            For more information about format and content of the parameter
152            see documentation.
153
154    @Return:
155        <nothing>
156    '''
157    import tarfile
158
159    ecd = _config.PATH_FLEXEXTRACT_DIR
160    tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
161    tar_file = os.path.join(ecd, tarball_name)
162
163    target_dirname = _config.FLEXEXTRACT_DIRNAME
164    fortran_executable = _config.FORTRAN_EXECUTABLE
165
166    if c.install_target.lower() != 'local': # ecgate or cca
167
168        mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
169                      c.flexpart_root_scripts)
170
171        mk_job_template(c.ecuid, c.ecgid, c.gateway,
172                        c.destination, c.flexpart_root_scripts)
173
174        mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
175
176        mk_tarball(tar_file, c.install_target)
177
178        put_file_to_ecserver(ecd, tarball_name, c.install_target,
179                             c.ecuid, c.ecgid)
180
181        submit_job_to_ecserver(c.install_target,
182                               os.path.join(_config.PATH_REL_JOBSCRIPTS,
183                                            _config.FILE_INSTALL_COMPILEJOB))
184
185        print('job compilation script has been submitted to ecgate for ' +
186              'installation in ' + c.flexpart_root_scripts +
187               '/' + target_dirname)
188        print('You should get an email with subject "flexcompile" within ' +
189              'the next few minutes!')
190
191    else: #local
192        if c.flexpart_root_scripts == _config.PATH_FLEXEXTRACT_DIR :
193            print('WARNING: FLEXPART_ROOT_SCRIPTS has not been specified')
194            print('flex_extract will be installed in here by compiling the ' +
195                  'Fortran source in ' + _config.PATH_FORTRAN_SRC)
196            os.chdir(_config.PATH_FORTRAN_SRC)
197        else: # creates the target working directory for flex_extract
198            c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
199                                        c.flexpart_root_scripts))
200            if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
201                mk_tarball(tar_file, c.install_target)
202                make_dir(os.path.join(c.flexpart_root_scripts,
203                                      target_dirname))
204                os.chdir(os.path.join(c.flexpart_root_scripts,
205                                      target_dirname))
206                un_tarball(tar_file)
207                os.chdir(os.path.join(c.flexpart_root_scripts,
208                                      target_dirname,
209                                      _config.PATH_REL_FORTRAN_SRC))
210
211        # Create Fortran executable - CONVERT2
212        print('Install ' + target_dirname + ' software at ' +
213              c.install_target + ' in directory ' +
214              os.path.abspath(c.flexpart_root_scripts) + '\n')
215
216        delete_convert_build('.')
217        make_convert_build('.', c.makefile)
218
219        os.chdir(ecd)
220        if os.path.isfile(tar_file):
221            os.remove(tar_file)
222
223    return
224
225def mk_tarball(tarball_path, target):
226    '''
227    @Description:
228        Creates a tarball with all necessary files which need to be sent to the
229        installation directory.
230        It does not matter if this is local or remote.
231        Collects all python files, the Fortran source and makefiles,
232        the ECMWF_ENV file, the CONTROL files as well as the
233        template files.
234
235    @Input:
236        tarball_path: string
237            The complete path to the tar file which will contain all
238            relevant data for flex_extract.
239
240        target: string
241            The queue where the job is submitted to.
242
243    @Return:
244        <nothing>
245    '''
246    import tarfile
247    from glob import glob
248
249    print('Create tarball ...')
250
251    # change to FLEXEXTRACT directory so that the tar can contain
252    # relative pathes to the files and directories
253    ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
254    os.chdir(ecd)
255
256    # get lists of the files to be added to the tar file
257    if target == 'local':
258        ECMWF_ENV_FILE = []
259    else:
260        ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
261
262    pyfiles = [os.path.relpath(x, ecd)
263               for x in UioFiles(_config.PATH_LOCAL_PYTHON, '*py').files]
264    controlfiles = [os.path.relpath(x, ecd)
265                    for x in UioFiles(_config.PATH_CONTROLFILES,
266                                      'CONTROL*').files]
267    tempfiles = [os.path.relpath(x, ecd)
268                 for x in UioFiles(_config.PATH_TEMPLATES , '*').files]
269    ffiles = [os.path.relpath(x, ecd)
270              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.f*').files]
271    hfiles = [os.path.relpath(x, ecd)
272              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.h').files]
273    makefiles = [os.path.relpath(x, ecd)
274                 for x in UioFiles(_config.PATH_FORTRAN_SRC, 'Makefile*').files]
275
276    # concatenate single lists to one for a better looping
277    filelist = pyfiles + controlfiles + tempfiles + ffiles + hfiles + \
278               makefiles + ECMWF_ENV_FILE
279
280    # create installation tar-file
281    try:
282        with tarfile.open(tarball_path, "w:gz") as tar_handle:
283            for file in filelist:
284                tar_handle.add(file)
285
286    except subprocess.CalledProcessError as e:
287        print('... ERROR CODE:\n ... ' + str(e.returncode))
288        print('... ERROR MESSAGE:\n ... ' + str(e))
289
290        sys.exit('... could not make installation tar ball!')
291
292    return
293
294
295def un_tarball(tarball_path):
296    '''
297    @Description:
298        Extracts the given tarball into current directory.
299
300    @Input:
301        tarball_path: string
302            The complete path to the tar file which will contain all
303            relevant data for flex_extract.
304
305    @Return:
306        <nothing>
307    '''
308    import tarfile
309
310    print('Untar ...')
311
312    with tarfile.open(tarball_path) as tar_handle:
313        tar_handle.extractall()
314
315    return
316
317def mk_env_vars(ecuid, ecgid, gateway, destination):
318    '''
319    @Description:
320        Creates a file named ECMWF_ENV which contains the
321        necessary environmental variables at ECMWF servers.
322        It is based on the template ECMWF_ENV.template.
323
324    @Input:
325        ecuid: string
326            The user id on ECMWF server.
327
328        ecgid: string
329            The group id on ECMWF server.
330
331        gateway: string
332            The gateway server the user is using.
333
334        destination: string
335            The remote destination which is used to transfer files
336            from ECMWF server to local gateway server.
337
338    @Return:
339        <nothing>
340    '''
341    from genshi.template.text import NewTextTemplate
342    from genshi.template import  TemplateLoader
343
344    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
345    ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
346                                     cls=NewTextTemplate)
347
348    stream = ecmwfvars_template.generate(user_name = ecuid,
349                                       user_group = ecgid,
350                                       gateway_name = gateway,
351                                       destination_name = destination
352                                       )
353
354    with open(_config.PATH_ECMWF_ENV, 'w') as f:
355        f.write(stream.render('text'))
356
357    return
358
359def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
360    '''
361    @Description:
362        Modifies the original job template file so that it is specified
363        for the user and the environment were it will be applied. Result
364        is stored in a new file "job.temp" in the python directory.
365
366    @Input:
367        makefile: string
368            Name of the makefile which should be used to compile FORTRAN
369            CONVERT2 program.
370
371        target: string
372            The target where the installation should be done, e.g. the queue.
373
374        ecuid: string
375            The user id on ECMWF server.
376
377        ecgid: string
378            The group id on ECMWF server.
379
380        fp_root: string
381           Path to the root directory of FLEXPART environment or flex_extract
382           environment.
383
384    @Return:
385        <nothing>
386    '''
387    from genshi.template.text import NewTextTemplate
388    from genshi.template import  TemplateLoader
389
390    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
391    compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
392                                   cls=NewTextTemplate)
393
394    if fp_root == '../':
395        fp_root = '$HOME'
396
397    stream = compile_template.generate(
398        usergroup = ecgid,
399        username = ecuid,
400        version_number = _config._VERSION_STR,
401        fp_root_scripts = fp_root,
402        makefile = makefile,
403        fortran_program = _config.FORTRAN_EXECUTABLE
404    )
405
406    compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
407                              _config.FILE_INSTALL_COMPILEJOB)
408
409    with open(compilejob, 'w') as f:
410        f.write(stream.render('text'))
411
412    return
413
414def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
415    '''
416    @Description:
417        Modifies the original job template file so that it is specified
418        for the user and the environment were it will be applied. Result
419        is stored in a new file.
420
421    @Input:
422        ecuid: string
423            The user id on ECMWF server.
424
425        ecgid: string
426            The group id on ECMWF server.
427
428        gateway: string
429            The gateway server the user is using.
430
431        destination: string
432            The remote destination which is used to transfer files
433            from ECMWF server to local gateway server.
434
435        fp_root: string
436           Path to the root directory of FLEXPART environment or flex_extract
437           environment.
438
439    @Return:
440        <nothing>
441    '''
442    from genshi.template.text import NewTextTemplate
443    from genshi.template import  TemplateLoader
444
445    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
446    compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
447                                   cls=NewTextTemplate)
448
449    fp_root_path_to_python = os.path.join(fp_root,
450                                          _config.FLEXEXTRACT_DIRNAME,
451                                          _config.PATH_REL_PYTHON)
452
453    stream = compile_template.generate(
454        usergroup = ecgid,
455        username = ecuid,
456        version_number = _config._VERSION_STR,
457        fp_root_path = fp_root_path_to_python,
458    )
459
460    tempjobfile = os.path.join(_config.PATH_TEMPLATES,
461                               _config.TEMPFILE_JOB)
462
463    with open(tempjobfile, 'w') as f:
464        f.write(stream.render('text'))
465
466    return
467
468def delete_convert_build(src_path):
469    '''
470    @Description:
471        Clean up the Fortran source directory and remove all
472        build files (e.g. *.o, *.mod and CONVERT2)
473
474    @Input:
475        src_path: string
476            Path to the fortran source directory.
477
478    @Return:
479        <nothing>
480    '''
481
482    modfiles = UioFiles(src_path, '*.mod')
483    objfiles = UioFiles(src_path, '*.o')
484    exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
485
486    modfiles.delete_files()
487    objfiles.delete_files()
488    exefile.delete_files()
489
490    return
491
492def make_convert_build(src_path, makefile):
493    '''
494    @Description:
495        Compiles the Fortran code and generates the executable.
496
497    @Input:
498        src_path: string
499            Path to the fortran source directory.
500
501        makefile: string
502            The name of the makefile which should be used.
503
504    @Return:
505        <nothing>
506    '''
507
508    try:
509        print('Using makefile: ' + makefile)
510        p = subprocess.Popen(['make', '-f',
511                              os.path.join(src_path, makefile)],
512                             stdin=subprocess.PIPE,
513                             stdout=subprocess.PIPE,
514                             stderr=subprocess.PIPE,
515                             bufsize=1)
516        pout, perr = p.communicate()
517        print(pout)
518        if p.returncode != 0:
519            print(perr)
520            print('Please edit ' + makefile +
521                  ' or try another Makefile in the src directory.')
522            print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
523                  'and EMOSLIB must be adapted.')
524            print('Available Makefiles:')
525            print(UioFiles(src_path, 'Makefile*'))
526            sys.exit('Compilation failed!')
527    except ValueError as e:
528        print('ERROR: Makefile call failed:')
529        print(e)
530    else:
531        subprocess.check_call(['ls', '-l',
532                               os.path.join(src_path,
533                                            _config.FORTRAN_EXECUTABLE)])
534
535    return
536
537
538if __name__ == "__main__":
539    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG