source: flex_extract.git/source/python/install.py @ 0aaeb04

ctbtodev
Last change on this file since 0aaeb04 was 0aaeb04, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

added writing of compilejob and ECMWF_ENV via templates and added tests
  • Property mode set to 100755
File size: 19.1 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Leopold Haimberger (University of Vienna)
5#
6# @Date: November 2015
7#
8# @Change History:
9#
10#    February 2018 - Anne Philipp (University of Vienna):
11#        - applied PEP8 style guide
12#        - added documentation
13#        - moved install_args_and_control in here
14#        - splitted code in smaller functions
15#        - delete convert build files in here instead of compile job script
16#        - changed static path names to Variables from config file
17#
18# @License:
19#    (C) Copyright 2015-2018.
20#
21#    This software is licensed under the terms of the Apache Licence Version 2.0
22#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
23#
24# @Program Functionality:
25#    Depending on the selected installation environment (locally or on the
26#    ECMWF server ecgate or cca) the program extracts the commandline
27#    arguments and the CONTROL file parameter and prepares the corresponding
28#    environment. The necessary files are collected in a tar-ball and placed
29#    at the target location. There its untared, the environment variables will
30#    be set and the Fortran code will be compiled. If the ECMWF environment is
31#    selected a job script is prepared and submitted for the remaining
32#    configurations after putting the tar-ball to the target ECMWF server.
33#
34# @Program Content:
35#    - main
36#    - get_install_cmdline_arguments
37#    - install_via_gateway
38#    - mk_tarball
39#    - un_tarball
40#    - mk_env_vars
41#    - mk_compilejob
42#    - mk_job_template
43#    - delete_convert_build
44#    - make_convert_build
45#
46#*******************************************************************************
47
48# ------------------------------------------------------------------------------
49# MODULES
50# ------------------------------------------------------------------------------
51import os
52import sys
53import glob
54import subprocess
55import inspect
56from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
57
58# software specific classes and modules from flex_extract
59import _config
60from classes.ControlFile import ControlFile
61from classes.UioFiles import UioFiles
62from mods.tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
63
64# ------------------------------------------------------------------------------
65# FUNCTIONS
66# ------------------------------------------------------------------------------
67def main():
68    '''
69    @Description:
70        Controls the installation process. Calls the installation function
71        if target is specified.
72
73    @Intput:
74        <nothing>
75
76    @Return:
77        <nothing>
78    '''
79
80    args = get_install_cmdline_arguments()
81    c = ControlFile(args.controlfile)
82    c.assign_args_to_control(args)
83    c.check_install_conditions()
84
85    install_via_gateway(c)
86
87    return
88
89def get_install_cmdline_arguments():
90    '''
91    @Description:
92        Decomposes the command line arguments and assigns them to variables.
93        Apply default values for non mentioned arguments.
94
95    @Input:
96        <nothing>
97
98    @Return:
99        args: instance of ArgumentParser
100            Contains the commandline arguments from script/program call.
101    '''
102    parser = ArgumentParser(description='Install flex_extract software locally or \
103                            on ECMWF machines',
104                            formatter_class=ArgumentDefaultsHelpFormatter)
105
106    parser.add_argument('--target', dest='install_target', default=None,
107                        help="Valid targets: local | ecgate | cca , \
108                        the latter two are at ECMWF")
109    parser.add_argument("--makefile", dest="makefile", default=None,
110                        help='Name of Makefile to use for compiling CONVERT2')
111    parser.add_argument("--ecuid", dest="ecuid", default=None,
112                        help='user id at ECMWF')
113    parser.add_argument("--ecgid", dest="ecgid", default=None,
114                        help='group id at ECMWF')
115    parser.add_argument("--gateway", dest="gateway", default=None,
116                        help='name of local gateway server')
117    parser.add_argument("--destination", dest="destination", default=None,
118                        help='ecaccess destination, e.g. leo@genericSftp')
119
120    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
121                        default=None, help="FLEXPART root directory on ECMWF \
122                        servers (to find grib2flexpart and COMMAND file)\n\
123                        Normally flex_extract resides in the scripts directory \
124                        of the FLEXPART distribution.")
125
126    # arguments for job submission to ECMWF, only needed by submit.py
127    parser.add_argument("--job_template", dest='job_template',
128                        default="job.temp.o",
129                        help="job template file for submission to ECMWF")
130
131    parser.add_argument("--controlfile", dest="controlfile",
132                        default='CONTROL.temp',
133                        help="file with CONTROL parameters")
134
135    args = parser.parse_args()
136
137    return args
138
139
140def install_via_gateway(c):
141    '''
142    @Description:
143        Perform the actual installation on local machine or prepare data
144        transfer to remote gate and submit a job script which will
145        install everything on the remote gate.
146
147    @Input:
148        c: instance of class ControlFile
149            Contains all the parameters of CONTROL file and
150            command line.
151            For more information about format and content of the parameter
152            see documentation.
153
154    @Return:
155        <nothing>
156    '''
157    import tarfile
158
159    ecd = _config.PATH_FLEXEXTRACT_DIR
160    tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
161    tar_file = os.path.join(ecd, tarball_name)
162
163    target_dirname = _config.FLEXEXTRACT_DIRNAME
164    fortran_executable = _config.FORTRAN_EXECUTABLE
165
166    if c.install_target.lower() != 'local': # ecgate or cca
167
168        mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
169                      c.flexpart_root_scripts)
170
171        mk_job_template(c.ecuid, c.ecgid, c.gateway,
172                        c.destination, c.flexpart_root_scripts)
173
174        mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
175
176        mk_tarball(tar_file, c.install_target)
177
178        put_file_to_ecserver(ecd, tarball_name, c.install_target,
179                             c.ecuid, c.ecgid)
180
181        submit_job_to_ecserver(c.install_target,
182                               os.path.join(_config.PATH_REL_JOBSCRIPTS,
183                                            _config.FILE_INSTALL_COMPILEJOB))
184
185        print('job compilation script has been submitted to ecgate for ' +
186              'installation in ' + c.flexpart_root_scripts +
187               '/' + target_dirname)
188        print('You should get an email with subject "flexcompile" within ' +
189              'the next few minutes!')
190
191    else: #local
192        if c.flexpart_root_scripts == _config.PATH_FLEXEXTRACT_DIR :
193            print('WARNING: FLEXPART_ROOT_SCRIPTS has not been specified')
194            print('flex_extract will be installed in here by compiling the ' +
195                  'Fortran source in ' + _config.PATH_FORTRAN_SRC)
196            os.chdir(_config.PATH_FORTRAN_SRC)
197        else: # creates the target working directory for flex_extract
198            c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
199                                        c.flexpart_root_scripts))
200            if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
201                mk_tarball(tar_file, c.install_target)
202                make_dir(os.path.join(c.flexpart_root_scripts,
203                                      target_dirname))
204                os.chdir(os.path.join(c.flexpart_root_scripts,
205                                      target_dirname))
206                un_tarball(tar_file)
207                os.chdir(os.path.join(c.flexpart_root_scripts,
208                                      target_dirname,
209                                      _config.PATH_REL_FORTRAN_SRC))
210
211        # Create Fortran executable - CONVERT2
212        print('Install ' + target_dirname + ' software at ' +
213              c.install_target + ' in directory ' +
214              os.path.abspath(c.flexpart_root_scripts) + '\n')
215
216        delete_convert_build('.')
217        make_convert_build('.', c.makefile)
218
219        os.chdir(ecd)
220        if os.path.isfile(tar_file):
221            os.remove(tar_file)
222
223    return
224
225def mk_tarball(tarball_path, target):
226    '''
227    @Description:
228        Creates a tarball with all necessary files which need to be sent to the
229        installation directory.
230        It does not matter if this is local or remote.
231        Collects all python files, the Fortran source and makefiles,
232        the ECMWF_ENV file, the CONTROL files as well as the
233        template files.
234
235    @Input:
236        tarball_path: string
237            The complete path to the tar file which will contain all
238            relevant data for flex_extract.
239
240        target: string
241            The queue where the job is submitted to.
242
243    @Return:
244        <nothing>
245    '''
246    import tarfile
247    from glob import glob
248
249    print('Create tarball ...')
250
251    # change to FLEXEXTRACT directory so that the tar can contain
252    # relative pathes to the files and directories
253    ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
254    os.chdir(ecd)
255
256    # get lists of the files to be added to the tar file
257    if target == 'local':
258        ECMWF_ENV_FILE = []
259    else:
260        ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
261
262    pyfiles = [os.path.relpath(x, ecd)
263               for x in UioFiles(_config.PATH_LOCAL_PYTHON, '*py').files]
264    controlfiles = [os.path.relpath(x, ecd)
265                    for x in UioFiles(_config.PATH_CONTROLFILES,
266                                      'CONTROL*').files]
267    tempfiles = [os.path.relpath(x, ecd)
268                 for x in UioFiles(_config.PATH_TEMPLATES , '*').files]
269    ffiles = [os.path.relpath(x, ecd)
270              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.f*').files]
271    hfiles = [os.path.relpath(x, ecd)
272              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.h').files]
273    makefiles = [os.path.relpath(x, ecd)
274                 for x in UioFiles(_config.PATH_FORTRAN_SRC, 'Makefile*').files]
275
276    # concatenate single lists to one for a better looping
277    filelist = pyfiles + controlfiles + tempfiles + ffiles + hfiles + \
278               makefiles + ECMWF_ENV_FILE
279
280    # create installation tar-file
281    try:
282        with tarfile.open(tarball_path, "w:gz") as tar_handle:
283            for file in filelist:
284                tar_handle.add(file)
285
286    except subprocess.CalledProcessError as e:
287        print('... ERROR CODE:\n ... ' + str(e.returncode))
288        print('... ERROR MESSAGE:\n ... ' + str(e))
289
290        sys.exit('... could not make installation tar ball!')
291
292    return
293
294
295def un_tarball(tarball_path):
296    '''
297    @Description:
298        Extracts the given tarball into current directory.
299
300    @Input:
301        tarball_path: string
302            The complete path to the tar file which will contain all
303            relevant data for flex_extract.
304
305    @Return:
306        <nothing>
307    '''
308    import tarfile
309
310    print('Untar ...')
311
312    with tarfile.open(tarball_path) as tar_handle:
313        tar_handle.extractall()
314
315    return
316
317def mk_env_vars(ecuid, ecgid, gateway, destination):
318    '''
319    @Description:
320        Creates a file named ECMWF_ENV which contains the
321        necessary environmental variables at ECMWF servers.
322        It is based on the template ECMWF_ENV.template.
323
324    @Input:
325        ecuid: string
326            The user id on ECMWF server.
327
328        ecgid: string
329            The group id on ECMWF server.
330
331        gateway: string
332            The gateway server the user is using.
333
334        destination: string
335            The remote destination which is used to transfer files
336            from ECMWF server to local gateway server.
337
338    @Return:
339        <nothing>
340    '''
341    from genshi.template.text import NewTextTemplate
342    from genshi.template import  TemplateLoader
343
344    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
345    ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
346                                     cls=NewTextTemplate)
347
348    stream = ecmwfvars_template.generate(user_name = ecuid,
349                                       user_group = ecgid,
350                                       gateway_name = gateway,
351                                       destination_name = destination
352                                       )
353
354    with open(_config.PATH_ECMWF_ENV, 'w') as f:
355        f.write(stream.render('text'))
356
357    return
358
359def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
360    '''
361    @Description:
362        Modifies the original job template file so that it is specified
363        for the user and the environment were it will be applied. Result
364        is stored in a new file "job.temp" in the python directory.
365
366    @Input:
367        makefile: string
368            Name of the makefile which should be used to compile FORTRAN
369            CONVERT2 program.
370
371        target: string
372            The target where the installation should be done, e.g. the queue.
373
374        ecuid: string
375            The user id on ECMWF server.
376
377        ecgid: string
378            The group id on ECMWF server.
379
380        fp_root: string
381           Path to the root directory of FLEXPART environment or flex_extract
382           environment.
383
384    @Return:
385        <nothing>
386    '''
387    from genshi.template.text import NewTextTemplate
388    from genshi.template import  TemplateLoader
389
390    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
391    compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
392                                   cls=NewTextTemplate)
393
394    if fp_root == '../':
395        fp_root = '$HOME'
396
397    stream = compile_template.generate(
398        username = ecuid,
399        usergroup = ecgid,
400        version_number = _config._VERSION_STR,
401        fp_root_scripts = fp_root,
402        makefile = makefile,
403        fortran_program = _config.FORTRAN_EXECUTABLE
404    )
405
406    compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
407                              _config.FILE_INSTALL_COMPILEJOB)
408
409    with open(compilejob, 'w') as f:
410        f.write(stream.render('text'))
411
412
413    # template = os.path.join(_config.PATH_REL_TEMPLATES,
414                            # _config.TEMPFILE_INSTALL_COMPILEJOB)
415    # with open(template) as f:
416        # fdata = f.read().split('\n')
417
418
419    # with open(compilejob, 'w') as fo:
420        # for data in fdata:
421            # if 'MAKEFILE=' in data:
422                # data = 'export MAKEFILE=' + makefile
423            # elif 'FLEXPART_ROOT_SCRIPTS=' in data:
424                # if fp_root != '../':
425                    # data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
426                # else:
427                    # data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
428            # elif target.lower() != 'local':
429                # if '--workdir' in data:
430                    # data = '#SBATCH --workdir=/scratch/ms/' + \
431                            # ecgid + '/' + ecuid
432                # elif '##PBS -o' in data:
433                    # data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \
434                           # 'flex_ecmwf.$Jobname.$Job_ID.out'
435                # elif 'FLEXPART_ROOT_SCRIPTS=' in data:
436                    # if fp_root != '../':
437                        # data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
438                    # else:
439                        # data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
440            # fo.write(data + '\n')
441
442    return
443
444def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
445    '''
446    @Description:
447        Modifies the original job template file so that it is specified
448        for the user and the environment were it will be applied. Result
449        is stored in a new file.
450
451    @Input:
452        ecuid: string
453            The user id on ECMWF server.
454
455        ecgid: string
456            The group id on ECMWF server.
457
458        gateway: string
459            The gateway server the user is using.
460
461        destination: string
462            The remote destination which is used to transfer files
463            from ECMWF server to local gateway server.
464
465        fp_root: string
466           Path to the root directory of FLEXPART environment or flex_extract
467           environment.
468
469    @Return:
470        <nothing>
471    '''
472    fp_root_path_to_python = os.path.join(fp_root, _config.FLEXEXTRACT_DIRNAME,
473                         _config.PATH_REL_PYTHON)
474
475    template = os.path.join(_config.PATH_REL_TEMPLATES,
476                            _config.TEMPFILE_INSTALL_JOB)
477    with open(template) as f:
478        fdata = f.read().split('\n')
479
480    jobfile_temp = os.path.join(_config.PATH_REL_TEMPLATES,
481                                _config.TEMPFILE_JOB)
482    with open(jobfile_temp, 'w') as fo:
483        for data in fdata:
484            if '--workdir' in data:
485                data = '#SBATCH --workdir=/scratch/ms/' + ecgid + '/' + ecuid
486            elif '##PBS -o' in data:
487                data = '##PBS -o /scratch/ms/' + ecgid + '/' + \
488                        ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out'
489            elif  'export PATH=${PATH}:' in data:
490                data += fp_root_path_to_python
491
492            fo.write(data + '\n')
493    return
494
495def delete_convert_build(src_path):
496    '''
497    @Description:
498        Clean up the Fortran source directory and remove all
499        build files (e.g. *.o, *.mod and CONVERT2)
500
501    @Input:
502        src_path: string
503            Path to the fortran source directory.
504
505    @Return:
506        <nothing>
507    '''
508
509    modfiles = UioFiles(src_path, '*.mod')
510    objfiles = UioFiles(src_path, '*.o')
511    exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
512
513    modfiles.delete_files()
514    objfiles.delete_files()
515    exefile.delete_files()
516
517    return
518
519def make_convert_build(src_path, makefile):
520    '''
521    @Description:
522        Compiles the Fortran code and generates the executable.
523
524    @Input:
525        src_path: string
526            Path to the fortran source directory.
527
528        makefile: string
529            The name of the makefile which should be used.
530
531    @Return:
532        <nothing>
533    '''
534
535    try:
536        print('Using makefile: ' + makefile)
537        p = subprocess.Popen(['make', '-f',
538                              os.path.join(src_path, makefile)],
539                             stdin=subprocess.PIPE,
540                             stdout=subprocess.PIPE,
541                             stderr=subprocess.PIPE,
542                             bufsize=1)
543        pout, perr = p.communicate()
544        print(pout)
545        if p.returncode != 0:
546            print(perr)
547            print('Please edit ' + makefile +
548                  ' or try another Makefile in the src directory.')
549            print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
550                  'and EMOSLIB must be adapted.')
551            print('Available Makefiles:')
552            print(UioFiles(src_path, 'Makefile*'))
553            sys.exit('Compilation failed!')
554    except ValueError as e:
555        print('ERROR: Makefile call failed:')
556        print(e)
557    else:
558        subprocess.check_call(['ls', '-l',
559                               os.path.join(src_path,
560                                            _config.FORTRAN_EXECUTABLE)])
561
562    return
563
564
565if __name__ == "__main__":
566    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG