source: flex_extract.git/source/python/install.py @ ff2a11c

ctbtodev
Last change on this file since ff2a11c was ff2a11c, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

changed generation of final job_scripts by using genshi templates

  • Property mode set to 100755
File size: 23.4 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Leopold Haimberger (University of Vienna)
5#
6# @Date: November 2015
7#
8# @Change History:
9#
10#    February 2018 - Anne Philipp (University of Vienna):
11#        - applied PEP8 style guide
12#        - added documentation
13#        - moved install_args_and_control in here
14#        - splitted code in smaller functions
15#        - delete convert build files in here instead of compile job script
16#        - changed static path names to Variables from config file
17#
18# @License:
19#    (C) Copyright 2015-2018.
20#
21#    This software is licensed under the terms of the Apache Licence Version 2.0
22#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
23#
24# @Program Functionality:
25#    Depending on the selected installation environment (locally or on the
26#    ECMWF server ecgate or cca) the program extracts the commandline
27#    arguments and the CONTROL file parameter and prepares the corresponding
28#    environment. The necessary files are collected in a tar-ball and placed
29#    at the target location. There its untared, the environment variables will
30#    be set and the Fortran code will be compiled. If the ECMWF environment is
31#    selected a job script is prepared and submitted for the remaining
32#    configurations after putting the tar-ball to the target ECMWF server.
33#
34# @Program Content:
35#    - main
36#    - get_install_cmdline_arguments
37#    - install_via_gateway
38#    - mk_tarball
39#    - un_tarball
40#    - mk_env_vars
41#    - mk_compilejob
42#    - mk_job_template
43#    - delete_convert_build
44#    - make_convert_build
45#
46#*******************************************************************************
47
48# ------------------------------------------------------------------------------
49# MODULES
50# ------------------------------------------------------------------------------
51import os
52import sys
53import glob
54import subprocess
55import inspect
56import tarfile
57from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
58
59# software specific classes and modules from flex_extract
60import _config
61from classes.ControlFile import ControlFile
62from classes.UioFiles import UioFiles
63from mods.tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
64
65# ------------------------------------------------------------------------------
66# FUNCTIONS
67# ------------------------------------------------------------------------------
68def main():
69    '''Controls the installation process. Calls the installation function
70    if target is specified.
71
72    Parameters
73    ----------
74
75    Return
76    ------
77    '''
78
79    args = get_install_cmdline_arguments()
80    c = ControlFile(args.controlfile)
81    c.assign_args_to_control(args)
82    check_install_conditions(c)
83
84    install_via_gateway(c)
85
86    return
87
88def get_install_cmdline_arguments():
89    '''Decomposes the command line arguments and assigns them to variables.
90    Apply default values for non mentioned arguments.
91
92    Parameters
93    ----------
94
95    Return
96    ------
97    args : :obj:`Namespace`
98        Contains the commandline arguments from script/program call.
99    '''
100    parser = ArgumentParser(description='Install flex_extract software locally or \
101                            on ECMWF machines',
102                            formatter_class=ArgumentDefaultsHelpFormatter)
103
104    parser.add_argument('--target', dest='install_target', default=None,
105                        help="Valid targets: local | ecgate | cca , \
106                        the latter two are at ECMWF")
107    parser.add_argument("--makefile", dest="makefile", default=None,
108                        help='Name of Makefile to use for compiling CONVERT2')
109    parser.add_argument("--ecuid", dest="ecuid", default=None,
110                        help='user id at ECMWF')
111    parser.add_argument("--ecgid", dest="ecgid", default=None,
112                        help='group id at ECMWF')
113    parser.add_argument("--gateway", dest="gateway", default=None,
114                        help='name of local gateway server')
115    parser.add_argument("--destination", dest="destination", default=None,
116                        help='ecaccess destination, e.g. leo@genericSftp')
117
118    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
119                        default=None, help="FLEXPART root directory on ECMWF \
120                        servers (to find grib2flexpart and COMMAND file)\n\
121                        Normally flex_extract resides in the scripts directory \
122                        of the FLEXPART distribution.")
123
124    # arguments for job submission to ECMWF, only needed by submit.py
125    parser.add_argument("--job_template", dest='job_template',
126                        default="job.temp.o",
127                        help="job template file for submission to ECMWF")
128
129    parser.add_argument("--controlfile", dest="controlfile",
130                        default='CONTROL.temp',
131                        help="file with CONTROL parameters")
132
133    args = parser.parse_args()
134
135    return args
136
137
138def install_via_gateway(c):
139    '''Perform the actual installation on local machine or prepare data
140    transfer to remote gate and submit a job script which will
141    install everything on the remote gate.
142
143    Parameters
144    ----------
145    c : :obj:`ControlFile`
146        Contains all the parameters of CONTROL file and
147        command line.
148
149    Return
150    ------
151
152    '''
153    import tarfile
154
155    ecd = _config.PATH_FLEXEXTRACT_DIR
156    tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
157    tar_file = os.path.join(ecd, tarball_name)
158
159    target_dirname = _config.FLEXEXTRACT_DIRNAME
160    fortran_executable = _config.FORTRAN_EXECUTABLE
161
162    if c.install_target.lower() != 'local': # ecgate or cca
163
164        mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
165                      c.flexpart_root_scripts)
166
167        mk_job_template(c.ecuid, c.ecgid, c.gateway,
168                        c.destination, c.flexpart_root_scripts)
169
170        mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
171
172        mk_tarball(tar_file, c.install_target)
173
174        put_file_to_ecserver(ecd, tarball_name, c.install_target,
175                             c.ecuid, c.ecgid)
176
177        submit_job_to_ecserver(c.install_target,
178                               os.path.join(_config.PATH_REL_JOBSCRIPTS,
179                                            _config.FILE_INSTALL_COMPILEJOB))
180
181        print('job compilation script has been submitted to ecgate for ' +
182              'installation in ' + c.flexpart_root_scripts +
183               '/' + target_dirname)
184        print('You should get an email with subject "flexcompile" within ' +
185              'the next few minutes!')
186
187    else: #local
188        if c.flexpart_root_scripts == _config.PATH_FLEXEXTRACT_DIR :
189            print('WARNING: FLEXPART_ROOT_SCRIPTS has not been specified')
190            print('flex_extract will be installed in here by compiling the ' +
191                  'Fortran source in ' + _config.PATH_FORTRAN_SRC)
192            os.chdir(_config.PATH_FORTRAN_SRC)
193        else: # creates the target working directory for flex_extract
194            c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
195                                        c.flexpart_root_scripts))
196            if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
197                mk_tarball(tar_file, c.install_target)
198                make_dir(os.path.join(c.flexpart_root_scripts,
199                                      target_dirname))
200                os.chdir(os.path.join(c.flexpart_root_scripts,
201                                      target_dirname))
202                un_tarball(tar_file)
203                os.chdir(os.path.join(c.flexpart_root_scripts,
204                                      target_dirname,
205                                      _config.PATH_REL_FORTRAN_SRC))
206
207        # Create Fortran executable - CONVERT2
208        print('Install ' + target_dirname + ' software at ' +
209              c.install_target + ' in directory ' +
210              os.path.abspath(c.flexpart_root_scripts) + '\n')
211
212        delete_convert_build('.')
213        make_convert_build('.', c.makefile)
214
215        os.chdir(ecd)
216        if os.path.isfile(tar_file):
217            os.remove(tar_file)
218
219    return
220
221def check_install_conditions(c):
222    '''Checks a couple of necessary attributes and conditions
223    for the installation such as if they exist and contain values.
224    Otherwise set default values.
225
226    Parameters
227    ----------
228    c : :obj:`ControlFile`
229        Contains all the parameters of CONTROL file and
230        command line.
231
232
233    Return
234    ------
235
236    '''
237
238    if c.install_target and \
239       c.install_target not in _config.INSTALL_TARGETS:
240        print('ERROR: unknown or missing installation target ')
241        print('target: ', c.install_target)
242        print('please specify correct installation target ' +
243              str(INSTALL_TARGETS))
244        print('use -h or --help for help')
245        sys.exit(1)
246
247    if c.install_target and c.install_target != 'local':
248        if not c.ecgid or not c.ecuid or \
249           not c.gateway or not c.destination:
250            print('Please enter your ECMWF user id and group id as well ' +
251                  'as the \nname of the local gateway and the ectrans ' +
252                  'destination ')
253            print('with command line options --ecuid --ecgid \
254                   --gateway --destination')
255            print('Try "' + sys.argv[0].split('/')[-1] + \
256                  ' -h" to print usage information')
257            print('Please consult ecaccess documentation or ECMWF user \
258                   support for further details')
259            sys.exit(1)
260
261        if not c.flexpart_root_scripts:
262            c.flexpart_root_scripts = '${HOME}'
263        else:
264            c.flexpart_root_scripts = c.flexpart_root_scripts
265    else: # local
266        if not c.flexpart_root_scripts:
267            c.flexpart_root_scripts = _config.PATH_FLEXEXTRACT_DIR
268
269    return
270
271
272def mk_tarball(tarball_path, target):
273    '''Creates a tarball with all necessary files which need to be sent to the
274    installation directory.
275    It does not matter if this is local or remote.
276    Collects all python files, the Fortran source and makefiles,
277    the ECMWF_ENV file, the CONTROL files as well as the
278    template files.
279
280    Parameters
281    ----------
282    tarball_path : :obj:`string`
283        The complete path to the tar file which will contain all
284        relevant data for flex_extract.
285
286    target : :obj:`string`
287        The queue where the job is submitted to.
288
289    Return
290    ------
291
292    '''
293    from glob import glob
294
295    print('Create tarball ...')
296
297    # change to FLEXEXTRACT directory so that the tar can contain
298    # relative pathes to the files and directories
299    ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
300    os.chdir(ecd)
301
302    # get lists of the files to be added to the tar file
303    if target == 'local':
304        ECMWF_ENV_FILE = []
305        runfile = [os.path.relpath(x, ecd)
306                   for x in UioFiles(_config.PATH_REL_RUN_DIR,
307                                     'run_local.sh').files]
308    else:
309        ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
310        runfile = [os.path.relpath(x, ecd)
311                       for x in UioFiles(_config.PATH_REL_RUN_DIR,
312                                         'run.sh').files]
313
314    pyfiles = [os.path.relpath(x, ecd)
315               for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
316    pytestfiles = [os.path.relpath(x, ecd)
317               for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
318    controlfiles = [os.path.relpath(x, ecd)
319                    for x in UioFiles(_config.PATH_REL_CONTROLFILES,
320                                      'CONTROL*').files]
321    tempfiles = [os.path.relpath(x, ecd)
322                 for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.temp').files]
323    nlfiles = [os.path.relpath(x, ecd)
324                 for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.nl').files]
325    gribtable = [os.path.relpath(x, ecd)
326                 for x in UioFiles(_config.PATH_REL_TEMPLATES , '*grib*').files]
327    ffiles = [os.path.relpath(x, ecd)
328              for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f*').files]
329    hfiles = [os.path.relpath(x, ecd)
330              for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
331    makefiles = [os.path.relpath(x, ecd)
332                 for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'Makefile*').files]
333    jobdir = [_config.PATH_REL_JOBSCRIPTS]
334
335    # concatenate single lists to one for a better looping
336    filelist = pyfiles + pytestfiles + controlfiles + tempfiles + nlfiles + \
337               ffiles + gribtable + hfiles + makefiles + ECMWF_ENV_FILE + \
338               runfile + jobdir + \
339               ['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
340
341    # create installation tar-file
342    exclude_files = [".ksh"]
343    try:
344        with tarfile.open(tarball_path, "w:gz") as tar_handle:
345            for file in filelist:
346                tar_handle.add(file, recursive=False,
347                               filter=lambda tarinfo: None
348                                      if os.path.splitext(tarinfo.name)[1]
349                                         in exclude_files
350                                      else tarinfo)
351    except subprocess.CalledProcessError as e:
352        print('... ERROR CODE:\n ... ' + str(e.returncode))
353        print('... ERROR MESSAGE:\n ... ' + str(e))
354
355        sys.exit('\n... could not make installation tar ball!')
356    except OSError as e:
357        print('... ERROR CODE: ' + str(e.errno))
358        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
359
360        sys.exit('\n... error occured while trying to read tar-file ' +
361                 str(tarball_path))
362
363    return
364
365
366def un_tarball(tarball_path):
367    '''Extracts the given tarball into current directory.
368
369    Parameters
370    ----------
371    tarball_path : :obj:`string`
372        The complete path to the tar file which will contain all
373        relevant data for flex_extract.
374
375    Return
376    ------
377
378    '''
379
380    print('Untar ...')
381
382    try:
383        with tarfile.open(tarball_path) as tar_handle:
384            tar_handle.extractall()
385    except tarfile.TarError as e:
386        sys.exit('\n... error occured while trying to read tar-file ' +
387                     str(tarball_path))
388    except OSError as e:
389        print('... ERROR CODE: ' + str(e.errno))
390        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
391
392        sys.exit('\n... error occured while trying to read tar-file ' +
393                 str(tarball_path))
394
395    return
396
397def mk_env_vars(ecuid, ecgid, gateway, destination):
398    '''Creates a file named ECMWF_ENV which contains the
399    necessary environmental variables at ECMWF servers.
400    It is based on the template ECMWF_ENV.template.
401
402    Parameters
403    ----------
404    ecuid : :obj:`string`
405        The user id on ECMWF server.
406
407    ecgid : :obj:`string`
408        The group id on ECMWF server.
409
410    gateway : :obj:`string`
411        The gateway server the user is using.
412
413    destination : :obj:`string`
414        The remote destination which is used to transfer files
415        from ECMWF server to local gateway server.
416
417    Return
418    ------
419
420    '''
421    from genshi.template.text import NewTextTemplate
422    from genshi.template import  TemplateLoader
423    from genshi.template.eval import UndefinedError
424
425    try:
426        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
427        ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
428                                         cls=NewTextTemplate)
429
430        stream = ecmwfvars_template.generate(user_name = ecuid,
431                                             user_group = ecgid,
432                                             gateway_name = gateway,
433                                             destination_name = destination
434                                             )
435    except UndefinedError as e:
436        print('... ERROR ' + str(e))
437
438        sys.exit('\n... error occured while trying to generate template ' +
439                 _config.PATH_ECMWF_ENV)
440    except OSError as e:
441        print('... ERROR CODE: ' + str(e.errno))
442        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
443
444        sys.exit('\n... error occured while trying to generate template ' +
445                 _config.PATH_ECMWF_ENV)
446
447    try:
448        with open(_config.PATH_ECMWF_ENV, 'w') as f:
449            f.write(stream.render('text'))
450    except OSError as e:
451        print('... ERROR CODE: ' + str(e.errno))
452        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
453
454        sys.exit('\n... error occured while trying to write ' +
455                 _config.PATH_ECMWF_ENV)
456
457    return
458
459def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
460    '''Modifies the original job template file so that it is specified
461    for the user and the environment were it will be applied. Result
462    is stored in a new file "job.temp" in the python directory.
463
464    Parameters
465    ----------
466    makefile : :obj:`string`
467        Name of the makefile which should be used to compile FORTRAN
468        CONVERT2 program.
469
470    target : :obj:`string`
471        The target where the installation should be done, e.g. the queue.
472
473    ecuid : :obj:`string`
474        The user id on ECMWF server.
475
476    ecgid : :obj:`string`
477        The group id on ECMWF server.
478
479    fp_root : :obj:`string`
480       Path to the root directory of FLEXPART environment or flex_extract
481       environment.
482
483    Return
484    ------
485
486    '''
487    from genshi.template.text import NewTextTemplate
488    from genshi.template import  TemplateLoader
489    from genshi.template.eval import  UndefinedError
490
491    if fp_root == '../':
492        fp_root = '$HOME'
493
494    try:
495        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
496        compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
497                                       cls=NewTextTemplate)
498
499        stream = compile_template.generate(
500            usergroup = ecgid,
501            username = ecuid,
502            version_number = _config._VERSION_STR,
503            fp_root_scripts = fp_root,
504            makefile = makefile,
505            fortran_program = _config.FORTRAN_EXECUTABLE
506        )
507    except UndefinedError as e:
508        print('... ERROR ' + str(e))
509
510        sys.exit('\n... error occured while trying to generate template ' +
511                 _config.TEMPFILE_INSTALL_COMPILEJOB)
512    except OSError as e:
513        print('... ERROR CODE: ' + str(e.errno))
514        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
515
516        sys.exit('\n... error occured while trying to generate template ' +
517                 _config.TEMPFILE_INSTALL_COMPILEJOB)
518
519    try:
520        compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
521                                  _config.FILE_INSTALL_COMPILEJOB)
522
523        with open(compilejob, 'w') as f:
524            f.write(stream.render('text'))
525    except OSError as e:
526        print('... ERROR CODE: ' + str(e.errno))
527        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
528
529        sys.exit('\n... error occured while trying to write ' +
530                 compilejob)
531
532    return
533
534def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
535    '''Modifies the original job template file so that it is specified
536    for the user and the environment were it will be applied. Result
537    is stored in a new file.
538
539    Parameters
540    ----------
541    ecuid : :obj:`string`
542        The user id on ECMWF server.
543
544    ecgid : :obj:`string`
545        The group id on ECMWF server.
546
547    gateway : :obj:`string`
548        The gateway server the user is using.
549
550    destination : :obj:`string`
551        The remote destination which is used to transfer files
552        from ECMWF server to local gateway server.
553
554    fp_root : :obj:`string`
555       Path to the root directory of FLEXPART environment or flex_extract
556       environment.
557
558    Return
559    ------
560
561    '''
562    from genshi.template.text import NewTextTemplate
563    from genshi.template import  TemplateLoader
564    from genshi.template.eval import  UndefinedError
565
566    fp_root_path_to_python = os.path.join(fp_root,
567                                          _config.FLEXEXTRACT_DIRNAME,
568                                          _config.PATH_REL_PYTHON_SRC)
569    if '$' in fp_root_path_to_python:
570        ind = fp_root_path_to_python.index('$')
571        fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
572                                 fp_root_path_to_python[ind:]
573
574    try:
575        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
576        compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
577                                       cls=NewTextTemplate)
578
579        stream = compile_template.generate(
580            usergroup = ecgid,
581            username = ecuid,
582            version_number = _config._VERSION_STR,
583            fp_root_path = fp_root_path_to_python,
584        )
585    except UndefinedError as e:
586        print('... ERROR ' + str(e))
587
588        sys.exit('\n... error occured while trying to generate template ' +
589                 _config.TEMPFILE_INSTALL_JOB)
590    except OSError as e:
591        print('... ERROR CODE: ' + str(e.errno))
592        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
593
594        sys.exit('\n... error occured while trying to generate template ' +
595                 _config.TEMPFILE_INSTALL_JOB)
596
597
598    try:
599        tempjobfile = os.path.join(_config.PATH_TEMPLATES,
600                                   _config.TEMPFILE_JOB)
601
602        with open(tempjobfile, 'w') as f:
603            f.write(stream.render('text'))
604    except OSError as e:
605        print('... ERROR CODE: ' + str(e.errno))
606        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
607
608        sys.exit('\n... error occured while trying to write ' +
609                 tempjobfile)
610
611    return
612
613def delete_convert_build(src_path):
614    '''Clean up the Fortran source directory and remove all
615    build files (e.g. \*.o, \*.mod and CONVERT2)
616
617    Parameters
618    ----------
619    src_path : :obj:`string`
620        Path to the fortran source directory.
621
622    Return
623    ------
624
625    '''
626
627    modfiles = UioFiles(src_path, '*.mod')
628    objfiles = UioFiles(src_path, '*.o')
629    exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
630
631    modfiles.delete_files()
632    objfiles.delete_files()
633    exefile.delete_files()
634
635    return
636
637def make_convert_build(src_path, makefile):
638    '''Compiles the Fortran code and generates the executable.
639
640    Parameters
641    ----------
642    src_path : :obj:`string`
643        Path to the fortran source directory.
644
645    makefile : :obj:`string`
646        The name of the makefile which should be used.
647
648    Return
649    ------
650
651    '''
652
653    try:
654        print('Using makefile: ' + makefile)
655        p = subprocess.Popen(['make', '-f',
656                              os.path.join(src_path, makefile)],
657                             stdin=subprocess.PIPE,
658                             stdout=subprocess.PIPE,
659                             stderr=subprocess.PIPE,
660                             bufsize=1)
661        pout, perr = p.communicate()
662        print(pout)
663        if p.returncode != 0:
664            print(perr)
665            print('Please edit ' + makefile +
666                  ' or try another Makefile in the src directory.')
667            print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
668                  'and EMOSLIB must be adapted.')
669            print('Available Makefiles:')
670            print(UioFiles(src_path, 'Makefile*'))
671            sys.exit('Compilation failed!')
672    except ValueError as e:
673        print('ERROR: Makefile call failed:')
674        print(e)
675    else:
676        subprocess.check_call(['ls', '-l',
677                               os.path.join(src_path,
678                                            _config.FORTRAN_EXECUTABLE)])
679
680    return
681
682
683if __name__ == "__main__":
684    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG