source: flex_extract.git/source/python/install.py @ 8652e7e

ctbtodev
Last change on this file since 8652e7e was 8652e7e, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

renamed some functions

  • Property mode set to 100755
File size: 23.2 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Leopold Haimberger (University of Vienna)
5#
6# @Date: November 2015
7#
8# @Change History:
9#
10#    February 2018 - Anne Philipp (University of Vienna):
11#        - applied PEP8 style guide
12#        - added documentation
13#        - moved install_args_and_control in here
14#        - splitted code in smaller functions
15#        - delete convert build files in here instead of compile job script
16#        - changed static path names to Variables from config file
17#
18# @License:
19#    (C) Copyright 2015-2018.
20#
21#    This software is licensed under the terms of the Apache Licence Version 2.0
22#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
23#
24# @Program Functionality:
25#    Depending on the selected installation environment (locally or on the
26#    ECMWF server ecgate or cca) the program extracts the commandline
27#    arguments and the CONTROL file parameter and prepares the corresponding
28#    environment. The necessary files are collected in a tar-ball and placed
29#    at the target location. There its untared, the environment variables will
30#    be set and the Fortran code will be compiled. If the ECMWF environment is
31#    selected a job script is prepared and submitted for the remaining
32#    configurations after putting the tar-ball to the target ECMWF server.
33#
34# @Program Content:
35#    - main
36#    - get_install_cmdline_args
37#    - install_via_gateway
38#    - mk_tarball
39#    - un_tarball
40#    - mk_env_vars
41#    - mk_compilejob
42#    - mk_job_template
43#    - del_convert_build
44#    - mk_convert_build
45#
46#*******************************************************************************
47
48# ------------------------------------------------------------------------------
49# MODULES
50# ------------------------------------------------------------------------------
51import os
52import sys
53import glob
54import subprocess
55import inspect
56import tarfile
57from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
58
59# software specific classes and modules from flex_extract
60import _config
61from classes.ControlFile import ControlFile
62from classes.UioFiles import UioFiles
63from mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
64                        silent_remove)
65
66# ------------------------------------------------------------------------------
67# FUNCTIONS
68# ------------------------------------------------------------------------------
69def main():
70    '''Controls the installation process. Calls the installation function
71    if target is specified.
72
73    Parameters
74    ----------
75
76    Return
77    ------
78    '''
79
80    args = get_install_cmdline_args()
81    c = ControlFile(args.controlfile)
82    c.assign_args_to_control(args)
83    check_install_conditions(c)
84
85    install_via_gateway(c)
86
87    return
88
89def get_install_cmdline_args():
90    '''Decomposes the command line arguments and assigns them to variables.
91    Apply default values for non mentioned arguments.
92
93    Parameters
94    ----------
95
96    Return
97    ------
98    args : :obj:`Namespace`
99        Contains the commandline arguments from script/program call.
100    '''
101    parser = ArgumentParser(description='Install flex_extract software locally or \
102                            on ECMWF machines',
103                            formatter_class=ArgumentDefaultsHelpFormatter)
104
105    parser.add_argument('--target', dest='install_target', default=None,
106                        help="Valid targets: local | ecgate | cca , \
107                        the latter two are at ECMWF")
108    parser.add_argument("--makefile", dest="makefile", default=None,
109                        help='Name of Makefile to use for compiling CONVERT2')
110    parser.add_argument("--ecuid", dest="ecuid", default=None,
111                        help='user id at ECMWF')
112    parser.add_argument("--ecgid", dest="ecgid", default=None,
113                        help='group id at ECMWF')
114    parser.add_argument("--gateway", dest="gateway", default=None,
115                        help='name of local gateway server')
116    parser.add_argument("--destination", dest="destination", default=None,
117                        help='ecaccess destination, e.g. leo@genericSftp')
118
119    parser.add_argument("--flexpartdir", dest="flexpartdir",
120                        default=None, help="FLEXPART root directory on ECMWF \
121                        servers (to find grib2flexpart and COMMAND file)\n\
122                        Normally flex_extract resides in the scripts directory \
123                        of the FLEXPART distribution.")
124
125    # arguments for job submission to ECMWF, only needed by submit.py
126    parser.add_argument("--job_template", dest='job_template',
127                        default="job.temp.o",
128                        help="job template file for submission to ECMWF")
129
130    parser.add_argument("--controlfile", dest="controlfile",
131                        default='CONTROL.temp',
132                        help="file with CONTROL parameters")
133
134    args = parser.parse_args()
135
136    return args
137
138
139def install_via_gateway(c):
140    '''Perform the actual installation on local machine or prepare data
141    transfer to remote gate and submit a job script which will
142    install everything on the remote gate.
143
144    Parameters
145    ----------
146    c : :obj:`ControlFile`
147        Contains all the parameters of CONTROL file and
148        command line.
149
150    Return
151    ------
152
153    '''
154    import tarfile
155
156    ecd = _config.PATH_FLEXEXTRACT_DIR
157    tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
158    tar_file = os.path.join(ecd, tarball_name)
159
160    target_dirname = _config.FLEXEXTRACT_DIRNAME
161    fortran_executable = _config.FORTRAN_EXECUTABLE
162
163    if c.install_target.lower() != 'local': # ecgate or cca
164
165        mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
166                      c.flexpartdir)
167
168        mk_job_template(c.ecuid, c.ecgid, c.gateway,
169                        c.destination, c.flexpartdir)
170
171        mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
172
173        mk_tarball(tar_file, c.install_target)
174
175        put_file_to_ecserver(ecd, tarball_name, c.install_target,
176                             c.ecuid, c.ecgid)
177
178        submit_job_to_ecserver(c.install_target,
179                               os.path.join(_config.PATH_REL_JOBSCRIPTS,
180                                            _config.FILE_INSTALL_COMPILEJOB))
181
182        silent_remove(tar_file)
183
184        print('job compilation script has been submitted to ecgate for ' +
185              'installation in ' + c.flexpartdir +
186               '/' + target_dirname)
187        print('You should get an email with subject "flexcompile" within ' +
188              'the next few minutes!')
189
190    else: #local
191        if c.flexpartdir == _config.PATH_FLEXEXTRACT_DIR :
192            print('WARNING: FLEXPARTDIR has not been specified')
193            print('flex_extract will be installed in here by compiling the ' +
194                  'Fortran source in ' + _config.PATH_FORTRAN_SRC)
195            os.chdir(_config.PATH_FORTRAN_SRC)
196        else: # creates the target working directory for flex_extract
197            c.flexpartdir = os.path.expandvars(os.path.expanduser(
198                                        c.flexpartdir))
199            if os.path.abspath(ecd) != os.path.abspath(c.flexpartdir):
200                mk_tarball(tar_file, c.install_target)
201                make_dir(os.path.join(c.flexpartdir,
202                                      target_dirname))
203                os.chdir(os.path.join(c.flexpartdir,
204                                      target_dirname))
205                un_tarball(tar_file)
206                os.chdir(os.path.join(c.flexpartdir,
207                                      target_dirname,
208                                      _config.PATH_REL_FORTRAN_SRC))
209
210        # Create Fortran executable - CONVERT2
211        print('Install ' + target_dirname + ' software at ' +
212              c.install_target + ' in directory ' +
213              os.path.abspath(c.flexpartdir) + '\n')
214
215        del_convert_build('.')
216        mk_convert_build('.', c.makefile)
217
218        os.chdir(ecd)
219        if os.path.isfile(tar_file):
220            os.remove(tar_file)
221
222    return
223
224def check_install_conditions(c):
225    '''Checks a couple of necessary attributes and conditions
226    for the installation such as if they exist and contain values.
227    Otherwise set default values.
228
229    Parameters
230    ----------
231    c : :obj:`ControlFile`
232        Contains all the parameters of CONTROL file and
233        command line.
234
235
236    Return
237    ------
238
239    '''
240
241    if c.install_target and \
242       c.install_target not in _config.INSTALL_TARGETS:
243        print('ERROR: unknown or missing installation target ')
244        print('target: ', c.install_target)
245        print('please specify correct installation target ' +
246              str(INSTALL_TARGETS))
247        print('use -h or --help for help')
248        sys.exit(1)
249
250    if c.install_target and c.install_target != 'local':
251        if not c.ecgid or not c.ecuid or \
252           not c.gateway or not c.destination:
253            print('Please enter your ECMWF user id and group id as well ' +
254                  'as the \nname of the local gateway and the ectrans ' +
255                  'destination ')
256            print('with command line options --ecuid --ecgid \
257                   --gateway --destination')
258            print('Try "' + sys.argv[0].split('/')[-1] + \
259                  ' -h" to print usage information')
260            print('Please consult ecaccess documentation or ECMWF user \
261                   support for further details')
262            sys.exit(1)
263
264        if not c.flexpartdir:
265            c.flexpartdir = '${HOME}'
266        else:
267            c.flexpartdir = c.flexpartdir
268    else: # local
269        if not c.flexpartdir:
270            c.flexpartdir = _config.PATH_FLEXEXTRACT_DIR
271
272    return
273
274
275def mk_tarball(tarball_path, target):
276    '''Creates a tarball with all necessary files which need to be sent to the
277    installation directory.
278    It does not matter if this is local or remote.
279    Collects all python files, the Fortran source and makefiles,
280    the ECMWF_ENV file, the CONTROL files as well as the
281    template files.
282
283    Parameters
284    ----------
285    tarball_path : :obj:`string`
286        The complete path to the tar file which will contain all
287        relevant data for flex_extract.
288
289    target : :obj:`string`
290        The queue where the job is submitted to.
291
292    Return
293    ------
294
295    '''
296    from glob import glob
297
298    print('Create tarball ...')
299
300    # change to FLEXEXTRACT directory so that the tar can contain
301    # relative pathes to the files and directories
302    ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
303    os.chdir(ecd)
304
305    # get lists of the files to be added to the tar file
306    if target == 'local':
307        ECMWF_ENV_FILE = []
308        runfile = [os.path.relpath(x, ecd)
309                   for x in UioFiles(_config.PATH_REL_RUN_DIR,
310                                     'run_local.sh').files]
311    else:
312        ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
313        runfile = [os.path.relpath(x, ecd)
314                       for x in UioFiles(_config.PATH_REL_RUN_DIR,
315                                         'run.sh').files]
316
317    pyfiles = [os.path.relpath(x, ecd)
318               for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
319    pytestfiles = [os.path.relpath(x, ecd)
320               for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
321    controlfiles = [os.path.relpath(x, ecd)
322                    for x in UioFiles(_config.PATH_REL_CONTROLFILES,
323                                      'CONTROL*').files]
324    tempfiles = [os.path.relpath(x, ecd)
325                 for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.temp').files]
326    nlfiles = [os.path.relpath(x, ecd)
327                 for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.nl').files]
328    gribtable = [os.path.relpath(x, ecd)
329                 for x in UioFiles(_config.PATH_REL_TEMPLATES , '*grib*').files]
330    ffiles = [os.path.relpath(x, ecd)
331              for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f*').files]
332    hfiles = [os.path.relpath(x, ecd)
333              for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
334    makefiles = [os.path.relpath(x, ecd)
335                 for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'Makefile*').files]
336    jobdir = [_config.PATH_REL_JOBSCRIPTS]
337
338    # concatenate single lists to one for a better looping
339    filelist = pyfiles + pytestfiles + controlfiles + tempfiles + nlfiles + \
340               ffiles + gribtable + hfiles + makefiles + ECMWF_ENV_FILE + \
341               runfile + jobdir + \
342               ['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
343
344    # create installation tar-file
345    exclude_files = [".ksh"]
346    try:
347        with tarfile.open(tarball_path, "w:gz") as tar_handle:
348            for file in filelist:
349                tar_handle.add(file, recursive=False,
350                               filter=lambda tarinfo: None
351                                      if os.path.splitext(tarinfo.name)[1]
352                                         in exclude_files
353                                      else tarinfo)
354    except subprocess.CalledProcessError as e:
355        print('... ERROR CODE:\n ... ' + str(e.returncode))
356        print('... ERROR MESSAGE:\n ... ' + str(e))
357
358        sys.exit('\n... could not make installation tar ball!')
359    except OSError as e:
360        print('... ERROR CODE: ' + str(e.errno))
361        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
362
363        sys.exit('\n... error occured while trying to read tar-file ' +
364                 str(tarball_path))
365
366    return
367
368
369def un_tarball(tarball_path):
370    '''Extracts the given tarball into current directory.
371
372    Parameters
373    ----------
374    tarball_path : :obj:`string`
375        The complete path to the tar file which will contain all
376        relevant data for flex_extract.
377
378    Return
379    ------
380
381    '''
382
383    print('Untar ...')
384
385    try:
386        with tarfile.open(tarball_path) as tar_handle:
387            tar_handle.extractall()
388    except tarfile.TarError as e:
389        sys.exit('\n... error occured while trying to read tar-file ' +
390                     str(tarball_path))
391    except OSError as e:
392        print('... ERROR CODE: ' + str(e.errno))
393        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
394
395        sys.exit('\n... error occured while trying to read tar-file ' +
396                 str(tarball_path))
397
398    return
399
400def mk_env_vars(ecuid, ecgid, gateway, destination):
401    '''Creates a file named ECMWF_ENV which contains the
402    necessary environmental variables at ECMWF servers.
403    It is based on the template ECMWF_ENV.template.
404
405    Parameters
406    ----------
407    ecuid : :obj:`string`
408        The user id on ECMWF server.
409
410    ecgid : :obj:`string`
411        The group id on ECMWF server.
412
413    gateway : :obj:`string`
414        The gateway server the user is using.
415
416    destination : :obj:`string`
417        The remote destination which is used to transfer files
418        from ECMWF server to local gateway server.
419
420    Return
421    ------
422
423    '''
424    from genshi.template.text import NewTextTemplate
425    from genshi.template import  TemplateLoader
426    from genshi.template.eval import UndefinedError
427
428    try:
429        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
430        ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
431                                         cls=NewTextTemplate)
432
433        stream = ecmwfvars_template.generate(user_name = ecuid,
434                                             user_group = ecgid,
435                                             gateway_name = gateway,
436                                             destination_name = destination
437                                             )
438    except UndefinedError as e:
439        print('... ERROR ' + str(e))
440
441        sys.exit('\n... error occured while trying to generate template ' +
442                 _config.PATH_ECMWF_ENV)
443    except OSError as e:
444        print('... ERROR CODE: ' + str(e.errno))
445        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
446
447        sys.exit('\n... error occured while trying to generate template ' +
448                 _config.PATH_ECMWF_ENV)
449
450    try:
451        with open(_config.PATH_ECMWF_ENV, 'w') as f:
452            f.write(stream.render('text'))
453    except OSError as e:
454        print('... ERROR CODE: ' + str(e.errno))
455        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
456
457        sys.exit('\n... error occured while trying to write ' +
458                 _config.PATH_ECMWF_ENV)
459
460    return
461
462def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
463    '''Modifies the original job template file so that it is specified
464    for the user and the environment were it will be applied. Result
465    is stored in a new file "job.temp" in the python directory.
466
467    Parameters
468    ----------
469    makefile : :obj:`string`
470        Name of the makefile which should be used to compile FORTRAN
471        CONVERT2 program.
472
473    target : :obj:`string`
474        The target where the installation should be done, e.g. the queue.
475
476    ecuid : :obj:`string`
477        The user id on ECMWF server.
478
479    ecgid : :obj:`string`
480        The group id on ECMWF server.
481
482    fp_root : :obj:`string`
483       Path to the root directory of FLEXPART environment or flex_extract
484       environment.
485
486    Return
487    ------
488
489    '''
490    from genshi.template.text import NewTextTemplate
491    from genshi.template import  TemplateLoader
492    from genshi.template.eval import  UndefinedError
493
494    if fp_root == '../':
495        fp_root = '$HOME'
496
497    try:
498        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
499        compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
500                                       cls=NewTextTemplate)
501
502        stream = compile_template.generate(
503            usergroup = ecgid,
504            username = ecuid,
505            version_number = _config._VERSION_STR,
506            fp_root_scripts = fp_root,
507            makefile = makefile,
508            fortran_program = _config.FORTRAN_EXECUTABLE
509        )
510    except UndefinedError as e:
511        print('... ERROR ' + str(e))
512
513        sys.exit('\n... error occured while trying to generate template ' +
514                 _config.TEMPFILE_INSTALL_COMPILEJOB)
515    except OSError as e:
516        print('... ERROR CODE: ' + str(e.errno))
517        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
518
519        sys.exit('\n... error occured while trying to generate template ' +
520                 _config.TEMPFILE_INSTALL_COMPILEJOB)
521
522    try:
523        compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
524                                  _config.FILE_INSTALL_COMPILEJOB)
525
526        with open(compilejob, 'w') as f:
527            f.write(stream.render('text'))
528    except OSError as e:
529        print('... ERROR CODE: ' + str(e.errno))
530        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
531
532        sys.exit('\n... error occured while trying to write ' +
533                 compilejob)
534
535    return
536
537def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
538    '''Modifies the original job template file so that it is specified
539    for the user and the environment were it will be applied. Result
540    is stored in a new file.
541
542    Parameters
543    ----------
544    ecuid : :obj:`string`
545        The user id on ECMWF server.
546
547    ecgid : :obj:`string`
548        The group id on ECMWF server.
549
550    gateway : :obj:`string`
551        The gateway server the user is using.
552
553    destination : :obj:`string`
554        The remote destination which is used to transfer files
555        from ECMWF server to local gateway server.
556
557    fp_root : :obj:`string`
558       Path to the root directory of FLEXPART environment or flex_extract
559       environment.
560
561    Return
562    ------
563
564    '''
565    from genshi.template.text import NewTextTemplate
566    from genshi.template import  TemplateLoader
567    from genshi.template.eval import  UndefinedError
568
569    fp_root_path_to_python = os.path.join(fp_root,
570                                          _config.FLEXEXTRACT_DIRNAME,
571                                          _config.PATH_REL_PYTHON_SRC)
572    if '$' in fp_root_path_to_python:
573        ind = fp_root_path_to_python.index('$')
574        fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
575                                 fp_root_path_to_python[ind:]
576
577    try:
578        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
579        compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
580                                       cls=NewTextTemplate)
581
582        stream = compile_template.generate(
583            usergroup = ecgid,
584            username = ecuid,
585            version_number = _config._VERSION_STR,
586            fp_root_path = fp_root_path_to_python,
587        )
588    except UndefinedError as e:
589        print('... ERROR ' + str(e))
590
591        sys.exit('\n... error occured while trying to generate template ' +
592                 _config.TEMPFILE_INSTALL_JOB)
593    except OSError as e:
594        print('... ERROR CODE: ' + str(e.errno))
595        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
596
597        sys.exit('\n... error occured while trying to generate template ' +
598                 _config.TEMPFILE_INSTALL_JOB)
599
600
601    try:
602        tempjobfile = os.path.join(_config.PATH_TEMPLATES,
603                                   _config.TEMPFILE_JOB)
604
605        with open(tempjobfile, 'w') as f:
606            f.write(stream.render('text'))
607    except OSError as e:
608        print('... ERROR CODE: ' + str(e.errno))
609        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
610
611        sys.exit('\n... error occured while trying to write ' +
612                 tempjobfile)
613
614    return
615
616def del_convert_build(src_path):
617    '''Clean up the Fortran source directory and remove all
618    build files (e.g. \*.o, \*.mod and CONVERT2)
619
620    Parameters
621    ----------
622    src_path : :obj:`string`
623        Path to the fortran source directory.
624
625    Return
626    ------
627
628    '''
629
630    modfiles = UioFiles(src_path, '*.mod')
631    objfiles = UioFiles(src_path, '*.o')
632    exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
633
634    modfiles.delete_files()
635    objfiles.delete_files()
636    exefile.delete_files()
637
638    return
639
640def mk_convert_build(src_path, makefile):
641    '''Compiles the Fortran code and generates the executable.
642
643    Parameters
644    ----------
645    src_path : :obj:`string`
646        Path to the fortran source directory.
647
648    makefile : :obj:`string`
649        The name of the makefile which should be used.
650
651    Return
652    ------
653
654    '''
655
656    try:
657        print('Using makefile: ' + makefile)
658        p = subprocess.Popen(['make', '-f',
659                              os.path.join(src_path, makefile)],
660                             stdin=subprocess.PIPE,
661                             stdout=subprocess.PIPE,
662                             stderr=subprocess.PIPE,
663                             bufsize=1)
664        pout, perr = p.communicate()
665        print(pout)
666        if p.returncode != 0:
667            print(perr)
668            print('Please edit ' + makefile +
669                  ' or try another Makefile in the src directory.')
670            print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
671                  'and EMOSLIB must be adapted.')
672            print('Available Makefiles:')
673            print(UioFiles(src_path, 'Makefile*'))
674            sys.exit('Compilation failed!')
675    except ValueError as e:
676        print('ERROR: Makefile call failed:')
677        print(e)
678    else:
679        subprocess.check_call(['ls', '-l',
680                               os.path.join(src_path,
681                                            _config.FORTRAN_EXECUTABLE)])
682
683    return
684
685
686if __name__ == "__main__":
687    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG