source: flex_extract.git/source/python/install.py @ 79729d5

dev
Last change on this file since 79729d5 was 79729d5, checked in by Anne Philipp <anne.philipp@…>, 9 months ago

switched from python2 to python3

  • Property mode set to 100755
File size: 23.2 KB
Line 
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Leopold Haimberger (University of Vienna)
5#
6# @Date: November 2015
7#
8# @Change History:
9#
10#    February 2018 - Anne Philipp (University of Vienna):
11#        - applied PEP8 style guide
12#        - added documentation
13#        - moved install_args_and_control in here
14#        - splitted code in smaller functions
15#        - delete convert build files in here instead of compile job script
16#        - changed static path names to variables from config file
17#        - splitted install function into several smaller pieces
18#        - use of tarfile package in python
19#
20# @License:
21#    (C) Copyright 2014-2019.
22#    Anne Philipp, Leopold Haimberger
23#
24#    This work is licensed under the Creative Commons Attribution 4.0
25#    International License. To view a copy of this license, visit
26#    http://creativecommons.org/licenses/by/4.0/ or send a letter to
27#    Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
28#
29# @Methods:
30#    main
31#    get_install_cmdline_args
32#    install_via_gateway
33#    check_install_conditions
34#    mk_tarball
35#    un_tarball
36#    mk_env_vars
37#    mk_compilejob
38#    mk_job_template
39#    del_convert_build
40#    mk_convert_build
41#
42#*******************************************************************************
43'''This script installs the flex_extract program.
44
45Depending on the selected installation environment (locally or on the
46ECMWF server ecgate or cca) the program extracts the commandline
47arguments and the CONTROL file parameter and prepares the corresponding
48environment.
49The necessary files are collected in a tar-ball and placed
50at the target location. There its untared, the environment variables will
51be set and the Fortran code will be compiled.
52If the ECMWF environment is selected a job script is prepared and submitted
53for the remaining configurations after putting the tar-ball to the
54target ECMWF server.
55
56Type: install.py --help
57to get information about command line parameters.
58Read the documentation for usage instructions.
59'''
60
61# ------------------------------------------------------------------------------
62# MODULES
63# ------------------------------------------------------------------------------
64import os
65import sys
66import glob
67import subprocess
68import inspect
69import tarfile
70from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
71
72# software specific classes and modules from flex_extract
73import _config
74from classes.ControlFile import ControlFile
75from classes.UioFiles import UioFiles
76from mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
77                        silent_remove, execute_subprocess)
78
79# ------------------------------------------------------------------------------
80# FUNCTIONS
81# ------------------------------------------------------------------------------
82def main():
83    '''Controls the installation process. Calls the installation function
84    if target is specified.
85
86    Parameters
87    ----------
88
89    Return
90    ------
91    '''
92
93    args = get_install_cmdline_args()
94    c = ControlFile(args.controlfile)
95    c.assign_args_to_control(args)
96    check_install_conditions(c)
97
98    install_via_gateway(c)
99
100    return
101
102def get_install_cmdline_args():
103    '''Decomposes the command line arguments and assigns them to variables.
104    Apply default values for non mentioned arguments.
105
106    Parameters
107    ----------
108
109    Return
110    ------
111    args : Namespace
112        Contains the commandline arguments from script/program call.
113    '''
114    parser = ArgumentParser(description='Install flex_extract software locally or \
115                            on ECMWF machines',
116                            formatter_class=ArgumentDefaultsHelpFormatter)
117
118    parser.add_argument('--target', dest='install_target', default=None,
119                        help="Valid targets: local | ecgate | cca , \
120                        the latter two are at ECMWF")
121    parser.add_argument("--makefile", dest="makefile", default=None,
122                        help='Name of Makefile to use for compiling CONVERT2')
123    parser.add_argument("--ecuid", dest="ecuid", default=None,
124                        help='user id at ECMWF')
125    parser.add_argument("--ecgid", dest="ecgid", default=None,
126                        help='group id at ECMWF')
127    parser.add_argument("--gateway", dest="gateway", default=None,
128                        help='name of local gateway server')
129    parser.add_argument("--destination", dest="destination", default=None,
130                        help='ecaccess destination, e.g. leo@genericSftp')
131
132    parser.add_argument("--flexpartdir", dest="flexpartdir",
133                        default=None, help="FLEXPART root directory on ECMWF \
134                        servers (to find grib2flexpart and COMMAND file)\n\
135                        Normally flex_extract resides in the scripts directory \
136                        of the FLEXPART distribution.")
137
138    # arguments for job submission to ECMWF, only needed by submit.py
139    parser.add_argument("--job_template", dest='job_template',
140                        default="job.temp.o",
141                        help="job template file for submission to ECMWF")
142
143    parser.add_argument("--controlfile", dest="controlfile",
144                        default='CONTROL.temp',
145                        help="file with CONTROL parameters")
146
147    args = parser.parse_args()
148
149    return args
150
151
152def install_via_gateway(c):
153    '''Perform the actual installation on local machine or prepare data
154    transfer to remote gate and submit a job script which will
155    install everything on the remote gate.
156
157    Parameters
158    ----------
159    c : ControlFile
160        Contains all the parameters of CONTROL file and
161        command line.
162
163    Return
164    ------
165
166    '''
167    import tarfile
168
169    ecd = _config.PATH_FLEXEXTRACT_DIR
170    tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
171    tar_file = os.path.join(ecd, tarball_name)
172
173    target_dirname = _config.FLEXEXTRACT_DIRNAME
174    fortran_executable = _config.FORTRAN_EXECUTABLE
175
176    if c.install_target.lower() != 'local': # ecgate or cca
177
178        mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
179                      c.flexpartdir)
180
181        mk_job_template(c.ecuid, c.ecgid, c.gateway,
182                        c.destination, c.flexpartdir)
183
184        mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
185
186        mk_tarball(tar_file, c.install_target)
187
188        put_file_to_ecserver(ecd, tarball_name, c.install_target,
189                             c.ecuid, c.ecgid)
190
191        submit_job_to_ecserver(c.install_target,
192                               os.path.join(_config.PATH_REL_JOBSCRIPTS,
193                                            _config.FILE_INSTALL_COMPILEJOB))
194
195        silent_remove(tar_file)
196
197        print('job compilation script has been submitted to ecgate for ' +
198              'installation in ' + c.flexpartdir +
199               '/' + target_dirname)
200        print('You should get an email with subject "flexcompile" within ' +
201              'the next few minutes!')
202
203    else: #local
204        if c.flexpartdir == _config.PATH_FLEXEXTRACT_DIR :
205            print('WARNING: FLEXPARTDIR has not been specified')
206            print('flex_extract will be installed in here by compiling the ' +
207                  'Fortran source in ' + _config.PATH_FORTRAN_SRC)
208            os.chdir(_config.PATH_FORTRAN_SRC)
209        else: # creates the target working directory for flex_extract
210            c.flexpartdir = os.path.expandvars(os.path.expanduser(
211                                        c.flexpartdir))
212            if os.path.abspath(ecd) != os.path.abspath(c.flexpartdir):
213                mk_tarball(tar_file, c.install_target)
214                make_dir(os.path.join(c.flexpartdir,
215                                      target_dirname))
216                os.chdir(os.path.join(c.flexpartdir,
217                                      target_dirname))
218                un_tarball(tar_file)
219                os.chdir(os.path.join(c.flexpartdir,
220                                      target_dirname,
221                                      _config.PATH_REL_FORTRAN_SRC))
222
223        # Create Fortran executable - CONVERT2
224        print('Install ' + target_dirname + ' software at ' +
225              c.install_target + ' in directory ' +
226              os.path.abspath(c.flexpartdir) + '\n')
227
228        del_convert_build('.')
229        mk_convert_build('.', c.makefile)
230
231        os.chdir(ecd)
232        if os.path.isfile(tar_file):
233            os.remove(tar_file)
234
235    return
236
237def check_install_conditions(c):
238    '''Checks a couple of necessary attributes and conditions
239    for the installation such as if they exist and contain values.
240    Otherwise set default values.
241
242    Parameters
243    ----------
244    c : ControlFile
245        Contains all the parameters of CONTROL file and
246        command line.
247
248
249    Return
250    ------
251
252    '''
253
254    if c.install_target and \
255       c.install_target not in _config.INSTALL_TARGETS:
256        print('ERROR: unknown or missing installation target ')
257        print('target: ', c.install_target)
258        print('please specify correct installation target ' +
259              str(INSTALL_TARGETS))
260        print('use -h or --help for help')
261        sys.exit(1)
262
263    if c.install_target and c.install_target != 'local':
264        if not c.ecgid or not c.ecuid or \
265           not c.gateway or not c.destination:
266            print('Please enter your ECMWF user id and group id as well ' +
267                  'as the \nname of the local gateway and the ectrans ' +
268                  'destination ')
269            print('with command line options --ecuid --ecgid \
270                   --gateway --destination')
271            print('Try "' + sys.argv[0].split('/')[-1] + \
272                  ' -h" to print usage information')
273            print('Please consult ecaccess documentation or ECMWF user \
274                   support for further details')
275            sys.exit(1)
276
277        if not c.flexpartdir:
278            c.flexpartdir = '${HOME}'
279        else:
280            c.flexpartdir = c.flexpartdir
281    else: # local
282        if not c.flexpartdir:
283            c.flexpartdir = _config.PATH_FLEXEXTRACT_DIR
284
285    return
286
287
288def mk_tarball(tarball_path, target):
289    '''Creates a tarball with all necessary files which need to be sent to the
290    installation directory.
291    It does not matter if this is local or remote.
292    Collects all python files, the Fortran source and makefiles,
293    the ECMWF_ENV file, the CONTROL files as well as the
294    template files.
295
296    Parameters
297    ----------
298    tarball_path : str
299        The complete path to the tar file which will contain all
300        relevant data for flex_extract.
301
302    target : str
303        The queue where the job is submitted to.
304
305    Return
306    ------
307
308    '''
309    from glob import glob
310
311    print('Create tarball ...')
312
313    # change to FLEXEXTRACT directory so that the tar can contain
314    # relative pathes to the files and directories
315    ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
316    os.chdir(ecd)
317
318    # get lists of the files to be added to the tar file
319    if target == 'local':
320        ECMWF_ENV_FILE = []
321        runfile = [os.path.relpath(x, ecd)
322                   for x in UioFiles(_config.PATH_REL_RUN_DIR,
323                                     'run_local.sh').files]
324    else:
325        ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
326        runfile = [os.path.relpath(x, ecd)
327                       for x in UioFiles(_config.PATH_REL_RUN_DIR,
328                                         'run.sh').files]
329
330    pyfiles = [os.path.relpath(x, ecd)
331               for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
332    pytestfiles = [os.path.relpath(x, ecd)
333               for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
334    controlfiles = [os.path.relpath(x, ecd)
335                    for x in UioFiles(_config.PATH_REL_CONTROLFILES,
336                                      'CONTROL*').files]
337    testfiles = [os.path.relpath(x, ecd)
338                 for x in UioFiles(_config.PATH_REL_TEST , '*').files]
339    tempfiles = [os.path.relpath(x, ecd)
340                 for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.temp').files]
341    nlfiles = [os.path.relpath(x, ecd)
342                 for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.nl').files]
343    gribtable = [os.path.relpath(x, ecd)
344                 for x in UioFiles(_config.PATH_REL_TEMPLATES , '*grib*').files]
345    ffiles = [os.path.relpath(x, ecd)
346              for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f*').files]
347    hfiles = [os.path.relpath(x, ecd)
348              for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
349    makefiles = [os.path.relpath(x, ecd)
350                 for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'Makefile*').files]
351    jobdir = [_config.PATH_REL_JOBSCRIPTS]
352
353    # concatenate single lists to one for a better looping
354    filelist = pyfiles + pytestfiles + controlfiles + tempfiles + nlfiles + \
355               ffiles + gribtable + hfiles + makefiles + ECMWF_ENV_FILE + \
356               runfile + jobdir + testfiles +\
357               ['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
358
359    # create installation tar-file
360    exclude_files = [".ksh"]
361    try:
362        with tarfile.open(tarball_path, "w:gz") as tar_handle:
363            for file in filelist:
364                tar_handle.add(file, recursive=False,
365                               filter=lambda tarinfo: None
366                                      if os.path.splitext(tarinfo.name)[1]
367                                         in exclude_files
368                                      else tarinfo)
369    except tarfile.TarError as e:
370        sys.exit('\n... error occured while trying to create the tar-file ' +
371                     str(tarball_path))
372
373    return
374
375
376def un_tarball(tarball_path):
377    '''Extracts the given tarball into current directory.
378
379    Parameters
380    ----------
381    tarball_path : str
382        The complete path to the tar file which will contain all
383        relevant data for flex_extract.
384
385    Return
386    ------
387
388    '''
389
390    print('Untar ...')
391
392    try:
393        with tarfile.open(tarball_path) as tar_handle:
394            tar_handle.extractall()
395    except tarfile.TarError as e:
396        sys.exit('\n... error occured while trying to read tar-file ' +
397                     str(tarball_path))
398    except OSError as e:
399        print('... ERROR CODE: ' + str(e.errno))
400        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
401
402        sys.exit('\n... error occured while trying to read tar-file ' +
403                 str(tarball_path))
404
405    return
406
407def mk_env_vars(ecuid, ecgid, gateway, destination):
408    '''Creates a file named ECMWF_ENV which contains the
409    necessary environmental variables at ECMWF servers.
410    It is based on the template ECMWF_ENV.template.
411
412    Parameters
413    ----------
414    ecuid : str
415        The user id on ECMWF server.
416
417    ecgid : str
418        The group id on ECMWF server.
419
420    gateway : str
421        The gateway server the user is using.
422
423    destination : str
424        The remote destination which is used to transfer files
425        from ECMWF server to local gateway server.
426
427    Return
428    ------
429
430    '''
431    from genshi.template.text import NewTextTemplate
432    from genshi.template import  TemplateLoader
433    from genshi.template.eval import UndefinedError
434
435    try:
436        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
437        ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
438                                         cls=NewTextTemplate)
439
440        stream = ecmwfvars_template.generate(user_name = ecuid,
441                                             user_group = ecgid,
442                                             gateway_name = gateway,
443                                             destination_name = destination
444                                             )
445    except UndefinedError as e:
446        print('... ERROR ' + str(e))
447
448        sys.exit('\n... error occured while trying to generate template ' +
449                 _config.PATH_ECMWF_ENV)
450    except OSError as e:
451        print('... ERROR CODE: ' + str(e.errno))
452        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
453
454        sys.exit('\n... error occured while trying to generate template ' +
455                 _config.PATH_ECMWF_ENV)
456
457    try:
458        with open(_config.PATH_ECMWF_ENV, 'w') as f:
459            f.write(stream.render('text'))
460    except OSError as e:
461        print('... ERROR CODE: ' + str(e.errno))
462        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
463
464        sys.exit('\n... error occured while trying to write ' +
465                 _config.PATH_ECMWF_ENV)
466
467    return
468
469def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
470    '''Modifies the original job template file so that it is specified
471    for the user and the environment were it will be applied. Result
472    is stored in a new file "job.temp" in the python directory.
473
474    Parameters
475    ----------
476    makefile : str
477        Name of the makefile which should be used to compile FORTRAN
478        CONVERT2 program.
479
480    target : str
481        The target where the installation should be done, e.g. the queue.
482
483    ecuid : str
484        The user id on ECMWF server.
485
486    ecgid : str
487        The group id on ECMWF server.
488
489    fp_root : str
490       Path to the root directory of FLEXPART environment or flex_extract
491       environment.
492
493    Return
494    ------
495
496    '''
497    from genshi.template.text import NewTextTemplate
498    from genshi.template import  TemplateLoader
499    from genshi.template.eval import  UndefinedError
500
501    if fp_root == '../':
502        fp_root = '$HOME'
503
504    try:
505        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
506        compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
507                                       cls=NewTextTemplate)
508
509        stream = compile_template.generate(
510            usergroup = ecgid,
511            username = ecuid,
512            version_number = _config._VERSION_STR,
513            fp_root_scripts = fp_root,
514            makefile = makefile,
515            fortran_program = _config.FORTRAN_EXECUTABLE
516        )
517    except UndefinedError as e:
518        print('... ERROR ' + str(e))
519
520        sys.exit('\n... error occured while trying to generate template ' +
521                 _config.TEMPFILE_INSTALL_COMPILEJOB)
522    except OSError as e:
523        print('... ERROR CODE: ' + str(e.errno))
524        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
525
526        sys.exit('\n... error occured while trying to generate template ' +
527                 _config.TEMPFILE_INSTALL_COMPILEJOB)
528
529    try:
530        compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
531                                  _config.FILE_INSTALL_COMPILEJOB)
532
533        with open(compilejob, 'w') as f:
534            f.write(stream.render('text'))
535    except OSError as e:
536        print('... ERROR CODE: ' + str(e.errno))
537        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
538
539        sys.exit('\n... error occured while trying to write ' +
540                 compilejob)
541
542    return
543
544def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
545    '''Modifies the original job template file so that it is specified
546    for the user and the environment were it will be applied. Result
547    is stored in a new file.
548
549    Parameters
550    ----------
551    ecuid : str
552        The user id on ECMWF server.
553
554    ecgid : str
555        The group id on ECMWF server.
556
557    gateway : str
558        The gateway server the user is using.
559
560    destination : str
561        The remote destination which is used to transfer files
562        from ECMWF server to local gateway server.
563
564    fp_root : str
565       Path to the root directory of FLEXPART environment or flex_extract
566       environment.
567
568    Return
569    ------
570
571    '''
572    from genshi.template.text import NewTextTemplate
573    from genshi.template import  TemplateLoader
574    from genshi.template.eval import  UndefinedError
575
576    fp_root_path_to_python = os.path.join(fp_root,
577                                          _config.FLEXEXTRACT_DIRNAME,
578                                          _config.PATH_REL_PYTHON_SRC)
579    if '$' in fp_root_path_to_python:
580        ind = fp_root_path_to_python.index('$')
581        fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \
582                                 fp_root_path_to_python[ind:]
583
584    try:
585        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
586        compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
587                                       cls=NewTextTemplate)
588
589        stream = compile_template.generate(
590            usergroup = ecgid,
591            username = ecuid,
592            version_number = _config._VERSION_STR,
593            fp_root_path = fp_root_path_to_python,
594        )
595    except UndefinedError as e:
596        print('... ERROR ' + str(e))
597
598        sys.exit('\n... error occured while trying to generate template ' +
599                 _config.TEMPFILE_INSTALL_JOB)
600    except OSError as e:
601        print('... ERROR CODE: ' + str(e.errno))
602        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
603
604        sys.exit('\n... error occured while trying to generate template ' +
605                 _config.TEMPFILE_INSTALL_JOB)
606
607
608    try:
609        tempjobfile = os.path.join(_config.PATH_TEMPLATES,
610                                   _config.TEMPFILE_JOB)
611
612        with open(tempjobfile, 'w') as f:
613            f.write(stream.render('text'))
614    except OSError as e:
615        print('... ERROR CODE: ' + str(e.errno))
616        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
617
618        sys.exit('\n... error occured while trying to write ' +
619                 tempjobfile)
620
621    return
622
623def del_convert_build(src_path):
624    '''Clean up the Fortran source directory and remove all
625    build files (e.g. \*.o, \*.mod and CONVERT2)
626
627    Parameters
628    ----------
629    src_path : str
630        Path to the fortran source directory.
631
632    Return
633    ------
634
635    '''
636
637    modfiles = UioFiles(src_path, '*.mod')
638    objfiles = UioFiles(src_path, '*.o')
639    exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
640
641    modfiles.delete_files()
642    objfiles.delete_files()
643    exefile.delete_files()
644
645    return
646
647def mk_convert_build(src_path, makefile):
648    '''Compiles the Fortran code and generates the executable.
649
650    Parameters
651    ----------
652    src_path : str
653        Path to the fortran source directory.
654
655    makefile : str
656        The name of the makefile which should be used.
657
658    Return
659    ------
660
661    '''
662
663    try:
664        print('Using makefile: ' + makefile)
665        p = subprocess.Popen(['make', '-f',
666                              os.path.join(src_path, makefile)],
667                             stdin=subprocess.PIPE,
668                             stdout=subprocess.PIPE,
669                             stderr=subprocess.PIPE,
670                             bufsize=1)
671        pout, perr = p.communicate()
672        print(pout.decode())
673        if p.returncode != 0:
674            print(perr.decode())
675            print('Please edit ' + makefile +
676                  ' or try another Makefile in the src directory.')
677            print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
678                  'and EMOSLIB must be adapted.')
679            print('Available Makefiles:')
680            print(UioFiles(src_path, 'Makefile*'))
681            sys.exit('Compilation failed!')
682    except ValueError as e:
683        print('ERROR: Makefile call failed:')
684        print(e)
685    else:
686        execute_subprocess(['ls', '-l', os.path.join(src_path,
687                            _config.FORTRAN_EXECUTABLE)], error_msg=
688                           'FORTRAN EXECUTABLE COULD NOT BE FOUND!')
689
690    return
691
692
693if __name__ == "__main__":
694    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG