source: flex_extract.git/source/python/install.py @ 2d56c04

ctbtodev
Last change on this file since 2d56c04 was 3946de5, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

added testing for most of the install functions; improved install functions with exception handling
  • Property mode set to 100755
File size: 21.5 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Leopold Haimberger (University of Vienna)
5#
6# @Date: November 2015
7#
8# @Change History:
9#
10#    February 2018 - Anne Philipp (University of Vienna):
11#        - applied PEP8 style guide
12#        - added documentation
13#        - moved install_args_and_control in here
14#        - splitted code in smaller functions
15#        - delete convert build files in here instead of compile job script
16#        - changed static path names to Variables from config file
17#
18# @License:
19#    (C) Copyright 2015-2018.
20#
21#    This software is licensed under the terms of the Apache Licence Version 2.0
22#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
23#
24# @Program Functionality:
25#    Depending on the selected installation environment (locally or on the
26#    ECMWF server ecgate or cca) the program extracts the commandline
27#    arguments and the CONTROL file parameter and prepares the corresponding
28#    environment. The necessary files are collected in a tar-ball and placed
29#    at the target location. There its untared, the environment variables will
30#    be set and the Fortran code will be compiled. If the ECMWF environment is
31#    selected a job script is prepared and submitted for the remaining
32#    configurations after putting the tar-ball to the target ECMWF server.
33#
34# @Program Content:
35#    - main
36#    - get_install_cmdline_arguments
37#    - install_via_gateway
38#    - mk_tarball
39#    - un_tarball
40#    - mk_env_vars
41#    - mk_compilejob
42#    - mk_job_template
43#    - delete_convert_build
44#    - make_convert_build
45#
46#*******************************************************************************
47
48# ------------------------------------------------------------------------------
49# MODULES
50# ------------------------------------------------------------------------------
51import os
52import sys
53import glob
54import subprocess
55import inspect
56import tarfile
57from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
58
59# software specific classes and modules from flex_extract
60import _config
61from classes.ControlFile import ControlFile
62from classes.UioFiles import UioFiles
63from mods.tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
64
65# ------------------------------------------------------------------------------
66# FUNCTIONS
67# ------------------------------------------------------------------------------
68def main():
69    '''Controls the installation process. Calls the installation function
70    if target is specified.
71
72    Parameters
73    ----------
74
75    Return
76    ------
77    '''
78
79    args = get_install_cmdline_arguments()
80    c = ControlFile(args.controlfile)
81    c.assign_args_to_control(args)
82    c.check_install_conditions()
83
84    install_via_gateway(c)
85
86    return
87
88def get_install_cmdline_arguments():
89    '''Decomposes the command line arguments and assigns them to variables.
90    Apply default values for non mentioned arguments.
91
92    Parameters
93    ----------
94
95    Return
96    ------
97    args : :obj:`Namespace`
98        Contains the commandline arguments from script/program call.
99    '''
100    parser = ArgumentParser(description='Install flex_extract software locally or \
101                            on ECMWF machines',
102                            formatter_class=ArgumentDefaultsHelpFormatter)
103
104    parser.add_argument('--target', dest='install_target', default=None,
105                        help="Valid targets: local | ecgate | cca , \
106                        the latter two are at ECMWF")
107    parser.add_argument("--makefile", dest="makefile", default=None,
108                        help='Name of Makefile to use for compiling CONVERT2')
109    parser.add_argument("--ecuid", dest="ecuid", default=None,
110                        help='user id at ECMWF')
111    parser.add_argument("--ecgid", dest="ecgid", default=None,
112                        help='group id at ECMWF')
113    parser.add_argument("--gateway", dest="gateway", default=None,
114                        help='name of local gateway server')
115    parser.add_argument("--destination", dest="destination", default=None,
116                        help='ecaccess destination, e.g. leo@genericSftp')
117
118    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
119                        default=None, help="FLEXPART root directory on ECMWF \
120                        servers (to find grib2flexpart and COMMAND file)\n\
121                        Normally flex_extract resides in the scripts directory \
122                        of the FLEXPART distribution.")
123
124    # arguments for job submission to ECMWF, only needed by submit.py
125    parser.add_argument("--job_template", dest='job_template',
126                        default="job.temp.o",
127                        help="job template file for submission to ECMWF")
128
129    parser.add_argument("--controlfile", dest="controlfile",
130                        default='CONTROL.temp',
131                        help="file with CONTROL parameters")
132
133    args = parser.parse_args()
134
135    return args
136
137
138def install_via_gateway(c):
139    '''Perform the actual installation on local machine or prepare data
140    transfer to remote gate and submit a job script which will
141    install everything on the remote gate.
142
143    Parameters
144    ----------
145    c : :obj:`ControlFile`
146        Contains all the parameters of CONTROL file and
147        command line.
148
149    Return
150    ------
151
152    '''
153    import tarfile
154
155    ecd = _config.PATH_FLEXEXTRACT_DIR
156    tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
157    tar_file = os.path.join(ecd, tarball_name)
158
159    target_dirname = _config.FLEXEXTRACT_DIRNAME
160    fortran_executable = _config.FORTRAN_EXECUTABLE
161
162    if c.install_target.lower() != 'local': # ecgate or cca
163
164        mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
165                      c.flexpart_root_scripts)
166
167        mk_job_template(c.ecuid, c.ecgid, c.gateway,
168                        c.destination, c.flexpart_root_scripts)
169
170        mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
171
172        mk_tarball(tar_file, c.install_target)
173
174        put_file_to_ecserver(ecd, tarball_name, c.install_target,
175                             c.ecuid, c.ecgid)
176
177        submit_job_to_ecserver(c.install_target,
178                               os.path.join(_config.PATH_REL_JOBSCRIPTS,
179                                            _config.FILE_INSTALL_COMPILEJOB))
180
181        print('job compilation script has been submitted to ecgate for ' +
182              'installation in ' + c.flexpart_root_scripts +
183               '/' + target_dirname)
184        print('You should get an email with subject "flexcompile" within ' +
185              'the next few minutes!')
186
187    else: #local
188        if c.flexpart_root_scripts == _config.PATH_FLEXEXTRACT_DIR :
189            print('WARNING: FLEXPART_ROOT_SCRIPTS has not been specified')
190            print('flex_extract will be installed in here by compiling the ' +
191                  'Fortran source in ' + _config.PATH_FORTRAN_SRC)
192            os.chdir(_config.PATH_FORTRAN_SRC)
193        else: # creates the target working directory for flex_extract
194            c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
195                                        c.flexpart_root_scripts))
196            if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
197                mk_tarball(tar_file, c.install_target)
198                make_dir(os.path.join(c.flexpart_root_scripts,
199                                      target_dirname))
200                os.chdir(os.path.join(c.flexpart_root_scripts,
201                                      target_dirname))
202                un_tarball(tar_file)
203                os.chdir(os.path.join(c.flexpart_root_scripts,
204                                      target_dirname,
205                                      _config.PATH_REL_FORTRAN_SRC))
206
207        # Create Fortran executable - CONVERT2
208        print('Install ' + target_dirname + ' software at ' +
209              c.install_target + ' in directory ' +
210              os.path.abspath(c.flexpart_root_scripts) + '\n')
211
212        delete_convert_build('.')
213        make_convert_build('.', c.makefile)
214
215        os.chdir(ecd)
216        if os.path.isfile(tar_file):
217            os.remove(tar_file)
218
219    return
220
221def mk_tarball(tarball_path, target):
222    '''Creates a tarball with all necessary files which need to be sent to the
223    installation directory.
224    It does not matter if this is local or remote.
225    Collects all python files, the Fortran source and makefiles,
226    the ECMWF_ENV file, the CONTROL files as well as the
227    template files.
228
229    Parameters
230    ----------
231    tarball_path : :obj:`string`
232        The complete path to the tar file which will contain all
233        relevant data for flex_extract.
234
235    target : :obj:`string`
236        The queue where the job is submitted to.
237
238    Return
239    ------
240
241    '''
242    from glob import glob
243
244    print('Create tarball ...')
245
246    # change to FLEXEXTRACT directory so that the tar can contain
247    # relative pathes to the files and directories
248    ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
249    os.chdir(ecd)
250
251    # get lists of the files to be added to the tar file
252    if target == 'local':
253        ECMWF_ENV_FILE = []
254        runfile = [os.path.relpath(x, ecd)
255                   for x in UioFiles(_config.PATH_REL_RUN_DIR,
256                                     'run_local.sh').files]
257    else:
258        ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
259        runfile = [os.path.relpath(x, ecd)
260                       for x in UioFiles(_config.PATH_REL_RUN_DIR,
261                                         'run.sh').files]
262
263    pyfiles = [os.path.relpath(x, ecd)
264               for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
265    pytestfiles = [os.path.relpath(x, ecd)
266               for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
267    controlfiles = [os.path.relpath(x, ecd)
268                    for x in UioFiles(_config.PATH_REL_CONTROLFILES,
269                                      'CONTROL*').files]
270    tempfiles = [os.path.relpath(x, ecd)
271                 for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.temp').files]
272    nlfiles = [os.path.relpath(x, ecd)
273                 for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.nl').files]
274    gribtable = [os.path.relpath(x, ecd)
275                 for x in UioFiles(_config.PATH_REL_TEMPLATES , '*grib*').files]
276    ffiles = [os.path.relpath(x, ecd)
277              for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f*').files]
278    hfiles = [os.path.relpath(x, ecd)
279              for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
280    makefiles = [os.path.relpath(x, ecd)
281                 for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'Makefile*').files]
282    jobdir = [_config.PATH_REL_JOBSCRIPTS]
283
284    # concatenate single lists to one for a better looping
285    filelist = pyfiles + pytestfiles + controlfiles + tempfiles + nlfiles + \
286               ffiles + gribtable + hfiles + makefiles + ECMWF_ENV_FILE + \
287               runfile + jobdir + \
288               ['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
289
290    # create installation tar-file
291    exclude_files = [".ksh"]
292    try:
293        with tarfile.open(tarball_path, "w:gz") as tar_handle:
294            for file in filelist:
295                tar_handle.add(file, recursive=False,
296                               filter=lambda tarinfo: None
297                                      if os.path.splitext(tarinfo.name)[1]
298                                         in exclude_files
299                                      else tarinfo)
300    except subprocess.CalledProcessError as e:
301        print('... ERROR CODE:\n ... ' + str(e.returncode))
302        print('... ERROR MESSAGE:\n ... ' + str(e))
303
304        sys.exit('\n... could not make installation tar ball!')
305    except OSError as e:
306        print('... ERROR CODE: ' + str(e.errno))
307        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
308
309        sys.exit('\n... error occured while trying to read tar-file ' +
310                 str(tarball_path))
311
312    return
313
314
315def un_tarball(tarball_path):
316    '''Extracts the given tarball into current directory.
317
318    Parameters
319    ----------
320    tarball_path : :obj:`string`
321        The complete path to the tar file which will contain all
322        relevant data for flex_extract.
323
324    Return
325    ------
326
327    '''
328
329    print('Untar ...')
330
331    try:
332        with tarfile.open(tarball_path) as tar_handle:
333            tar_handle.extractall()
334    except tarfile.TarError as e:
335        sys.exit('\n... error occured while trying to read tar-file ' +
336                     str(tarball_path))
337    except OSError as e:
338        print('... ERROR CODE: ' + str(e.errno))
339        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
340
341        sys.exit('\n... error occured while trying to read tar-file ' +
342                 str(tarball_path))
343
344    return
345
346def mk_env_vars(ecuid, ecgid, gateway, destination):
347    '''Creates a file named ECMWF_ENV which contains the
348    necessary environmental variables at ECMWF servers.
349    It is based on the template ECMWF_ENV.template.
350
351    Parameters
352    ----------
353    ecuid : :obj:`string`
354        The user id on ECMWF server.
355
356    ecgid : :obj:`string`
357        The group id on ECMWF server.
358
359    gateway : :obj:`string`
360        The gateway server the user is using.
361
362    destination : :obj:`string`
363        The remote destination which is used to transfer files
364        from ECMWF server to local gateway server.
365
366    Return
367    ------
368
369    '''
370    from genshi.template.text import NewTextTemplate
371    from genshi.template import  TemplateLoader
372    from genshi.template.eval import UndefinedError
373
374    try:
375        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
376        ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
377                                         cls=NewTextTemplate)
378
379        stream = ecmwfvars_template.generate(user_name = ecuid,
380                                             user_group = ecgid,
381                                             gateway_name = gateway,
382                                             destination_name = destination
383                                             )
384    except UndefinedError as e:
385        print('... ERROR ' + str(e))
386
387        sys.exit('\n... error occured while trying to generate template ' +
388                 _config.PATH_ECMWF_ENV)
389    except OSError as e:
390        print('... ERROR CODE: ' + str(e.errno))
391        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
392
393        sys.exit('\n... error occured while trying to generate template ' +
394                 _config.PATH_ECMWF_ENV)
395
396    try:
397        with open(_config.PATH_ECMWF_ENV, 'w') as f:
398            f.write(stream.render('text'))
399    except OSError as e:
400        print('... ERROR CODE: ' + str(e.errno))
401        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
402
403        sys.exit('\n... error occured while trying to write ' +
404                 _config.PATH_ECMWF_ENV)
405
406    return
407
408def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
409    '''Modifies the original job template file so that it is specified
410    for the user and the environment were it will be applied. Result
411    is stored in a new file "job.temp" in the python directory.
412
413    Parameters
414    ----------
415    makefile : :obj:`string`
416        Name of the makefile which should be used to compile FORTRAN
417        CONVERT2 program.
418
419    target : :obj:`string`
420        The target where the installation should be done, e.g. the queue.
421
422    ecuid : :obj:`string`
423        The user id on ECMWF server.
424
425    ecgid : :obj:`string`
426        The group id on ECMWF server.
427
428    fp_root : :obj:`string`
429       Path to the root directory of FLEXPART environment or flex_extract
430       environment.
431
432    Return
433    ------
434
435    '''
436    from genshi.template.text import NewTextTemplate
437    from genshi.template import  TemplateLoader
438    from genshi.template.eval import  UndefinedError
439
440    if fp_root == '../':
441        fp_root = '$HOME'
442
443    try:
444        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
445        compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
446                                       cls=NewTextTemplate)
447
448        stream = compile_template.generate(
449            usergroup = ecgid,
450            username = ecuid,
451            version_number = _config._VERSION_STR,
452            fp_root_scripts = fp_root,
453            makefile = makefile,
454            fortran_program = _config.FORTRAN_EXECUTABLE
455        )
456    except UndefinedError as e:
457        print('... ERROR ' + str(e))
458
459        sys.exit('\n... error occured while trying to generate template ' +
460                 _config.TEMPFILE_INSTALL_COMPILEJOB)
461    except OSError as e:
462        print('... ERROR CODE: ' + str(e.errno))
463        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
464
465        sys.exit('\n... error occured while trying to generate template ' +
466                 _config.TEMPFILE_INSTALL_COMPILEJOB)
467
468    try:
469        compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
470                                  _config.FILE_INSTALL_COMPILEJOB)
471
472        with open(compilejob, 'w') as f:
473            f.write(stream.render('text'))
474    except OSError as e:
475        print('... ERROR CODE: ' + str(e.errno))
476        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
477
478        sys.exit('\n... error occured while trying to write ' +
479                 compilejob)
480
481    return
482
483def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
484    '''Modifies the original job template file so that it is specified
485    for the user and the environment were it will be applied. Result
486    is stored in a new file.
487
488    Parameters
489    ----------
490    ecuid : :obj:`string`
491        The user id on ECMWF server.
492
493    ecgid : :obj:`string`
494        The group id on ECMWF server.
495
496    gateway : :obj:`string`
497        The gateway server the user is using.
498
499    destination : :obj:`string`
500        The remote destination which is used to transfer files
501        from ECMWF server to local gateway server.
502
503    fp_root : :obj:`string`
504       Path to the root directory of FLEXPART environment or flex_extract
505       environment.
506
507    Return
508    ------
509
510    '''
511    from genshi.template.text import NewTextTemplate
512    from genshi.template import  TemplateLoader
513    from genshi.template.eval import  UndefinedError
514
515    fp_root_path_to_python = os.path.join(fp_root,
516                                          _config.FLEXEXTRACT_DIRNAME,
517                                          _config.PATH_REL_PYTHON_SRC)
518
519    try:
520        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
521        compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
522                                       cls=NewTextTemplate)
523
524        stream = compile_template.generate(
525            usergroup = ecgid,
526            username = ecuid,
527            version_number = _config._VERSION_STR,
528            fp_root_path = fp_root_path_to_python,
529        )
530    except UndefinedError as e:
531        print('... ERROR ' + str(e))
532
533        sys.exit('\n... error occured while trying to generate template ' +
534                 _config.TEMPFILE_INSTALL_JOB)
535    except OSError as e:
536        print('... ERROR CODE: ' + str(e.errno))
537        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
538
539        sys.exit('\n... error occured while trying to generate template ' +
540                 _config.TEMPFILE_INSTALL_JOB)
541
542
543    try:
544        tempjobfile = os.path.join(_config.PATH_TEMPLATES,
545                                   _config.TEMPFILE_JOB)
546
547        with open(tempjobfile, 'w') as f:
548            f.write(stream.render('text'))
549    except OSError as e:
550        print('... ERROR CODE: ' + str(e.errno))
551        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
552
553        sys.exit('\n... error occured while trying to write ' +
554                 tempjobfile)
555
556    return
557
558def delete_convert_build(src_path):
559    '''Clean up the Fortran source directory and remove all
560    build files (e.g. \*.o, \*.mod and CONVERT2)
561
562    Parameters
563    ----------
564    src_path : :obj:`string`
565        Path to the fortran source directory.
566
567    Return
568    ------
569
570    '''
571
572    modfiles = UioFiles(src_path, '*.mod')
573    objfiles = UioFiles(src_path, '*.o')
574    exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
575
576    modfiles.delete_files()
577    objfiles.delete_files()
578    exefile.delete_files()
579
580    return
581
582def make_convert_build(src_path, makefile):
583    '''Compiles the Fortran code and generates the executable.
584
585    Parameters
586    ----------
587    src_path : :obj:`string`
588        Path to the fortran source directory.
589
590    makefile : :obj:`string`
591        The name of the makefile which should be used.
592
593    Return
594    ------
595
596    '''
597
598    try:
599        print('Using makefile: ' + makefile)
600        p = subprocess.Popen(['make', '-f',
601                              os.path.join(src_path, makefile)],
602                             stdin=subprocess.PIPE,
603                             stdout=subprocess.PIPE,
604                             stderr=subprocess.PIPE,
605                             bufsize=1)
606        pout, perr = p.communicate()
607        print(pout)
608        if p.returncode != 0:
609            print(perr)
610            print('Please edit ' + makefile +
611                  ' or try another Makefile in the src directory.')
612            print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
613                  'and EMOSLIB must be adapted.')
614            print('Available Makefiles:')
615            print(UioFiles(src_path, 'Makefile*'))
616            sys.exit('Compilation failed!')
617    except ValueError as e:
618        print('ERROR: Makefile call failed:')
619        print(e)
620    else:
621        subprocess.check_call(['ls', '-l',
622                               os.path.join(src_path,
623                                            _config.FORTRAN_EXECUTABLE)])
624
625    return
626
627
628if __name__ == "__main__":
629    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG