source: flex_extract.git/source/python/install.py @ 27fe969

devfeature/makefilesorigin/task/language-editingtask/language-editing
Last change on this file since 27fe969 was 27fe969, checked in by Anne Philipp <anne.philipp@…>, 22 months ago

put grib2flexpart part of code into its own function; changed function documentation of input controlfile parameter

  • Property mode set to 100755
File size: 17.8 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Leopold Haimberger (University of Vienna)
5#
6# @Date: November 2015
7#
8# @Change History:
9#
10#    February 2018 - Anne Philipp (University of Vienna):
11#        - applied PEP8 style guide
12#        - added documentation
13#        - moved install_args_and_control in here
14#        - splitted code in smaller functions
15#        - delete convert build files in here instead of compile job script
16#        - changed static path names to Variables from config file
17#
18# @License:
19#    (C) Copyright 2015-2018.
20#
21#    This software is licensed under the terms of the Apache Licence Version 2.0
22#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
23#
24# @Program Functionality:
25#    Depending on the selected installation environment (locally or on the
26#    ECMWF server ecgate or cca) the program extracts the commandline
27#    arguments and the CONTROL file parameter and prepares the corresponding
28#    environment. The necessary files are collected in a tar-ball and placed
29#    at the target location. There its untared, the environment variables will
30#    be set and the Fortran code will be compiled. If the ECMWF environment is
31#    selected a job script is prepared and submitted for the remaining
32#    configurations after putting the tar-ball to the target ECMWF server.
33#
34# @Program Content:
35#    - main
36#    - get_install_cmdline_arguments
37#    - install_via_gateway
38#    - mk_tarball
39#    - un_tarball
40#    - mk_env_vars
41#    - mk_compilejob
42#    - mk_job_template
43#    - delete_convert_build
44#    - make_convert_build
45#
46#*******************************************************************************
47
48# ------------------------------------------------------------------------------
49# MODULES
50# ------------------------------------------------------------------------------
51import os
52import sys
53import glob
54import subprocess
55import inspect
56from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
57
58# software specific classes and modules from flex_extract
59import _config
60from classes.ControlFile import ControlFile
61from classes.UioFiles import UioFiles
62from mods.tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
63
64# ------------------------------------------------------------------------------
65# FUNCTIONS
66# ------------------------------------------------------------------------------
67def main():
68    '''
69    @Description:
70        Controls the installation process. Calls the installation function
71        if target is specified.
72
73    @Intput:
74        <nothing>
75
76    @Return:
77        <nothing>
78    '''
79
80    args = get_install_cmdline_arguments()
81    c = ControlFile(args.controlfile)
82    c.assign_args_to_control(args)
83    c.check_install_conditions()
84
85    install_via_gateway(c)
86
87    return
88
89def get_install_cmdline_arguments():
90    '''
91    @Description:
92        Decomposes the command line arguments and assigns them to variables.
93        Apply default values for non mentioned arguments.
94
95    @Input:
96        <nothing>
97
98    @Return:
99        args: instance of ArgumentParser
100            Contains the commandline arguments from script/program call.
101    '''
102    parser = ArgumentParser(description='Install flex_extract software locally or \
103                            on ECMWF machines',
104                            formatter_class=ArgumentDefaultsHelpFormatter)
105
106    parser.add_argument('--target', dest='install_target', default=None,
107                        help="Valid targets: local | ecgate | cca , \
108                        the latter two are at ECMWF")
109    parser.add_argument("--makefile", dest="makefile", default=None,
110                        help='Name of Makefile to use for compiling CONVERT2')
111    parser.add_argument("--ecuid", dest="ecuid", default=None,
112                        help='user id at ECMWF')
113    parser.add_argument("--ecgid", dest="ecgid", default=None,
114                        help='group id at ECMWF')
115    parser.add_argument("--gateway", dest="gateway", default=None,
116                        help='name of local gateway server')
117    parser.add_argument("--destination", dest="destination", default=None,
118                        help='ecaccess destination, e.g. leo@genericSftp')
119
120    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
121                        default=None, help="FLEXPART root directory on ECMWF \
122                        servers (to find grib2flexpart and COMMAND file)\n\
123                        Normally flex_extract resides in the scripts directory \
124                        of the FLEXPART distribution.")
125
126    # arguments for job submission to ECMWF, only needed by submit.py
127    parser.add_argument("--job_template", dest='job_template',
128                        default="job.temp.o",
129                        help="job template file for submission to ECMWF")
130
131    parser.add_argument("--controlfile", dest="controlfile",
132                        default='CONTROL.temp',
133                        help="file with CONTROL parameters")
134
135    args = parser.parse_args()
136
137    return args
138
139
140def install_via_gateway(c):
141    '''
142    @Description:
143        Perform the actual installation on local machine or prepare data
144        transfer to remote gate and submit a job script which will
145        install everything on the remote gate.
146
147    @Input:
148        c: instance of class ControlFile
149            Contains all the parameters of CONTROL file and
150            command line.
151            For more information about format and content of the parameter
152            see documentation.
153
154    @Return:
155        <nothing>
156    '''
157    import tarfile
158
159    ecd = _config.PATH_FLEXEXTRACT_DIR
160    tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
161    tar_file = os.path.join(ecd, tarball_name)
162
163    target_dirname = _config.FLEXEXTRACT_DIRNAME
164    fortran_executable = _config.FORTRAN_EXECUTABLE
165
166    if c.install_target.lower() != 'local': # ecgate or cca
167
168        mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
169                      c.flexpart_root_scripts)
170
171        mk_job_template(c.ecuid, c.ecgid, c.gateway,
172                        c.destination, c.flexpart_root_scripts)
173
174        mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
175
176        mk_tarball(tar_file, c.install_target)
177
178        put_file_to_ecserver(ecd, tarball_name, c.install_target,
179                             c.ecuid, c.ecgid)
180
181        submit_job_to_ecserver(c.install_target,
182                               os.path.join(_config.PATH_REL_JOBSCRIPTS,
183                                            _config.FILE_INSTALL_COMPILEJOB))
184
185        print('job compilation script has been submitted to ecgate for ' +
186              'installation in ' + c.flexpart_root_scripts +
187               '/' + target_dirname)
188        print('You should get an email with subject "flexcompile" within ' +
189              'the next few minutes!')
190
191    else: #local
192        if c.flexpart_root_scripts == _config.PATH_FLEXEXTRACT_DIR :
193            print('WARNING: FLEXPART_ROOT_SCRIPTS has not been specified')
194            print('flex_extract will be installed in here by compiling the ' +
195                  'Fortran source in ' + _config.PATH_FORTRAN_SRC)
196            os.chdir(_config.PATH_FORTRAN_SRC)
197        else: # creates the target working directory for flex_extract
198            c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
199                                        c.flexpart_root_scripts))
200            if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
201                mk_tarball(tar_file, c.install_target)
202                make_dir(os.path.join(c.flexpart_root_scripts,
203                                      target_dirname))
204                os.chdir(os.path.join(c.flexpart_root_scripts,
205                                      target_dirname))
206                un_tarball(tar_file)
207                os.chdir(os.path.join(c.flexpart_root_scripts,
208                                      target_dirname,
209                                      _config.PATH_REL_FORTRAN_SRC))
210
211        # Create Fortran executable - CONVERT2
212        print('Install ' + target_dirname + ' software at ' +
213              c.install_target + ' in directory ' +
214              os.path.abspath(c.flexpart_root_scripts) + '\n')
215
216        delete_convert_build('.')
217        make_convert_build('.', c.makefile)
218
219        os.chdir(ecd)
220        if os.path.isfile(tar_file):
221            os.remove(tar_file)
222
223    return
224
225def mk_tarball(tarball_path, target):
226    '''
227    @Description:
228        Creates a tarball with all necessary files which need to be sent to the
229        installation directory.
230        It does not matter if this is local or remote.
231        Collects all python files, the Fortran source and makefiles,
232        the ECMWF_ENV file, the CONTROL files as well as the
233        template files.
234
235    @Input:
236        tarball_path: string
237            The complete path to the tar file which will contain all
238            relevant data for flex_extract.
239
240        target: string
241            The queue where the job is submitted to.
242
243    @Return:
244        <nothing>
245    '''
246    import tarfile
247    from glob import glob
248
249    print('Create tarball ...')
250
251    # change to FLEXEXTRACT directory so that the tar can contain
252    # relative pathes to the files and directories
253    ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
254    os.chdir(ecd)
255
256    # get lists of the files to be added to the tar file
257    if target == 'local':
258        ECMWF_ENV_FILE = []
259    else:
260        ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
261
262    pyfiles = [os.path.relpath(x, ecd)
263               for x in UioFiles(_config.PATH_LOCAL_PYTHON, '*py').files]
264    controlfiles = [os.path.relpath(x, ecd)
265                    for x in UioFiles(_config.PATH_CONTROLFILES,
266                                      'CONTROL*').files]
267    tempfiles = [os.path.relpath(x, ecd)
268                 for x in UioFiles(_config.PATH_TEMPLATES , '*').files]
269    ffiles = [os.path.relpath(x, ecd)
270              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.f*').files]
271    hfiles = [os.path.relpath(x, ecd)
272              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.h').files]
273    makefiles = [os.path.relpath(x, ecd)
274                 for x in UioFiles(_config.PATH_FORTRAN_SRC, 'Makefile*').files]
275
276    # concatenate single lists to one for a better looping
277    filelist = pyfiles + controlfiles + tempfiles + ffiles + hfiles + \
278               makefiles + ECMWF_ENV_FILE
279
280    # create installation tar-file
281    try:
282        with tarfile.open(tarball_path, "w:gz") as tar_handle:
283            for file in filelist:
284                tar_handle.add(file)
285
286    except subprocess.CalledProcessError as e:
287        print('... ERROR CODE:\n ... ' + str(e.returncode))
288        print('... ERROR MESSAGE:\n ... ' + str(e))
289
290        sys.exit('... could not make installation tar ball!')
291
292    return
293
294
295def un_tarball(tarball_path):
296    '''
297    @Description:
298        Extracts the given tarball into current directory.
299
300    @Input:
301        tarball_path: string
302            The complete path to the tar file which will contain all
303            relevant data for flex_extract.
304
305    @Return:
306        <nothing>
307    '''
308    import tarfile
309
310    print('Untar ...')
311
312    with tarfile.open(tarball_path) as tar_handle:
313        tar_handle.extractall()
314
315    return
316
317def mk_env_vars(ecuid, ecgid, gateway, destination):
318    '''
319    @Description:
320        Creates a file named ECMWF_ENV which contains the
321        necessary environmental variables at ECMWF servers.
322
323    @Input:
324        ecuid: string
325            The user id on ECMWF server.
326
327        ecgid: string
328            The group id on ECMWF server.
329
330        gateway: string
331            The gateway server the user is using.
332
333        destination: string
334            The remote destination which is used to transfer files
335            from ECMWF server to local gateway server.
336
337    @Return:
338        <nothing>
339    '''
340
341    with open(_config.PATH_REL_ECMWF_ENV, 'w') as fo:
342        fo.write('ECUID ' + ecuid + '\n')
343        fo.write('ECGID ' + ecgid + '\n')
344        fo.write('GATEWAY ' + gateway + '\n')
345        fo.write('DESTINATION ' + destination + '\n')
346
347    return
348
349def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
350    '''
351    @Description:
352        Modifies the original job template file so that it is specified
353        for the user and the environment were it will be applied. Result
354        is stored in a new file "job.temp" in the python directory.
355
356    @Input:
357        makefile: string
358            Name of the makefile which should be used to compile FORTRAN
359            CONVERT2 program.
360
361        target: string
362            The target where the installation should be done, e.g. the queue.
363
364        ecuid: string
365            The user id on ECMWF server.
366
367        ecgid: string
368            The group id on ECMWF server.
369
370        fp_root: string
371           Path to the root directory of FLEXPART environment or flex_extract
372           environment.
373
374    @Return:
375        <nothing>
376    '''
377
378    template = os.path.join(_config.PATH_REL_TEMPLATES,
379                            _config.TEMPFILE_INSTALL_COMPILEJOB)
380    with open(template) as f:
381        fdata = f.read().split('\n')
382
383    compilejob = os.path.join(_config.PATH_REL_JOBSCRIPTS,
384                              _config.FILE_INSTALL_COMPILEJOB)
385    with open(compilejob, 'w') as fo:
386        for data in fdata:
387            if 'MAKEFILE=' in data:
388                data = 'export MAKEFILE=' + makefile
389            elif 'FLEXPART_ROOT_SCRIPTS=' in data:
390                if fp_root != '../':
391                    data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
392                else:
393                    data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
394            elif target.lower() != 'local':
395                if '--workdir' in data:
396                    data = '#SBATCH --workdir=/scratch/ms/' + \
397                            ecgid + '/' + ecuid
398                elif '##PBS -o' in data:
399                    data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \
400                           'flex_ecmwf.$Jobname.$Job_ID.out'
401                elif 'FLEXPART_ROOT_SCRIPTS=' in data:
402                    if fp_root != '../':
403                        data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
404                    else:
405                        data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
406            fo.write(data + '\n')
407
408    return
409
410def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
411    '''
412    @Description:
413        Modifies the original job template file so that it is specified
414        for the user and the environment were it will be applied. Result
415        is stored in a new file.
416
417    @Input:
418        ecuid: string
419            The user id on ECMWF server.
420
421        ecgid: string
422            The group id on ECMWF server.
423
424        gateway: string
425            The gateway server the user is using.
426
427        destination: string
428            The remote destination which is used to transfer files
429            from ECMWF server to local gateway server.
430
431        fp_root: string
432           Path to the root directory of FLEXPART environment or flex_extract
433           environment.
434
435    @Return:
436        <nothing>
437    '''
438    fp_root_path_to_python = os.path.join(fp_root, _config.FLEXEXTRACT_DIRNAME,
439                         _config.PATH_REL_PYTHON)
440
441    template = os.path.join(_config.PATH_REL_TEMPLATES,
442                            _config.TEMPFILE_INSTALL_JOB)
443    with open(template) as f:
444        fdata = f.read().split('\n')
445
446    jobfile_temp = os.path.join(_config.PATH_REL_TEMPLATES,
447                                _config.TEMPFILE_JOB)
448    with open(jobfile_temp, 'w') as fo:
449        for data in fdata:
450            if '--workdir' in data:
451                data = '#SBATCH --workdir=/scratch/ms/' + ecgid + '/' + ecuid
452            elif '##PBS -o' in data:
453                data = '##PBS -o /scratch/ms/' + ecgid + '/' + \
454                        ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out'
455            elif  'export PATH=${PATH}:' in data:
456                data += fp_root_path_to_python
457
458            fo.write(data + '\n')
459    return
460
461def delete_convert_build(src_path):
462    '''
463    @Description:
464        Clean up the Fortran source directory and remove all
465        build files (e.g. *.o, *.mod and CONVERT2)
466
467    @Input:
468        src_path: string
469            Path to the fortran source directory.
470
471    @Return:
472        <nothing>
473    '''
474
475    modfiles = UioFiles(src_path, '*.mod')
476    objfiles = UioFiles(src_path, '*.o')
477    exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
478
479    modfiles.delete_files()
480    objfiles.delete_files()
481    exefile.delete_files()
482
483    return
484
485def make_convert_build(src_path, makefile):
486    '''
487    @Description:
488        Compiles the Fortran code and generates the executable.
489
490    @Input:
491        src_path: string
492            Path to the fortran source directory.
493
494        makefile: string
495            The name of the makefile which should be used.
496
497    @Return:
498        <nothing>
499    '''
500
501    try:
502        print('Using makefile: ' + makefile)
503        p = subprocess.Popen(['make', '-f',
504                              os.path.join(src_path, makefile)],
505                             stdin=subprocess.PIPE,
506                             stdout=subprocess.PIPE,
507                             stderr=subprocess.PIPE,
508                             bufsize=1)
509        pout, perr = p.communicate()
510        print(pout)
511        if p.returncode != 0:
512            print(perr)
513            print('Please edit ' + makefile +
514                  ' or try another Makefile in the src directory.')
515            print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
516                  'and EMOSLIB must be adapted.')
517            print('Available Makefiles:')
518            print(UioFiles(src_path, 'Makefile*'))
519            sys.exit('Compilation failed!')
520    except ValueError as e:
521        print('ERROR: Makefile call failed:')
522        print(e)
523    else:
524        subprocess.check_call(['ls', '-l',
525                               os.path.join(src_path,
526                                            _config.FORTRAN_EXECUTABLE)])
527
528    return
529
530
531if __name__ == "__main__":
532    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG