source: flex_extract.git/source/python/install.py @ 4971f63

dev
Last change on this file since 4971f63 was 4971f63, checked in by Anne Philipp <anne.philipp@…>, 16 months ago

eliminated some redundancy and exchanged CONTROL2 with the config-filename

  • Property mode set to 100755
File size: 18.2 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Leopold Haimberger (University of Vienna)
5#
6# @Date: November 2015
7#
8# @Change History:
9#
10#    February 2018 - Anne Philipp (University of Vienna):
11#        - applied PEP8 style guide
12#        - added documentation
13#        - moved install_args_and_control in here
14#        - splitted code in smaller functions
15#        - delete convert build files in here instead of compile job script
16#        - changed static path names to Variables from config file
17#
18# @License:
19#    (C) Copyright 2015-2018.
20#
21#    This software is licensed under the terms of the Apache Licence Version 2.0
22#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
23#
24# @Program Functionality:
25#    Depending on the selected installation environment (locally or on the
26#    ECMWF server ecgate or cca) the program extracts the commandline
27#    arguments and the CONTROL file parameter and prepares the corresponding
28#    environment. The necessary files are collected in a tar-ball and placed
29#    at the target location. There its untared, the environment variables will
30#    be set and the Fortran code will be compiled. If the ECMWF environment is
31#    selected a job script is prepared and submitted for the remaining
32#    configurations after putting the tar-ball to the target ECMWF server.
33#
34# @Program Content:
35#    - main
36#    - get_install_cmdline_arguments
37#    - install_via_gateway
38#    - mk_tarball
39#    - un_tarball
40#    - mk_env_vars
41#    - mk_compilejob
42#    - mk_job_template
43#    - delete_convert_build
44#    - make_convert_build
45#
46#*******************************************************************************
47
48# ------------------------------------------------------------------------------
49# MODULES
50# ------------------------------------------------------------------------------
51import os
52import sys
53import glob
54import subprocess
55import inspect
56from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
57
58# software specific classes and modules from flex_extract
59import _config
60from classes.ControlFile import ControlFile
61from classes.UioFiles import UioFiles
62from mods.tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
63
64# ------------------------------------------------------------------------------
65# FUNCTIONS
66# ------------------------------------------------------------------------------
67def main():
68    '''
69    @Description:
70        Controls the installation process. Calls the installation function
71        if target is specified.
72
73    @Intput:
74        <nothing>
75
76    @Return:
77        <nothing>
78    '''
79
80    args = get_install_cmdline_arguments()
81    c = ControlFile(args.controlfile)
82    c.assign_args_to_control(args)
83    c.check_install_conditions()
84
85    install_via_gateway(c)
86
87    return
88
89def get_install_cmdline_arguments():
90    '''
91    @Description:
92        Decomposes the command line arguments and assigns them to variables.
93        Apply default values for non mentioned arguments.
94
95    @Input:
96        <nothing>
97
98    @Return:
99        args: instance of ArgumentParser
100            Contains the commandline arguments from script/program call.
101    '''
102    parser = ArgumentParser(description='Install flex_extract software locally or \
103                            on ECMWF machines',
104                            formatter_class=ArgumentDefaultsHelpFormatter)
105
106    parser.add_argument('--target', dest='install_target', default=None,
107                        help="Valid targets: local | ecgate | cca , \
108                        the latter two are at ECMWF")
109    parser.add_argument("--makefile", dest="makefile", default=None,
110                        help='Name of Makefile to use for compiling CONVERT2')
111    parser.add_argument("--ecuid", dest="ecuid", default=None,
112                        help='user id at ECMWF')
113    parser.add_argument("--ecgid", dest="ecgid", default=None,
114                        help='group id at ECMWF')
115    parser.add_argument("--gateway", dest="gateway", default=None,
116                        help='name of local gateway server')
117    parser.add_argument("--destination", dest="destination", default=None,
118                        help='ecaccess destination, e.g. leo@genericSftp')
119
120    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
121                        default=None, help="FLEXPART root directory on ECMWF \
122                        servers (to find grib2flexpart and COMMAND file)\n\
123                        Normally flex_extract resides in the scripts directory \
124                        of the FLEXPART distribution.")
125
126    # arguments for job submission to ECMWF, only needed by submit.py
127    parser.add_argument("--job_template", dest='job_template',
128                        default="job.temp.o",
129                        help="job template file for submission to ECMWF")
130
131    parser.add_argument("--controlfile", dest="controlfile",
132                        default='CONTROL.temp',
133                        help="file with CONTROL parameters")
134
135    args = parser.parse_args()
136
137    return args
138
139
140def install_via_gateway(c):
141    '''
142    @Description:
143        Perform the actual installation on local machine or prepare data
144        transfer to remote gate and submit a job script which will
145        install everything on the remote gate.
146
147    @Input:
148        c: instance of class ControlFile
149            Contains all necessary information of a CONTROL file. The parameters
150            are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
151            NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
152            RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
153            SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
154            ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR
155            For more information about format and content of the parameter see
156            documentation.
157
158    @Return:
159        <nothing>
160    '''
161    import tarfile
162
163    ecd = _config.PATH_FLEXEXTRACT_DIR
164    tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
165    tar_file = os.path.join(ecd, tarball_name)
166
167    target_dirname = _config.FLEXEXTRACT_DIRNAME
168    fortran_executable = _config.FORTRAN_EXECUTABLE
169
170    if c.install_target.lower() != 'local': # ecgate or cca
171
172        mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
173                      c.flexpart_root_scripts)
174
175        mk_job_template(c.ecuid, c.ecgid, c.gateway,
176                        c.destination, c.flexpart_root_scripts)
177
178        mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
179
180        mk_tarball(tar_file, c.install_target)
181
182        put_file_to_ecserver(ecd, tarball_name, c.install_target,
183                             c.ecuid, c.ecgid)
184
185        submit_job_to_ecserver(c.install_target,
186                               os.path.join(_config.PATH_REL_JOBSCRIPTS,
187                                            _config.FILE_INSTALL_COMPILEJOB))
188
189        print('job compilation script has been submitted to ecgate for ' +
190              'installation in ' + c.flexpart_root_scripts +
191               '/' + target_dirname)
192        print('You should get an email with subject "flexcompile" within ' +
193              'the next few minutes!')
194
195    else: #local
196        if c.flexpart_root_scripts == _config.PATH_FLEXEXTRACT_DIR :
197            print('WARNING: FLEXPART_ROOT_SCRIPTS has not been specified')
198            print('flex_extract will be installed in here by compiling the ' +
199                  'Fortran source in ' + _config.PATH_FORTRAN_SRC)
200            os.chdir(_config.PATH_FORTRAN_SRC)
201        else: # creates the target working directory for flex_extract
202            c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
203                                        c.flexpart_root_scripts))
204            if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
205                mk_tarball(tar_file, c.install_target)
206                make_dir(os.path.join(c.flexpart_root_scripts,
207                                      target_dirname))
208                os.chdir(os.path.join(c.flexpart_root_scripts,
209                                      target_dirname))
210                un_tarball(tar_file)
211                os.chdir(os.path.join(c.flexpart_root_scripts,
212                                      target_dirname,
213                                      _config.PATH_REL_FORTRAN_SRC))
214
215        # Create Fortran executable - CONVERT2
216        print('Install ' + target_dirname + ' software at ' +
217              c.install_target + ' in directory ' +
218              os.path.abspath(c.flexpart_root_scripts) + '\n')
219
220        delete_convert_build('.')
221        make_convert_build('.', c.makefile)
222
223        os.chdir(ecd)
224        if os.path.isfile(tar_file):
225            os.remove(tar_file)
226
227    return
228
229def mk_tarball(tarball_path, target):
230    '''
231    @Description:
232        Creates a tarball with all necessary files which need to be sent to the
233        installation directory.
234        It does not matter if this is local or remote.
235        Collects all python files, the Fortran source and makefiles,
236        the ECMWF_ENV file, the CONTROL files as well as the
237        template files.
238
239    @Input:
240        tarball_path: string
241            The complete path to the tar file which will contain all
242            relevant data for flex_extract.
243
244        target: string
245            The queue where the job is submitted to.
246
247    @Return:
248        <nothing>
249    '''
250    import tarfile
251    from glob import glob
252
253    print('Create tarball ...')
254
255    # change to FLEXEXTRACT directory so that the tar can contain
256    # relative pathes to the files and directories
257    ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
258    os.chdir(ecd)
259
260    # get lists of the files to be added to the tar file
261    if target == 'local':
262        ECMWF_ENV_FILE = []
263    else:
264        ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
265
266    pyfiles = [os.path.relpath(x, ecd)
267               for x in UioFiles(_config.PATH_LOCAL_PYTHON, '*py').files]
268    controlfiles = [os.path.relpath(x, ecd)
269                    for x in UioFiles(_config.PATH_CONTROLFILES,
270                                      'CONTROL*').files]
271    tempfiles = [os.path.relpath(x, ecd)
272                 for x in UioFiles(_config.PATH_TEMPLATES , '*').files]
273    ffiles = [os.path.relpath(x, ecd)
274              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.f*').files]
275    hfiles = [os.path.relpath(x, ecd)
276              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.h').files]
277    makefiles = [os.path.relpath(x, ecd)
278                 for x in UioFiles(_config.PATH_FORTRAN_SRC, 'Makefile*').files]
279
280    # concatenate single lists to one for a better looping
281    filelist = pyfiles + controlfiles + tempfiles + ffiles + hfiles + \
282               makefiles + ECMWF_ENV_FILE
283
284    # create installation tar-file
285    try:
286        with tarfile.open(tarball_path, "w:gz") as tar_handle:
287            for file in filelist:
288                tar_handle.add(file)
289
290    except subprocess.CalledProcessError as e:
291        print('... ERROR CODE:\n ... ' + str(e.returncode))
292        print('... ERROR MESSAGE:\n ... ' + str(e))
293
294        sys.exit('... could not make installation tar ball!')
295
296    return
297
298
299def un_tarball(tarball_path):
300    '''
301    @Description:
302        Extracts the given tarball into current directory.
303
304    @Input:
305        tarball_path: string
306            The complete path to the tar file which will contain all
307            relevant data for flex_extract.
308
309    @Return:
310        <nothing>
311    '''
312    import tarfile
313
314    print('Untar ...')
315
316    with tarfile.open(tarball_path) as tar_handle:
317        tar_handle.extractall()
318
319    return
320
321def mk_env_vars(ecuid, ecgid, gateway, destination):
322    '''
323    @Description:
324        Creates a file named ECMWF_ENV which contains the
325        necessary environmental variables at ECMWF servers.
326
327    @Input:
328        ecuid: string
329            The user id on ECMWF server.
330
331        ecgid: string
332            The group id on ECMWF server.
333
334        gateway: string
335            The gateway server the user is using.
336
337        destination: string
338            The remote destination which is used to transfer files
339            from ECMWF server to local gateway server.
340
341    @Return:
342        <nothing>
343    '''
344
345    with open(_config.PATH_REL_ECMWF_ENV, 'w') as fo:
346        fo.write('ECUID ' + ecuid + '\n')
347        fo.write('ECGID ' + ecgid + '\n')
348        fo.write('GATEWAY ' + gateway + '\n')
349        fo.write('DESTINATION ' + destination + '\n')
350
351    return
352
353def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
354    '''
355    @Description:
356        Modifies the original job template file so that it is specified
357        for the user and the environment were it will be applied. Result
358        is stored in a new file "job.temp" in the python directory.
359
360    @Input:
361        makefile: string
362            Name of the makefile which should be used to compile FORTRAN
363            CONVERT2 program.
364
365        target: string
366            The target where the installation should be done, e.g. the queue.
367
368        ecuid: string
369            The user id on ECMWF server.
370
371        ecgid: string
372            The group id on ECMWF server.
373
374        fp_root: string
375           Path to the root directory of FLEXPART environment or flex_extract
376           environment.
377
378    @Return:
379        <nothing>
380    '''
381
382    template = os.path.join(_config.PATH_REL_TEMPLATES,
383                            _config.TEMPFILE_INSTALL_COMPILEJOB)
384    with open(template) as f:
385        fdata = f.read().split('\n')
386
387    compilejob = os.path.join(_config.PATH_REL_JOBSCRIPTS,
388                              _config.FILE_INSTALL_COMPILEJOB)
389    with open(compilejob, 'w') as fo:
390        for data in fdata:
391            if 'MAKEFILE=' in data:
392                data = 'export MAKEFILE=' + makefile
393            elif 'FLEXPART_ROOT_SCRIPTS=' in data:
394                if fp_root != '../':
395                    data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
396                else:
397                    data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
398            elif target.lower() != 'local':
399                if '--workdir' in data:
400                    data = '#SBATCH --workdir=/scratch/ms/' + \
401                            ecgid + '/' + ecuid
402                elif '##PBS -o' in data:
403                    data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \
404                           'flex_ecmwf.$Jobname.$Job_ID.out'
405                elif 'FLEXPART_ROOT_SCRIPTS=' in data:
406                    if fp_root != '../':
407                        data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
408                    else:
409                        data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
410            fo.write(data + '\n')
411
412    return
413
414def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
415    '''
416    @Description:
417        Modifies the original job template file so that it is specified
418        for the user and the environment were it will be applied. Result
419        is stored in a new file.
420
421    @Input:
422        ecuid: string
423            The user id on ECMWF server.
424
425        ecgid: string
426            The group id on ECMWF server.
427
428        gateway: string
429            The gateway server the user is using.
430
431        destination: string
432            The remote destination which is used to transfer files
433            from ECMWF server to local gateway server.
434
435        fp_root: string
436           Path to the root directory of FLEXPART environment or flex_extract
437           environment.
438
439    @Return:
440        <nothing>
441    '''
442    fp_root_path_to_python = os.path.join(fp_root, _config.FLEXEXTRACT_DIRNAME,
443                         _config.PATH_REL_PYTHON)
444
445    template = os.path.join(_config.PATH_REL_TEMPLATES,
446                            _config.TEMPFILE_INSTALL_JOB)
447    with open(template) as f:
448        fdata = f.read().split('\n')
449
450    jobfile_temp = os.path.join(_config.PATH_REL_TEMPLATES,
451                                _config.TEMPFILE_JOB)
452    with open(jobfile_temp, 'w') as fo:
453        for data in fdata:
454            if '--workdir' in data:
455                data = '#SBATCH --workdir=/scratch/ms/' + ecgid + '/' + ecuid
456            elif '##PBS -o' in data:
457                data = '##PBS -o /scratch/ms/' + ecgid + '/' + \
458                        ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out'
459            elif  'export PATH=${PATH}:' in data:
460                data += fp_root_path_to_python
461
462            fo.write(data + '\n')
463    return
464
465def delete_convert_build(src_path):
466    '''
467    @Description:
468        Clean up the Fortran source directory and remove all
469        build files (e.g. *.o, *.mod and CONVERT2)
470
471    @Input:
472        src_path: string
473            Path to the fortran source directory.
474
475    @Return:
476        <nothing>
477    '''
478
479    modfiles = UioFiles(src_path, '*.mod')
480    objfiles = UioFiles(src_path, '*.o')
481    exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
482
483    modfiles.delete_files()
484    objfiles.delete_files()
485    exefile.delete_files()
486
487    return
488
489def make_convert_build(src_path, makefile):
490    '''
491    @Description:
492        Compiles the Fortran code and generates the executable.
493
494    @Input:
495        src_path: string
496            Path to the fortran source directory.
497
498        makefile: string
499            The name of the makefile which should be used.
500
501    @Return:
502        <nothing>
503    '''
504
505    try:
506        print('Using makefile: ' + makefile)
507        p = subprocess.Popen(['make', '-f',
508                              os.path.join(src_path, makefile)],
509                             stdin=subprocess.PIPE,
510                             stdout=subprocess.PIPE,
511                             stderr=subprocess.PIPE,
512                             bufsize=1)
513        pout, perr = p.communicate()
514        print(pout)
515        if p.returncode != 0:
516            print(perr)
517            print('Please edit ' + makefile +
518                  ' or try another Makefile in the src directory.')
519            print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
520                  'and EMOSLIB must be adapted.')
521            print('Available Makefiles:')
522            print(UioFiles(src_path, 'Makefile*'))
523            sys.exit('Compilation failed!')
524    except ValueError as e:
525        print('ERROR: Makefile call failed:')
526        print(e)
527    else:
528        subprocess.check_call(['ls', '-l',
529                               os.path.join(src_path,
530                                            _config.FORTRAN_EXECUTABLE)])
531
532    return
533
534
535if __name__ == "__main__":
536    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG