source: flex_extract.git/source/pythontest/TestInstallTar/flex_extract_v7.1_ecgate/source/python/install.py @ 25b14be

ctbtodev
Last change on this file since 25b14be was 25b14be, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

changed whole tree structure of flex_extract to have better overview

  • Property mode set to 100755
File size: 18.5 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Leopold Haimberger (University of Vienna)
5#
6# @Date: November 2015
7#
8# @Change History:
9#
10#    February 2018 - Anne Philipp (University of Vienna):
11#        - applied PEP8 style guide
12#        - added documentation
13#        - moved install_args_and_control in here
14#        - splitted code in smaller functions
15#        - delete convert build files in here instead of compile job script
16#        - changed static path names to Variables from config file
17#
18# @License:
19#    (C) Copyright 2015-2018.
20#
21#    This software is licensed under the terms of the Apache Licence Version 2.0
22#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
23#
24# @Program Functionality:
25#    Depending on the selected installation environment (locally or on the
26#    ECMWF server ecgate or cca) the program extracts the commandline
27#    arguments and the CONTROL file parameter and prepares the corresponding
28#    environment. The necessary files are collected in a tar-ball and placed
29#    at the target location. There its untared, the environment variables will
30#    be set and the Fortran code will be compiled. If the ECMWF environment is
31#    selected a job script is prepared and submitted for the remaining
32#    configurations after putting the tar-ball to the target ECMWF server.
33#
34# @Program Content:
35#    - main
36#    - get_install_cmdline_arguments
37#    - install_via_gateway
38#    - mk_tarball
39#    - un_tarball
40#    - mk_env_vars
41#    - mk_compilejob
42#    - mk_job_template
43#    - delete_convert_build
44#    - make_convert_build
45#
46#*******************************************************************************
47
48# ------------------------------------------------------------------------------
49# MODULES
50# ------------------------------------------------------------------------------
51import os
52import sys
53import glob
54import subprocess
55import inspect
56from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
57
58# software specific classes and modules from flex_extract
59import _config
60from classes.ControlFile import ControlFile
61from classes.UioFiles import UioFiles
62from mods.tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
63
64# ------------------------------------------------------------------------------
65# FUNCTIONS
66# ------------------------------------------------------------------------------
67def main():
68    '''
69    @Description:
70        Controls the installation process. Calls the installation function
71        if target is specified.
72
73    @Intput:
74        <nothing>
75
76    @Return:
77        <nothing>
78    '''
79
80    #os.chdir(_config.PATH_LOCAL_PYTHON)
81
82    args = get_install_cmdline_arguments()
83
84    try:
85        c = ControlFile(args.controlfile)
86    except IOError:
87        print('Could not read CONTROL file "' + args.controlfile + '"')
88        print('Either it does not exist or its syntax is wrong.')
89        print('Try "' + sys.argv[0].split('/')[-1] +
90              ' -h" to print usage information')
91        exit(1)
92
93    c.assign_args_to_control(args)
94    c.check_install_conditions()
95
96    install_via_gateway(c)
97
98    return
99
100def get_install_cmdline_arguments():
101    '''
102    @Description:
103        Decomposes the command line arguments and assigns them to variables.
104        Apply default values for non mentioned arguments.
105
106    @Input:
107        <nothing>
108
109    @Return:
110        args: instance of ArgumentParser
111            Contains the commandline arguments from script/program call.
112    '''
113    parser = ArgumentParser(description='Install flex_extract software locally or \
114                            on ECMWF machines',
115                            formatter_class=ArgumentDefaultsHelpFormatter)
116
117    parser.add_argument('--target', dest='install_target', default=None,
118                        help="Valid targets: local | ecgate | cca , \
119                        the latter two are at ECMWF")
120    parser.add_argument("--makefile", dest="makefile", default=None,
121                        help='Name of Makefile to use for compiling CONVERT2')
122    parser.add_argument("--ecuid", dest="ecuid", default=None,
123                        help='user id at ECMWF')
124    parser.add_argument("--ecgid", dest="ecgid", default=None,
125                        help='group id at ECMWF')
126    parser.add_argument("--gateway", dest="gateway", default=None,
127                        help='name of local gateway server')
128    parser.add_argument("--destination", dest="destination", default=None,
129                        help='ecaccess destination, e.g. leo@genericSftp')
130
131    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
132                        default=None, help="FLEXPART root directory on ECMWF \
133                        servers (to find grib2flexpart and COMMAND file)\n\
134                        Normally flex_extract resides in the scripts directory \
135                        of the FLEXPART distribution.")
136
137    # arguments for job submission to ECMWF, only needed by submit.py
138    parser.add_argument("--job_template", dest='job_template',
139                        default="job.temp.o",
140                        help="job template file for submission to ECMWF")
141
142    parser.add_argument("--controlfile", dest="controlfile",
143                        default='CONTROL.temp',
144                        help="file with CONTROL parameters")
145
146    args = parser.parse_args()
147
148    return args
149
150
151def install_via_gateway(c):
152    '''
153    @Description:
154        Perform the actual installation on local machine or prepare data
155        transfer to remote gate and submit a job script which will
156        install everything on the remote gate.
157
158    @Input:
159        c: instance of class ControlFile
160            Contains all necessary information of a CONTROL file. The parameters
161            are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
162            NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
163            RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
164            SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
165            ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR
166            For more information about format and content of the parameter see
167            documentation.
168
169    @Return:
170        <nothing>
171    '''
172    import tarfile
173
174    ecd = _config.PATH_FLEXEXTRACT_DIR
175    tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
176    tar_file = os.path.join(ecd, tarball_name)
177
178    target_dirname = _config.FLEXEXTRACT_DIRNAME
179    fortran_executable = _config.FORTRAN_EXECUTABLE
180
181    if c.install_target.lower() != 'local': # ecgate or cca
182
183        mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
184                      c.flexpart_root_scripts)
185
186        mk_job_template(c.ecuid, c.ecgid, c.gateway,
187                        c.destination, c.flexpart_root_scripts)
188
189        mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
190
191        mk_tarball(tar_file, c.install_target)
192
193        put_file_to_ecserver(ecd, tarball_name, c.install_target,
194                             c.ecuid, c.ecgid)
195
196        submit_job_to_ecserver(c.install_target,
197                               os.path.join(_config.PATH_REL_JOBSCRIPTS,
198                                            _config.FILE_INSTALL_COMPILEJOB))
199
200        print('job compilation script has been submitted to ecgate for ' +
201              'installation in ' + c.flexpart_root_scripts +
202               '/' + target_dirname)
203        print('You should get an email with subject "flexcompile" within ' +
204              'the next few minutes!')
205
206    else: #local
207        if c.flexpart_root_scripts == _config.PATH_FLEXEXTRACT_DIR :
208            print('WARNING: FLEXPART_ROOT_SCRIPTS has not been specified')
209            print('flex_extract will be installed in here by compiling the ' +
210                  'Fortran source in ' + _config.PATH_FORTRAN_SRC)
211            os.chdir(_config.PATH_FORTRAN_SRC)
212        else: # creates the target working directory for flex_extract
213            c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
214                                        c.flexpart_root_scripts))
215            if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
216                mk_tarball(tar_file, c.install_target)
217                make_dir(os.path.join(c.flexpart_root_scripts,
218                                      target_dirname))
219                os.chdir(os.path.join(c.flexpart_root_scripts,
220                                      target_dirname))
221                un_tarball(tar_file)
222                os.chdir(os.path.join(c.flexpart_root_scripts,
223                                      target_dirname,
224                                      _config.PATH_REL_FORTRAN_SRC))
225
226        # Create Fortran executable - CONVERT2
227        print('Install ' + target_dirname + ' software at ' +
228              c.install_target + ' in directory ' +
229              os.path.abspath(c.flexpart_root_scripts) + '\n')
230
231        delete_convert_build('.')
232        make_convert_build('.', c.makefile)
233
234        os.chdir(ecd)
235#        if os.path.isfile(tar_file):
236#            os.remove(tar_file)
237
238    return
239
240def mk_tarball(tarball_path, target):
241    '''
242    @Description:
243        Creates a tarball with all necessary files which need to be sent to the
244        installation directory.
245        It does not matter if this is local or remote.
246        Collects all python files, the Fortran source and makefiles,
247        the ECMWF_ENV file, the CONTROL files as well as the
248        template files.
249
250    @Input:
251        tarball_path: string
252            The complete path to the tar file which will contain all
253            relevant data for flex_extract.
254
255        target: string
256            The queue where the job is submitted to.
257
258    @Return:
259        <nothing>
260    '''
261    import tarfile
262    from glob import glob
263
264    print('Create tarball ...')
265
266    # change to FLEXEXTRACT directory so that the tar can contain
267    # relative pathes to the files and directories
268    ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
269    os.chdir(ecd)
270
271    # get lists of the files to be added to the tar file
272    if target == 'local':
273        ECMWF_ENV_FILE = []
274    else:
275        ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
276
277    pyfiles = [os.path.relpath(x, ecd)
278               for x in UioFiles(_config.PATH_LOCAL_PYTHON, '*py').files]
279    controlfiles = [os.path.relpath(x, ecd)
280                    for x in UioFiles(_config.PATH_CONTROLFILES,
281                                      'CONTROL*').files]
282    tempfiles = [os.path.relpath(x, ecd)
283                 for x in UioFiles(_config.PATH_TEMPLATES , '*').files]
284    ffiles = [os.path.relpath(x, ecd)
285              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.f*').files]
286    hfiles = [os.path.relpath(x, ecd)
287              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.h').files]
288    makefiles = [os.path.relpath(x, ecd)
289                 for x in UioFiles(_config.PATH_FORTRAN_SRC, 'Makefile*').files]
290
291    # concatenate single lists to one for a better looping
292    filelist = pyfiles + controlfiles + tempfiles + ffiles + hfiles + \
293               makefiles + ECMWF_ENV_FILE
294
295    # create installation tar-file
296    try:
297        with tarfile.open(tarball_path, "w:gz") as tar_handle:
298            for file in filelist:
299                tar_handle.add(file)
300
301    except subprocess.CalledProcessError as e:
302        print('... ERROR CODE:\n ... ' + str(e.returncode))
303        print('... ERROR MESSAGE:\n ... ' + str(e))
304
305        sys.exit('... could not make installation tar ball!')
306
307    return
308
309
310def un_tarball(tarball_path):
311    '''
312    @Description:
313        Extracts the given tarball into current directory.
314
315    @Input:
316        tarball_path: string
317            The complete path to the tar file which will contain all
318            relevant data for flex_extract.
319
320    @Return:
321        <nothing>
322    '''
323    import tarfile
324
325    print('Untar ...')
326
327    with tarfile.open(tarball_path) as tar_handle:
328        tar_handle.extractall()
329
330    return
331
332def mk_env_vars(ecuid, ecgid, gateway, destination):
333    '''
334    @Description:
335        Creates a file named ECMWF_ENV which contains the
336        necessary environmental variables at ECMWF servers.
337
338    @Input:
339        ecuid: string
340            The user id on ECMWF server.
341
342        ecgid: string
343            The group id on ECMWF server.
344
345        gateway: string
346            The gateway server the user is using.
347
348        destination: string
349            The remote destination which is used to transfer files
350            from ECMWF server to local gateway server.
351
352    @Return:
353        <nothing>
354    '''
355
356    with open(_config.PATH_REL_ECMWF_ENV, 'w') as fo:
357        fo.write('ECUID ' + ecuid + '\n')
358        fo.write('ECGID ' + ecgid + '\n')
359        fo.write('GATEWAY ' + gateway + '\n')
360        fo.write('DESTINATION ' + destination + '\n')
361
362    return
363
364def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
365    '''
366    @Description:
367        Modifies the original job template file so that it is specified
368        for the user and the environment were it will be applied. Result
369        is stored in a new file "job.temp" in the python directory.
370
371    @Input:
372        makefile: string
373            Name of the makefile which should be used to compile FORTRAN
374            CONVERT2 program.
375
376        target: string
377            The target where the installation should be done, e.g. the queue.
378
379        ecuid: string
380            The user id on ECMWF server.
381
382        ecgid: string
383            The group id on ECMWF server.
384
385        fp_root: string
386           Path to the root directory of FLEXPART environment or flex_extract
387           environment.
388
389    @Return:
390        <nothing>
391    '''
392
393    template = os.path.join(_config.PATH_REL_TEMPLATES,
394                            _config.TEMPFILE_INSTALL_COMPILEJOB)
395    with open(template) as f:
396        fdata = f.read().split('\n')
397
398    compilejob = os.path.join(_config.PATH_REL_JOBSCRIPTS,
399                              _config.FILE_INSTALL_COMPILEJOB)
400    with open(compilejob, 'w') as fo:
401        for data in fdata:
402            if 'MAKEFILE=' in data:
403                data = 'export MAKEFILE=' + makefile
404            elif 'FLEXPART_ROOT_SCRIPTS=' in data:
405                if fp_root != '../':
406                    data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
407                else:
408                    data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
409            elif target.lower() != 'local':
410                if '--workdir' in data:
411                    data = '#SBATCH --workdir=/scratch/ms/' + \
412                            ecgid + '/' + ecuid
413                elif '##PBS -o' in data:
414                    data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \
415                           'flex_ecmwf.$Jobname.$Job_ID.out'
416                elif 'FLEXPART_ROOT_SCRIPTS=' in data:
417                    if fp_root != '../':
418                        data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
419                    else:
420                        data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
421            fo.write(data + '\n')
422
423    return
424
425def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
426    '''
427    @Description:
428        Modifies the original job template file so that it is specified
429        for the user and the environment were it will be applied. Result
430        is stored in a new file.
431
432    @Input:
433        ecuid: string
434            The user id on ECMWF server.
435
436        ecgid: string
437            The group id on ECMWF server.
438
439        gateway: string
440            The gateway server the user is using.
441
442        destination: string
443            The remote destination which is used to transfer files
444            from ECMWF server to local gateway server.
445
446        fp_root: string
447           Path to the root directory of FLEXPART environment or flex_extract
448           environment.
449
450    @Return:
451        <nothing>
452    '''
453    fp_root_path_to_python = os.path.join(fp_root, _config.FLEXEXTRACT_DIRNAME,
454                         _config.PATH_REL_PYTHON)
455
456    template = os.path.join(_config.PATH_REL_TEMPLATES,
457                            _config.TEMPFILE_INSTALL_JOB)
458    with open(template) as f:
459        fdata = f.read().split('\n')
460
461    jobfile_temp = os.path.join(_config.PATH_REL_TEMPLATES,
462                                _config.TEMPFILE_JOB)
463    with open(jobfile_temp, 'w') as fo:
464        for data in fdata:
465            if '--workdir' in data:
466                data = '#SBATCH --workdir=/scratch/ms/' + ecgid + '/' + ecuid
467            elif '##PBS -o' in data:
468                data = '##PBS -o /scratch/ms/' + ecgid + '/' + \
469                        ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out'
470            elif  'export PATH=${PATH}:' in data:
471                data += fp_root_path_to_python
472
473            fo.write(data + '\n')
474    return
475
476def delete_convert_build(src_path):
477    '''
478    @Description:
479        Clean up the Fortran source directory and remove all
480        build files (e.g. *.o, *.mod and CONVERT2)
481
482    @Input:
483        src_path: string
484            Path to the fortran source directory.
485
486    @Return:
487        <nothing>
488    '''
489
490    modfiles = UioFiles(src_path, '*.mod')
491    objfiles = UioFiles(src_path, '*.o')
492    exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
493
494    modfiles.delete_files()
495    objfiles.delete_files()
496    exefile.delete_files()
497
498    return
499
500def make_convert_build(src_path, makefile):
501    '''
502    @Description:
503        Compiles the Fortran code and generates the executable.
504
505    @Input:
506        src_path: string
507            Path to the fortran source directory.
508
509        makefile: string
510            The name of the makefile which should be used.
511
512    @Return:
513        <nothing>
514    '''
515
516    try:
517        print('Using makefile: ' + makefile)
518        p = subprocess.Popen(['make', '-f',
519                              os.path.join(src_path, makefile)],
520                             stdin=subprocess.PIPE,
521                             stdout=subprocess.PIPE,
522                             stderr=subprocess.PIPE,
523                             bufsize=1)
524        pout, perr = p.communicate()
525        print(pout)
526        if p.returncode != 0:
527            print(perr)
528            print('Please edit ' + makefile +
529                  ' or try another Makefile in the src directory.')
530            print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
531                  'and EMOSLIB must be adapted.')
532            print('Available Makefiles:')
533            print(UioFiles(src_path, 'Makefile*'))
534            sys.exit('Compilation failed!')
535    except ValueError as e:
536        print('ERROR: Makefile call failed:')
537        print(e)
538    else:
539        subprocess.check_call(['ls', '-l',
540                               os.path.join(src_path,
541                                            _config.FORTRAN_EXECUTABLE)])
542
543    return
544
545
546if __name__ == "__main__":
547    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG