source: flex_extract.git/python/pythontest/TestInstallTar/test_untar/python/install.py @ 2fb99de

ctbtodev
Last change on this file since 2fb99de was 2fb99de, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

introduced config with path definitions and changed py files accordingly; Installation works; some tests were added for tarball making; Problems in submission to ecgate
  • Property mode set to 100755
File size: 18.7 KB
RevLine 
[2fb99de]1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Leopold Haimberger (University of Vienna)
5#
6# @Date: November 2015
7#
8# @Change History:
9#
10#    February 2018 - Anne Philipp (University of Vienna):
11#        - applied PEP8 style guide
12#        - added documentation
13#        - moved install_args_and_control in here
14#
15# @License:
16#    (C) Copyright 2015-2018.
17#
18#    This software is licensed under the terms of the Apache Licence Version 2.0
19#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
20#
21# @Program Functionality:
22#    Depending on the selected installation environment (locally or on the
23#    ECMWF server ecgate or cca) the program extracts the commandline
24#    arguments and the CONTROL file parameter and prepares the corresponding
25#    environment. The necessary files are collected in a tar-ball and placed
26#    at the target location. There its untared, the environment variables will
27#    be set and the Fortran code will be compiled. If the ECMWF environment is
28#    selected a job script is prepared and submitted for the remaining
29#    configurations after putting the tar-ball to the target ECMWF server.
30#
31# @Program Content:
32#    - main
33#    - get_install_cmdline_arguments
34#    - install_via_gateway
35#    - mk_tarball
36#    - un_tarball
37#    - mk_env_vars
38#    - mk_compilejob
39#    - mk_job_template
40#    - delete_convert_build
41#    - make_convert_build
42#
43#*******************************************************************************
44
45# ------------------------------------------------------------------------------
46# MODULES
47# ------------------------------------------------------------------------------
48import os
49import sys
50import glob
51import subprocess
52import inspect
53from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
54
55# software specific classes and modules from flex_extract
56import _config
57from ControlFile import ControlFile
58from UioFiles import UioFiles
59from tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
60
61
62# ------------------------------------------------------------------------------
63# FUNCTIONS
64# ------------------------------------------------------------------------------
65def main():
66    '''
67    @Description:
68        Controls the installation process. Calls the installation function
69        if target is specified.
70
71    @Intput:
72        <nothing>
73
74    @Return:
75        <nothing>
76    '''
77
78    os.chdir(_config.PATH_LOCAL_PYTHON)
79    args = get_install_cmdline_arguments()
80
81    try:
82        c = ControlFile(args.controlfile)
83    except IOError:
84        print 'Could not read CONTROL file "' + args.controlfile + '"'
85        print 'Either it does not exist or its syntax is wrong.'
86        print 'Try "' + sys.argv[0].split('/')[-1] + \
87              ' -h" to print usage information'
88        exit(1)
89
90    c.assign_args_to_control(args)
91    c.check_install_conditions()
92
93    install_via_gateway(c)
94
95    return
96
97def get_install_cmdline_arguments():
98    '''
99    @Description:
100        Decomposes the command line arguments and assigns them to variables.
101        Apply default values for non mentioned arguments.
102
103    @Input:
104        <nothing>
105
106    @Return:
107        args: instance of ArgumentParser
108            Contains the commandline arguments from script/program call.
109    '''
110    parser = ArgumentParser(description='Install flex_extract software locally or \
111                            on ECMWF machines',
112                            formatter_class=ArgumentDefaultsHelpFormatter)
113
114    parser.add_argument('--target', dest='install_target', default=None,
115                        help="Valid targets: local | ecgate | cca , \
116                        the latter two are at ECMWF")
117    parser.add_argument("--makefile", dest="makefile", default=None,
118                        help='Name of Makefile to use for compiling CONVERT2')
119    parser.add_argument("--ecuid", dest="ecuid", default=None,
120                        help='user id at ECMWF')
121    parser.add_argument("--ecgid", dest="ecgid", default=None,
122                        help='group id at ECMWF')
123    parser.add_argument("--gateway", dest="gateway", default=None,
124                        help='name of local gateway server')
125    parser.add_argument("--destination", dest="destination", default=None,
126                        help='ecaccess destination, e.g. leo@genericSftp')
127
128    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
129                        default=None, help="FLEXPART root directory on ECMWF \
130                        servers (to find grib2flexpart and COMMAND file)\n\
131                        Normally flex_extract resides in the scripts directory \
132                        of the FLEXPART distribution, thus the:")
133
134    # arguments for job submission to ECMWF, only needed by submit.py
135    parser.add_argument("--job_template", dest='job_template',
136                        default="job.temp.o",
137                        help="job template file for submission to ECMWF")
138
139    parser.add_argument("--controlfile", dest="controlfile",
140                        default='CONTROL.temp',
141                        help="file with CONTROL parameters")
142
143    args = parser.parse_args()
144
145    return args
146
147
148def install_via_gateway(c):
149    '''
150    @Description:
151        Perform the actual installation on local machine or prepare data
152        transfer to remote gate and submit a job script which will
153        install everything on the remote gate.
154
155    @Input:
156        c: instance of class ControlFile
157            Contains all necessary information of a CONTROL file. The parameters
158            are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
159            NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
160            RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
161            SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
162            ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR
163            For more information about format and content of the parameter see
164            documentation.
165
166    @Return:
167        <nothing>
168    '''
169    import tarfile
170
171    ecd = _config.PATH_FLEXEXTRACT_DIR
172    tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
173    tar_file = os.path.join(ecd + os.path.sep + tarball_name)
174
175    target_dirname = _config.FLEXEXTRACT_DIRNAME
176    fortran_executable = _config.FORTRAN_EXECUTABLE
177
178    if c.install_target.lower() != 'local': # ecgate or cca
179
180        mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
181                      c.flexpart_root_scripts)
182
183        mk_job_template(c.ecuid, c.ecgid, c.gateway,
184                        c.destination, c.flexpart_root_scripts)
185
186        mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
187
188        mk_tarball(tar_file)
189
190        put_file_to_ecserver(ecd, tarball_name, c.install_target,
191                             c.ecuid, c.ecgid)
192
193        submit_job_to_ecserver(c.install_target,
194                               os.path.join(_config.PATH_JOBSCRIPTS +
195                                            os.path.sep +
196                                            _config.FILE_INSTALL_COMPILEJOB))
197
198        print('job compilation script has been submitted to ecgate for ' +
199              'installation in ' + c.flexpart_root_scripts +
200               '/' + target_dirname)
201        print('You should get an email with subject "flexcompile" within ' +
202              'the next few minutes!')
203
204    else: #local
205        if not c.flexpart_root_scripts or c.flexpart_root_scripts == '../':
206            #install_dir = c.flexpart_root_scripts
207            print('WARNING: FLEXPART_ROOT_SCRIPTS has not been specified')
208            print('There will be only the compilation of the Fortran program' +
209                  ' in ' + _config.PATH_FORTRAN_SRC)
210            os.chdir(_config.PATH_FORTRAN_SRC)
211        else: # creates the target working directory for flex_extract
212            c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
213                                        c.flexpart_root_scripts))
214            if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
215                mk_tarball(tar_file)
216                make_dir(os.path.join(c.flexpart_root_scripts + os.path.sep +
217                                      target_dirname))
218                os.chdir(os.path.join(c.flexpart_root_scripts + os.path.sep +
219                                      target_dirname))
220                un_tarball(tar_file)
221                os.chdir(os.path.join(c.flexpart_root_scripts + os.path.sep +
222                                      target_dirname + os.path.sep +
223                                      _config.PATH_RELATIVE_FORTRAN_SRC))
224
225        # Create Fortran executable - CONVERT2
226        print('Install ' + target_dirname + ' software at ' +
227              c.install_target + ' in directory ' +
228              os.path.abspath(c.flexpart_root_scripts) + '\n')
229
230        delete_convert_build('.')
231        make_convert_build('.', c.makefile)
232
233        os.chdir(ecd)
234        if os.path.isfile(tar_file):
235            os.remove(tar_file)
236
237    return
238
239def mk_tarball(tarball_path):
240    '''
241    @Description:
242        Creates a tarball with all necessary files which need to be sent to the
243        installation directory.
244        It does not matter if this is local or remote.
245        Collects all python files, the Fortran source and makefiles,
246        the ECMWF_ENV file, the CONTROL files as well as the
247        template files.
248
249    @Input:
250        tarball_path: string
251            The complete path to the tar file which will contain all
252            relevant data for flex_extract.
253
254    @Return:
255        <nothing>
256    '''
257    import tarfile
258    from glob import glob
259
260    print('Create tarball ...')
261
262    # change to FLEXEXTRACT directory so that the tar can contain
263    # relative pathes to the files and directories
264    ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
265    os.chdir(ecd)
266
267    # get lists of the files to be added to the tar file
268    ECMWF_ENV_FILE = [os.path.join(_config.PATH_RELATIVE_PYTHON +
269                                   os.path.sep + _config.FILE_USER_ENVVARS)]
270    pyfiles = [os.path.relpath(x,ecd)
271               for x in glob(_config.PATH_LOCAL_PYTHON +
272                             os.path.sep + '*py')]
273    controlfiles = [os.path.relpath(x,ecd)
274                    for x in glob(_config.PATH_CONTROLFILES +
275                                  os.path.sep + 'CONTROL*')]
276    tempfiles = [os.path.relpath(x,ecd)
277                 for x in glob(_config.PATH_TEMPLATES)]
278    ffiles = [os.path.relpath(x,ecd)
279              for x in glob(_config.PATH_FORTRAN_SRC +
280                            os.path.sep + '*.f*')]
281    hfiles = [os.path.relpath(x,ecd)
282              for x in glob(_config.PATH_FORTRAN_SRC +
283                            os.path.sep + '*.h')]
284    makefiles = [os.path.relpath(x,ecd)
285                 for x in glob(_config.PATH_FORTRAN_SRC +
286                               os.path.sep + 'Makefile*')]
287
288    # concatenate single lists to one for a better looping
289    filelist = pyfiles + controlfiles + tempfiles + ffiles + hfiles + \
290               makefiles + ECMWF_ENV_FILE
291
292    # create installation tar-file
293    try:
294        with tarfile.open(tarball_path, "w:gz") as tar_handle:
295            for file in filelist:
296                tar_handle.add(file)
297
298    except subprocess.CalledProcessError as e:
299        print('... ERROR CODE:\n ... ' + str(e.returncode))
300        print('... ERROR MESSAGE:\n ... ' + str(e))
301
302        sys.exit('... could not make installation tar ball!')
303
304    return
305
306
307def un_tarball(tarball_path):
308    '''
309    @Description:
310        Extracts the given tarball into current directory.
311
312    @Input:
313        tarball_path: string
314            The complete path to the tar file which will contain all
315            relevant data for flex_extract.
316
317    @Return:
318        <nothing>
319    '''
320    import tarfile
321
322    print('Untar ...')
323
324    with tarfile.open(tarball_path) as tar_handle:
325        tar_handle.extractall()
326
327    return
328
329def mk_env_vars(ecuid, ecgid, gateway, destination):
330    '''
331    @Description:
332        Creates a file named ECMWF_ENV which contains the
333        necessary environmental variables at ECMWF servers.
334
335    @Input:
336        ecuid: string
337            The user id on ECMWF server.
338
339        ecgid: string
340            The group id on ECMWF server.
341
342        gateway: string
343            The gateway server the user is using.
344
345        destination: string
346            The remote destination which is used to transfer files
347            from ECMWF server to local gateway server.
348
349    @Return:
350        <nothing>
351    '''
352
353    with open(os.path.join(_config.PATH_LOCAL_PYTHON + os.path.sep +
354                           _config.FILE_USER_ENVVARS), 'w') as fo:
355        fo.write('ECUID ' + ecuid + '\n')
356        fo.write('ECGID ' + ecgid + '\n')
357        fo.write('GATEWAY ' + gateway + '\n')
358        fo.write('DESTINATION ' + destination + '\n')
359
360    return
361
362def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
363    '''
364    @Description:
365        Modifies the original job template file so that it is specified
366        for the user and the environment were it will be applied. Result
367        is stored in a new file "job.temp" in the python directory.
368
369    @Input:
370        makefile: string
371            Name of the makefile which should be used to compile FORTRAN
372            CONVERT2 program.
373
374        target: string
375            The target where the installation should be done, e.g. the queue.
376
377        ecuid: string
378            The user id on ECMWF server.
379
380        ecgid: string
381            The group id on ECMWF server.
382
383        fp_root: string
384           Path to the root directory of FLEXPART environment or flex_extract
385           environment.
386
387    @Return:
388        <nothing>
389    '''
390
391    template = os.path.join(_config.PATH_TEMPLATES + os.path.sep +
392                            _config.TEMPFILE_INSTALL_COMPILEJOB)
393    with open(template) as f:
394        fdata = f.read().split('\n')
395
396    compilejob = os.path.join(_config.PATH_JOBSCRIPTS + os.path.sep +
397                              _config.FILE_INSTALL_COMPILEJOB)
398    with open(compilejob, 'w') as fo:
399        for data in fdata:
400            if 'MAKEFILE=' in data:
401                data = 'export MAKEFILE=' + makefile
402            elif 'FLEXPART_ROOT_SCRIPTS=' in data:
403                if fp_root != '../':
404                    data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
405                else:
406                    data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
407            elif target.lower() != 'local':
408                if '--workdir' in data:
409                    data = '#SBATCH --workdir=/scratch/ms/' + \
410                            ecgid + '/' + ecuid
411                elif '##PBS -o' in data:
412                    data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \
413                           'flex_ecmwf.$Jobname.$Job_ID.out'
414                elif 'FLEXPART_ROOT_SCRIPTS=' in data:
415                    if fp_root != '../':
416                        data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
417                    else:
418                        data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
419            fo.write(data + '\n')
420
421    return
422
423def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
424    '''
425    @Description:
426        Modifies the original job template file so that it is specified
427        for the user and the environment were it will be applied. Result
428        is stored in a new file "job.temp" in the python directory.
429
430    @Input:
431        ecuid: string
432            The user id on ECMWF server.
433
434        ecgid: string
435            The group id on ECMWF server.
436
437        gateway: string
438            The gateway server the user is using.
439
440        destination: string
441            The remote destination which is used to transfer files
442            from ECMWF server to local gateway server.
443
444        fp_root: string
445           Path to the root directory of FLEXPART environment or flex_extract
446           environment.
447
448    @Return:
449        <nothing>
450    '''
451    ec_python_rel_path = _config.FLEXEXTRACT_DIRNAME + '/' + \
452                         _config.PATH_RELATIVE_PYTHON
453
454    template = os.path.join(_config.PATH_TEMPLATES + os.path.sep +
455                            _config.TEMPFILE_INSTALL_JOB)
456    with open(template) as f:
457        fdata = f.read().split('\n')
458
459    jobfile_temp = os.path.join(_config.PATH_JOBSCRIPTS + os.path.sep +
460                                _config.TEMPFILE_JOB)
461    with open(jobfile_temp, 'w') as fo:
462        for data in fdata:
463            if '--workdir' in data:
464                data = '#SBATCH --workdir=/scratch/ms/' + ecgid + \
465                        '/' + ecuid
466            elif '##PBS -o' in data:
467                data = '##PBS -o /scratch/ms/' + ecgid + '/' + \
468                        ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out'
469            elif  'export PATH=${PATH}:' in data:
470                data += fp_root + '/' + ec_python_rel_path
471
472            fo.write(data + '\n')
473    return
474
475def delete_convert_build(src_path):
476    '''
477    @Description:
478        Clean up the Fortran source directory and remove all
479        build files (e.g. *.o, *.mod and CONVERT2)
480
481    @Input:
482        src_path: string
483            Path to the fortran source directory.
484
485    @Return:
486        <nothing>
487    '''
488
489    modfiles = UioFiles(src_path, '*.mod')
490    objfiles = UioFiles(src_path, '*.o')
491    exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
492
493    modfiles.delete_files()
494    objfiles.delete_files()
495    exefile.delete_files()
496
497    return
498
499def make_convert_build(src_path, makefile):
500    '''
501    @Description:
502        Compiles the Fortran code and generates the executable.
503
504    @Input:
505        src_path: string
506            Path to the fortran source directory.
507
508        makefile: string
509            The name of the makefile which should be used.
510
511    @Return:
512        <nothing>
513    '''
514
515    try:
516        print('Using makefile: ' + makefile)
517        p = subprocess.Popen(['make', '-f',
518                              os.path.join(src_path + os.path.sep + makefile)],
519                             stdin=subprocess.PIPE,
520                             stdout=subprocess.PIPE,
521                             stderr=subprocess.PIPE,
522                             bufsize=1)
523        pout, perr = p.communicate()
524        print(pout)
525        if p.returncode != 0:
526            print(perr)
527            print('Please edit ' + makefile +
528                  ' or try another Makefile in the src directory.')
529            print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
530                  'and EMOSLIB must be adapted.')
531            print('Available Makefiles:')
532            print(UioFiles(src_path, 'Makefile*'))
533            sys.exit('Compilation failed!')
534    except ValueError as e:
535        print('ERROR: Makefile call failed:')
536        print(e)
537    else:
538        subprocess.check_call(['ls', '-l',
539                               os.path.join(src_path + os.path.sep +
540                                            _config.FORTRAN_EXECUTABLE)])
541
542    return
543
544
545if __name__ == "__main__":
546    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG