source: flex_extract.git/python/install.py @ 2fb99de

dev
Last change on this file since 2fb99de was 2fb99de, checked in by Anne Philipp <anne.philipp@…>, 16 months ago

introduced config with path definitions and changed py files accordingly; Installation works; some tests were added for tarball making; Problems in submission to ecgate

  • Property mode set to 100755
File size: 18.5 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Leopold Haimberger (University of Vienna)
5#
6# @Date: November 2015
7#
8# @Change History:
9#
10#    February 2018 - Anne Philipp (University of Vienna):
11#        - applied PEP8 style guide
12#        - added documentation
13#        - moved install_args_and_control in here
14#        - splitted code in smaller functions
15#        - delete convert build files in here instead of compile job script
16#        - changed static path names to Variables from config file
17#
18# @License:
19#    (C) Copyright 2015-2018.
20#
21#    This software is licensed under the terms of the Apache Licence Version 2.0
22#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
23#
24# @Program Functionality:
25#    Depending on the selected installation environment (locally or on the
26#    ECMWF server ecgate or cca) the program extracts the commandline
27#    arguments and the CONTROL file parameter and prepares the corresponding
28#    environment. The necessary files are collected in a tar-ball and placed
29#    at the target location. There its untared, the environment variables will
30#    be set and the Fortran code will be compiled. If the ECMWF environment is
31#    selected a job script is prepared and submitted for the remaining
32#    configurations after putting the tar-ball to the target ECMWF server.
33#
34# @Program Content:
35#    - main
36#    - get_install_cmdline_arguments
37#    - install_via_gateway
38#    - mk_tarball
39#    - un_tarball
40#    - mk_env_vars
41#    - mk_compilejob
42#    - mk_job_template
43#    - delete_convert_build
44#    - make_convert_build
45#
46#*******************************************************************************
47
48# ------------------------------------------------------------------------------
49# MODULES
50# ------------------------------------------------------------------------------
51import os
52import sys
53import glob
54import subprocess
55import inspect
56from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
57
58# software specific classes and modules from flex_extract
59import _config
60from ControlFile import ControlFile
61from UioFiles import UioFiles
62from tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
63
64# ------------------------------------------------------------------------------
65# FUNCTIONS
66# ------------------------------------------------------------------------------
67def main():
68    '''
69    @Description:
70        Controls the installation process. Calls the installation function
71        if target is specified.
72
73    @Intput:
74        <nothing>
75
76    @Return:
77        <nothing>
78    '''
79
80    #os.chdir(_config.PATH_LOCAL_PYTHON)
81
82    args = get_install_cmdline_arguments()
83
84    try:
85        c = ControlFile(args.controlfile)
86    except IOError:
87        print('Could not read CONTROL file "' + args.controlfile + '"')
88        print('Either it does not exist or its syntax is wrong.')
89        print('Try "' + sys.argv[0].split('/')[-1] +
90              ' -h" to print usage information')
91        exit(1)
92
93    c.assign_args_to_control(args)
94    c.check_install_conditions()
95
96    install_via_gateway(c)
97
98    return
99
100def get_install_cmdline_arguments():
101    '''
102    @Description:
103        Decomposes the command line arguments and assigns them to variables.
104        Apply default values for non mentioned arguments.
105
106    @Input:
107        <nothing>
108
109    @Return:
110        args: instance of ArgumentParser
111            Contains the commandline arguments from script/program call.
112    '''
113    parser = ArgumentParser(description='Install flex_extract software locally or \
114                            on ECMWF machines',
115                            formatter_class=ArgumentDefaultsHelpFormatter)
116
117    parser.add_argument('--target', dest='install_target', default=None,
118                        help="Valid targets: local | ecgate | cca , \
119                        the latter two are at ECMWF")
120    parser.add_argument("--makefile", dest="makefile", default=None,
121                        help='Name of Makefile to use for compiling CONVERT2')
122    parser.add_argument("--ecuid", dest="ecuid", default=None,
123                        help='user id at ECMWF')
124    parser.add_argument("--ecgid", dest="ecgid", default=None,
125                        help='group id at ECMWF')
126    parser.add_argument("--gateway", dest="gateway", default=None,
127                        help='name of local gateway server')
128    parser.add_argument("--destination", dest="destination", default=None,
129                        help='ecaccess destination, e.g. leo@genericSftp')
130
131    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
132                        default=None, help="FLEXPART root directory on ECMWF \
133                        servers (to find grib2flexpart and COMMAND file)\n\
134                        Normally flex_extract resides in the scripts directory \
135                        of the FLEXPART distribution.")
136
137    # arguments for job submission to ECMWF, only needed by submit.py
138    parser.add_argument("--job_template", dest='job_template',
139                        default="job.temp.o",
140                        help="job template file for submission to ECMWF")
141
142    parser.add_argument("--controlfile", dest="controlfile",
143                        default='CONTROL.temp',
144                        help="file with CONTROL parameters")
145
146    args = parser.parse_args()
147
148    return args
149
150
151def install_via_gateway(c):
152    '''
153    @Description:
154        Perform the actual installation on local machine or prepare data
155        transfer to remote gate and submit a job script which will
156        install everything on the remote gate.
157
158    @Input:
159        c: instance of class ControlFile
160            Contains all necessary information of a CONTROL file. The parameters
161            are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
162            NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
163            RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
164            SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
165            ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR
166            For more information about format and content of the parameter see
167            documentation.
168
169    @Return:
170        <nothing>
171    '''
172    import tarfile
173
174    ecd = _config.PATH_FLEXEXTRACT_DIR
175    tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
176    tar_file = os.path.join(ecd, tarball_name)
177
178    target_dirname = _config.FLEXEXTRACT_DIRNAME
179    fortran_executable = _config.FORTRAN_EXECUTABLE
180
181    if c.install_target.lower() != 'local': # ecgate or cca
182
183        mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
184                      c.flexpart_root_scripts)
185
186        mk_job_template(c.ecuid, c.ecgid, c.gateway,
187                        c.destination, c.flexpart_root_scripts)
188
189        mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
190
191        mk_tarball(tar_file)
192
193        put_file_to_ecserver(ecd, tarball_name, c.install_target,
194                             c.ecuid, c.ecgid)
195
196        submit_job_to_ecserver(c.install_target,
197                               os.path.join(_config.PATH_RELATIVE_JOBSCRIPTS,
198                                            _config.FILE_INSTALL_COMPILEJOB))
199
200        print('job compilation script has been submitted to ecgate for ' +
201              'installation in ' + c.flexpart_root_scripts +
202               '/' + target_dirname)
203        print('You should get an email with subject "flexcompile" within ' +
204              'the next few minutes!')
205
206    else: #local
207        if c.flexpart_root_scripts == _config.PATH_FLEXEXTRACT_DIR :
208            print('WARNING: FLEXPART_ROOT_SCRIPTS has not been specified')
209            print('flex_extract will be installed in here by compiling the ' +
210                  'Fortran source in ' + _config.PATH_FORTRAN_SRC)
211            os.chdir(_config.PATH_FORTRAN_SRC)
212        else: # creates the target working directory for flex_extract
213            c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
214                                        c.flexpart_root_scripts))
215            if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
216                mk_tarball(tar_file)
217                make_dir(os.path.join(c.flexpart_root_scripts,
218                                      target_dirname))
219                os.chdir(os.path.join(c.flexpart_root_scripts,
220                                      target_dirname))
221                un_tarball(tar_file)
222                os.chdir(os.path.join(c.flexpart_root_scripts,
223                                      target_dirname,
224                                      _config.PATH_RELATIVE_FORTRAN_SRC))
225
226        # Create Fortran executable - CONVERT2
227        print('Install ' + target_dirname + ' software at ' +
228              c.install_target + ' in directory ' +
229              os.path.abspath(c.flexpart_root_scripts) + '\n')
230
231        delete_convert_build('.')
232        make_convert_build('.', c.makefile)
233
234        os.chdir(ecd)
235        if os.path.isfile(tar_file):
236            os.remove(tar_file)
237
238    return
239
240def mk_tarball(tarball_path):
241    '''
242    @Description:
243        Creates a tarball with all necessary files which need to be sent to the
244        installation directory.
245        It does not matter if this is local or remote.
246        Collects all python files, the Fortran source and makefiles,
247        the ECMWF_ENV file, the CONTROL files as well as the
248        template files.
249
250    @Input:
251        tarball_path: string
252            The complete path to the tar file which will contain all
253            relevant data for flex_extract.
254
255    @Return:
256        <nothing>
257    '''
258    import tarfile
259    from glob import glob
260
261    print('Create tarball ...')
262
263    # change to FLEXEXTRACT directory so that the tar can contain
264    # relative pathes to the files and directories
265    ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
266    os.chdir(ecd)
267
268    # get lists of the files to be added to the tar file
269    ECMWF_ENV_FILE = [_config.PATH_RELATIVE_ECMWF_ENV]
270    pyfiles = [os.path.relpath(x, ecd)
271               for x in glob(_config.PATH_LOCAL_PYTHON +
272                             os.path.sep + '*py')]
273    controlfiles = [os.path.relpath(x, ecd)
274                    for x in glob(_config.PATH_CONTROLFILES +
275                                  os.path.sep + 'CONTROL*')]
276    tempfiles = [os.path.relpath(x, ecd)
277                 for x in glob(_config.PATH_TEMPLATES +
278                               os.path.sep + '*')]
279    ffiles = [os.path.relpath(x, ecd)
280              for x in glob(_config.PATH_FORTRAN_SRC +
281                            os.path.sep + '*.f*')]
282    hfiles = [os.path.relpath(x, ecd)
283              for x in glob(_config.PATH_FORTRAN_SRC +
284                            os.path.sep + '*.h')]
285    makefiles = [os.path.relpath(x, ecd)
286                 for x in glob(_config.PATH_FORTRAN_SRC +
287                               os.path.sep + 'Makefile*')]
288
289    # concatenate single lists to one for a better looping
290    filelist = pyfiles + controlfiles + tempfiles + ffiles + hfiles + \
291               makefiles + ECMWF_ENV_FILE
292
293    # create installation tar-file
294    try:
295        with tarfile.open(tarball_path, "w:gz") as tar_handle:
296            for file in filelist:
297                tar_handle.add(file)
298
299    except subprocess.CalledProcessError as e:
300        print('... ERROR CODE:\n ... ' + str(e.returncode))
301        print('... ERROR MESSAGE:\n ... ' + str(e))
302
303        sys.exit('... could not make installation tar ball!')
304
305    return
306
307
308def un_tarball(tarball_path):
309    '''
310    @Description:
311        Extracts the given tarball into current directory.
312
313    @Input:
314        tarball_path: string
315            The complete path to the tar file which will contain all
316            relevant data for flex_extract.
317
318    @Return:
319        <nothing>
320    '''
321    import tarfile
322
323    print('Untar ...')
324
325    with tarfile.open(tarball_path) as tar_handle:
326        tar_handle.extractall()
327
328    return
329
330def mk_env_vars(ecuid, ecgid, gateway, destination):
331    '''
332    @Description:
333        Creates a file named ECMWF_ENV which contains the
334        necessary environmental variables at ECMWF servers.
335
336    @Input:
337        ecuid: string
338            The user id on ECMWF server.
339
340        ecgid: string
341            The group id on ECMWF server.
342
343        gateway: string
344            The gateway server the user is using.
345
346        destination: string
347            The remote destination which is used to transfer files
348            from ECMWF server to local gateway server.
349
350    @Return:
351        <nothing>
352    '''
353
354    with open(_config.PATH_RELATIVE_ECMWF_ENV, 'w') as fo:
355        fo.write('ECUID ' + ecuid + '\n')
356        fo.write('ECGID ' + ecgid + '\n')
357        fo.write('GATEWAY ' + gateway + '\n')
358        fo.write('DESTINATION ' + destination + '\n')
359
360    return
361
362def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
363    '''
364    @Description:
365        Modifies the original job template file so that it is specified
366        for the user and the environment were it will be applied. Result
367        is stored in a new file "job.temp" in the python directory.
368
369    @Input:
370        makefile: string
371            Name of the makefile which should be used to compile FORTRAN
372            CONVERT2 program.
373
374        target: string
375            The target where the installation should be done, e.g. the queue.
376
377        ecuid: string
378            The user id on ECMWF server.
379
380        ecgid: string
381            The group id on ECMWF server.
382
383        fp_root: string
384           Path to the root directory of FLEXPART environment or flex_extract
385           environment.
386
387    @Return:
388        <nothing>
389    '''
390
391    template = os.path.join(_config.PATH_RELATIVE_TEMPLATES,
392                            _config.TEMPFILE_INSTALL_COMPILEJOB)
393    with open(template) as f:
394        fdata = f.read().split('\n')
395
396    compilejob = os.path.join(_config.PATH_RELATIVE_JOBSCRIPTS,
397                              _config.FILE_INSTALL_COMPILEJOB)
398    with open(compilejob, 'w') as fo:
399        for data in fdata:
400            if 'MAKEFILE=' in data:
401                data = 'export MAKEFILE=' + makefile
402            elif 'FLEXPART_ROOT_SCRIPTS=' in data:
403                if fp_root != '../':
404                    data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
405                else:
406                    data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
407            elif target.lower() != 'local':
408                if '--workdir' in data:
409                    data = '#SBATCH --workdir=/scratch/ms/' + \
410                            ecgid + '/' + ecuid
411                elif '##PBS -o' in data:
412                    data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \
413                           'flex_ecmwf.$Jobname.$Job_ID.out'
414                elif 'FLEXPART_ROOT_SCRIPTS=' in data:
415                    if fp_root != '../':
416                        data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
417                    else:
418                        data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
419            fo.write(data + '\n')
420
421    return
422
423def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
424    '''
425    @Description:
426        Modifies the original job template file so that it is specified
427        for the user and the environment were it will be applied. Result
428        is stored in a new file.
429
430    @Input:
431        ecuid: string
432            The user id on ECMWF server.
433
434        ecgid: string
435            The group id on ECMWF server.
436
437        gateway: string
438            The gateway server the user is using.
439
440        destination: string
441            The remote destination which is used to transfer files
442            from ECMWF server to local gateway server.
443
444        fp_root: string
445           Path to the root directory of FLEXPART environment or flex_extract
446           environment.
447
448    @Return:
449        <nothing>
450    '''
451    fp_root_path_to_python = os.path.join(fp_root, _config.FLEXEXTRACT_DIRNAME,
452                         _config.PATH_RELATIVE_PYTHON)
453
454    template = os.path.join(_config.PATH_RELATIVE_TEMPLATES,
455                            _config.TEMPFILE_INSTALL_JOB)
456    with open(template) as f:
457        fdata = f.read().split('\n')
458
459    jobfile_temp = os.path.join(_config.PATH_RELATIVE_TEMPLATES,
460                                _config.TEMPFILE_JOB)
461    with open(jobfile_temp, 'w') as fo:
462        for data in fdata:
463            if '--workdir' in data:
464                data = '#SBATCH --workdir=/scratch/ms/' + ecgid + '/' + ecuid
465            elif '##PBS -o' in data:
466                data = '##PBS -o /scratch/ms/' + ecgid + '/' + \
467                        ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out'
468            elif  'export PATH=${PATH}:' in data:
469                data += fp_root_path_to_python
470
471            fo.write(data + '\n')
472    return
473
474def delete_convert_build(src_path):
475    '''
476    @Description:
477        Clean up the Fortran source directory and remove all
478        build files (e.g. *.o, *.mod and CONVERT2)
479
480    @Input:
481        src_path: string
482            Path to the fortran source directory.
483
484    @Return:
485        <nothing>
486    '''
487
488    modfiles = UioFiles(src_path, '*.mod')
489    objfiles = UioFiles(src_path, '*.o')
490    exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)
491
492    modfiles.delete_files()
493    objfiles.delete_files()
494    exefile.delete_files()
495
496    return
497
498def make_convert_build(src_path, makefile):
499    '''
500    @Description:
501        Compiles the Fortran code and generates the executable.
502
503    @Input:
504        src_path: string
505            Path to the fortran source directory.
506
507        makefile: string
508            The name of the makefile which should be used.
509
510    @Return:
511        <nothing>
512    '''
513
514    try:
515        print('Using makefile: ' + makefile)
516        p = subprocess.Popen(['make', '-f',
517                              os.path.join(src_path, makefile)],
518                             stdin=subprocess.PIPE,
519                             stdout=subprocess.PIPE,
520                             stderr=subprocess.PIPE,
521                             bufsize=1)
522        pout, perr = p.communicate()
523        print(pout)
524        if p.returncode != 0:
525            print(perr)
526            print('Please edit ' + makefile +
527                  ' or try another Makefile in the src directory.')
528            print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
529                  'and EMOSLIB must be adapted.')
530            print('Available Makefiles:')
531            print(UioFiles(src_path, 'Makefile*'))
532            sys.exit('Compilation failed!')
533    except ValueError as e:
534        print('ERROR: Makefile call failed:')
535        print(e)
536    else:
537        subprocess.check_call(['ls', '-l',
538                               os.path.join(src_path,
539                                            _config.FORTRAN_EXECUTABLE)])
540
541    return
542
543
544if __name__ == "__main__":
545    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG