source: flex_extract.git/python/install.py @ 97e09f4

devfeature/makefilesorigin/task/language-editingtask/language-editing
Last change on this file since 97e09f4 was 97e09f4, checked in by Anne Philipp <anne.philipp@…>, 2 years ago

completed testing and adaptations of ecacces job-submit and file-put functions

  • Property mode set to 100755
File size: 16.7 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Leopold Haimberger (University of Vienna)
5#
6# @Date: November 2015
7#
8# @Change History:
9#
10#    February 2018 - Anne Philipp (University of Vienna):
11#        - applied PEP8 style guide
12#        - added documentation
13#        - moved install_args_and_control in here
14#
15# @License:
16#    (C) Copyright 2015-2018.
17#
18#    This software is licensed under the terms of the Apache Licence Version 2.0
19#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
20#
21# @Program Functionality:
22#    Depending on the selected installation environment (locally or on the
23#    ECMWF server ecgate or cca) the program extracts the commandline
24#    arguments and the CONTROL file parameter and prepares the corresponding
25#    environment. The necessary files are collected in a tar-ball and placed
26#    at the target location. There its untared, the environment variables will
27#    be set and the Fortran code will be compiled. If the ECMWF environment is
28#    selected a job script is prepared and submitted for the remaining
29#    configurations after putting the tar-ball to the target ECMWF server.
30#
31# @Program Content:
32#    - main
33#    - get_install_cmdline_arguments
34#    - install_via_gateway
35#    - mk_tarball
36#    - mk_env_vars
37#    - mk_compilejob
38#    - mk_job_template
39#    - delete_convert_build
40#    - make_convert_build
41#
42#*******************************************************************************
43
44# ------------------------------------------------------------------------------
45# MODULES
46# ------------------------------------------------------------------------------
47import os
48import sys
49import subprocess
50import inspect
51from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
52
53# software specific classes and modules from flex_extract
54from ControlFile import ControlFile
55from UioFiles import UioFiles
56from tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
57
58# add path to pythonpath so that python finds its buddies
59LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
60    inspect.getfile(inspect.currentframe())))
61if LOCAL_PYTHON_PATH not in sys.path:
62    sys.path.append(LOCAL_PYTHON_PATH)
63
64_VERSION_STR = '7.1'
65
66# ------------------------------------------------------------------------------
67# FUNCTIONS
68# ------------------------------------------------------------------------------
69def main():
70    '''
71    @Description:
72        Controls the installation process. Calls the installation function
73        if target is specified.
74
75    @Intput:
76        <nothing>
77
78    @Return:
79        <nothing>
80    '''
81
82    os.chdir(LOCAL_PYTHON_PATH)
83    args = get_install_cmdline_arguments()
84
85    try:
86        c = ControlFile(args.controlfile)
87    except IOError:
88        print 'Could not read CONTROL file "' + args.controlfile + '"'
89        print 'Either it does not exist or its syntax is wrong.'
90        print 'Try "' + sys.argv[0].split('/')[-1] + \
91              ' -h" to print usage information'
92        exit(1)
93
94    c.assign_args_to_control(args)
95    c.check_install_conditions()
96
97    install_via_gateway(c)
98
99    return
100
101def get_install_cmdline_arguments():
102    '''
103    @Description:
104        Decomposes the command line arguments and assigns them to variables.
105        Apply default values for non mentioned arguments.
106
107    @Input:
108        <nothing>
109
110    @Return:
111        args: instance of ArgumentParser
112            Contains the commandline arguments from script/program call.
113    '''
114    parser = ArgumentParser(description='Install flex_extract software locally or \
115                            on ECMWF machines',
116                            formatter_class=ArgumentDefaultsHelpFormatter)
117
118    parser.add_argument('--target', dest='install_target', default=None,
119                        help="Valid targets: local | ecgate | cca , \
120                        the latter two are at ECMWF")
121    parser.add_argument("--makefile", dest="makefile", default=None,
122                        help='Name of Makefile to use for compiling CONVERT2')
123    parser.add_argument("--ecuid", dest="ecuid", default=None,
124                        help='user id at ECMWF')
125    parser.add_argument("--ecgid", dest="ecgid", default=None,
126                        help='group id at ECMWF')
127    parser.add_argument("--gateway", dest="gateway", default=None,
128                        help='name of local gateway server')
129    parser.add_argument("--destination", dest="destination", default=None,
130                        help='ecaccess destination, e.g. leo@genericSftp')
131
132    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
133                        default=None, help="FLEXPART root directory on ECMWF \
134                        servers (to find grib2flexpart and COMMAND file)\n\
135                        Normally flex_extract resides in the scripts directory \
136                        of the FLEXPART distribution, thus the:")
137
138    # arguments for job submission to ECMWF, only needed by submit.py
139    parser.add_argument("--job_template", dest='job_template',
140                        default="job.temp.o",
141                        help="job template file for submission to ECMWF")
142
143    parser.add_argument("--controlfile", dest="controlfile",
144                        default='CONTROL.temp',
145                        help="file with CONTROL parameters")
146
147    args = parser.parse_args()
148
149    return args
150
151
152def install_via_gateway(c):
153    '''
154    @Description:
155        Perform the actual installation on local machine or prepare data
156        transfer to remote gate and submit a job script which will
157        install everything on the remote gate.
158
159    @Input:
160        c: instance of class ControlFile
161            Contains all necessary information of a CONTROL file. The parameters
162            are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
163            NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
164            RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
165            SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
166            ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR
167            For more information about format and content of the parameter see
168            documentation.
169
170    @Return:
171        <nothing>
172    '''
173
174    ecd = c.ecmwfdatadir
175    tarball_name = 'flex_extract_v' + _VERSION_STR + '.tar'
176    target_dir = 'flex_extract_v' + _VERSION_STR
177    fortran_executable = 'CONVERT2'
178
179    if c.install_target.lower() != 'local': # ecgate or cca
180
181        mk_compilejob(ecd + 'python/compilejob.temp', c.makefile,
182                      c.install_target, c.ecuid, c.ecgid,
183                      c.flexpart_root_scripts)
184
185        mk_job_template(ecd + 'python/job.temp.o', c.ecuid, c.ecgid, c.gateway,
186                        c.destination, c.flexpart_root_scripts)
187
188        mk_env_vars(ecd, c.ecuid, c.ecgid, c.gateway, c.destination)
189
190        #os.chdir('/')
191
192        mk_tarball(ecd, tarball_name)
193
194        put_file_to_ecserver(ecd, tarball_name, c.install_target,
195                             c.ecuid, c.ecgid)
196
197        submit_job_to_ecserver(ecd + '/python/', c.install_target,
198                               'compilejob.ksh')
199
200        print 'job compilation script has been submitted to ecgate for ' + \
201              'installation in ' + c.flexpart_root_scripts + \
202               '/' + target_dir
203        print 'You should get an email with subject "flexcompile" within ' + \
204              'the next few minutes!'
205
206    else: #local
207        if not c.flexpart_root_scripts or c.flexpart_root_scripts == '../':
208            print 'WARNING: FLEXPART_ROOT_SCRIPTS has not been specified'
209            print 'There will be only the compilation of ' + \
210                  ' in ' + ecd + '/src'
211            os.chdir(ecd + '/src')
212        else: # creates the target working directory for flex_extract
213            c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
214                c.flexpart_root_scripts))
215            if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
216                os.chdir('/')
217                mk_tarball(ecd, tarball_name)
218                make_dir(c.flexpart_root_scripts + '/' + target_dir)
219                os.chdir(c.flexpart_root_scripts + '/' + target_dir)
220                print 'Untar ...'
221                subprocess.check_call(['tar', '-xvf',
222                                       ecd + '../' + tarball_name])
223                os.chdir(c.flexpart_root_scripts + '/' + target_dir + '/src')
224
225        # Create Fortran executable - CONVERT2
226        print 'Install ' + target_dir + ' software on ' + \
227              c.install_target + ' in directory ' + \
228              os.path.abspath(os.getcwd() + '/../') + '\n'
229
230        delete_convert_build('')
231        make_convert_build('', c.makefile, fortran_executable)
232
233    return
234
235def mk_tarball(ecd, tarname):
236    '''
237    @Description:
238        Creates a tarball from all files which need to be sent to the
239        installation directory.
240        It does not matter if this is local or remote.
241        Collects all python files, the Fortran source and makefiles,
242        the ECMWF_ENV file, the CONTROL files as well as
243        the korn shell and template files.
244
245    @Input:
246        ecd: string
247            The path were the file is to be stored.
248
249        tarname: string
250            The name of the file to send to the ECMWF server.
251
252    @Return:
253        <nothing>
254    '''
255
256    print 'Create tarball ...'
257    try:
258        subprocess.check_call(['tar -cvf '+
259                               ecd + '../' + tarname + ' ' +
260                               ecd + 'python/*py ' +
261                               ecd + 'python/CONTROL* ' +
262                               ecd + 'python/*ksh ' +
263                               ecd + 'python/*temp* ' +
264                               ecd + 'python/ECMWF_ENV ' +
265                               ecd + 'grib_templates ' +
266                               ecd + 'src/*.f ' +
267                               ecd + 'src/*.f90 ' +
268                               ecd + 'src/*.h ' +
269                               ecd + 'src/Makefile*'], shell=True)
270    except subprocess.CalledProcessError as e:
271        print 'ERROR:'
272        print e.output
273        sys.exit('could not make installation tar ball!')
274
275    return
276
277def mk_env_vars(ecd, ecuid, ecgid, gateway, destination):
278    '''
279    @Description:
280        Creates a file named ECMWF_ENV which contains the
281        necessary environmental variables at ECMWF servers.
282
283    @Input:
284        ecd: string
285            The path were the file is to be stored.
286
287        ecuid: string
288            The user id on ECMWF server.
289
290        ecgid: string
291            The group id on ECMWF server.
292
293        gateway: string
294            The gateway server the user is using.
295
296        destination: string
297            The remote destination which is used to transfer files
298            from ECMWF server to local gateway server.
299
300    @Return:
301        <nothing>
302    '''
303
304    with open(ecd + 'python/ECMWF_ENV', 'w') as fo:
305        fo.write('ECUID ' + ecuid + '\n')
306        fo.write('ECGID ' + ecgid + '\n')
307        fo.write('GATEWAY ' + gateway + '\n')
308        fo.write('DESTINATION ' + destination + '\n')
309
310    return
311
312def mk_compilejob(template, makefile, target, ecuid, ecgid, fp_root):
313    '''
314    @Description:
315        Modifies the original job template file so that it is specified
316        for the user and the environment were it will be applied. Result
317        is stored in a new file "job.temp" in the python directory.
318
319    @Input:
320        template: string
321            File which contains the original text for the job template.
322            It must contain the complete path to the file.
323
324        makefile: string
325            Name of the makefile which should be used to compile FORTRAN
326            CONVERT2 program.
327
328        target: string
329            The target where the installation should be done, e.g. the queue.
330
331        ecuid: string
332            The user id on ECMWF server.
333
334        ecgid: string
335            The group id on ECMWF server.
336
337        fp_root: string
338           Path to the root directory of FLEXPART environment or flex_extract
339           environment.
340
341    @Return:
342        <nothing>
343    '''
344
345    with open(template) as f:
346        fdata = f.read().split('\n')
347
348    with open(template[:-4] + 'ksh', 'w') as fo:
349        for data in fdata:
350            if 'MAKEFILE=' in data:
351                data = 'export MAKEFILE=' + makefile
352            elif 'FLEXPART_ROOT_SCRIPTS=' in data:
353                if fp_root != '../':
354                    data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
355                else:
356                    data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
357            elif target.lower() != 'local':
358                if '--workdir' in data:
359                    data = '#SBATCH --workdir=/scratch/ms/' + \
360                            ecgid + '/' + ecuid
361                elif '##PBS -o' in data:
362                    data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \
363                           'flex_ecmwf.$Jobname.$Job_ID.out'
364                elif 'FLEXPART_ROOT_SCRIPTS=' in data:
365                    if fp_root != '../':
366                        data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
367                    else:
368                        data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
369            fo.write(data + '\n')
370
371    return
372
373def mk_job_template(template, ecuid, ecgid, gateway, destination, fp_root):
374    '''
375    @Description:
376        Modifies the original job template file so that it is specified
377        for the user and the environment were it will be applied. Result
378        is stored in a new file "job.temp" in the python directory.
379
380    @Input:
381        template: string
382            File which contains the original text for the job template.
383            It must contain the complete path to the file.
384
385        ecuid: string
386            The user id on ECMWF server.
387
388        ecgid: string
389            The group id on ECMWF server.
390
391        gateway: string
392            The gateway server the user is using.
393
394        destination: string
395            The remote destination which is used to transfer files
396            from ECMWF server to local gateway server.
397
398        fp_root: string
399           Path to the root directory of FLEXPART environment or flex_extract
400           environment.
401
402    @Return:
403        <nothing>
404    '''
405
406    with open(template) as f:
407        fdata = f.read().split('\n')
408
409    with open(template[:-2], 'w') as fo:
410        for data in fdata:
411            if '--workdir' in data:
412                data = '#SBATCH --workdir=/scratch/ms/' + ecgid + \
413                        '/' + ecuid
414            elif '##PBS -o' in data:
415                data = '##PBS -o /scratch/ms/' + ecgid + '/' + \
416                        ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out'
417            elif  'export PATH=${PATH}:' in data:
418                data += fp_root + '/flex_extract_v7.1/python'
419
420            fo.write(data + '\n')
421    return
422
423def delete_convert_build(ecd):
424    '''
425    @Description:
426        Clean up the Fortran source directory and remove all
427        build files (e.g. *.o, *.mod and CONVERT2)
428
429    @Input:
430        ecd: string
431            The path to the Fortran program.
432
433    @Return:
434        <nothing>
435    '''
436
437    modfiles = UioFiles(ecd, '*.mod')
438    objfiles = UioFiles(ecd, '*.o')
439    exefile = UioFiles(ecd, 'CONVERT2')
440
441    modfiles.delete_files()
442    objfiles.delete_files()
443    exefile.delete_files()
444
445    return
446
447def make_convert_build(ecd, makefile, f_executable):
448    '''
449    @Description:
450        Compiles the Fortran code and generates the executable.
451
452    @Input:
453        ecd: string
454            The path were the file is to be stored.
455
456        makefile: string
457            The name of the makefile which should be used.
458
459        f_executable: string
460            The name of the executable the Fortran program generates after
461            compilation.
462
463    @Return:
464        <nothing>
465    '''
466
467    try:
468        print 'Using makefile: ' + makefile
469        p = subprocess.Popen(['make', '-f', ecd + makefile],
470                             stdin=subprocess.PIPE,
471                             stdout=subprocess.PIPE,
472                             stderr=subprocess.PIPE,
473                             bufsize=1)
474        pout, perr = p.communicate()
475        print pout
476        if p.returncode != 0:
477            print perr
478            print 'Please edit ' + makefile + \
479                  ' or try another Makefile in the src directory.'
480            print 'Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB ' \
481                  'and EMOSLIB must be adapted.'
482            print 'Available Makefiles:'
483            print UioFiles('.', 'Makefile*')
484            sys.exit('Compilation failed!')
485    except ValueError as e:
486        print 'ERROR: Makefile call failed:'
487        print e
488    else:
489        subprocess.check_call(['ls', '-l', ecd + f_executable])
490
491    return
492
493
494if __name__ == "__main__":
495    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG