source: flex_extract.git/python/install.py @ efdb01a

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#************************************************************************
# TODO AP
#AP
# - Functionality Provided is not correct for this file
# - localpythonpath should not be set in module load section!
# - create a class Installation and divide installation in 3 subdefs for
#   ecgate, local and cca seperatly
# - Change History ist nicht angepasst ans File! Original geben lassen
#************************************************************************
"""
@Author: Anne Fouilloux (University of Oslo)

@Date: October 2014

@ChangeHistory:
    November 2015 - Leopold Haimberger (University of Vienna):
        - using the WebAPI also for general MARS retrievals
        - job submission on ecgate and cca
        - job templates suitable for twice daily operational dissemination
        - dividing retrievals of longer periods into digestable chunks
        - retrieve also longer term forecasts, not only analyses and
          short term forecast data
        - conversion into GRIB2
        - conversion into .fp format for faster execution of FLEXPART

    February 2018 - Anne Philipp (University of Vienna):
        - applied PEP8 style guide
        - added documentation

@License:
    (C) Copyright 2014 UIO.

    This software is licensed under the terms of the Apache Licence Version 2.0
    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.

@Requirements:
    - A standard python 2.6 or 2.7 installation
    - dateutils
    - matplotlib (optional, for debugging)
    - ECMWF specific packages, all available from https://software.ecmwf.int/
        ECMWF WebMARS, gribAPI with python enabled, emoslib and
        ecaccess web toolkit

@Description:
    Further documentation may be obtained from www.flexpart.eu.

    Functionality provided:
        Prepare input 3D-wind fields in hybrid coordinates +
        surface fields for FLEXPART runs
"""
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
import calendar
import shutil
import datetime
import time
import os,sys,glob
import subprocess
import inspect
# add path to submit.py to pythonpath so that python finds its buddies
localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
sys.path.append(localpythonpath)
from UIOFiles import UIOFiles
from string import strip
from argparse import ArgumentParser,ArgumentDefaultsHelpFormatter
from GribTools import GribTools
from Control import Control
from getMARSdata import getMARSdata
from prepareFLEXPART import prepareFLEXPART
from ECFlexpart import ECFlexpart

# ------------------------------------------------------------------------------
# FUNCTIONS
# ------------------------------------------------------------------------------
def install_args_and_control():
    '''
    @Description:
        Assigns the command line arguments for installation and reads
        control file content. Apply default values for non mentioned arguments.

    @Input:
        <nothing>

    @Return:
        args: instance of ArgumentParser
            Contains the commandline arguments from script/program call.

        c: instance of class Control
            Contains all necessary information of a control file. The parameters
            are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
            NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
            RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
            SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
            ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR
            For more information about format and content of the parameter see
            documentation.
    '''
    parser = ArgumentParser(description='Install ECMWFDATA software locally or \
                            on ECMWF machines',
                            formatter_class=ArgumentDefaultsHelpFormatter)

    parser.add_argument('--target', dest='install_target',
                        help="Valid targets: local | ecgate | cca , \
                        the latter two are at ECMWF")
    parser.add_argument("--makefile", dest="makefile",
                        help='Name of Makefile to use for compiling CONVERT2')
    parser.add_argument("--ecuid", dest="ecuid",
                        help='user id at ECMWF')
    parser.add_argument("--ecgid", dest="ecgid",
                        help='group id at ECMWF')
    parser.add_argument("--gateway", dest="gateway",
                        help='name of local gateway server')
    parser.add_argument("--destination", dest="destination",
                        help='ecaccess destination, e.g. leo@genericSftp')

    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
                        help="FLEXPART root directory on ECMWF servers \
                        (to find grib2flexpart and COMMAND file)\n\
                        Normally ECMWFDATA resides in the scripts directory \
                        of the FLEXPART distribution, thus the:")

# arguments for job submission to ECMWF, only needed by submit.py
    parser.add_argument("--job_template", dest='job_template',
                        default="job.temp.o",
                        help="job template file for submission to ECMWF")

    parser.add_argument("--controlfile", dest="controlfile",
                        default='CONTROL.temp',
                        help="file with control parameters")

    args = parser.parse_args()

    try:
        c = Control(args.controlfile)
    except:
        print('Could not read control file "' + args.controlfile + '"')
        print('Either it does not exist or its syntax is wrong.')
        print('Try "' + sys.argv[0].split('/')[-1] +
              ' -h" to print usage information')
        exit(1)

    if args.install_target != 'local':
        if (args.ecgid is None or args.ecuid is None or args.gateway is None
            or args.destination is None):
            print('Please enter your ECMWF user id and group id as well as \
                   the \nname of the local gateway and the ectrans \
                   destination ')
            print('with command line options --ecuid --ecgid \
                   --gateway --destination')
            print('Try "' + sys.argv[0].split('/')[-1] +
                  ' -h" to print usage information')
            print('Please consult ecaccess documentation or ECMWF user support \
                   for further details')
            sys.exit(1)
        else:
            c.ecuid = args.ecuid
            c.ecgid = args.ecgid
            c.gateway = args.gateway
            c.destination = args.destination

    try:
        c.makefile = args.makefile
    except:
        pass

    if args.install_target == 'local':
        if args.flexpart_root_scripts is None:
            c.flexpart_root_scripts = '../'
        else:
            c.flexpart_root_scripts = args.flexpart_root_scripts

    if args.install_target != 'local':
        if args.flexpart_root_scripts is None:
            c.ec_flexpart_root_scripts = '${HOME}'
        else:
            c.ec_flexpart_root_scripts = args.flexpart_root_scripts

    return args, c


def main():
    '''
    '''
    os.chdir(localpythonpath)
    args, c = install_args_and_control()
    if args.install_target is not None:
        install_via_gateway(c, args.install_target)
    else:
        print('Please specify installation target (local|ecgate|cca)')
        print('use -h or --help for help')
    sys.exit()

def install_via_gateway(c, target):

    ecd = c.ecmwfdatadir
    template = ecd + 'python/compilejob.temp'
    job = ecd + 'python/compilejob.ksh'
    fo = open(job, 'w')
#AP could do with open(template) as f, open(job, 'w') as fo:
#AP or nested with statements
    with open(template) as f:
        fdata = f.read().split('\n')
        for data in fdata:
            if 'MAKEFILE=' in data:
                if c.makefile is not None:
                    data = 'export MAKEFILE=' + c.makefile
            if 'FLEXPART_ROOT_SCRIPTS=' in data:
                if c.flexpart_root_scripts != '../':
                    data = 'export FLEXPART_ROOT_SCRIPTS=' + \
                            c.flexpart_root_scripts
                else:
                    data='export FLEXPART_ROOT_SCRIPTS=$HOME'
            if target.lower() != 'local':
                if '--workdir' in data:
                    data = '#SBATCH --workdir=/scratch/ms/' + c.ecgid + \
                            '/' + c.ecuid
                if '##PBS -o' in data:
                    data = '##PBS -o /scratch/ms/' + c.ecgid + '/' + c.ecuid + \
                            'flex_ecmwf.$Jobname.$Job_ID.out'
                if 'FLEXPART_ROOT_SCRIPTS=' in data:
                    if c.ec_flexpart_root_scripts != '../':
                        data = 'export FLEXPART_ROOT_SCRIPTS=' + \
                                c.ec_flexpart_root_scripts
                    else:
                        data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
            fo.write(data + '\n')
    f.close()
    fo.close()

    if target.lower() != 'local':
        template = ecd + 'python/job.temp.o'
#AP hier eventuell Zeile für Zeile lesen und dann if Entscheidung
        with open(template) as f:
            fdata = f.read().split('\n')
        f.close()
        fo = open(template[:-2], 'w')
        for data in fdata:
            if '--workdir' in data:
                data = '#SBATCH --workdir=/scratch/ms/' + c.ecgid + \
                        '/' + c.ecuid
            if '##PBS -o' in data:
                data = '##PBS -o /scratch/ms/' + c.ecgid + '/' + \
                        c.ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out'
            if  'export PATH=${PATH}:' in data:
                data += c.ec_flexpart_root_scripts + '/ECMWFDATA7.1/python'
            if 'cat>>' in data or 'cat >>' in data:
                i = data.index('>')
                fo.write(data[:i] + data[i+1:] + '\n')
                fo.write('GATEWAY ' + c.gateway + '\n')
                fo.write('DESTINATION ' + c.destination + '\n')
                fo.write('EOF\n')

            fo.write(data + '\n')
        fo.close()

        job = ecd + 'python/ECMWF_ENV'
        with open(job, 'w') as fo:
            fo.write('ECUID ' + c.ecuid + '\n')
            fo.write('ECGID ' + c.ecgid + '\n')
            fo.write('GATEWAY ' + c.gateway + '\n')
            fo.write('DESTINATION ' + c.destination + '\n')
        fo.close()



    if target.lower() == 'local':
        # compile CONVERT2
        if c.flexpart_root_scripts is None or c.flexpart_root_scripts == '../':
            print('Warning: FLEXPART_ROOT_SCRIPTS has not been specified')
            print('Only CONVERT2 will be compiled in ' + ecd + '/../src')
        else:
            c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
                                        c.flexpart_root_scripts))
            if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
                os.chdir('/')
                p = subprocess.check_call(['tar', '-cvf',
                                           ecd + '../ECMWFDATA7.1.tar',
                                           ecd + 'python',
                                           ecd + 'grib_templates',
                                           ecd + 'src'])
                try:
                    os.makedirs(c.flexpart_root_scripts + '/ECMWFDATA7.1')
                except:
                    pass
                os.chdir(c.flexpart_root_scripts + '/ECMWFDATA7.1')
                p = subprocess.check_call(['tar', '-xvf',
                                           ecd + '../ECMWFDATA7.1.tar'])
                os.chdir(c.flexpart_root_scripts + '/ECMWFDATA7.1/src')

        os.chdir('../src')
        print(('install ECMWFDATA7.1 software on ' + target + ' in directory '
               + os.getcwd()))
        if c.makefile is None:
            makefile = 'Makefile.local.ifort'
        else:
            makefile = c.makefile
        flist = glob.glob('*.mod') + glob.glob('*.o')
        if flist:
            p = subprocess.check_call(['rm'] + flist)
        try:
            print(('Using makefile: ' + makefile))
            p = subprocess.check_call(['make', '-f', makefile])
            p = subprocess.check_call(['ls', '-l','CONVERT2'])
        except:
            print('compile failed - please edit ' + makefile +
                  ' or try another Makefile in the src directory.')
            print('most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
                    'and EMOSLIB must be adapted.')
            print('Available Makefiles:')
            print(glob.glob('Makefile*'))

    elif target.lower() == 'ecgate':
        os.chdir('/')
        p = subprocess.check_call(['tar', '-cvf',
                                   ecd + '../ECMWFDATA7.1.tar',
                                   ecd + 'python',
                                   ecd + 'grib_templates',
                                   ecd + 'src'])
        try:
            p = subprocess.check_call(['ecaccess-file-put',
                                       ecd + '../ECMWFDATA7.1.tar',
                                       'ecgate:/home/ms/' + c.ecgid + '/' +
                                       c.ecuid + '/ECMWFDATA7.1.tar'])
        except:
            print('ecaccess-file-put failed! Probably the eccert key has expired.')
            exit(1)
        p = subprocess.check_call(['ecaccess-job-submit',
                                   '-queueName',
                                   target,
                                   ecd + 'python/compilejob.ksh'])
        print('compilejob.ksh has been submitted to ecgate for '
                'installation in ' + c.ec_flexpart_root_scripts +
                '/ECMWFDATA7.1')
        print('You should get an email with subject flexcompile within '
                'the next few minutes')

    elif target.lower() == 'cca':
        os.chdir('/')
        p = subprocess.check_call(['tar', '-cvf',
                                   ecd + '../ECMWFDATA7.1.tar',
                                   ecd + 'python',
                                   ecd + 'grib_templates',
                                   ecd + 'src'])
        try:
            p = subprocess.check_call(['ecaccess-file-put',
                                       ecd + '../ECMWFDATA7.1.tar',
                                       'cca:/home/ms/' + c.ecgid + '/' +
                                       c.ecuid + '/ECMWFDATA7.1.tar'])
        except:
            print('ecaccess-file-put failed! '
                    'Probably the eccert key has expired.')
            exit(1)

        p=subprocess.check_call(['ecaccess-job-submit',
                                '-queueName',
                                target,
                                ecd + 'python/compilejob.ksh'])
        print('compilejob.ksh has been submitted to cca for installation in ' +
              c.ec_flexpart_root_scripts + '/ECMWFDATA7.1')
        print('You should get an email with subject flexcompile '
                'within the next few minutes')

    else:
        print('ERROR: unknown installation target ', target)
        print('Valid targets: ecgate, cca, local')

    return


if __name__ == "__main__":
    main()