source: flex_extract.git/source/python/classes/ControlFile.py @ 96e1533

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Leopold Haimberger (University of Vienna)
#
# @Date: November 2015
#
# @Change History:
#
#   February 2018 - Anne Philipp (University of Vienna):
#        - applied PEP8 style guide
#        - added documentation
#        - applied some minor modifications in programming style/structure
#        - changed name of class Control to ControlFile for more
#          self-explanation naming
#        - outsource of class ControlFile
#        - initialisation of class attributes ( to avoid high number of
#          conditional statements and set default values )
#        - divided assignment of attributes and the check of conditions
#        - outsourced the commandline argument assignments to control attributes
#
# @License:
#    (C) Copyright 2015-2018.
#
#    This software is licensed under the terms of the Apache Licence Version 2.0
#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# @Class Description:
#    The CONTROL file is the steering part of the FLEXPART extraction
#    software. All necessary parameters needed to retrieve the data fields
#    from the MARS archive for driving FLEXPART are set in a CONTROL file.
#    Some specific parameters like the start and end dates can be overwritten
#    by the command line parameters, but in generel all parameters needed
#    for a complete set of fields for FLEXPART can be set in the CONTROL file.
#
# @Class Content:
#    - __init__
#    - __read_controlfile__
#    - __str__
#    - assign_args_to_control
#    - assign_envs_to_control
#    - check_conditions
#    - check_install_conditions
#    - to_list
#
# @Class Attributes:
#
#
#*******************************************************************************

# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
import os
import re
import sys
import inspect

# software specific classes and modules from flex_extract
sys.path.append('../')
import _config
from mods.tools import my_error, silent_remove

# ------------------------------------------------------------------------------
# CLASS
# ------------------------------------------------------------------------------
class ControlFile(object):
    '''
    Contains the information which are stored in the CONTROL files.
    '''

    def __init__(self, filename):
        '''Initialises the instance of ControlFile class and defines
        all class attributes with default values. Afterwards calls
        function __read_controlfile__ to read parameter from Control file.

        Parameters
        ----------
        filename : :obj:`string`
            Name of CONTROL file.

        Return
        ------

        '''

        # list of all possible class attributes and their default values
        self.controlfile = filename
        self.start_date = None
        self.end_date = None
        self.date_chunk = 3
        self.dtime = None
        self.basetime = None
        self.maxstep = None
        self.type = None
        self.time = None
        self.step = None
        self.acctype = None
        self.acctime = None
        self.accmaxstep = None
        self.marsclass = None
        self.dataset = None
        self.stream = None
        self.number = 'OFF'
        self.expver = '1'
        self.gaussian = ''
        self.grid = None
        self.area = ''
        self.left = None
        self.lower = None
        self.upper = None
        self.right = None
        self.level = None
        self.levelist = None
        self.resol = None
        self.gauss = 0
        self.accuracy = 24
        self.omega = 0
        self.omegadiff = 0
        self.eta = 0
        self.etadiff = 0
        self.etapar = 77
        self.dpdeta = 1
        self.smooth = 0
        self.format = 'GRIB1'
        self.addpar = None
        self.prefix = 'EN'
        self.cwc = 0
        self.wrf = 0
        self.ecfsdir = 'ectmp:/${USER}/econdemand/'
        self.mailfail = ['${USER}']
        self.mailops = ['${USER}']
        self.grib2flexpart = 0
        self.ecstorage = 0
        self.ectrans = 0
        self.inputdir = _config.PATH_INPUT_DIR
        self.outputdir = None
        self.ecmwfdatadir = _config.PATH_FLEXEXTRACT_DIR
        self.exedir = _config.PATH_FORTRAN_SRC
        self.flexpart_root_scripts = None
        self.makefile = 'Makefile.gfortran'
        self.destination = None
        self.gateway = None
        self.ecuid = None
        self.ecgid = None
        self.install_target = None
        self.debug = 0
        self.request = 0
        self.public = 0
        self.ecapi = None

        self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
                         'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
                         'ectrans', 'debug', 'request', 'public']

        self.__read_controlfile__()

        return

    def __read_controlfile__(self):
        '''Read CONTROL file and assign all CONTROL file variables.

        Parameters
        ----------

        Return
        ------

        '''

        try:
            cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
            with open(cfile) as f:
                fdata = f.read().split('\n')
        except IOError:
            print('Could not read CONTROL file "' + cfile + '"')
            print('Either it does not exist or its syntax is wrong.')
            print('Try "' + sys.argv[0].split('/')[-1] + \
                      ' -h" to print usage information')
            sys.exit(1)

        # go through every line and store parameter
        for ldata in fdata:
            data = ldata.split()
            if len(data) > 1:
                if 'm_' in data[0].lower():
                    data[0] = data[0][2:]
                if data[0].lower() == 'class':
                    data[0] = 'marsclass'
                if data[0].lower() == 'day1':
                    data[0] = 'start_date'
                if data[0].lower() == 'day2':
                    data[0] = 'end_date'
                if data[0].lower() == 'addpar':
                    if '/' in data[1]:
                        # remove leading '/' sign from addpar content
                        if data[1][0] == '/':
                            data[1] = data[1][1:]
                        dd = data[1].split('/')
                        data = [data[0]]
                        for d in dd:
                            data.append(d)
                if len(data) == 2:
                    if '$' in data[1]:
                        setattr(self, data[0].lower(), data[1])
                        while '$' in data[1]:
                            i = data[1].index('$')
                            j = data[1].find('{')
                            k = data[1].find('}')
                            var = os.getenv(data[1][j+1:k])
                            if var is not None:
                                data[1] = data[1][:i] + var + data[1][k+1:]
                            else:
                                my_error(self.mailfail,
                                         'Could not find variable '
                                         + data[1][j+1:k] + ' while reading ' +
                                         self.controlfile)
                        setattr(self, data[0].lower() + '_expanded', data[1])
                    else:
                        if data[1].lower() != 'none':
                            setattr(self, data[0].lower(), data[1])
                        else:
                            setattr(self, data[0].lower(), None)
                elif len(data) > 2:
                    setattr(self, data[0].lower(), (data[1:]))
            else:
                pass

        return

    def __str__(self):
        '''Prepares a string which have all the ControlFile class attributes
        with its associated values. Each attribute is printed in one line and
        in alphabetical order.

        Example
        -------
        'age': 10
        'color': 'Spotted'
        'kids': 0
        'legs': 2
        'name': 'Dog'
        'smell': 'Alot'

        Parameters
        ----------

        Return
        ------
        string
            Single string of concatenated ControlFile class attributes
            with their values
        '''
        import collections

        attrs = vars(self).copy()
        attrs = collections.OrderedDict(sorted(attrs.items()))

        return '\n'.join("%s: %s" % item for item in attrs.items())

    def assign_args_to_control(self, args):
        '''Overwrites the existing ControlFile instance attributes with
        the command line arguments.

        Parameters
        ----------
        args : :obj:`Namespace`
            Contains the commandline arguments from script/program call.

        Return
        ------

        '''

        # get dictionary of command line parameters and eliminate all
        # parameters which are None (were not specified)
        args_dict = vars(args)
        arguments = {k : args_dict[k] for k in args_dict
                     if args_dict[k] != None}

        # assign all passed command line arguments to ControlFile instance
        for k, v in arguments.iteritems():
            setattr(self, str(k), v)

        return

    def assign_envs_to_control(self, envs):
        '''Assigns the ECMWF environment parameter.

        Parameters
        ----------
        envs : :obj:`dictionary` of :obj:`strings`
            Contains the ECMWF environment parameternames "ECUID", "ECGID",
            "DESTINATION" and "GATEWAY" with its corresponding values.
            They were read from the file "ECMWF_ENV".

        Return
        ------

        '''

        for k, v in envs.iteritems():
            setattr(self, str(k).lower(), str(v))

        return

    def check_conditions(self, queue):
        '''Checks a couple of necessary attributes and conditions,
        such as if they exist and contain values.
        Otherwise set default values.

        Parameters
        ----------
        queue : :obj:`string`
            Name of the queue if submitted to the ECMWF servers.
            Used to check if ecuid, ecgid, gateway and destination
            are set correctly and are not empty.

        Return
        ------

        '''
        from mods.tools import my_error
        import numpy as np

        # check for having at least a starting date
        # otherwise program is not allowed to run
        if not self.start_date:
            print('start_date specified neither in command line nor \
                   in CONTROL file ' +  self.controlfile)
            print('Try "' + sys.argv[0].split('/')[-1] +
                  ' -h" to print usage information')
            sys.exit(1)

        # retrieve just one day if end_date isn't set
        if not self.end_date:
            self.end_date = self.start_date

        # basetime has only two possible values
        if self.basetime:
            if int(self.basetime) != 0 and int(self.basetime) != 12:
                print('Basetime has an invalid value!')
                print('Basetime = ' + str(self.basetime))
                sys.exit(1)

        # assure consistency of levelist and level
        # up-to-date available maximum level numbers at ECMWF, 05.10.2018
        max_level_list = [16, 19, 31, 40, 50, 60, 62, 91, 137]
        if not self.levelist and not self.level:
            print('Warning: neither levelist nor level \
                               specified in CONTROL file')
            sys.exit(1)
        elif not self.levelist and self.level:
            self.levelist = '1/to/' + self.level
        elif (self.levelist and not self.level) or \
             (self.levelist[-1] != self.level[-1]):
            self.level = self.levelist.split('/')[-1]
        else:
            pass

        # check if max level is a valid level
        if int(self.level) not in max_level_list:
            print('ERROR: ')
            print('LEVEL must be the maximum level of a specified '
                  'level list from ECMWF, e.g.')
            print('[16, 19, 31, 40, 50, 60, 62, 91 or 137]')
            print('Check parameter "LEVEL" or the max level of "LEVELIST"!')
            sys.exit(1)

        # if area was provided (only from commandline)
        # decompose area into its 4 components
        if self.area:
            components = self.area.split('/')
            # convert float to integer coordinates
            if '.' in self.area:
                components = [str(int(float(item) * 1000))
                              for i, item in enumerate(components)]
            self.upper, self.left, self.lower, self.right = components

        # prepare step list if "/" signs are found
        if '/' in self.step:
            steps = self.step.split('/')
            if 'to' in self.step.lower() and 'by' in self.step.lower():
                ilist = np.arange(int(steps[0]),
                                  int(steps[2]) + 1,
                                  int(steps[4]))
                self.step = ['{:0>3}'.format(i) for i in ilist]
            elif 'to' in self.step.lower() and 'by' not in self.step.lower():
                my_error(self.mailfail, self.step + ':\n' +
                         'if "to" is used in steps parameter, \
                         please use "by" as well')
            else:
                self.step = steps

        # if maxstep wasn't provided
        # search for it in the "step" parameter
        if not self.maxstep:
            self.maxstep = 0
            for s in self.step:
                if int(s) > self.maxstep:
                    self.maxstep = int(s)
        else:
            self.maxstep = int(self.maxstep)

        # set root scripts since it is needed later on
        if not self.flexpart_root_scripts:
            self.flexpart_root_scripts = self.ecmwfdatadir

        if not self.outputdir:
            self.outputdir = self.inputdir

        if not isinstance(self.mailfail, list):
            if ',' in self.mailfail:
                self.mailfail = self.mailfail.split(',')
            elif ' ' in self.mailfail:
                self.mailfail = self.mailfail.split()
            else:
                self.mailfail = [self.mailfail]

        if not isinstance(self.mailops, list):
            if ',' in self.mailops:
                self.mailops = self.mailops.split(',')
            elif ' ' in self.mailops:
                self.mailops = self.mailops.split()
            else:
                self.mailops = [self.mailops]

        if queue in _config.QUEUES_LIST and \
           not self.gateway or not self.destination or \
           not self.ecuid or not self.ecgid:
            print('\nEnvironment variables GATEWAY, DESTINATION, ECUID and \
                   ECGID were not set properly!')
            print('Please check for existence of file "ECMWF_ENV" in the \
                   python directory!')
            sys.exit(1)

        if self.request != 0:
            marsfile = os.path.join(self.inputdir,
                                    _config.FILE_MARS_REQUESTS)
            if os.path.isfile(marsfile):
                silent_remove(marsfile)

        # check all logical variables for data type
        # if its a string change to integer
        for var in self.logicals:
            if not isinstance(getattr(self, var), int):
                setattr(self, var, int(getattr(self, var)))

        if self.public and not self.dataset:
            print('ERROR: ')
            print('If public mars data wants to be retrieved, '
                  'the "dataset"-parameter has to be set in the control file!')
            sys.exit(1)

        if not isinstance(self.type, list):
            self.type = [self.type]

        for i, val in enumerate(self.type):
            if self.type[i] == 'AN' and int(self.step[i]) != 0:
                print('Analysis retrievals must have STEP = 0 (is set to 0)')
                self.type[i] = 0

        if not isinstance(self.time, list):
            self.time = [self.time]

        if not isinstance(self.step, list):
            self.step = [self.step]

        if not self.acctype:
            print('... Control paramter ACCTYPE was not defined.')
            try:
                if len(self.type) > 1 and self.type[1] != 'AN':
                    print('Use old setting by using TYPE[1] for flux forecast!')
                    self.acctype = self.type[1]
            except:
                print('Use default value "FC" for flux forecast!')
                self.acctype='FC'

        if not self.acctime:
            print('... Control paramter ACCTIME was not defined.')
            print('Use default value "00/12" for flux forecast!')
            self.acctime='00/12'

        if not self.accmaxstep:
            print('... Control paramter ACCMAXSTEP was not defined.')
            print('Use default value "12" for flux forecast!')
            self.accmaxstep='12'

        return

    def check_install_conditions(self):
        '''Checks a couple of necessary attributes and conditions
        for the installation such as if they exist and contain values.
        Otherwise set default values.

        Parameters
        ----------

        Return
        ------

        '''

        if self.install_target and \
           self.install_target not in ['local', 'ecgate', 'cca']:
            print('ERROR: unknown or missing installation target ')
            print('target: ', self.install_target)
            print('please specify correct installation target ' +
                  '(local | ecgate | cca)')
            print('use -h or --help for help')
            sys.exit(1)

        if self.install_target and self.install_target != 'local':
            if not self.ecgid or not self.ecuid or \
               not self.gateway or not self.destination:
                print('Please enter your ECMWF user id and group id as well ' +
                      'as the \nname of the local gateway and the ectrans ' +
                      'destination ')
                print('with command line options --ecuid --ecgid \
                       --gateway --destination')
                print('Try "' + sys.argv[0].split('/')[-1] + \
                      ' -h" to print usage information')
                print('Please consult ecaccess documentation or ECMWF user \
                       support for further details')
                sys.exit(1)

            if not self.flexpart_root_scripts:
                self.flexpart_root_scripts = '${HOME}'
            else:
                self.flexpart_root_scripts = self.flexpart_root_scripts
        else: # local
            if not self.flexpart_root_scripts:
                self.flexpart_root_scripts = _config.PATH_FLEXEXTRACT_DIR

        return

    def to_list(self):
        '''Just generates a list of strings containing the attributes and
        assigned values except the attributes "_expanded", "exedir",
        "ecmwfdatadir" and "flexpart_root_scripts".

        Parameters
        ----------

        Return
        ------
        l : :obj:`list`
            A sorted list of the all ControlFile class attributes with
            their values except the attributes "_expanded", "exedir",
            "ecmwfdatadir" and "flexpart_root_scripts".
        '''

        import collections

        attrs = collections.OrderedDict(sorted(vars(self).copy().items()))

        l = list()

        for item in attrs.items():
            if '_expanded' in item[0]:
                pass
            elif 'exedir' in item[0]:
                pass
            elif 'flexpart_root_scripts' in item[0]:
                pass
            elif 'ecmwfdatadir' in item[0]:
                pass
            else:
                if isinstance(item[1], list):
                    stot = ''
                    for s in item[1]:
                        stot += s + ' '

                    l.append("%s %s" % (item[0], stot))
                else:
                    l.append("%s %s" % item)

        return sorted(l)