Index: python/pythontest/TestInstallTar/test_untar/python/ControlFile.py
===================================================================
--- python/pythontest/TestInstallTar/test_untar/python/ControlFile.py	(revision 2fb99de9b96e65c8b5caa85c276ad8e0be540ed0)
+++ python/pythontest/TestInstallTar/test_untar/python/ControlFile.py	(revision 2fb99de9b96e65c8b5caa85c276ad8e0be540ed0)
@@ -0,0 +1,534 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#*******************************************************************************
+# @Author: Leopold Haimberger (University of Vienna)
+#
+# @Date: November 2015
+#
+# @Change History:
+#
+#   February 2018 - Anne Philipp (University of Vienna):
+#        - applied PEP8 style guide
+#        - added documentation
+#        - applied some minor modifications in programming style/structure
+#        - changed name of class Control to ControlFile for more
+#          self-explanation naming
+#        - outsource of class ControlFile
+#        - initialisation of class attributes ( to avoid high number of
+#          conditional statements and set default values )
+#        - divided assignment of attributes and the check of conditions
+#        - outsourced the commandline argument assignments to control attributes
+#
+# @License:
+#    (C) Copyright 2015-2018.
+#
+#    This software is licensed under the terms of the Apache Licence Version 2.0
+#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# @Class Description:
+#    The CONTROL file is the steering part of the FLEXPART extraction
+#    software. All necessary parameters needed to retrieve the data fields
+#    from the MARS archive for driving FLEXPART are set in a CONTROL file.
+#    Some specific parameters like the start and end dates can be overwritten
+#    by the command line parameters, but in generel all parameters needed
+#    for a complete set of fields for FLEXPART can be set in the CONTROL file.
+#
+# @Class Content:
+#    - __init__
+#    - __read_controlfile__
+#    - __str__
+#    - assign_args_to_control
+#    - assign_envs_to_control
+#    - check_conditions
+#    - check_install_conditions
+#    - to_list
+#
+# @Class Attributes:
+#
+#
+#*******************************************************************************
+
+# ------------------------------------------------------------------------------
+# MODULES
+# ------------------------------------------------------------------------------
+import os
+import re
+import sys
+import inspect
+
+import _config
+
+# ------------------------------------------------------------------------------
+# CLASS
+# ------------------------------------------------------------------------------
+class ControlFile(object):
+    '''
+    Class containing the information of the flex_extract CONTROL file.
+
+    Contains all the parameters of CONTROL file, which are e.g.:
+    DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
+    STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
+    LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
+    OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
+    ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
+    MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR,
+    BASETIME, DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
+
+    For more information about format and content of the parameter
+    see documentation.
+
+    '''
+
+    def __init__(self, filename):
+        '''
+        @Description:
+            Initialises the instance of ControlFile class and defines
+            all class attributes with default values. Afterwards calls
+            function __read_controlfile__ to read parameter from
+            Control file.
+
+        @Input:
+            self: instance of ControlFile class
+                Description see class documentation.
+
+            filename: string
+                Name of CONTROL file.
+
+        @Return:
+            <nothing>
+        '''
+
+        # list of all possible class attributes and their default values
+        self.controlfile = filename
+        self.start_date = None
+        self.end_date = None
+        self.date_chunk = 3
+        self.dtime = None
+        self.basetime = None
+        self.maxstep = None
+        self.type = None
+        self.time = None
+        self.step = None
+        self.marsclass = None
+        self.stream = None
+        self.number = 'OFF'
+        self.expver = None
+        self.grid = None
+        self.area = ''
+        self.left = None
+        self.lower = None
+        self.upper = None
+        self.right = None
+        self.level = None
+        self.levelist = None
+        self.resol = None
+        self.gauss = 0
+        self.accuracy = 24
+        self.omega = 0
+        self.omegadiff = 0
+        self.eta = 0
+        self.etadiff = 0
+        self.etapar = 77
+        self.dpdeta = 1
+        self.smooth = 0
+        self.format = 'GRIB1'
+        self.addpar = None
+        self.prefix = 'EN'
+        self.cwc = 0
+        self.wrf = 0
+        self.ecfsdir = 'ectmp:/${USER}/econdemand/'
+        self.mailfail = ['${USER}']
+        self.mailops = ['${USER}']
+        self.grib2flexpart = 0
+        self.ecstorage = 0
+        self.ectrans = 0
+        self.inputdir = '../work'
+        self.outputdir = self.inputdir
+        self.ecmwfdatadir = None
+        self.exedir = None
+        self.flexpart_root_scripts = None
+        self.makefile = None
+        self.destination = None
+        self.gateway = None
+        self.ecuid = None
+        self.ecgid = None
+        self.install_target = None
+        self.debug = 0
+        self.request = 0
+
+        self.__read_controlfile__()
+
+        return
+
+    def __read_controlfile__(self):
+        '''
+        @Description:
+            Read CONTROL file and assign all CONTROL file variables.
+
+        @Input:
+            self: instance of ControlFile class
+                Description see class documentation.
+
+        @Return:
+            <nothing>
+        '''
+        from tools import my_error
+
+        # read whole CONTROL file
+        with open(self.controlfile) as f:
+            fdata = f.read().split('\n')
+
+        # go through every line and store parameter
+        for ldata in fdata:
+            data = ldata.split()
+            if len(data) > 1:
+                if 'm_' in data[0].lower():
+                    data[0] = data[0][2:]
+                if data[0].lower() == 'class':
+                    data[0] = 'marsclass'
+                if data[0].lower() == 'day1':
+                    data[0] = 'start_date'
+                if data[0].lower() == 'day2':
+                    data[0] = 'end_date'
+                if data[0].lower() == 'addpar':
+                    if '/' in data[1]:
+                        # remove leading '/' sign from addpar content
+                        if data[1][0] == '/':
+                            data[1] = data[1][1:]
+                        dd = data[1].split('/')
+                        data = [data[0]]
+                        for d in dd:
+                            data.append(d)
+                if len(data) == 2:
+                    if '$' in data[1]:
+                        setattr(self, data[0].lower(), data[1])
+                        while '$' in data[1]:
+                            i = data[1].index('$')
+                            j = data[1].find('{')
+                            k = data[1].find('}')
+                            var = os.getenv(data[1][j+1:k])
+                            if var is not None:
+                                data[1] = data[1][:i] + var + data[1][k+1:]
+                            else:
+                                my_error(self.mailfail,
+                                         'Could not find variable '
+                                         + data[1][j+1:k] + ' while reading ' +
+                                         self.controlfile)
+                        setattr(self, data[0].lower() + '_expanded', data[1])
+                    else:
+                        if data[1].lower() != 'none':
+                            setattr(self, data[0].lower(), data[1])
+                        else:
+                            setattr(self, data[0].lower(), None)
+                elif len(data) > 2:
+                    setattr(self, data[0].lower(), (data[1:]))
+            else:
+                pass
+
+        # script directory
+        self.ecmwfdatadir = os.path.dirname(os.path.abspath(inspect.getfile(
+            inspect.currentframe()))) + '/../'
+
+        # Fortran source directory
+        self.exedir = self.ecmwfdatadir + 'src/'
+
+        return
+
+    def __str__(self):
+        '''
+        @Description:
+            Prepares a string which have all the ControlFile
+            class attributes with its associated values.
+            Each attribute is printed in one line and in
+            alphabetical order.
+
+            Example:
+            'age': 10
+            'color': 'Spotted'
+            'kids': 0
+            'legs': 2
+            'name': 'Dog'
+            'smell': 'Alot'
+
+        @Input:
+            self: instance of ControlFile class
+                Description see class documentation.
+
+        @Return:
+            string of ControlFile class attributes with their values
+        '''
+        import collections
+
+        attrs = vars(self)
+        attrs = collections.OrderedDict(sorted(attrs.items()))
+
+        return '\n'.join("%s: %s" % item for item in attrs.items())
+
+    def assign_args_to_control(self, args):
+        '''
+        @Description:
+            Overwrites the existing ControlFile instance attributes with
+            the command line arguments.
+
+        @Input:
+            self: instance of ControlFile class
+                Description see class documentation.
+
+            args: instance of ArgumentParser
+                Contains the commandline arguments from script/program call.
+
+        @Return:
+            <nothing>
+        '''
+
+        # get dictionary of command line parameters and eliminate all
+        # parameters which are None (were not specified)
+        args_dict = vars(args)
+        arguments = {k : args_dict[k] for k in args_dict
+                     if args_dict[k] != None}
+
+        # assign all passed command line arguments to ControlFile instance
+        for k, v in arguments.iteritems():
+            setattr(self, str(k), v)
+
+        return
+
+    def assign_envs_to_control(self, envs):
+        '''
+        @Description:
+            Assigns the ECMWF environment parameter.
+
+        @Input:
+            envs: dict of strings
+                Contains the ECMWF environment parameternames "ECUID", "ECGID",
+                "DESTINATION" and "GATEWAY" with its corresponding values.
+                They were read from the file "ECMWF_ENV".
+
+        @Return:
+            <nothing>
+        '''
+
+        for k, v in envs.iteritems():
+            setattr(self, str(k).lower(), str(v))
+
+        return
+
+    def check_conditions(self):
+        '''
+        @Description:
+            Checks a couple of necessary attributes and conditions,
+            such as if they exist and contain values.
+            Otherwise set default values.
+
+        @Input:
+            self: instance of ControlFile class
+                Description see class documentation.
+
+        @Return:
+            <nothing>
+        '''
+        from tools import my_error
+        import numpy as np
+
+        # check for having at least a starting date
+        # otherwise program is not allowed to run
+        if self.start_date is None:
+            print 'start_date specified neither in command line nor ' + \
+                  'in CONTROL file ' +  self.controlfile
+            print 'Try "' + sys.argv[0].split('/')[-1] + \
+                  ' -h" to print usage information'
+            sys.exit(1)
+
+        # retrieve just one day if end_date isn't set
+        if self.end_date is None:
+            self.end_date = self.start_date
+
+        # assure consistency of levelist and level
+        if self.levelist is None:
+            if self.level is None:
+                print 'Warning: neither levelist nor level ' + \
+                      'specified in CONTROL file'
+                sys.exit(1)
+            else:
+                self.levelist = '1/to/' + self.level
+        else:
+            if 'to' in self.levelist.lower():
+                self.level = self.levelist.split('/')[2]
+            else:
+                self.level = self.levelist.split('/')[-1]
+
+        # if area was provided at command line
+        # decompse area into its 4 components
+        if self.area:
+            afloat = '.' in self.area
+            l = self.area.split('/')
+            if afloat:
+                for i, item in enumerate(l):
+                    item = str(int(float(item) * 1000))
+            self.upper, self.left, self.lower, self.right = l
+
+        # prepare step for correct usage
+        if '/' in self.step:
+            l = self.step.split('/')
+            if 'to' in self.step.lower():
+                if 'by' in self.step.lower():
+                    ilist = np.arange(int(l[0]), int(l[2]) + 1, int(l[4]))
+                    self.step = ['{:0>3}'.format(i) for i in ilist]
+                else:
+                    my_error(self.mailfail, self.step + ':\n' +
+                             'if "to" is used, please use "by" as well')
+            else:
+                self.step = l
+
+        # if maxstep wasn't provided
+        # search for it in the "step" parameter
+        if self.maxstep is None:
+            self.maxstep = 0
+            for s in self.step:
+                if int(s) > self.maxstep:
+                    self.maxstep = int(s)
+        else:
+            self.maxstep = int(self.maxstep)
+
+        # set root scripts since it is needed later on
+        if not self.flexpart_root_scripts:
+            self.flexpart_root_scripts = self.ecmwfdatadir
+
+        if not isinstance(self.mailfail, list):
+            if ',' in self.mailfail:
+                self.mailfail = self.mailfail.split(',')
+            elif ' ' in self.mailfail:
+                self.mailfail = self.mailfail.split()
+            else:
+                self.mailfail = [self.mailfail]
+
+        if not isinstance(self.mailops, list):
+            if ',' in self.mailops:
+                self.mailops = self.mailops.split(',')
+            elif ' ' in self.mailops:
+                self.mailops = self.mailops.split()
+            else:
+                self.mailops = [self.mailops]
+
+        if not self.gateway or not self.destination or \
+           not self.ecuid or not self.ecgid:
+            print '\nEnvironment variables GATWAY, DESTINATION, ECUID and ' + \
+                  'ECGID were not set properly!'
+            print 'Please check for excistence of file "ECMWF_ENV" in the ' + \
+                  'python directory!'
+            sys.exit(1)
+
+        if self.request != 0:
+            marsfile = os.path.join(_config.PATH_RUN_DIR + os.path.sep +
+                                    _config.FILE_MARS_REQUESTS)
+            if os.path.isfile(marsfile):
+                os.remove(marsfile)
+
+        # check logical variables for data type
+        # if its a string change to integer
+        logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
+                    'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
+                    'ectrans', 'debug', 'request']
+
+        for var in logicals:
+            if not isinstance(getattr(self, var), int):
+                setattr(self, var, int(getattr(self, var)))
+
+        return
+
+    def check_install_conditions(self):
+        '''
+        @Description:
+            Checks a couple of necessary attributes and conditions
+            for the installation such as if they exist and contain values.
+            Otherwise set default values.
+
+        @Input:
+            self: instance of ControlFile class
+                Description see class documentation.
+
+        @Return:
+            <nothing>
+        '''
+
+        if self.install_target and \
+           self.install_target not in ['local', 'ecgate', 'cca']:
+            print('ERROR: unknown or missing installation target ')
+            print('target: ', self.install_target)
+            print('please specify correct installation target ' +
+                  '(local | ecgate | cca)')
+            print('use -h or --help for help')
+            sys.exit(1)
+
+        if self.install_target and self.install_target != 'local':
+            if not self.ecgid or not self.ecuid or \
+               not self.gateway or not self.destination:
+                print('Please enter your ECMWF user id and group id as well ' +
+                      'as the \nname of the local gateway and the ectrans ' +
+                      'destination ')
+                print('with command line options --ecuid --ecgid \
+                       --gateway --destination')
+                print('Try "' + sys.argv[0].split('/')[-1] + \
+                      ' -h" to print usage information')
+                print('Please consult ecaccess documentation or ECMWF user \
+                       support for further details')
+                sys.exit(1)
+
+            if not self.flexpart_root_scripts:
+                self.flexpart_root_scripts = '${HOME}'
+            else:
+                self.flexpart_root_scripts = self.flexpart_root_scripts
+        else: # local
+            if not self.flexpart_root_scripts:
+                self.flexpart_root_scripts = '../'
+
+        if not self.makefile:
+            self.makefile = 'Makefile.gfortran'
+
+        return
+
+    def to_list(self):
+        '''
+        @Description:
+            Just generates a list of strings containing the attributes and
+            assigned values except the attributes "_expanded", "exedir",
+            "ecmwfdatadir" and "flexpart_root_scripts".
+
+        @Input:
+            self: instance of ControlFile class
+                Description see class documentation.
+
+        @Return:
+            l: list
+                A sorted list of the all ControlFile class attributes with
+                their values except the attributes "_expanded", "exedir",
+                "ecmwfdatadir" and "flexpart_root_scripts".
+        '''
+
+        import collections
+
+        attrs = collections.OrderedDict(sorted(vars(self).items()))
+
+        l = list()
+
+        for item in attrs.items():
+            if '_expanded' in item[0]:
+                pass
+            elif 'exedir' in item[0]:
+                pass
+            elif 'flexpart_root_scripts' in item[0]:
+                pass
+            elif 'ecmwfdatadir' in item[0]:
+                pass
+            else:
+                if isinstance(item[1], list):
+                    stot = ''
+                    for s in item[1]:
+                        stot += s + ' '
+
+                    l.append("%s %s" % (item[0], stot))
+                else:
+                    l.append("%s %s" % item)
+
+        return sorted(l)
+
