#!/usr/bin/env python
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Fouilloux (University of Oslo)
#
# @Date: October 2014
#
# @Change History:
#
#    November 2015 - Leopold Haimberger (University of Vienna):
#        - using the WebAPI also for general MARS retrievals
#        - job submission on ecgate and cca
#        - job templates suitable for twice daily operational dissemination
#        - dividing retrievals of longer periods into digestable chunks
#        - retrieve also longer term forecasts, not only analyses and
#          short term forecast data
#        - conversion into GRIB2
#        - conversion into .fp format for faster execution of FLEXPART
#
#    February 2018 - Anne Philipp (University of Vienna):
#        - applied PEP8 style guide
#        - added documentation
#        - minor changes in programming style for consistence
#        - BUG: removed call of clean_up-Function after call of
#               prepareFlexpart in main since it is already called in
#               prepareFlexpart at the end!
#        - created function main and moved the two function calls for
#          arguments and prepare_flexpart into it
#
# @License:
#    (C) Copyright 2014-2018.
#
#    This software is licensed under the terms of the Apache Licence Version 2.0
#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# @Program Functionality:
#    This program prepares the final version of the grib files which are
#    then used by FLEXPART. It converts the bunch of grib files extracted
#    via get_mars_data by doing for example the necessary conversion to get
#    consistent grids or the disaggregation of flux data. Finally, the
#    program combines the data fields in files per available hour with the
#    naming convention xxYYMMDDHH, where xx should be 2 arbitrary letters
#    (mostly xx is chosen to be "EN").
#
# @Program Content:
#    - main
#    - prepare_flexpart
#

# ------------------------------------------------------------------------------
# MODULES
import datetime
import os
import inspect
import sys
import socket

# software specific classes and modules from flex_extract

sys.path.append(os.path.dirname(os.path.abspath(
    inspect.getfile(inspect.currentframe()))) + '/../')
import _config
from classes.UioFiles import UioFiles
from classes.ControlFile import ControlFile
from tools import clean_up, get_cmdline_arguments, read_ecenv, make_dir
from classes.EcFlexpart import EcFlexpart

ecapi = 'ecmwf' not in socket.gethostname()
try:
    if ecapi:
        import ecmwfapi
except ImportError:
    ecapi = False

# FUNCTION
def main():
    '''Controls the program to prepare flexpart input files from mars data.

    This is done if it is called directly from command line.
    Then it also takes program call arguments and control file input.

    Parameters
    ----------

    Return
    ------

    '''

    args = get_cmdline_arguments()
    c = ControlFile(args.controlfile)

    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
    c.assign_args_to_control(args)
    c.assign_envs_to_control(env_parameter)
    c.check_conditions(args.queue)

    prepare_flexpart(args.ppid, c)

    return

def prepare_flexpart(ppid, c):
    '''Converts the mars data into flexpart ready input files.

    Specific data fields are converted to a different grid and the flux
    data are going to be disaggregated. The data fields are collected by
    hour and stored in a file with a specific FLEXPART relevant naming
    convention.

    ppid : :obj:`int`
        Contains the ppid number of the current ECMWF job. It will be None if
        the method was called within this module.

    c : :obj:`ControlFile`
        Contains all the parameters of CONTROL file and
        command line.

    Return
    ------

    '''

    if not ppid:
        c.ppid = str(os.getppid())
    else:
        c.ppid = ppid

    c.ecapi = ecapi

    # create the start and end date
    start = datetime.date(year=int(c.start_date[:4]),
                          month=int(c.start_date[4:6]),
                          day=int(c.start_date[6:]))

    end = datetime.date(year=int(c.end_date[:4]),
                        month=int(c.end_date[4:6]),
                        day=int(c.end_date[6:]))

    # assign starting date minus 1 day
    # since for basetime 00 we need the 12 hours upfront
    # (the day before from 12 UTC to current day 00 UTC)
    if c.basetime == '00':
        start = start - datetime.timedelta(days=1)

    print('Prepare ' + start.strftime("%Y%m%d") +
           "/to/" + end.strftime("%Y%m%d"))

    # create output dir if necessary
    if not os.path.exists(c.outputdir):
        make_dir(c.outputdir)

    # get all files with flux data to be deaccumulated
    inputfiles = UioFiles(c.inputdir, '*OG_acc_SL*.' + str(c.ppid) + '.*')

    # deaccumulate the flux data
    flexpart = EcFlexpart(c, fluxes=True)
    flexpart.write_namelist(c)
    flexpart.deacc_fluxes(inputfiles, c)

    # get a list of all files from the root inputdir
    inputfiles = UioFiles(c.inputdir, '????__??.*' + str(c.ppid) + '.*')

    # produce FLEXPART-ready GRIB files and process them -
    # copy/transfer/interpolate them or make them GRIB2
    flexpart = EcFlexpart(c, fluxes=False)
    flexpart.create(inputfiles, c)
    flexpart.process_output(c)
    if c.grib2flexpart:
        # prepare environment for a FLEXPART run
        # to convert grib to flexpart binary format
        flexpart.prepare_fp_files(c)

    # check if in debugging mode, then store all files
    # otherwise delete temporary files
    if c.debug:
        print('\nTemporary files left intact')
    else:
        clean_up(c)

    return

if __name__ == "__main__":
    main()

#
#
#
#        - moved the getEIdata program into a function "get_mars_data"
#        - moved the AgurmentParser into a seperate function
#        - adatpted the function for the use in flex_extract
#        - renamed file to get_mars_data
#
#        - added structured documentation
#        - added function main and moved function calls vom __main__ there
#          (necessary for better documentation with docstrings for later
#          online documentation)
#        - use of UIFiles class for file selection and deletion
#
#
#
#
#    This program can be used as a module in the whole flex_extract process
#    or can be run by itself to just extract MARS data from ECMWF. To do so,
#    a couple of necessary parameters has to be passed with the program call.
#    See documentation for more details.
#
#    - main
#    - get_mars_data
#    - do_retrievement
#

# MODULES
import os
import sys
from datetime import datetime, timedelta

from tools import (my_error, normal_exit, get_cmdline_arguments,
                   read_ecenv, make_dir)

try:
    ecapi = True
    import ecmwfapi
# FUNCTION
    '''Controls the program to get data out of mars.

    Return
    ------

    '''

    get_mars_data(c)
    normal_exit(c.mailfail, 'Done!')

    return

def get_mars_data(c):
    '''Retrieves the EC data needed for a FLEXPART simulation.

    Start and end dates for retrieval period is set. Retrievals
    are divided into smaller periods if necessary and datechunk parameter
    is set.

    Return
    ------

    '''

    if not os.path.exists(c.inputdir):
        make_dir(c.inputdir)

    if c.request == 0 or c.request == 2:
        print("Retrieving EC data!")
    elif c.request == 1:
        print("Printing mars requests!")

    print("start date %s " % (c.start_date))
    print("end date %s " % (c.end_date))

    if ecapi:
        if c.public:
            server = ecmwfapi.ECMWFDataServer()
        else:
            server = ecmwfapi.ECMWFService("mars")
    else:
        server = False

    print('Using ECMWF WebAPI: ' + str(c.ecapi))

    # basetime geht rückwärts

    # if basetime 00
    # dann wird von 12 am vortag bis 00 am start tag geholt
    # aber ohne 12 selbst sondern 12 + step

    # if basetime 12
    # dann wird von 00 + step bis 12 am start tag geholt

    # purer forecast wird vorwärts bestimmt.
    # purer forecast mode ist dann wenn  größer 24 stunden
    # wie kann das noch festgestellt werden ????
    # nur FC und steps mehr als 24 ?
    # die einzige problematik beim reinen forecast ist die benennung der files!
    # also sobald es Tagesüberschneidungen gibt
    # allerdings ist das relevant und ersichtlich an den NICHT FLUSS DATEN

    start = datetime.strptime(c.start_date, '%Y%m%d')
    end = datetime.strptime(c.end_date, '%Y%m%d')
    # time period for one single retrieval
    datechunk = timedelta(days=int(c.date_chunk))

        start = start - timedelta(days=1)

    if c.maxstep <= 24:
        startm1 = start - timedelta(days=1)

    if c.basetime == '00' or c.basetime == '12':
        # endp1 = end + timedelta(days=1)
        endp1 = end
    else:
        # endp1 = end + timedelta(days=2)
        endp1 = end + timedelta(days=1)

    # --------------  flux data ------------------------------------------------
        print('... removing old flux content of ' + c.inputdir)
        tobecleaned = UioFiles(c.inputdir,
                               '*_acc_*.' + str(os.getppid()) + '.*.grb')
        tobecleaned.delete_files()

    # if forecast for maximum one day (upto 24h) are to be retrieved,
    # collect accumulation data (flux data)
    # with additional days in the beginning and at the end
    # (used for complete disaggregation of original period)
        do_retrievement(c, server, startm1, endp1, datechunk, fluxes=True)

    # if forecast data longer than 24h are to be retrieved,
    # with the exact start and end date
    # (disaggregation will be done for the
    # exact time period with boundary conditions)
    else:
        do_retrievement(c, server, start, end, datechunk, fluxes=True)

    # --------------  non flux data --------------------------------------------
        print('... removing old non flux content of ' + c.inputdir)
                               '*__*.' + str(os.getppid()) + '.*.grb')

    do_retrievement(c, server, start, end, datechunk, fluxes=False)

    return

def do_retrievement(c, server, start, end, delta_t, fluxes=False):
    '''Divides the complete retrieval period in smaller chunks and
    retrieves the data from MARS.

    server : :obj:`ECMWFService`
            The server connection to ECMWF.

    start : :obj:`datetime`
        The start date of the retrieval.

    end : :obj:`datetime`
        The end date of the retrieval.

    delta_t : :obj:`datetime`
        Delta_t + 1 is the maximal time period of a single
        retrieval.

    fluxes : :obj:`boolean`, optional
        Decides if the flux parameters are to be retrieved or
        the rest of the parameter list.
        Default value is False.

    Return
    ------

    '''

    # since actual day also counts as one day,
    # we only need to add datechunk - 1 days to retrieval
    # for a period
    delta_t_m1 = delta_t - timedelta(days=1)

    day = start
    while day <= end:
        flexpart = EcFlexpart(c, fluxes)
        tmpday = day + delta_t_m1
        if tmpday < end:
            dates = day.strftime("%Y%m%d") + "/to/" + \
                    tmpday.strftime("%Y%m%d")
        else:
                    end.strftime("%Y%m%d")

        print("... retrieve " + dates + " in dir " + c.inputdir)

        try:
            flexpart.retrieve(server, dates, c.public, c.request, c.inputdir)
        except IOError:
            my_error(c.mailfail, 'MARS request failed')

        day += delta_t

    return

    main()

#
# @Date: July 2014
#
#   February 2018 - Anne Philipp (University of Vienna):
#        - changed some naming
#
#
#
# @Class Description:
#    The GRIB API provides all necessary tools to work directly with the
#    grib files. Nevertheless, the GRIB API tools are very basic and are in
#    direct connection with the grib files. This class provides some higher
#    functions which apply a set of GRIB API tools together in the respective
#    context. So, the class initially contains a list of grib files (their
#    names) and the using program then applies the methods directly on the
#    class objects without having to think about how the actual GRIB API
#    tools have to be arranged.
#
# @Class Content:
#    - __init__
#    - get_keys
#    - set_keys
#    - copy
#    - index
#
# @Class Attributes:
#    - filenames
#

# MODULES
import os
from gribapi import grib_new_from_file, grib_is_defined, grib_get, \
     grib_release, grib_set, grib_write, grib_index_read, \
     grib_index_new_from_file, grib_index_add_file,  \
     grib_index_write

# from eccodes import (codes_grib_new_from_file, codes_is_defined, codes_get,
                     # codes_release, codes_set, codes_write, codes_index_read,
                     # codes_index_new_from_file, codes_index_add_file,
                     # codes_index_write)

# CLASS
class GribTools(object):
    '''
    Class for GRIB utilities (new methods) based on GRIB API
    '''
    # --------------------------------------------------------------------------
    # CLASS FUNCTIONS
    def __init__(self, filenames):
        '''Initialise an object of GribTools and assign a list of filenames.

        Parameters
        ----------
        filenames : :obj:`list` of :obj:`strings`
             A list of filenames.

        Return
        ------

        '''

        self.filenames = filenames

        return

    def get_keys(self, keynames, wherekeynames=[], wherekeyvalues=[]):
        '''Get keyvalues for a given list of keynames a where statement
        can be given (list of key and list of values)

        keynames : :obj:`list` of :obj:`string`
            List of keynames.

        wherekeynames : :obj:`list` of :obj:`string`, optional
            Default value is an empty list.

        wherekeyvalues : :obj:`list` of :obj:`string`, optional

        Return
        ------
        return_list : :obj:`list` of :obj:`string`
            List of keyvalues for given keynames.
        '''

        fileid = open(self.filenames, 'r')

        return_list = []

        while 1:
            gid = grib_new_from_file(fileid)

            if gid is None:
                break

            if len(wherekeynames) != len(wherekeyvalues):
                raise Exception("Number of key values and key names must be \
                                 the same. Give a value for each keyname!")

            select = True
            i = 0
            for wherekey in wherekeynames:
                if not grib_is_defined(gid, wherekey):
                    raise Exception("where key was not defined")

                select = (select and (str(wherekeyvalues[i]) ==
                                      str(grib_get(gid, wherekey))))
                i += 1

            if select:
                llist = []
                for key in keynames:
                    llist.extend([str(grib_get(gid, key))])
                return_list.append(llist)

            grib_release(gid)

        fileid.close()

        return return_list

    def set_keys(self, fromfile, keynames, keyvalues, wherekeynames=[],
                 wherekeyvalues=[], strict=False, filemode='w'):
        '''Opens the file to read the grib messages and then write
        them to a new output file. By default all messages are
        written out. Also, the keyvalues of the passed list of
        keynames are set or only those meeting the where statement.
        (list of key and list of values).

        fromfile : :obj:`string`
            Filename of the input file to read the grib messages from.

            List of keynames. Default is an empty list.

        keyvalues : :obj:`list` of :obj:`string`

        strict : :obj:`boolean`, optional
            Decides if everything from keynames and keyvalues
            is written out the grib file (False) or only those
            meeting the where statement (True). Default is False.

        filemode : :obj:`string`, optional
            Sets the mode for the output file. Default is "w".

        Return
        ------

        '''
        fout = open(self.filenames, filemode)
        fin = open(fromfile)

        while 1:
            gid = grib_new_from_file(fin)

                break

                raise Exception("Give a value for each keyname!")

            i = 0
                    raise Exception("where Key was not defined")

                i += 1

            if select:
                i = 0
                    grib_set(gid, key, keyvalues[i])
                    i += 1

            grib_write(gid, fout)

        fin.close()
        fout.close()

        return

    def copy(self, filename_in, selectWhere=True,
             keynames=[], keyvalues=[], filemode='w'):
        '''Add the content of another input grib file to the objects file but
        only messages corresponding to keys/values passed to the function.
        The selectWhere switch decides if to copy the keys equal to (True) or
        different to (False) the keynames/keyvalues list passed to the function.

        filename_in : :obj:`string`

        selectWhere : :obj:`boolean`, optional
            Decides if to copy the keynames and values equal to (True) or
            different to (False) the keynames/keyvalues list passed to the
            function. Default is True.

        keynames : :obj:`list` of :obj:`string`, optional

        keyvalues : :obj:`list` of :obj:`string`, optional

        Return
        ------

        '''

        fin = open(filename_in)

        while 1:

                break

            if len(keynames) != len(keyvalues):

            i = 0
            for key in keynames:
                if not grib_is_defined(gid, key):
                    raise Exception("Key was not defined")

                if selectWhere:
                    select = (select and (str(keyvalues[i]) ==
                                          str(grib_get(gid, key))))
                else:
                    select = (select and (str(keyvalues[i]) !=
                i += 1

            if select:
                grib_write(gid, fout)

        return

    def index(self, index_keys=["mars"], index_file="my.idx"):
        '''Create index file from a list of files if it does not exist or
        read an index file.

        index_keys: :obj:`list` of :obj:`string`, optional
            Contains the list of key parameter names from
            which the index is to be created.
            Default is a list with a single entry string "mars".

        index_file: :obj:`string`, optional
            Filename where the indices are stored.
            Default is "my.idx".

        Return
        ------
        iid: :obj:`integer`
            Grib index id.
        '''
        print("... index will be done")
        iid = None

        if os.path.exists(index_file):
            iid = grib_index_read(index_file)
            print("Use existing index file: %s " % (index_file))
        else:
            for filename in self.filenames:
                print("Inputfile: %s " % (filename))
                if iid is None:
                    iid = grib_index_new_from_file(filename, index_keys)
                else:
                    grib_index_add_file(iid, filename)

            if iid is not None:
                grib_index_write(iid, index_file)

        print('... index done')

        return iid

#
#
#
#        - extended with class Control
#        - removed functions mkdir_p, daterange, years_between, months_between
#        - added functions darain, dapoly, to_param_id, init128, normal_exit,
#          my_error, clean_up, install_args_and_control,
#          interpret_args_and_control,
#        - removed function __del__ in class EIFLexpart
#        - added the following functions in EIFlexpart:
#            - create_namelist
#            - process_output
#            - deacc_fluxes
#        - modified existing EIFlexpart - functions for the use in
#          flex_extract
#        - added conversion into GRIB2
#        - added conversion into .fp format for faster execution of FLEXPART
#          (see https://www.flexpart.eu/wiki/FpCtbtoWo4FpFormat)
#
#        - removed function getFlexpartTime in class EcFlexpart
#        - outsourced class ControlFile
#        - outsourced class MarsRetrieval
#        - changed class name from EIFlexpart to EcFlexpart
#        - applied minor code changes (style)
#        - removed "dead code" , e.g. retrieval of Q since it is not needed
#        - removed "times" parameter from retrieve-method since it is not used
#        - seperated function "retrieve" into smaller functions (less code
#          duplication, easier testing)
#
#
#
#    FLEXPART needs grib files in a specifc format. All necessary data fields
#    for one time step are stored in a single file. The class represents an
#    instance with all the parameter and settings necessary for retrieving
#    MARS data and modifing them so they are fitting FLEXPART need. The class
#    is able to disaggregate the fluxes and convert grid types to the one needed
#    by FLEXPART, therefore using the FORTRAN program.
#
#    - write_namelist
#    - retrieve
#    - process_output
#    - create
#    - deacc_fluxes
#
#
#  TODO
#
#pylint: disable=unsupported-assignment-operation
# this is disabled because for this specific case its an error in pylint
#pylint: disable=consider-using-enumerate
# this is not useful in this case
# MODULES
import os
import sys
import glob
import shutil
import subprocess
import numpy as np

from gribapi import (grib_set, grib_index_select, grib_new_from_index, grib_get,
                     grib_write, grib_get_values, grib_set_values, grib_release,
                     grib_index_release, grib_index_get)

# from eccodes import (codes_index_select, codes_new_from_index, codes_get,
                     # codes_get_values, codes_set_values, codes_set,
                     # codes_write, codes_release, codes_new_from_index,
                     # codes_index_release, codes_index_get)

sys.path.append('../')
from GribTools import GribTools
from mods.tools import (init128, to_param_id, silent_remove, product,
                        my_error, make_dir)
from MarsRetrieval import MarsRetrieval
import mods.disaggregation as disaggregation

# CLASS
class EcFlexpart(object):
    '''
    Class to retrieve FLEXPART specific ECMWF data.
    '''
    def __init__(self, c, fluxes=False):
        '''Creates an object/instance of EcFlexpart with the associated
        settings of its attributes for the retrieval.

        Parameters:
        -----------
        c : :obj:`ControlFile`
            Contains all the parameters of CONTROL file and
            command line.

        fluxes : :obj:`boolean`, optional
            Decides if the flux parameter settings are stored or
            the rest of the parameter list.
            Default value is False.

        Return
        ------

        '''
        # set a counter for the number of mars requests generated
        self.mreq_count = 0

        # different mars types for retrieving data for flexpart
        self.types = dict()

        # Pure forecast mode
        if c.maxstep > len(c.type) and 'AN' not in c.type:
            c.type = [c.type[0]]
            c.step = ['{:0>3}'.format(int(c.step[0]))]
            c.time = [c.time[0]]
            for i in range(1, c.maxstep + 1):
                c.type.append(c.type[0])
                c.step.append('{:0>3}'.format(i))
                c.time.append(c.time[0])

        self.inputdir = c.inputdir
        self.dataset = c.dataset
        self.basetime = c.basetime
        self.dtime = c.dtime
        i = 0
        if fluxes and c.maxstep <= 24:
            # no forecast beyond one day is needed!
            # Thus, prepare flux data manually as usual
            # with only forecast fields with start times at 00/12
            # (but without 00/12 fields since these are
            # the initialisation times of the flux fields
            # and therefore are zero all the time)
            self.types[str(c.acctype)] = {'times': str(c.acctime),
                                          'steps': '{}/to/{}/by/{}'.format(
                                              c.dtime, c.accmaxstep, c.dtime)}
        else:
            for ty, st, ti in zip(c.type, c.step, c.time):
                btlist = range(24)
                if c.basetime == '12':
                    btlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
                if c.basetime == '00':
                    btlist = [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 0]

                if ((ty.upper() == 'AN' and (int(c.time[i]) % int(c.dtime)) == 0) or
                    (ty.upper() != 'AN' and (int(c.step[i]) % int(c.dtime)) == 0 and
                     (int(c.step[i]) % int(c.dtime) == 0)) ) and \
                    (int(c.time[i]) in btlist or c.maxstep > 24):

                    if ty not in self.types.keys():
                        self.types[ty] = {'times': '', 'steps': ''}

                    if ti not in self.types[ty]['times']:
                        if self.types[ty]['times']:
                            self.types[ty]['times'] += '/'
                        self.types[ty]['times'] += ti

                    if st not in self.types[ty]['steps']:
                        if self.types[ty]['steps']:
                            self.types[ty]['steps'] += '/'
                        self.types[ty]['steps'] += st
                i += 1

        self.marsclass = c.marsclass
        self.stream = c.stream
        self.number = c.number
        self.resol = c.resol
        self.accuracy = c.accuracy
        self.level = c.level
        self.expver = c.expver
        self.levelist = c.levelist
        # for gaussian grid retrieval
        self.glevelist = '1/to/' + c.level
        self.gaussian = c.gaussian

        if 'N' in c.grid:  # Gaussian output grid
            self.grid = c.grid
            self.area = 'G'
        else:
            self.grid = '{}/{}'.format(int(c.grid) / 1000., int(c.grid) / 1000.)
            self.area = '{}/{}/{}/{}'.format(int(c.upper) / 1000.,
                                             int(c.left) / 1000.,
                                             int(c.lower) / 1000.,
                                             int(c.right) / 1000.)

        self.outputfilelist = []

        # Now comes the nasty part that deals with the different
        # scenarios we have:
        # 1) Calculation of etadot on
        #    a) Gaussian grid
        #    b) Output grid
        #    c) Output grid using parameter 77 retrieved from MARS
        # 3) Calculation/Retrieval of omega
        # 4) Download also data for WRF

        # Different grids need different retrievals
        # SH = Spherical Harmonics, GG = Gaussian Grid,
        # OG = Output Grid, ML = MultiLevel, SL = SingleLevel
        self.params = {'SH__ML': '', 'SH__SL': '',
                       'GG__ML': '', 'GG__SL': '',
                       'OG__ML': '', 'OG__SL': '',
                       'OG_OROLSM_SL': '', 'OG_acc_SL': ''}
        # the self.params dictionary stores a list of
        # [param, levtype, levelist, grid] per key

        if fluxes is False:
            self.params['SH__SL'] = ['LNSP', 'ML', '1', 'OFF']
            #                        "SD/MSL/TCC/10U/10V/2T/2D/129/172"
            self.params['OG__SL'] = ["141/151/164/165/166/167/168/129/172", \
                                     'SFC', '1', self.grid]
            if c.addpar:
                if c.addpar[0] == '/':
                    c.addpar = c.addpar[1:]
                self.params['OG__SL'][0] += '/' + '/'.join(c.addpar)

            if c.marsclass.upper() == 'EA' or c.marsclass.upper() == 'EP':
                self.params['OG_OROLSM__SL'] = ["160/27/28/244",
                                                'SFC', '1', self.grid]
            else:
                self.params['OG_OROLSM__SL'] = ["160/27/28/173", \

            self.params['OG__ML'] = ['T/Q', 'ML', self.levelist, self.grid]

            #if c.gauss == '0' and c.eta == '1':
            if not c.gauss and c.eta:
                # the simplest case
                self.params['OG__ML'][0] += '/U/V/77'
            #elif c.gauss == '0' and c.eta == '0':
            elif not c.gauss and not c.eta:
            # this is not recommended (inaccurate)
                self.params['OG__ML'][0] += '/U/V'
            #elif c.gauss == '1' and c.eta == '0':
            elif c.gauss and not c.eta:
                # this is needed for data before 2008, or for reanalysis data
                self.params['GG__SL'] = ['Q', 'ML', '1', \
                                         '{}'.format((int(self.resol) + 1) / 2)]
                self.params['SH__ML'] = ['U/V/D', 'ML', self.glevelist, 'OFF']
            else:
                print('Warning: This is a very costly parameter combination, \
                      use only for debugging!')
                self.params['GG__ML'] = ['U/V/D/77', 'ML', self.glevelist, \

            if c.omega:
                self.params['OG__ML'][0] += '/W'

            # add cloud water content if necessary
            if c.cwc:
                self.params['OG__ML'][0] += '/CLWC/CIWC'

            # add vorticity and geopotential height for WRF if necessary
            if c.wrf:
                self.params['OG__ML'][0] += '/Z/VO'
                if '/D' not in self.params['OG__ML'][0]:
                    self.params['OG__ML'][0] += '/D'
                #wrf_sfc = 'sp/msl/skt/2t/10u/10v/2d/z/lsm/sst/ci/sd/stl1/ /
                #           stl2/stl3/stl4/swvl1/swvl2/swvl3/swvl4'.upper()
                wrf_sfc = '134/235/167/165/166/168/129/172/34/31/141/ \
                           139/170/183/236/39/40/41/42'
                lwrt_sfc = wrf_sfc.split('/')
                for par in lwrt_sfc:
                    if par not in self.params['OG__SL'][0]:
                        self.params['OG__SL'][0] += '/' + par

        else:
            self.params['OG_acc_SL'] = ["LSP/CP/SSHF/EWSS/NSSS/SSR", \
                                        'SFC', '1', self.grid]

        return

    def _mk_targetname(self, ftype, param, date):
        '''Creates the filename for the requested grib data to be stored in.
        This name is passed as the "target" parameter in the request.

        ftype : :obj:`string`
            Shortcut name of the type of the field. E.g. AN, FC, PF, ...

        param : :obj:`string`
            Shortcut of the grid type. E.g. SH__ML, SH__SL, GG__ML,
            GG__SL, OG__ML, OG__SL, OG_OROLSM_SL, OG_acc_SL

        date : :obj:`string`
            The date period of the grib data to be stored in this file.

        Return
        ------
        targetname : :obj:`string`
            The target filename for the grib data.
        '''
        targetname = (self.inputdir + '/' + ftype + param + '.' + date + '.' +
                      str(os.getppid()) + '.' + str(os.getpid()) + '.grb')

        return targetname

    def _start_retrievement(self, request, par_dict):
        '''Creates the Mars Retrieval and prints or submits the request
        depending on the status of the request variable.

        request : :obj:`integer`
            Selects the mode of retrieval.
            0: Retrieves the data from ECMWF.
            1: Prints the mars requests to an output file.
            2: Retrieves the data and prints the mars request.

        par_dict : :obj:`dictionary`
            Contains all parameter which have to be set for creating the
            Mars Retrievals. The parameter are:
            marsclass, dataset, stream, type, levtype, levelist, resol,
            gaussian, accuracy, grid, target, area, date, time, number,
            step, expver, param

        Return
        ------

        '''
        # increase number of mars requests
        self.mreq_count += 1

        MR = MarsRetrieval(self.server,
                           self.public,
                           marsclass=par_dict['marsclass'],
                           dataset=par_dict['dataset'],
                           stream=par_dict['stream'],
                           type=par_dict['type'],
                           levtype=par_dict['levtype'],
                           levelist=par_dict['levelist'],
                           resol=par_dict['resol'],
                           gaussian=par_dict['gaussian'],
                           accuracy=par_dict['accuracy'],
                           grid=par_dict['grid'],
                           target=par_dict['target'],
                           area=par_dict['area'],
                           date=par_dict['date'],
                           time=par_dict['time'],
                           number=par_dict['number'],
                           step=par_dict['step'],
                           expver=par_dict['expver'],
                           param=par_dict['param'])

        if request == 0:
            MR.display_info()
            MR.data_retrieve()
        elif request == 1:
            MR.print_infodata_csv(self.inputdir, self.mreq_count)
        elif request == 2:
        else:
            print('Failure')

        return

    def _mk_index_values(self, inputdir, inputfiles, keys):
        '''Creates an index file for a set of grib parameter keys.
        The values from the index keys are returned in a list.

        keys : :obj:`dictionary`
            List of parameter names which serves as index.

        inputfiles : :obj:`UioFiles`
            Contains a list of files.

        Return
        ------
        iid : :obj:`grib_index`
            This is a grib specific index structure to access
            messages in a file.

        index_vals : :obj:`list`
            Contains the values from the keys used for a distinct selection
            of grib messages in processing  the grib files.
            Content looks like e.g.:
            index_vals[0]: ('20171106', '20171107', '20171108') ; date
            index_vals[1]: ('0', '1200', '1800', '600') ; time
            index_vals[2]: ('0', '12', '3', '6', '9') ; stepRange
        '''
        iid = None
        index_keys = keys

        indexfile = os.path.join(inputdir, _config.FILE_GRIB_INDEX)
        silent_remove(indexfile)
        grib = GribTools(inputfiles.files)
        # creates new index file
        iid = grib.index(index_keys=index_keys, index_file=indexfile)

        # read the values of index keys
        index_vals = []
        for key in index_keys:
            #index_vals.append(grib_index_get(iid, key))
            #print(index_vals[-1])
            key_vals = grib_index_get(iid, key)
            print(key_vals)
            # have to sort the steps for disaggregation,
            # therefore convert to int first
            if key == 'step':
                key_vals = [int(k) for k in key_vals]
                key_vals.sort()
                key_vals = [str(k) for k in key_vals]
            index_vals.append(key_vals)
            # index_vals looks for example like:
            # index_vals[0]: ('20171106', '20171107', '20171108') ; date
            # index_vals[1]: ('0', '1200') ; time
            # index_vals[2]: (3', '6', '9', '12') ; stepRange

        return iid, index_vals

    def retrieve(self, server, dates, public, request, inputdir='.'):
        '''Finalizing the retrieval information by setting final details
        depending on grid type.
        Prepares MARS retrievals per grid type and submits them.

        server : :obj:`ECMWFService` or :obj:`ECMWFDataServer`
            The connection to the ECMWF server. This is different
            for member state users which have full access and non
            member state users which have only access to the public
            data sets. The decision is made from command line argument
            "public"; for public access its True (ECMWFDataServer)
            for member state users its False (ECMWFService)

        dates : :obj:`string`
            Contains start and end date of the retrieval in the format
            "YYYYMMDD/to/YYYYMMDD"

        inputdir : :obj:`string`, optional
            Path to the directory where the retrieved data is about
            to be stored. The default is the current directory ('.').

        Return
        ------

        '''
        self.dates = dates
        self.server = server
        self.public = public
        self.inputdir = inputdir
        oro = False

        # define times with datetime module
        t12h = timedelta(hours=12)
        t24h = timedelta(hours=24)

        # dictionary which contains all parameter for the mars request,
        # entries with a "None" will change in different requests and will
        # therefore be set in each request seperately
        retr_param_dict = {'marsclass':self.marsclass,
                           'dataset':self.dataset,
                           'stream':None,
                           'type':None,
                           'levtype':None,
                           'levelist':None,
                           'resol':self.resol,
                           'gaussian':None,
                           'accuracy':self.accuracy,
                           'grid':None,
                           'target':None,
                           'area':None,
                           'date':None,
                           'time':None,
                           'number':self.number,
                           'step':None,
                           'expver':self.expver,
                           'param':None}

        for ftype in self.types:
            # fk contains field types such as
            #     [AN, FC, PF, CV]
            # fv contains all of the items of the belonging key
            #     [times, steps]
            for pk, pv in self.params.iteritems():
                # pk contains one of these keys of params
                #     [SH__ML, SH__SL, GG__ML, GG__SL, OG__ML, OG__SL,
                #      OG_OROLSM_SL, OG_acc_SL]
                # pv contains all of the items of the belonging key
                #     [param, levtype, levelist, grid]
                if isinstance(pv, str):
                    continue
                retr_param_dict['type'] = ftype
                retr_param_dict['time'] = self.types[ftype]['times']
                retr_param_dict['step'] = self.types[ftype]['steps']
                retr_param_dict['date'] = self.dates
                retr_param_dict['stream'] = self.stream
                retr_param_dict['target'] = \
                    self._mk_targetname(ftype,
                                        pk,
                                        retr_param_dict['date'].split('/')[0])
                retr_param_dict['param'] = pv[0]
                retr_param_dict['levtype'] = pv[1]
                retr_param_dict['levelist'] = pv[2]
                retr_param_dict['grid'] = pv[3]
                retr_param_dict['area'] = self.area
                retr_param_dict['gaussian'] = self.gaussian

                if pk == 'OG__SL':
                    pass
                if pk == 'OG_OROLSM__SL' and not oro:
                    oro = True
                    # in CERA20C (class EP) there is no stream "OPER"!
                    if self.marsclass.upper() != 'EP':
                        retr_param_dict['stream'] = 'OPER'
                    retr_param_dict['type'] = 'AN'
                    retr_param_dict['time'] = '00'
                    retr_param_dict['step'] = '000'
                    retr_param_dict['date'] = self.dates.split('/')[0]
                    retr_param_dict['target'] = self._mk_targetname('',
                                            pk, retr_param_dict['date'])
                elif pk == 'OG_OROLSM__SL' and oro:
                    continue
                if pk == 'GG__SL' and pv[0] == 'Q':
                    retr_param_dict['area'] = ""
                    retr_param_dict['gaussian'] = 'reduced'

    # ------  on demand path  --------------------------------------------------
                if not self.basetime:
                    # ******* start retrievement
                    self._start_retrievement(request, retr_param_dict)
    # ------  operational path  ------------------------------------------------
                else:
                    # check if mars job requests fields beyond basetime.
                    # if yes eliminate those fields since they may not
                    # be accessible with user's credentials

                    enddate = retr_param_dict['date'].split('/')[-1]
                    elimit = datetime.strptime(enddate + self.basetime,
                                               '%Y%m%d%H')

                    if self.basetime == '12':
                        # --------------  flux data ----------------------------
                        if 'acc' in pk:

                        # Strategy:
                        # if maxtime-elimit >= 24h reduce date by 1,
                        # if 12h <= maxtime-elimit<12h reduce time for last date
                        # if maxtime-elimit<12h reduce step for last time
                        # A split of the MARS job into 2 is likely necessary.

                            startdate = retr_param_dict['date'].split('/')[0]
                            enddate = datetime.strftime(elimit - t24h,'%Y%m%d')
                            retr_param_dict['date'] = '/'.join([startdate,
                                                                'to',
                                                                enddate])

                            # ******* start retrievement
                            self._start_retrievement(request, retr_param_dict)

                            retr_param_dict['date'] = \
                                datetime.strftime(elimit - t12h, '%Y%m%d')
                            retr_param_dict['time'] = '00'
                            retr_param_dict['target'] = \
                                self._mk_targetname(ftype, pk,
                                                    retr_param_dict['date'])

                        # --------------  non flux data ------------------------
                        else:

                    else: # basetime = 0
                        retr_param_dict['date'] = \
                            datetime.strftime(elimit - t24h, '%Y%m%d')

                        timesave = ''.join(retr_param_dict['time'])

                        if '/' in retr_param_dict['time']:
                            times = retr_param_dict['time'].split('/')
                            steps = retr_param_dict['step'].split('/')
                            while (pk != 'OG_OROLSM__SL' and
                                   'acc' not in pk and
                                   (int(times[0]) + int(steps[0])) <= 12):
                                times = times[1:]

                            if len(times) > 1:
                                retr_param_dict['time'] = '/'.join(times)
                            else:
                                retr_param_dict['time'] = times[0]

                        # ******* start retrievement
                        self._start_retrievement(request, retr_param_dict)

                        if (pk != 'OG_OROLSM__SL' and
                            int(retr_param_dict['step'].split('/')[0]) == 0 and
                            int(timesave.split('/')[0]) == 0):

                                datetime.strftime(elimit, '%Y%m%d')
                            retr_param_dict['step'] = '000'

        if request == 0 or request == 2:
            print('MARS retrieve done ... ')
            print('MARS request printed ...')

        return

    def write_namelist(self, c):
        '''Creates a namelist file in the temporary directory and writes
        the following values to it: maxl, maxb, mlevel,
        mlevelist, mnauf, metapar, rlo0, rlo1, rla0, rla1,
        momega, momegadiff, mgauss, msmooth, meta, metadiff, mdpdeta

        filename : :obj:`string`
                Name of the namelist file.

        Return
        ------

        '''

        from genshi.template.text import NewTextTemplate
        from genshi.template import  TemplateLoader

        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
        namelist_template = loader.load(_config.TEMPFILE_NAMELIST,
                                       cls=NewTextTemplate)

        area = np.asarray(self.area.split('/')).astype(float)
        grid = np.asarray(self.grid.split('/')).astype(float)

        if area[1] > area[3]:
            area[1] -= 360
        maxl = int((area[3] - area[1]) / grid[1]) + 1
        maxb = int((area[0] - area[2]) / grid[0]) + 1

        stream = namelist_template.generate(
            maxl = str(maxl),
            maxb = str(maxb),
            mlevel = str(self.level),
            mlevelist = str(self.levelist),
            mnauf = str(self.resol),
            metapar = '77',
            rlo0 = str(area[1]),
            rlo1 = str(area[3]),
            rla0 = str(area[2]),
            rla1 = str(area[0]),
            momega = str(c.omega),
            momegadiff = str(c.omegadiff),
            mgauss = str(c.gauss),
            msmooth = str(c.smooth),
            meta = str(c.eta),
            metadiff = str(c.etadiff),
            mdpdeta = str(c.dpdeta)
        )

        namelistfile = os.path.join(self.inputdir, _config.FILE_NAMELIST)

        with open(namelistfile, 'w') as f:
            f.write(stream.render('text'))

        return

    def deacc_fluxes(self, inputfiles, c):
        '''Goes through all flux fields in ordered time and de-accumulate
        the fields. Afterwards the fields are disaggregated in time.
        Different versions of disaggregation is provided for rainfall
        data (darain, modified linear) and the surface fluxes and
        stress data (dapoly, cubic polynomial).

        Return
        ------

        '''

        table128 = init128(_config.PATH_GRIBTABLE)
        pars = to_param_id(self.params['OG_acc_SL'][0], table128)

        iid = None
        index_vals = None

        # get the values of the keys which are used for distinct access
        # of grib messages via product
        index_keys = ["date", "time", "step"]
        iid, index_vals = self._mk_index_values(c.inputdir,
                                                inputfiles,
                                                index_keys)
        # index_vals looks like e.g.:
        # index_vals[0]: ('20171106', '20171107', '20171108') ; date
        # index_vals[1]: ('0', '1200', '1800', '600') ; time
        # index_vals[2]: ('0', '12', '3', '6', '9') ; stepRange

        valsdict = {}
        svalsdict = {}
#        stepsdict = {}
        for p in pars:
            valsdict[str(p)] = []
            svalsdict[str(p)] = []
#            stepsdict[str(p)] = []

        print('maxstep: ', c.maxstep)

        # "product" genereates each possible combination between the
        # values of the index keys
        for prod in product(*index_vals):
            # e.g. prod = ('20170505', '0', '12')
            #             (  date    ,time, step)

            print('current product: ', prod)

            for i in range(len(index_keys)):
                grib_index_select(iid, index_keys[i], prod[i])

            # get first id from current product
            gid = grib_new_from_index(iid)

            # if there is no data for this specific time combination / product
            # skip the rest of the for loop and start with next timestep/product
            if not gid:
                continue

            # create correct timestamp from the three time informations
            cdate = str(grib_get(gid, 'date'))
            ctime = '{:0>2}'.format(grib_get(gid, 'time')/100)
            cstep = '{:0>3}'.format(grib_get(gid, 'step'))
            t_date = datetime.strptime(cdate + ctime, '%Y%m%d%H')
            t_dt = t_date + timedelta(hours=int(cstep))
            t_m1dt = t_date + timedelta(hours=int(cstep)-int(c.dtime))
            t_m2dt = t_date + timedelta(hours=int(cstep)-2*int(c.dtime))
            t_enddate = None

            if c.maxstep > 12:
                fnout = os.path.join(c.inputdir, 'flux' +
                                     t_date.strftime('%Y%m%d.%H') +
                                     '.{:0>3}'.format(step-2*int(c.dtime)))
                gnout = os.path.join(c.inputdir, 'flux' +
                                     '.{:0>3}'.format(step-int(c.dtime)))
                hnout = os.path.join(c.inputdir, 'flux' +
                                     '.{:0>3}'.format(step))
            else:
                                     t_m2dt.strftime('%Y%m%d%H'))
                                     t_m1dt.strftime('%Y%m%d%H'))
                                     t_dt.strftime('%Y%m%d%H'))

            print("outputfile = " + fnout)

            # read message for message and store relevant data fields
            # data keywords are stored in pars
            while 1:
                if not gid:
                    break
                cparamId = str(grib_get(gid, 'paramId'))
                step = grib_get(gid, 'step')
                time = grib_get(gid, 'time')
                ni = grib_get(gid, 'Ni')
                nj = grib_get(gid, 'Nj')
                if cparamId in valsdict.keys():
                    values = grib_get_values(gid)
                    vdp = valsdict[cparamId]
                    svdp = svalsdict[cparamId]
 #                   sd = stepsdict[cparamId]

                    if cparamId == '142' or cparamId == '143':
                        fak = 1. / 1000.
                    else:
                        fak = 3600.

                    values = (np.reshape(values, (nj, ni))).flatten() / fak
                    vdp.append(values[:])  # save the accumulated values
                    if c.marsclass.upper() == 'EA' or \
                       step <= int(c.dtime):
                        svdp.append(values[:] / int(c.dtime))
                    else:  # deaccumulate values
                        svdp.append((vdp[-1] - vdp[-2]) / int(c.dtime))

                    print(cparamId, time, step, len(values),
                          values[0], np.std(values))

                    # len(svdp) correspond to the time
                    if len(svdp) >= 3:
                        if len(svdp) > 3:
                            if cparamId == '142' or cparamId == '143':
                                values = disaggregation.darain(svdp)
                            else:
                                values = disaggregation.dapoly(svdp)

                            if not (step == c.maxstep and c.maxstep > 12 \
                                    or t_dt == t_enddate):
                                vdp.pop(0)
                                svdp.pop(0)
                        else:
                            if c.maxstep > 12:
                                values = svdp[1]
                            else:
                                values = svdp[0]

                        grib_set_values(gid, values)

                        if c.maxstep > 12:
                            grib_set(gid, 'step', max(0, step-2*int(c.dtime)))
                        else:
                            grib_set(gid, 'step', 0)
                            grib_set(gid, 'time', t_m2dt.hour*100)
                            grib_set(gid, 'date', int(t_m2dt.strftime('%Y%m%d')))

                        with open(fnout, 'w') as f_handle:
                            grib_write(gid, f_handle)

                        if c.basetime:
                            t_enddate = datetime.strptime(c.end_date +
                                                          c.basetime,
                                                          '%Y%m%d%H')
                        else:
                            t_enddate = t_date + timedelta(2*int(c.dtime))

                        # squeeze out information of last two steps contained
                        # in svdp
                        # if step+int(c.dtime) == c.maxstep and c.maxstep>12
                        # or t_dt+timedelta(hours = int(c.dtime))
                        # >= t_enddate:
                        # Note that svdp[0] has not been popped in this case

                        if step == c.maxstep and c.maxstep > 12 or \
                           t_dt == t_enddate:

                            values = svdp[3]
                            grib_set_values(gid, values)
                            truedatetime = t_m2dt + timedelta(hours=
                                                              2*int(c.dtime))
                            grib_set(gid, 'time', truedatetime.hour * 100)
                            grib_set(gid, 'date', truedatetime.year * 10000 +
                                      truedatetime.month * 100 +
                                      truedatetime.day)
                            with open(hnout, 'w') as h_handle:
                                grib_write(gid, h_handle)

                            #values = (svdp[1]+svdp[2])/2.
                                values = disaggregation.darain(list(reversed(svdp)))
                            else:
                                values = disaggregation.dapoly(list(reversed(svdp)))

                            truedatetime = t_m2dt + timedelta(hours=int(c.dtime))
                                     truedatetime.month * 100 +
                                     truedatetime.day)
                            with open(gnout, 'w') as g_handle:
                                grib_write(gid, g_handle)

                grib_release(gid)

                gid = grib_new_from_index(iid)

        grib_index_release(iid)

        return

    def create(self, inputfiles, c):
        '''An index file will be created which depends on the combination
        of "date", "time" and "stepRange" values. This is used to iterate
        over all messages in each grib file which were passed through the
        parameter "inputfiles" to seperate specific parameters into fort.*
        files. Afterwards the FORTRAN program is called to convert
        the data fields all to the same grid and put them in one file
        per unique time step (combination of "date", "time" and
        "stepRange").

        Note
        ----
        This method is based on the ECMWF example index.py
        https://software.ecmwf.int/wiki/display/GRIB/index.py

        Return
        ------

        '''

        if c.wrf:
            table128 = init128(_config.PATH_GRIBTABLE)
            wrfpars = to_param_id('sp/mslp/skt/2t/10u/10v/2d/z/lsm/sst/ci/sd/\
                                   stl1/stl2/stl3/stl4/swvl1/swvl2/swvl3/swvl4',
                                  table128)

        # these numbers are indices for the temporary files "fort.xx"
        # which are used to seperate the grib fields to,
        # for the Fortran program input
        # 10: U,V | 11: T | 12: lnsp | 13: D | 16: sfc fields
        # 17: Q | 18: Q , gaussian| 19: w | 21: etadot | 22: clwc+ciwc
        fdict = {'10':None, '11':None, '12':None, '13':None, '16':None,
                 '17':None, '18':None, '19':None, '21':None, '22':None}

        iid = None

            if not gid:
                continue

            # remove old fort.* files and open new ones
            # they are just valid for a single product
            for k, f in fdict.iteritems():
                fortfile = os.path.join(c.inputdir, 'fort.' + k)
                silent_remove(fortfile)
                fdict[k] = open(fortfile, 'w')

            timestamp = datetime.strptime(cdate + ctime, '%Y%m%d%H')
            timestamp += timedelta(hours=int(cstep))
            cdate_hour = datetime.strftime(timestamp, '%Y%m%d%H')

            # if the timestamp is out of basetime start/end date period,
            # skip this specific product
            if c.basetime:
                start_time = datetime.strptime(c.end_date + c.basetime,
                                                '%Y%m%d%H') - time_delta
                end_time = datetime.strptime(c.end_date + c.basetime,
                                             '%Y%m%d%H')
                if timestamp < start_time or timestamp > end_time:
                    continue

            if c.wrf:
                if 'olddate' not in locals() or cdate != olddate:
                    fwrf = open(os.path.join(c.outputdir,
                                'WRF' + cdate + '.' + ctime + '.000.grb2'), 'w')
                    olddate = cdate[:]

            # savedfields remembers which fields were already used.
            savedfields = []
            # sum of cloud liquid and ice water content
            scwc = None
            while 1:
                if not gid:
                    break
                paramId = grib_get(gid, 'paramId')
                gridtype = grib_get(gid, 'gridType')
                levtype = grib_get(gid, 'typeOfLevel')
                if paramId == 77: # ETADOT
                    grib_write(gid, fdict['21'])
                elif paramId == 130: # T
                    grib_write(gid, fdict['11'])
                elif paramId == 131 or paramId == 132: # U, V wind component
                    grib_write(gid, fdict['10'])
                elif paramId == 133 and gridtype != 'reduced_gg': # Q
                    grib_write(gid, fdict['17'])
                elif paramId == 133 and gridtype == 'reduced_gg': # Q, gaussian
                    grib_write(gid, fdict['18'])
                elif paramId == 135: # W
                    grib_write(gid, fdict['19'])
                elif paramId == 152: # LNSP
                    grib_write(gid, fdict['12'])
                elif paramId == 155 and gridtype == 'sh': # D
                    grib_write(gid, fdict['13'])
                elif paramId == 246 or paramId == 247: # CLWC, CIWC
                    # sum cloud liquid water and ice
                    if scwc is None:
                        scwc = grib_get_values(gid)
                    else:
                        scwc += grib_get_values(gid)
                        grib_set_values(gid, scwc)
                        grib_set(gid, 'paramId', 201031)
                        grib_write(gid, fdict['22'])
                elif c.wrf and paramId in [129, 138, 155] and \
                      levtype == 'hybrid': # Z, VO, D
                    # do not do anything right now
                    # these are specific parameter for WRF
                    pass
                else:
                    if paramId not in savedfields:
                        # SD/MSL/TCC/10U/10V/2T/2D/Z/LSM/SDOR/CVL/CVH/SR
                        # and all ADDPAR parameter
                        grib_write(gid, fdict['16'])
                        savedfields.append(paramId)
                    else:
                        print('duplicate ' + str(paramId) + ' not written')

                try:
                    if c.wrf:
                        # model layer
                        if levtype == 'hybrid' and \
                           paramId in [129, 130, 131, 132, 133, 138, 155]:
                            grib_write(gid, fwrf)
                        # sfc layer
                        elif paramId in wrfpars:
                except AttributeError:
                    pass

            for f in fdict.values():
                f.close()

            # call for Fortran program to convert e.g. reduced_gg grids to
            # regular_ll and calculate detadot/dp
            pwd = os.getcwd()
            os.chdir(c.inputdir)
            if os.stat('fort.21').st_size == 0 and c.eta:
                print('Parameter 77 (etadot) is missing, most likely it is \
                       not available for this type or date/time\n')
                print('Check parameters CLASS, TYPE, STREAM, START_DATE\n')
                my_error(c.mailfail, 'fort.21 is empty while parameter eta \
                         is set to 1 in CONTROL file')

            # Fortran program creates file fort.15 (with u,v,etadot,t,sp,q)
            p = subprocess.check_call([os.path.join(
                c.exedir, _config.FORTRAN_EXECUTABLE)], shell=True)
            os.chdir(pwd)

            # create name of final output file, e.g. EN13040500 (ENYYMMDDHH)
                suffix = cdate[2:8] + '.' + ctime + '.' + cstep
            else:
                suffix = cdate_hour[2:10]
            fnout = os.path.join(c.inputdir, c.prefix + suffix)
            # collect for final processing
            self.outputfilelist.append(os.path.basename(fnout))

            # create outputfile and copy all data from intermediate files
            # to the outputfile (final GRIB input files for FLEXPART)
            orolsm = os.path.basename(glob.glob(c.inputdir +
                                        '/OG_OROLSM__SL.*.' + c.ppid + '*')[0])
            fluxfile = 'flux' + cdate[0:2] + suffix
            if not c.cwc:
                flist = ['fort.15', fluxfile, 'fort.16', orolsm]
            else:
                flist = ['fort.15', 'fort.22', fluxfile, 'fort.16', orolsm]

            with open(fnout, 'wb') as fout:
                for f in flist:
                    shutil.copyfileobj(open(os.path.join(c.inputdir, f), 'rb'),
                                       fout)

                with open(os.path.join(c.outputdir, 'OMEGA'), 'wb') as fout:
                    shutil.copyfileobj(open(os.path.join(c.inputdir, 'fort.25'),
                                            'rb'), fout)

        if c.wrf:
            fwrf.close()

        return

    def process_output(self, c):
        '''The grib files are postprocessed depending on the selection in
        CONTROL file. The resulting files are moved to the output
        directory if its not equal to the input directory.
        The following modifications might be done if
        properly switched in CONTROL file:
        GRIB2 - Conversion to GRIB2
        ECTRANS - Transfer of files to gateway server
        ECSTORAGE - Storage at ECMWF server

        Return
        ------

        '''

        print('\n\nPostprocessing:\n Format: {}\n'.format(c.format))

        if not c.ecapi:
            print('ecstorage: {}\n ecfsdir: {}\n'.
                  format(c.ecstorage, c.ecfsdir))
            print('ectrans: {}\n gateway: {}\n destination: {}\n '
                  .format(c.ectrans, c.gateway, c.destination))

        print('Output filelist: ')
        print(self.outputfilelist)

        for ofile in self.outputfilelist:
            ofile = os.path.join(self.inputdir, ofile)

            if c.format.lower() == 'grib2':
                p = subprocess.check_call(['grib_set', '-s', 'edition=2, \
                                           productDefinitionTemplateNumber=8',
                                           ofile, ofile + '_2'])
                p = subprocess.check_call(['mv', ofile + '_2', ofile])

            if c.ectrans and not c.ecapi:
                p = subprocess.check_call(['ectrans', '-overwrite', '-gateway',
                                           c.gateway, '-remote', c.destination,
                                           '-source', ofile])

            if c.ecstorage and not c.ecapi:
                p = subprocess.check_call(['ecp', '-o', ofile,
                                           os.path.expandvars(c.ecfsdir)])

            if c.outputdir != c.inputdir:
                p = subprocess.check_call(['mv',
                                           os.path.join(c.inputdir, ofile),
                                           c.outputdir])

        return

    def prepare_fp_files(self, c):
        '''Conversion of GRIB files to FLEXPART binary format.

        Return
        ------

        '''
        # generate AVAILABLE file
        # Example of AVAILABLE file data:
        # 20131107 000000      EN13110700              ON DISC
        clist = []
            fname = ofile.split('/')
            if '.' in fname[-1]:
                l = fname[-1].split('.')
                timestamp = datetime.strptime(l[0][-6:] + l[1],
                                              '%y%m%d%H')
                timestamp += timedelta(hours=int(l[2]))
                cdate = datetime.strftime(timestamp, '%Y%m%d')
                chms = datetime.strftime(timestamp, '%H%M%S')
            else:
                cdate = '20' + fname[-1][-8:-2]
                chms = fname[-1][-2:] + '0000'
            clist.append(cdate + ' ' + chms + ' '*6 +
                         fname[-1] + ' '*14 + 'ON DISC')
        clist.sort()
        with open(c.outputdir + '/' + 'AVAILABLE', 'w') as f:
            f.write('\n'.join(clist) + '\n')

        # generate pathnames file
        pwd = os.path.abspath(c.outputdir)
        with open(pwd + '/pathnames', 'w') as f:
            f.write(pwd + '/Options/\n')
            f.write(pwd + '/\n')
            f.write(pwd + '/AVAILABLE\n')
            f.write(' = == = == = == = == = == ==  = \n')

        # create Options dir if necessary
        if not os.path.exists(pwd + '/Options'):
            make_dir(pwd+'/Options')

        # read template COMMAND file
        with open(os.path.expandvars(os.path.expanduser(
            c.flexpart_root_scripts)) + '/../Options/COMMAND', 'r') as f:
            lflist = f.read().split('\n')

        # find index of list where to put in the
        # date and time information
        # usually after the LDIRECT parameter
        i = 0
        for l in lflist:
            if 'LDIRECT' in l.upper():
                break
            i += 1

        # insert the date and time information of run start and end
        # into the list of lines of COMMAND file
        lflist = lflist[:i+1] + \
                 [clist[0][:16], clist[-1][:16]] + \
                 lflist[i+3:]

        # write the new COMMAND file
        with open(pwd + '/Options/COMMAND', 'w') as g:
            g.write('\n'.join(lflist) + '\n')

        # change to outputdir and start the grib2flexpart run
        # afterwards switch back to the working dir
        os.chdir(c.outputdir)
        p = subprocess.check_call([
            os.path.expandvars(os.path.expanduser(c.flexpart_root_scripts))
            + '/../FLEXPART_PROGRAM/grib2flexpart', 'useAvailable', '.'])
        os.chdir(pwd)

        return

# @Author: Anne Philipp (University of Vienna)
#
# @Date: March 2018
#
#
#        - migration of the methods dapoly and darain from Fortran
#          (flex_extract_v6 and earlier) to Python
#
#    April 2018 - Anne Philipp (University of Vienna):
#        - outsourced the disaggregation functions dapoly and darain
#          to a new module named disaggregation
#
#    (C) Copyright 2015-2018.
#
#
# @Module Description:
#    disaggregation of deaccumulated flux data from an ECMWF model FG field.
#    Initially the flux data to be concerned are:
#    - large-scale precipitation
#    - convective precipitation
#    - surface sensible heat flux
#    - surface solar radiation
#    - u stress
#    - v stress
#    Different versions of disaggregation is provided for rainfall
#    data (darain, modified linear) and the surface fluxes and
#    stress data (dapoly, cubic polynomial).
#
# @Module Content:
#    - dapoly
#    - darain
#    - IA3
#

# MODULES

# FUNCTIONS
def dapoly(alist):
    """Cubic polynomial interpolation of deaccumulated fluxes.

    Interpolation of deaccumulated fluxes of an ECMWF model FG field
    using a cubic polynomial solution which conserves the integrals
    of the fluxes within each timespan.
    Disaggregation is done for 4 accumluated timespans which
    generates a new, disaggregated value which is output at the
    central point of the 4 accumulation timespans.
    This new point is used for linear interpolation of the complete
    timeseries afterwards.

    alist : :obj:`list` of :obj:`array` of :obj:`float`
        List of 4 timespans as 2-dimensional, horizontal fields.
        E.g. [[array_t1], [array_t2], [array_t3], [array_t4]]

    Return
    ------
    nfield : :obj:`array` of :obj:`float`
        Interpolated flux at central point of accumulation timespan.

    Note
    ----
    March 2000    : P. JAMES
        Original author

    June 2003     : A. BECK
        Adaptations

    November 2015 : Leopold Haimberger (University of Vienna)
        Migration from Fortran to Python

    """

    pya = (alist[3] - alist[0] + 3. * (alist[1] - alist[2])) / 6.
    pyb = (alist[2] + alist[0]) / 2. - alist[1] - 9. * pya / 2.
    pyc = alist[1] - alist[0] - 7. * pya / 2. - 2. * pyb
    pyd = alist[0] - pya / 4. - pyb / 3. - pyc / 2.
    nfield = 8. * pya + 4. * pyb + 2. * pyc + pyd

    return nfield

def darain(alist):
    """Linear interpolation of deaccumulated fluxes.

    Interpolation of deaccumulated fluxes of an ECMWF model FG rainfall
    field using a modified linear solution which conserves the integrals
    Disaggregation is done for 4 accumluated timespans which generates
    a new, disaggregated value which is output at the central point
    of the 4 accumulation timespans. This new point is used for linear
    interpolation of the complete timeseries afterwards.

    Return
    ------

    Note
    ----

    """

    xa = alist[0]
    xb = alist[1]
    xc = alist[2]
    xd = alist[3]
    xa[xa < 0.] = 0.
    xb[xb < 0.] = 0.
    xc[xc < 0.] = 0.
    xd[xd < 0.] = 0.

    xac = 0.5 * xb
    mask = xa + xc > 0.
    xac[mask] = xb[mask] * xc[mask] / (xa[mask] + xc[mask])
    xbd = 0.5 * xc
    mask = xb + xd > 0.
    xbd[mask] = xb[mask] * xc[mask] / (xb[mask] + xd[mask])
    nfield = xac + xbd

def IA3(g):
    """ Interpolation with a non-negative geometric mean based algorithm.

    The original grid is reconstructed by adding two sampling points in each
    data series interval. This subgrid is used to keep all information during
    the interpolation within the associated interval. Additionally, an advanced
    monotonicity filter is applied to improve the monotonicity properties of
    the series.

    Note
    ----
    Copyright 2017
    Sabine Hittmeir, Anne Philipp, Petra Seibert

    This work is licensed under the Creative Commons Attribution 4.0
    International License. To view a copy of this license, visit
    http://creativecommons.org/licenses/by/4.0/ or send a letter to
    Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.

    g : :obj:`list` of :obj:`float`
        Complete data series that will be interpolated having
        the dimension of the original raw series.

    Return
    ------
    f : :obj:`list` of :obj:`float`
        The interpolated data series with additional subgrid points.
        Its dimension is equal to the length of the input data series
        times three.

    References
    For more information see article:
    Hittmeir, S.; Philipp, A.; Seibert, P. (2017): A conservative
    interpolation scheme for extensive quantities with application to the
    Lagrangian particle dispersion model FLEXPART.,
    Geoscientific Model Development
    """

    #######################  variable description #############################
    #                                                                         #
    # i      - index variable for looping over the data series                #
    # g      - input data series                                              #
    # f      - interpolated and filtered data series with additional          #
    #          grid points                                                    #
    # fi     - function value at position i, f_i                              #
    # fi1    - first  sub-grid function value f_i^1                           #
    # fi2    - second sub-grid function value f_i^2                           #
    # fip1   - next function value at position i+1, f_(i+1)                   #
    # dt     - time step                                                      #
    # fmon   - monotonicity filter                                            #
    #                                                                         #
    ###########################################################################

    import numpy as np

    # time step
    dt=1.0

    ############### Non-negative Geometric Mean Based Algorithm ###############

    # for the left boundary the following boundary condition is valid:
    # the value at t=0 of the interpolation algorithm coincides with the
    # first data value according to the persistence hypothesis
    f=[g[0]]

    # compute two first sub-grid intervals without monotonicity check
    # go through the data series and extend each interval by two sub-grid
    # points and interpolate the corresponding data values
    # except for the last interval due to boundary conditions
    for i in range(0,2):

        # as a requirement:
        # if there is a zero data value such that g[i]=0, then the whole
        # interval in f has to be zero to such that f[i+1]=f[i+2]=f[i+3]=0
        # according to Eq. (6)
        if g[i]==0.:
            f.extend([0.,0.,0.])

        # otherwise the sub-grid values are calculated and added to the list
        else:
            # temporal save of last value in interpolated list
            # since it is the left boundary and hence the new (fi) value
            fi = f[-1]

            # the value at the end of the interval (fip1) is prescribed by the
            # geometric mean, restricted such that non-negativity is guaranteed
            # according to Eq. (25)
            fip1=min( 3.*g[i] , 3.*g[i+1] , np.sqrt(g[i+1]*g[i]) )

            # the function value at the first sub-grid point (fi1) is determined
            # according to the equal area condition with Eq. (19)
            fi1=3./2.*g[i]-5./12.*fip1-1./12.*fi

            # the function value at the second sub-grid point (fi2) is determined
            # according Eq. (18)
            fi2=fi1+1./3.*(fip1-fi)

            # add next interval of interpolated (sub-)grid values
            f.append(fi1)
            f.append(fi2)
            f.append(fip1)

    # compute rest of the data series intervals
    for i in range(2,len(g)-1):

            # apply monotonicity filter for interval before
            # check if there is "M" or "W" shape
            if     np.sign(f[-5]-f[-6]) * np.sign(f[-4]-f[-5])==-1 \
               and np.sign(f[-4]-f[-5]) * np.sign(f[-3]-f[-4])==-1 \
               and np.sign(f[-3]-f[-4]) * np.sign(f[-2]-f[-3])==-1:

                # the monotonicity filter corrects the value at (fim1) by
                # substituting (fim1) with (fmon), see Eq. (27), (28) and (29)
                fmon = min(3.*g[i-2], \
                           3.*g[i-1], \
                           np.sqrt(max(0,(18./13.*g[i-2] - 5./13.*f[-7]) *
                                         (18./13.*g[i-1] - 5./13.*f[-1]))))

                # recomputation of the sub-grid interval values while the
                # interval boundaries (fi) and (fip2) remains unchanged
                # see Eq. (18) and (19)
                f[-4]=fmon
                f[-6]=3./2.*g[i-2]-5./12.*fmon-1./12.*f[-7]
                f[-5]=f[-6]+(fmon-f[-7])/3.
                f[-3]=3./2.*g[i-1]-5./12.*f[-1]-1./12.*fmon
                f[-2]=f[-3]+(f[-1]-fmon)/3.

        else:
            fi = f[-1]

                # substituting (fim1) with fmon, see Eq. (27), (28) and (29)

    # separate treatment of the final interval

    # as a requirement:
    # if there is a zero data value such that g[i]=0, then the whole
    # interval in f has to be zero to such that f[i+1]=f[i+2]=f[i+3]=0
    # according to Eq. (6)
    if g[-1]==0.:
        # apply monotonicity filter for interval before
        # check if there is "M" or "W" shape
        if     np.sign(f[-5]-f[-6]) * np.sign(f[-4]-f[-5])==-1 \
           and np.sign(f[-4]-f[-5]) * np.sign(f[-3]-f[-4])==-1 \
           and np.sign(f[-3]-f[-4]) * np.sign(f[-2]-f[-3])==-1:

            # the monotonicity filter corrects the value at (fim1) by
            # substituting (fim1) with (fmon), see Eq. (27), (28) and (29)
            fmon = min(3.*g[-3], \
                       3.*g[-2], \
                       np.sqrt(max(0,(18./13.*g[-3] - 5./13.*f[-7]) *
                                     (18./13.*g[-2] - 5./13.*f[-1]))))

            # recomputation of the sub-grid interval values while the
            # interval boundaries (fi) and (fip2) remains unchanged
            # see Eq. (18) and (19)
            f[-4]=fmon
            f[-6]=3./2.*g[-3]-5./12.*fmon-1./12.*f[-7]
            f[-5]=f[-6]+(fmon-f[-7])/3.
            f[-3]=3./2.*g[-2]-5./12.*f[-1]-1./12.*fmon
            f[-2]=f[-3]+(f[-1]-fmon)/3.

        f.extend([0.,0.,0.])

    # otherwise the sub-grid values are calculated and added to the list
    # using the persistence hypothesis as boundary condition
    else:
        # temporal save of last value in interpolated list
        # since it is the left boundary and hence the new (fi) value
        fi = f[-1]
        # since last interval in series, last value is also fip1
        fip1 = g[-1]
        # the function value at the first sub-grid point (fi1) is determined
        # according to the equal area condition with Eq. (19)
        fi1 = 3./2.*g[-1]-5./12.*fip1-1./12.*fi
        # the function value at the second sub-grid point (fi2) is determined
        # according Eq. (18)
        fi2 = fi1+dt/3.*(fip1-fi)

        # add next interval of interpolated (sub-)grid values
        f.append(fi1)
        f.append(fi2)
        f.append(fip1)

    return f

#
#
#
#        - modified method list_files to work with glob instead of listdir
#        - added pattern search in method list_files
#
#        - optimisation of method list_files since it didn't work correctly
#          for sub directories
#        - additional speed up of method list_files
#        - modified the class so that it is initiated with a pattern instead
#          of suffixes. Gives more precision in selection of files.
#
#
#
# @Class Decription:
#    The class is for file manipulation. It is initiated with a regular
#    expression pattern for this instance and can produce a list of Files
#    from the given file pattern. These files can be deleted.
#
#    - __str__
#    - __list_files__
#    - delete_files
#
#    - pattern
#    - files
#

# MODULES
import os
import sys
import fnmatch

# software specific module from flex_extract
#import profiling
from mods.tools import silent_remove, get_list_as_string

# CLASS

class UioFiles(object):
    '''Class to manipulate files. At initialisation it has the pattern
    which stores a regular expression pattern for the files, the path
    to the files and the files already.
    '''
    def __init__(self, path, pattern):
        '''Assignes a specific pattern for these files.

        path : :obj:`string`
            Directory where to list the files.

        pattern : :obj:`string`
            Regular expression pattern. For example: '\*.grb'

        Return
        ------

        '''

        self.path = path
        self.pattern = pattern
        self.files = []

        self.__list_files__(self.path)

        return

    #@profiling.timefn
    def __list_files__(self, path):
        '''Lists all files in the directory with the matching
        regular expression pattern.

            Path to the files.

        Return
        ------

        '''
        # Get the absolute path
        path = os.path.abspath(path)

        # get all files in the dir and subdir as absolut path
        for root, dirnames, filenames in os.walk(path):
            for filename in fnmatch.filter(filenames, self.pattern):
                self.files.append(os.path.join(root, filename))

        return

    def __str__(self):
        '''Converts the list of files into a single string.
        The entries are sepereated by "," sign.

        Return
        ------
        files_string : :obj:`string`
            The content of the list as a single string.
        '''

        filenames = [os.path.basename(f) for f in self.files]
        files_string = get_list_as_string(filenames, concatenate_sign=', ')

        return files_string

    def delete_files(self):
        '''Deletes the files.

        Return
        ------

        '''

        for old_file in self.files:
            silent_remove(old_file)

        return

#
#
#
#
#        - minor changes in programming style (for consistence)
#        - changed path names to variables from config file
#        - added option for writing mars requests to extra file
#          additionally,as option without submitting the mars jobs
#
#
#
#    This program is the main program of flex_extract and controls the
#    program flow.
#    If it is supposed to work locally then it works through the necessary
#    functions get_mars_data and prepareFlexpart. Otherwise it prepares
#    a shell job script which will do the necessary work on the
#    ECMWF server and is submitted via ecaccess-job-submit.
#
#    - main
#    - submit
#

# MODULES
import os
import sys
import collections

from mods.tools import (normal_exit, get_cmdline_arguments,
                        submit_job_to_ecserver, read_ecenv)
from mods.get_mars_data import get_mars_data
from mods.prepare_flexpart import prepare_flexpart

    '''Get the arguments from script call and from CONTROL file.
    Decides from the argument "queue" if the local version
    is done "queue=None" or the gateway version with "queue=ecgate"
    or "queue=cca".

    Return
    ------

    '''

    # on local side
    # on ECMWF server this would also be the local side
    called_from_dir = os.getcwd()
    if args.queue is None:
        if c.inputdir[0] != '/':
            c.inputdir = os.path.join(called_from_dir, c.inputdir)
        if c.outputdir[0] != '/':
            c.outputdir = os.path.join(called_from_dir, c.outputdir)
        get_mars_data(c)
        if c.request == 0 or c.request == 2:
            prepare_flexpart(args.ppid, c)
            normal_exit(c.mailfail, 'FLEX_EXTRACT IS DONE!')
        else:
            normal_exit(c.mailfail, 'PRINTING MARS_REQUESTS DONE!')
    # send files to ECMWF server and install there
    else:
        submit(args.job_template, c, args.queue)

    return

def submit(jtemplate, c, queue):
    '''Prepares the job script and submit it to the specified queue.

    jtemplate : :obj:`string`
        Job template file from sub-directory "_templates" for
        submission to ECMWF. It contains all necessary
        module and variable settings for the ECMWF environment as well as
        the job call and mail report instructions.
        Default is "job.temp".

    queue : :obj:`string`
        Name of queue for submission to ECMWF (e.g. ecgate or cca )

    Return
    ------

    '''

    # read template file and get index for CONTROL input
    with open(os.path.join(_config.PATH_TEMPLATES, jtemplate)) as f:
        lftext = f.read().split('\n')
    insert_point = lftext.index('EOF')

    if not c.basetime:
    # --------- create on demand job script ------------------------------------
        if c.maxstep > 24:
            print('---- Pure forecast mode! ----')
        else:
            print('---- On-demand mode! ----')
        job_file = os.path.join(_config.PATH_JOBSCRIPTS,
                                jtemplate[:-4] + 'ksh')
        clist = c.to_list()

        lftextondemand = lftext[:insert_point] + clist + lftext[insert_point:]

        with open(job_file, 'w') as f:
            f.write('\n'.join(lftextondemand))

        submit_job_to_ecserver(queue, job_file)

    else:
    # --------- create operational job script ----------------------------------
        print('---- Operational mode! ----')
                                jtemplate[:-5] + 'oper.ksh')

        if c.maxstep:
            mt = int(c.maxstep)
        else:
            mt = 0

        c.start_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
        c.end_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
        c.base_time = '${MSJ_BASETIME}'
        if mt > 24:
            c.time = '${MSJ_BASETIME} {MSJ_BASETIME}'

        colist = c.to_list()

        lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:]

            f.write('\n'.join(lftextoper))

    print('You should get an email with subject flex.hostname.pid')

    return

    main()

# @Author: Leopold Haimberger (University of Vienna)
#
# @Date: November 2015
#
#
#        - moved install_args_and_control in here
#        - splitted code in smaller functions
#        - delete convert build files in here instead of compile job script
#        - changed static path names to Variables from config file
#
#
#
#    Depending on the selected installation environment (locally or on the
#    ECMWF server ecgate or cca) the program extracts the commandline
#    arguments and the CONTROL file parameter and prepares the corresponding
#    environment. The necessary files are collected in a tar-ball and placed
#    at the target location. There its untared, the environment variables will
#    be set and the Fortran code will be compiled. If the ECMWF environment is
#    selected a job script is prepared and submitted for the remaining
#    configurations after putting the tar-ball to the target ECMWF server.
#
#    - main
#    - get_install_cmdline_arguments
#    - install_via_gateway
#    - mk_tarball
#    - un_tarball
#    - mk_env_vars
#    - mk_compilejob
#    - mk_job_template
#    - delete_convert_build
#    - make_convert_build
#

# MODULES
import os
import sys
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter

from mods.tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver

    '''Controls the installation process. Calls the installation function
    if target is specified.

    Return
    ------
    '''

    args = get_install_cmdline_arguments()
    c.check_install_conditions()

    install_via_gateway(c)

    return

def get_install_cmdline_arguments():
    '''Decomposes the command line arguments and assigns them to variables.
    Apply default values for non mentioned arguments.

    Return
    ------
    args : :obj:`Namespace`
        Contains the commandline arguments from script/program call.
    '''
    parser = ArgumentParser(description='Install flex_extract software locally or \
                            on ECMWF machines',
                            formatter_class=ArgumentDefaultsHelpFormatter)

    parser.add_argument('--target', dest='install_target', default=None,
                        help="Valid targets: local | ecgate | cca , \
                        the latter two are at ECMWF")
    parser.add_argument("--makefile", dest="makefile", default=None,
                        help='Name of Makefile to use for compiling CONVERT2')
    parser.add_argument("--ecuid", dest="ecuid", default=None,
                        help='user id at ECMWF')
    parser.add_argument("--ecgid", dest="ecgid", default=None,
                        help='group id at ECMWF')
    parser.add_argument("--gateway", dest="gateway", default=None,
                        help='name of local gateway server')
    parser.add_argument("--destination", dest="destination", default=None,
                        help='ecaccess destination, e.g. leo@genericSftp')

    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
                        default=None, help="FLEXPART root directory on ECMWF \
                        servers (to find grib2flexpart and COMMAND file)\n\
                        Normally flex_extract resides in the scripts directory \
                        of the FLEXPART distribution.")

    # arguments for job submission to ECMWF, only needed by submit.py
    parser.add_argument("--job_template", dest='job_template',
                        default="job.temp.o",
                        help="job template file for submission to ECMWF")

    parser.add_argument("--controlfile", dest="controlfile",
                        default='CONTROL.temp',
                        help="file with CONTROL parameters")

    args = parser.parse_args()

    return args

def install_via_gateway(c):
    '''Perform the actual installation on local machine or prepare data
    transfer to remote gate and submit a job script which will
    install everything on the remote gate.

    Return
    ------

    '''
    import tarfile

    ecd = _config.PATH_FLEXEXTRACT_DIR
    tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
    tar_file = os.path.join(ecd, tarball_name)

    target_dirname = _config.FLEXEXTRACT_DIRNAME
    fortran_executable = _config.FORTRAN_EXECUTABLE

    if c.install_target.lower() != 'local': # ecgate or cca

        mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
                      c.flexpart_root_scripts)

        mk_job_template(c.ecuid, c.ecgid, c.gateway,
                        c.destination, c.flexpart_root_scripts)

        mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)

        mk_tarball(tar_file, c.install_target)

        put_file_to_ecserver(ecd, tarball_name, c.install_target,
                             c.ecuid, c.ecgid)

        submit_job_to_ecserver(c.install_target,
                               os.path.join(_config.PATH_REL_JOBSCRIPTS,
                                            _config.FILE_INSTALL_COMPILEJOB))

        print('job compilation script has been submitted to ecgate for ' +
              'installation in ' + c.flexpart_root_scripts +
               '/' + target_dirname)
        print('You should get an email with subject "flexcompile" within ' +
              'the next few minutes!')

    else: #local
        if c.flexpart_root_scripts == _config.PATH_FLEXEXTRACT_DIR :
            print('WARNING: FLEXPART_ROOT_SCRIPTS has not been specified')
            print('flex_extract will be installed in here by compiling the ' +
                  'Fortran source in ' + _config.PATH_FORTRAN_SRC)
            os.chdir(_config.PATH_FORTRAN_SRC)
        else: # creates the target working directory for flex_extract
            c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
                                        c.flexpart_root_scripts))
            if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
                mk_tarball(tar_file, c.install_target)
                make_dir(os.path.join(c.flexpart_root_scripts,
                                      target_dirname))
                os.chdir(os.path.join(c.flexpart_root_scripts,
                un_tarball(tar_file)
                                      target_dirname,
                                      _config.PATH_REL_FORTRAN_SRC))

        # Create Fortran executable - CONVERT2
        print('Install ' + target_dirname + ' software at ' +
              c.install_target + ' in directory ' +
              os.path.abspath(c.flexpart_root_scripts) + '\n')

        delete_convert_build('.')
        make_convert_build('.', c.makefile)

        os.chdir(ecd)
        if os.path.isfile(tar_file):
            os.remove(tar_file)

    return

def mk_tarball(tarball_path, target):
    '''Creates a tarball with all necessary files which need to be sent to the
    installation directory.
    It does not matter if this is local or remote.
    Collects all python files, the Fortran source and makefiles,
    the ECMWF_ENV file, the CONTROL files as well as the
    template files.

    tarball_path : :obj:`string`
        The complete path to the tar file which will contain all
        relevant data for flex_extract.

    target : :obj:`string`
        The queue where the job is submitted to.

    Return
    ------

    '''
    from glob import glob

    print('Create tarball ...')

    # change to FLEXEXTRACT directory so that the tar can contain
    # relative pathes to the files and directories
    ecd = _config.PATH_FLEXEXTRACT_DIR + '/'
    os.chdir(ecd)

    # get lists of the files to be added to the tar file
    if target == 'local':
        ECMWF_ENV_FILE = []
    else:
        ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]

    pyfiles = [os.path.relpath(x, ecd)
               for x in UioFiles(_config.PATH_LOCAL_PYTHON, '*py').files]
    controlfiles = [os.path.relpath(x, ecd)
                    for x in UioFiles(_config.PATH_CONTROLFILES,
                                      'CONTROL*').files]
    tempfiles = [os.path.relpath(x, ecd)
                 for x in UioFiles(_config.PATH_TEMPLATES , '*').files]
    ffiles = [os.path.relpath(x, ecd)
              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.f*').files]
    hfiles = [os.path.relpath(x, ecd)
              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.h').files]
    makefiles = [os.path.relpath(x, ecd)
                 for x in UioFiles(_config.PATH_FORTRAN_SRC, 'Makefile*').files]

    # concatenate single lists to one for a better looping
    filelist = pyfiles + controlfiles + tempfiles + ffiles + hfiles + \
               makefiles + ECMWF_ENV_FILE

    # create installation tar-file
    try:
        with tarfile.open(tarball_path, "w:gz") as tar_handle:
            for file in filelist:
                tar_handle.add(file)

    except subprocess.CalledProcessError as e:
        print('... ERROR CODE:\n ... ' + str(e.returncode))
        print('... ERROR MESSAGE:\n ... ' + str(e))

        sys.exit('... could not make installation tar ball!')

    return

def un_tarball(tarball_path):
    '''Extracts the given tarball into current directory.

    Return
    ------

    '''

    print('Untar ...')

    with tarfile.open(tarball_path) as tar_handle:
        tar_handle.extractall()

    return

def mk_env_vars(ecuid, ecgid, gateway, destination):
    '''Creates a file named ECMWF_ENV which contains the
    necessary environmental variables at ECMWF servers.
    It is based on the template ECMWF_ENV.template.

    ecuid : :obj:`string`
        The user id on ECMWF server.

    ecgid : :obj:`string`
        The group id on ECMWF server.

    gateway : :obj:`string`
        The gateway server the user is using.

    destination : :obj:`string`
        The remote destination which is used to transfer files
        from ECMWF server to local gateway server.

    Return
    ------

    '''
    from genshi.template.text import NewTextTemplate
    from genshi.template import  TemplateLoader

    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
    ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
                                     cls=NewTextTemplate)

    stream = ecmwfvars_template.generate(user_name = ecuid,
                                       user_group = ecgid,
                                       gateway_name = gateway,
                                       destination_name = destination
                                       )

    with open(_config.PATH_ECMWF_ENV, 'w') as f:
        f.write(stream.render('text'))

    return

def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
    '''Modifies the original job template file so that it is specified
    for the user and the environment were it will be applied. Result
    is stored in a new file "job.temp" in the python directory.

    makefile : :obj:`string`
        Name of the makefile which should be used to compile FORTRAN
        CONVERT2 program.

        The target where the installation should be done, e.g. the queue.

    fp_root : :obj:`string`
       Path to the root directory of FLEXPART environment or flex_extract
       environment.

    Return
    ------

    '''

    compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
                                   cls=NewTextTemplate)

    if fp_root == '../':
        fp_root = '$HOME'

    stream = compile_template.generate(
        usergroup = ecgid,
        username = ecuid,
        version_number = _config._VERSION_STR,
        fp_root_scripts = fp_root,
        makefile = makefile,
        fortran_program = _config.FORTRAN_EXECUTABLE
    )

    compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
                              _config.FILE_INSTALL_COMPILEJOB)

    with open(compilejob, 'w') as f:

    return

def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
    is stored in a new file.

    Return
    ------

    '''

    compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,

    fp_root_path_to_python = os.path.join(fp_root,
                                          _config.FLEXEXTRACT_DIRNAME,
                                          _config.PATH_REL_PYTHON)

        fp_root_path = fp_root_path_to_python,
    )

    tempjobfile = os.path.join(_config.PATH_TEMPLATES,
                               _config.TEMPFILE_JOB)

    with open(tempjobfile, 'w') as f:

    return

def delete_convert_build(src_path):
    '''Clean up the Fortran source directory and remove all
    build files (e.g. \*.o, \*.mod and CONVERT2)

    src_path : :obj:`string`
        Path to the fortran source directory.

    Return
    ------

    '''

    modfiles = UioFiles(src_path, '*.mod')
    objfiles = UioFiles(src_path, '*.o')
    exefile = UioFiles(src_path, _config.FORTRAN_EXECUTABLE)

    modfiles.delete_files()
    objfiles.delete_files()
    exefile.delete_files()

    return

def make_convert_build(src_path, makefile):
    '''Compiles the Fortran code and generates the executable.

        The name of the makefile which should be used.

    Return
    ------

    '''

    try:
        print('Using makefile: ' + makefile)
        p = subprocess.Popen(['make', '-f',
                              os.path.join(src_path, makefile)],
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             bufsize=1)
        pout, perr = p.communicate()
        print(pout)
        if p.returncode != 0:
            print(perr)
            print('Please edit ' + makefile +
                  ' or try another Makefile in the src directory.')
            print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
                  'and EMOSLIB must be adapted.')
            print('Available Makefiles:')
            print(UioFiles(src_path, 'Makefile*'))
            sys.exit('Compilation failed!')
    except ValueError as e:
        print('ERROR: Makefile call failed:')
        print(e)
    else:
        subprocess.check_call(['ls', '-l',
                               os.path.join(src_path,
                                            _config.FORTRAN_EXECUTABLE)])

    return

    main()

#
#
#
#   November 2015 - Leopold Haimberger (University of Vienna):
#        - optimized display_info
#        - optimized data_retrieve and seperate between python and shell
#          script call
#
#        - applied some minor modifications in programming style/structure
#
#
#
#    A MARS revtrieval has a specific syntax with a selection of keywords and
#    their corresponding values. This class provides the necessary functions
#    by displaying the selected parameters and their values and the actual
#    retrievement of the data through a mars request or a Python web api
#    interface. The initialization already expects all the keyword values.
#
#    - display_info
#    - data_retrieve
#
#    - server
#    - marsclass
#    - dtype
#    - levtype
#    - levelist
#    - repres
#    - date
#    - resol
#    - stream
#    - area
#    - time
#    - step
#    - expver
#    - number
#    - accuracy
#    - grid
#    - gaussian
#    - target
#    - param
#

# MODULES
import os
import sys

# CLASS
class MarsRetrieval(object):
    '''Class for submitting MARS retrievals.

    A description of MARS keywords/arguments and examples of their
    values can be found here:
    https://software.ecmwf.int/wiki/display/UDOC/\
                   Identification+keywords#Identificationkeywords-class
    '''

    def __init__(self, server, public, marsclass="ei", dataset="", type="",
                 levtype="", levelist="", repres="", date="", resol="",
                 stream="", area="", time="", step="", expver="1",
                 number="", accuracy="", grid="", gaussian="", target="",
                 param=""):
        '''Initialises the instance of the MarsRetrieval class and
        defines and assigns a set of the necessary retrieval parameters
        for the FLEXPART input data.
        A description of MARS keywords/arguments, their dependencies
        on each other and examples of their values can be found here:

        https://software.ecmwf.int/wiki/display/UDOC/MARS+keywords

        server : :obj:`ECMWFService`
            This is the connection to the ECMWF data servers.
            It is needed for the pythonic access of ECMWF data.

        public : :obj:`integer`
            Decides which Web API version is used:
            0: member-state users and full archive access
            1: public access and limited access to the public server and
               datasets. Needs the parameter dataset.
            Default is "0" and for member-state users.

        marsclass : :obj:`string`, optional
            Characterisation of dataset. E.g. EI (ERA-Interim),
            E4 (ERA40), OD (Operational archive), ea (ERA5).
            Default is the ERA-Interim dataset "ei".

        dataset : :obj:`string`, optional
            For public datasets there is the specific naming and parameter
            dataset which has to be used to characterize the type of
            data. Usually there is less data available, either in times,
            domain or parameter.
            Default is an empty string.

        type : :obj:`string`, optional
            Determines the type of fields to be retrieved.
            Selects between observations, images or fields.
            Examples for fields: Analysis (an), Forecast (fc),
            Perturbed Forecast (pf), Control Forecast (cf) and so on.

        levtype : :obj:`string`, optional
            Denotes type of level. Has a direct implication on valid
            levelist values!
            E.g. model level (ml), pressure level (pl), surface (sfc),
            potential vorticity (pv), potential temperature (pt)
            and depth (dp).

        levelist : :obj:`string`, optional
            Specifies the required levels. It has to have a valid
            correspondence to the selected levtype.
            Examples: model level: 1/to/137, pressure levels: 500/to/1000

        repres : :obj:`string`, optional
            Selects the representation of the archived data.
            E.g. sh - spherical harmonics, gg - Gaussian grid,
            ll - latitude/longitude, ...

        date : :obj:`string`, optional
            Specifies the Analysis date, the Forecast base date or
            Observations date. Valid formats are:
            Absolute as YYYY-MM-DD or YYYYMMDD.

        resol : :obj:`string`, optional
            Specifies the desired triangular truncation of retrieved data,
            before carrying out any other selected post-processing.
            The default is automatic truncation (auto), by which the lowest
            resolution compatible with the value specified in grid is
            automatically selected for the retrieval.
            Users wanting to perform post-processing from full spectral
            resolution should specify Archived Value (av).
            The following are examples of existing resolutions found in
            the archive: 63, 106, 159, 213, 255, 319, 399, 511, 799 or 1279.
            This keyword has no meaning/effect if the archived data is
            not in spherical harmonics representation.
            The best selection can be found here:
            https://software.ecmwf.int/wiki/display/UDOC/\
                  Retrieve#Retrieve-Truncationbeforeinterpolation

        stream : :obj:`string`, optional
            Identifies the forecasting system used to generate the data.
            E.g. oper (Atmospheric model), enfo (Ensemble forecats), ...

        area : :obj:`string`, optional
            Specifies the desired sub-area of data to be extracted.
            Areas can be defined to wrap around the globe.

            Latitude values must be given as signed numbers, with:
                north latitudes (i.e. north of the equator)
                    being positive (e.g: 40.5)
                south latitutes (i.e. south of the equator)
                    being negative (e.g: -50.5)
            Longtitude values must be given as signed numbers, with:
                east longitudes (i.e. east of the 0 degree meridian)
                    being positive (e.g: 35.0)
                west longitudes (i.e. west of the 0 degree meridian)
                    being negative (e.g: -20.5)

            E.g.: North/West/South/East

        time : :obj:`string`, optional
            Specifies the time of the data in hours and minutes.
            Valid values depend on the type of data: Analysis time,
            Forecast base time or First guess verification time
            (all usually at synoptic hours: 00, 06, 12 and 18 ).
            Observation time (any combination in hours and minutes is valid,
            subject to data availability in the archive).
            The syntax is HHMM or HH:MM. If MM is omitted it defaults to 00.

        step : :obj:`string`, optional
            Specifies the forecast time step from forecast base time.
            Valid values are hours (HH) from forecast base time. It also
            specifies the length of the forecast which verifies at
            First Guess time.
            E.g. 1/3/6-hourly

        expver : :obj:`string`, optional
            The version of the dataset. Each experiment is assigned a
            unique code (version). Production data is assigned 1 or 2,
            and experimental data in Operations 11, 12 ,...
            Research or Member State's experiments have a four letter
            experiment identifier.
            Default is "1".

        number : :obj:`string`, optional
            Selects the member in ensemble forecast run. (Only then it
            is necessary.) It has a different meaning depending on
            the type of data.
            E.g. Perturbed Forecasts: specifies the Ensemble forecast member

        accuracy : :obj:`string`, optional
            Specifies the number of bits per value to be used in the
            generated GRIB coded fields.
            A positive integer may be given to specify the preferred number
            of bits per packed value. This must not be greater than the
            number of bits normally used for a Fortran integer on the
            processor handling the request (typically 32 or 64 bit).
            Within a compute request the accuracy of the original fields
            can be passed to the result field by specifying accuracy=av.

        grid : :obj:`string`, optional
            Specifies the output grid which can be either a Gaussian grid
            or a Latitude/Longitude grid. MARS requests specifying
            grid=av will return the archived model grid.

            Lat/Lon grid: The grid spacing needs to be an integer
            fraction of 90 degrees e.g. grid = 0.5/0.5

            Gaussian grid: specified by a letter denoting the type of
            Gaussian grid followed by an integer (the grid number)
            representing the number of lines between the Pole and Equator,
            e.g.
            grid = F160 - full (or regular) Gaussian grid with
                   160 latitude lines between the pole and equator
            grid = N320 - ECMWF original reduced Gaussian grid with
                   320 latitude lines between the pole and equator,
                   see Reduced Gaussian Grids for grid numbers used at ECMWF
            grid = O640 - ECMWF octahedral (reduced) Gaussian grid with
                   640 latitude lines between the pole and equator

        gaussian : :obj:`string`, optional
            This parameter is deprecated and should no longer be used.
            Specifies the desired type of Gaussian grid for the output.
            Valid Gaussian grids are quasi-regular (reduced) or regular.
            Keyword gaussian can only be specified together with
            keyword grid. Gaussian without grid has no effect.

        target : :obj:`string`, optional
            Specifies a file into which data is to be written after
            retrieval or manipulation. Path names should always be
            enclosed in double quotes. The MARS client supports automatic
            generation of multiple target files using MARS keywords
            enclosed in square brackets [ ].  If the environment variable
            MARS_MULTITARGET_STRICT_FORMAT is set to 1 before calling mars,
            the keyword values will be used in the filename as shown by
            the ecCodes GRIB tool grib_ls -m, e.g. with
            MARS_MULTITARGET_STRICT_FORMAT set to 1 the keywords time,
            expver and param will be formatted as 0600, 0001 and 129.128
            rather than 600, 1 and 129.

        param : :obj:`string`, optional
            Specifies the meteorological parameter.
            The list of meteorological parameters in MARS is extensive.
            Their availability is directly related to their meteorological
            meaning and, therefore, the rest of directives specified
            in the MARS request.
            Meteorological parameters can be specified by their
            GRIB code (param=130), their mnemonic (param=t) or
            full name (param=temperature).
            The list of parameter should be seperated by a "/"-sign.
            E.g. 130/131/133

        Return
        ------

        '''

        self.marsclass = marsclass
        self.dataset = dataset
        self.type = type
        self.levtype = levtype
        self.levelist = levelist
        self.repres = repres
        self.date = date
        self.resol = resol
        self.stream = stream
        self.area = area
        self.time = time
        self.step = step
        self.expver = expver
        self.number = number
        self.accuracy = accuracy
        self.grid = grid
        self.gaussian = gaussian
        self.target = target
        self.param = param

        return

    def display_info(self):
        '''Prints all class attributes and their values to the
        standard output.

        Return
        ------

        '''
        # Get all class attributes and their values as a dictionary
        attrs = vars(self).copy()

        # iterate through all attributes and print them
        # with their corresponding values
        for item in attrs.items():
            if item[0] in ['server', 'public']:
                pass
            else:
                print(item[0] + ': ' + str(item[1]))

        return

    def print_info(self, inputdir, request_number):
        '''Prints all mars requests to an extra file for debugging and
        information.

        inputdir : :obj:`string`
            The path where all data from the retrievals are stored.

        request_number : :obj:`integer`
            Number of mars requests for flux and non-flux data.

        Return
        ------

        '''

        # open a file to store all requests to
        with open(os.path.join(inputdir,
                               _config.FILE_MARS_REQUESTS), 'a') as f:
            f.write('mars_request #' + str(request_number) + '\n')
            # iterate through all attributes and print them
            # with their corresponding values
            for item in attrs.items():
                if item[0] in ['server', 'public']:
                    pass
                else:
                    f.write(item[0] + ': ' + str(item[1]) + '\n')
            f.write('\n\n')

        return

    def print_infodata_csv(self, inputdir, request_number):
        '''Write all request parameter in alpabetical order into a "csv" file.

        Return
        ------

        '''

        del attrs['server']
        del attrs['public']

            f.write(str(request_number) + ', ')
            f.write(', '.join(str(attrs[key])
                              for key in sorted(attrs.iterkeys())))
            f.write('\n')

        return

    def data_retrieve(self):
        '''Submits a MARS retrieval. Depending on the existence of
        ECMWF Web-API it is submitted via Python or a
        subprocess in the Shell. The parameter for the mars retrieval
        are taken from the defined class attributes.

        Return
        ------

        '''

        # eliminate unnecessary attributes from the dictionary attrs

        # exchange parameter name for marsclass
        mclass = attrs.get('marsclass')
        del attrs['marsclass']
        attrs['class'] = mclass

        # prepare target variable as needed for the Web API mode
        # within the dictionary for full access
        # as a single variable for public access
        target = attrs.get('target')
        if not int(self.public):
            del attrs['target']
        print('target: ' + target)

        # find all keys without a value and convert all other values to strings
        empty_keys = []
        for key, value in attrs.iteritems():
            if value == '':
                empty_keys.append(str(key))
            else:
                attrs[key] = str(value)

        # delete all empty parameter from the dictionary
        for key in empty_keys:
            del attrs[key]

        # MARS request via Python script
        if self.server:
            try:
                if self.public:
                    print('RETRIEVE PUBLIC DATA!')
                    self.server.retrieve(attrs)
                else:
                    print('EXECUTE NON-PUBLIC RETRIEVAL!')
                    self.server.execute(attrs, target)
            except:
                e = sys.exc_info()[0]
                print("ERROR: ", e)
                print('MARS Request failed!')
                if not self.public and os.stat(target).st_size == 0:
                    print('MARS Request returned no data - '
                          'please check request')
                    raise IOError
                elif self.public and os.stat(target).st_size == 0:
                    print('Public MARS Request returned no data - '
                else:
        # MARS request via extra process in shell
        else:
            request_str = 'ret'
            for key, value in attrs.iteritems():
                request_str = request_str + ',' + key + '=' + str(value)
            request_str += ',target="' + target + '"'
            p = subprocess.Popen(['mars'],
                                 stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE,
                                 bufsize=1)
            pout = p.communicate(input=request_str)[0]
            print(pout.decode())

            if 'Some errors reported' in pout.decode():
                print('MARS Request failed - please check request')
                raise IOError
            elif os.stat(target).st_size == 0:
                print('MARS Request returned no data - please check request')

        return

#
#
#
#        - changed name of class Control to ControlFile for more
#          self-explanation naming
#        - outsource of class ControlFile
#        - initialisation of class attributes ( to avoid high number of
#          conditional statements and set default values )
#        - divided assignment of attributes and the check of conditions
#        - outsourced the commandline argument assignments to control attributes
#
#
#
#    The CONTROL file is the steering part of the FLEXPART extraction
#    software. All necessary parameters needed to retrieve the data fields
#    from the MARS archive for driving FLEXPART are set in a CONTROL file.
#    Some specific parameters like the start and end dates can be overwritten
#    by the command line parameters, but in generel all parameters needed
#    for a complete set of fields for FLEXPART can be set in the CONTROL file.
#
#    - __read_controlfile__
#    - __str__
#    - assign_args_to_control
#    - assign_envs_to_control
#    - check_conditions
#    - check_install_conditions
#    - to_list
#
#
#

# MODULES
import os
import re
import sys

from mods.tools import my_error, silent_remove

# CLASS
class ControlFile(object):
    '''
    Contains the information which are stored in the CONTROL files.
    '''

    def __init__(self, filename):
        '''Initialises the instance of ControlFile class and defines
        all class attributes with default values. Afterwards calls
        function __read_controlfile__ to read parameter from Control file.

            Name of CONTROL file.

        Return
        ------

        '''

        # list of all possible class attributes and their default values
        self.controlfile = filename
        self.start_date = None
        self.end_date = None
        self.date_chunk = 3
        self.dtime = None
        self.basetime = None
        self.maxstep = None
        self.type = None
        self.time = None
        self.step = None
        self.acctype = None
        self.acctime = None
        self.accmaxstep = None
        self.marsclass = None
        self.dataset = None
        self.stream = None
        self.number = 'OFF'
        self.expver = '1'
        self.gaussian = ''
        self.grid = None
        self.area = ''
        self.left = None
        self.lower = None
        self.upper = None
        self.right = None
        self.level = None
        self.levelist = None
        self.resol = None
        self.gauss = 0
        self.accuracy = 24
        self.omega = 0
        self.omegadiff = 0
        self.eta = 0
        self.etadiff = 0
        self.etapar = 77
        self.dpdeta = 1
        self.smooth = 0
        self.format = 'GRIB1'
        self.addpar = None
        self.prefix = 'EN'
        self.cwc = 0
        self.wrf = 0
        self.ecfsdir = 'ectmp:/${USER}/econdemand/'
        self.mailfail = ['${USER}']
        self.mailops = ['${USER}']
        self.grib2flexpart = 0
        self.ecstorage = 0
        self.ectrans = 0
        self.inputdir = _config.PATH_INPUT_DIR
        self.outputdir = None
        self.ecmwfdatadir = _config.PATH_FLEXEXTRACT_DIR
        self.exedir = _config.PATH_FORTRAN_SRC
        self.flexpart_root_scripts = None
        self.makefile = 'Makefile.gfortran'
        self.destination = None
        self.gateway = None
        self.ecuid = None
        self.ecgid = None
        self.install_target = None
        self.debug = 0
        self.request = 0
        self.public = 0

        self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
                         'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
                         'ectrans', 'debug', 'request', 'public']

        self.__read_controlfile__()

        return

    def __read_controlfile__(self):
        '''Read CONTROL file and assign all CONTROL file variables.

        Return
        ------

        '''

        try:
            cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
            with open(cfile) as f:
                fdata = f.read().split('\n')
            print('Could not read CONTROL file "' + cfile + '"')
            print('Either it does not exist or its syntax is wrong.')
            print('Try "' + sys.argv[0].split('/')[-1] + \
                      ' -h" to print usage information')
            sys.exit(1)

        # go through every line and store parameter
        for ldata in fdata:
            data = ldata.split()
            if len(data) > 1:
                if 'm_' in data[0].lower():
                    data[0] = data[0][2:]
                if data[0].lower() == 'class':
                    data[0] = 'marsclass'
                if data[0].lower() == 'day1':
                    data[0] = 'start_date'
                if data[0].lower() == 'day2':
                    data[0] = 'end_date'
                if data[0].lower() == 'addpar':
                    if '/' in data[1]:
                        # remove leading '/' sign from addpar content
                        if data[1][0] == '/':
                            data[1] = data[1][1:]
                        dd = data[1].split('/')
                        data = [data[0]]
                        for d in dd:
                            data.append(d)
                if len(data) == 2:
                    if '$' in data[1]:
                        setattr(self, data[0].lower(), data[1])
                        while '$' in data[1]:
                            i = data[1].index('$')
                            j = data[1].find('{')
                            k = data[1].find('}')
                            var = os.getenv(data[1][j+1:k])
                            if var is not None:
                                data[1] = data[1][:i] + var + data[1][k+1:]
                            else:
                                my_error(self.mailfail,
                                         'Could not find variable '
                                         + data[1][j+1:k] + ' while reading ' +
                                         self.controlfile)
                        setattr(self, data[0].lower() + '_expanded', data[1])
                    else:
                        if data[1].lower() != 'none':
                            setattr(self, data[0].lower(), data[1])
                        else:
                            setattr(self, data[0].lower(), None)
                elif len(data) > 2:
                    setattr(self, data[0].lower(), (data[1:]))
            else:
                pass

        return

        '''Prepares a string which have all the ControlFile class attributes
        with its associated values. Each attribute is printed in one line and
        in alphabetical order.

        Example
        -------
        'age': 10
        'color': 'Spotted'
        'kids': 0
        'legs': 2
        'name': 'Dog'
        'smell': 'Alot'

        Return
        ------
        string
            Single string of concatenated ControlFile class attributes
            with their values
        '''
        import collections

        attrs = collections.OrderedDict(sorted(attrs.items()))

        return '\n'.join("%s: %s" % item for item in attrs.items())

    def assign_args_to_control(self, args):
        '''Overwrites the existing ControlFile instance attributes with
        the command line arguments.

        args : :obj:`Namespace`
            Contains the commandline arguments from script/program call.

        Return
        ------

        '''

        # get dictionary of command line parameters and eliminate all
        # parameters which are None (were not specified)
        args_dict = vars(args)
        arguments = {k : args_dict[k] for k in args_dict
                     if args_dict[k] != None}

        # assign all passed command line arguments to ControlFile instance
        for k, v in arguments.iteritems():
            setattr(self, str(k), v)

        return

    def assign_envs_to_control(self, envs):
        '''Assigns the ECMWF environment parameter.

        envs : :obj:`dictionary` of :obj:`strings`
            Contains the ECMWF environment parameternames "ECUID", "ECGID",
            "DESTINATION" and "GATEWAY" with its corresponding values.
            They were read from the file "ECMWF_ENV".

        Return
        ------

        '''

        for k, v in envs.iteritems():
            setattr(self, str(k).lower(), str(v))

        return

    def check_conditions(self, queue):
        '''Checks a couple of necessary attributes and conditions,
        such as if they exist and contain values.
        Otherwise set default values.

        queue : :obj:`string`
            Name of the queue if submitted to the ECMWF servers.
            Used to check if ecuid, ecgid, gateway and destination
            are set correctly and are not empty.

        Return
        ------

        '''
        from mods.tools import my_error
        import numpy as np

        # check for having at least a starting date
        # otherwise program is not allowed to run
        if not self.start_date:
            print('start_date specified neither in command line nor \
                   in CONTROL file ' +  self.controlfile)
            print('Try "' + sys.argv[0].split('/')[-1] +
                  ' -h" to print usage information')

        # retrieve just one day if end_date isn't set
        if not self.end_date:
            self.end_date = self.start_date

        # basetime has only two possible values
        if self.basetime:
            if int(self.basetime) != 0 and int(self.basetime) != 12:
                print('Basetime has an invalid value!')
                print('Basetime = ' + str(self.basetime))
                sys.exit(1)

        # assure consistency of levelist and level
        # up-to-date available maximum level numbers at ECMWF, 05.10.2018
        max_level_list = [16, 19, 31, 40, 50, 60, 62, 91, 137]
        if not self.levelist and not self.level:
            print('Warning: neither levelist nor level \
                               specified in CONTROL file')
        elif not self.levelist and self.level:
            self.levelist = '1/to/' + self.level
        elif (self.levelist and not self.level) or \
             (self.levelist[-1] != self.level[-1]):
            self.level = self.levelist.split('/')[-1]
        else:
            pass

        # check if max level is a valid level
        if int(self.level) not in max_level_list:
            print('ERROR: ')
            print('LEVEL must be the maximum level of a specified '
                  'level list from ECMWF, e.g.')
            print('[16, 19, 31, 40, 50, 60, 62, 91 or 137]')
            print('Check parameter "LEVEL" or the max level of "LEVELIST"!')

        # if area was provided (only from commandline)
        # decompose area into its 4 components
        if self.area:
            components = self.area.split('/')
            # convert float to integer coordinates
            if '.' in self.area:
                components = [str(int(float(item) * 1000))
                              for i, item in enumerate(components)]
            self.upper, self.left, self.lower, self.right = components

        # prepare step list if "/" signs are found
        if '/' in self.step:
            steps = self.step.split('/')
            if 'to' in self.step.lower() and 'by' in self.step.lower():
                ilist = np.arange(int(steps[0]),
                                  int(steps[2]) + 1,
                                  int(steps[4]))
                self.step = ['{:0>3}'.format(i) for i in ilist]
            elif 'to' in self.step.lower() and 'by' not in self.step.lower():
                my_error(self.mailfail, self.step + ':\n' +
                         'if "to" is used in steps parameter, \
                         please use "by" as well')
            else:
                self.step = steps

        # if maxstep wasn't provided
        # search for it in the "step" parameter
        if not self.maxstep:
            self.maxstep = 0
            for s in self.step:
                if int(s) > self.maxstep:
                    self.maxstep = int(s)
        else:
            self.maxstep = int(self.maxstep)

        # set root scripts since it is needed later on
        if not self.flexpart_root_scripts:
            self.flexpart_root_scripts = self.ecmwfdatadir

        if not self.outputdir:
            self.outputdir = self.inputdir

        if not isinstance(self.mailfail, list):
            if ',' in self.mailfail:
                self.mailfail = self.mailfail.split(',')
            elif ' ' in self.mailfail:
                self.mailfail = self.mailfail.split()
            else:
                self.mailfail = [self.mailfail]

        if not isinstance(self.mailops, list):
            if ',' in self.mailops:
                self.mailops = self.mailops.split(',')
            elif ' ' in self.mailops:
                self.mailops = self.mailops.split()
            else:
                self.mailops = [self.mailops]

        if queue in _config.QUEUES_LIST and \
           not self.gateway or not self.destination or \
           not self.ecuid or not self.ecgid:
            print('\nEnvironment variables GATEWAY, DESTINATION, ECUID and \
                   ECGID were not set properly!')
            print('Please check for existence of file "ECMWF_ENV" in the \
                   python directory!')

        if self.request != 0:
            marsfile = os.path.join(self.inputdir,
                                    _config.FILE_MARS_REQUESTS)
            if os.path.isfile(marsfile):
                silent_remove(marsfile)

        # check all logical variables for data type
        # if its a string change to integer
        for var in self.logicals:
            if not isinstance(getattr(self, var), int):
                setattr(self, var, int(getattr(self, var)))

        if self.public and not self.dataset:
            print('If public mars data wants to be retrieved, '
                  'the "dataset"-parameter has to be set in the control file!')

        if not isinstance(self.type, list):
            self.type = [self.type]

        for i, val in enumerate(self.type):
            if self.type[i] == 'AN' and int(self.step[i]) != 0:
                print('Analysis retrievals must have STEP = 0 (is set to 0)')
                self.type[i] = 0

        if not isinstance(self.time, list):
            self.time = [self.time]

        if not isinstance(self.step, list):
            self.step = [self.step]

        if not self.acctype:
            print('... Control paramter ACCTYPE was not defined.')
            try:
                if len(self.type) > 1 and self.type[1] != 'AN':
                    print('Use old setting by using TYPE[1] for flux forecast!')
                    self.acctype = self.type[1]
            except:
                print('Use default value "FC" for flux forecast!')
                self.acctype='FC'

        if not self.acctime:
            print('... Control paramter ACCTIME was not defined.')
            print('Use default value "00/12" for flux forecast!')
            self.acctime='00/12'

        if not self.accmaxstep:
            print('... Control paramter ACCMAXSTEP was not defined.')
            print('Use default value "12" for flux forecast!')
            self.accmaxstep='12'

        return

    def check_install_conditions(self):
        '''Checks a couple of necessary attributes and conditions
        for the installation such as if they exist and contain values.

        Return
        ------

        '''

        if self.install_target and \
           self.install_target not in ['local', 'ecgate', 'cca']:
            print('ERROR: unknown or missing installation target ')
            print('target: ', self.install_target)
            print('please specify correct installation target ' +
                  '(local | ecgate | cca)')
            print('use -h or --help for help')

        if self.install_target and self.install_target != 'local':
            if not self.ecgid or not self.ecuid or \
               not self.gateway or not self.destination:
                print('Please enter your ECMWF user id and group id as well ' +
                      'as the \nname of the local gateway and the ectrans ' +
                      'destination ')
                print('with command line options --ecuid --ecgid \
                       --gateway --destination')
                print('Try "' + sys.argv[0].split('/')[-1] + \
                print('Please consult ecaccess documentation or ECMWF user \
                       support for further details')

            if not self.flexpart_root_scripts:
                self.flexpart_root_scripts = '${HOME}'
            else:
                self.flexpart_root_scripts = self.flexpart_root_scripts
        else: # local
                self.flexpart_root_scripts = _config.PATH_FLEXEXTRACT_DIR

        return

    def to_list(self):
        '''Just generates a list of strings containing the attributes and
        assigned values except the attributes "_expanded", "exedir",
        "ecmwfdatadir" and "flexpart_root_scripts".

        Return
        ------
        l : :obj:`list`
            A sorted list of the all ControlFile class attributes with
            their values except the attributes "_expanded", "exedir",
            "ecmwfdatadir" and "flexpart_root_scripts".
        '''

        attrs = collections.OrderedDict(sorted(vars(self).copy().items()))

        l = list()

            if '_expanded' in item[0]:
                pass
            elif 'exedir' in item[0]:
                pass
            elif 'flexpart_root_scripts' in item[0]:
                pass
            elif 'ecmwfdatadir' in item[0]:
                pass
            else:
                if isinstance(item[1], list):
                    stot = ''
                    for s in item[1]:
                        stot += s + ' '

                    l.append("%s %s" % (item[0], stot))
                else:
                    l.append("%s %s" % item)

        return sorted(l)

#
# @Date: May 2018
#
#    October 2014 - Anne Fouilloux (University of Oslo)
#        - created functions silent_remove and product (taken from ECMWF)
#
#    November 2015 - Leopold Haimberger (University of Vienna)
#        - created functions: interpret_args_and_control, clean_up
#          my_error, normal_exit, init128, to_param_id
#
#        - moved all functions from file Flexparttools to this file tools
#        - added function get_list_as_string
#        - seperated args and control interpretation
#
#
#
# @Modul Description:
#    This module contains a couple of helpful functions which are
#    used in different places in flex_extract.
#
#    - get_cmdline_arguments
#    - clean_up
#    - my_error
#    - normal_exit
#    - product
#    - silent_remove
#    - init128
#    - to_param_id
#    - get_list_as_string
#    - make_dir
#

# MODULES
import os
import errno
import sys
import traceback

def none_or_str(value):
    '''Converts the input string into pythons None-type if the string
    contains string "None".

    value : :obj:`string`
        String to be checked for the "None" word.

    Return
    ------
    None or value:
        Return depends on the content of the input value. If it was "None",
        then the python type None is returned. Otherwise the string itself.
    '''
    if value == 'None':
        return None
    return value

def none_or_int(value):
    contains string "None". Otherwise it is converted to an integer value.

    Return
    ------
    None or int(value):
        then the python type None is returned. Otherwise the string is
        converted into an integer value.
    '''
    return int(value)

def get_cmdline_arguments():

    Return
    ------
    '''

    parser = ArgumentParser(description='Retrieve FLEXPART input from \
                                ECMWF MARS archive',

    # the most important arguments
    parser.add_argument("--start_date", dest="start_date",
                        type=none_or_str, default=None,
                        help="start date YYYYMMDD")
    parser.add_argument("--end_date", dest="end_date",
                        help="end_date YYYYMMDD")
    parser.add_argument("--date_chunk", dest="date_chunk",
                        type=none_or_int, default=None,
                        help="# of days to be retrieved at once")
                        type=none_or_str, default='CONTROL.temp',

    # parameter for extra output information
    parser.add_argument("--debug", dest="debug",
                        help="debug mode - leave temporary files intact")
    parser.add_argument("--request", dest="request",
                        help="list all mars request in file mars_requests.dat \
                        and skip submission to mars")
    parser.add_argument("--public", dest="public",
                        help="public mode - retrieves the public datasets")

    # some arguments that override the default in the CONTROL file
    parser.add_argument("--basetime", dest="basetime",
                        help="base such as 00 or 12 (for half day retrievals)")
    parser.add_argument("--step", dest="step",
                        help="steps such as 00/to/48")
    parser.add_argument("--levelist", dest="levelist",
                        help="Vertical levels to be retrieved, e.g. 30/to/60")
    parser.add_argument("--area", dest="area",
                        help="area defined as north/west/south/east")

    # set the working directories
    parser.add_argument("--inputdir", dest="inputdir",
                        help="root directory for storing intermediate files")
    parser.add_argument("--outputdir", dest="outputdir",
                        help="root directory for storing output files")
                        help="FLEXPART root directory (to find grib2flexpart \
                        and COMMAND file)\n Normally flex_extract resides in \
                        the scripts directory of the FLEXPART distribution")

    # this is only used by prepare_flexpart.py to rerun a postprocessing step
    parser.add_argument("--ppid", dest="ppid",
                        help="specify parent process id for \
                        rerun of prepare_flexpart")

                        type=none_or_str, default="job.temp",
    parser.add_argument("--queue", dest="queue",
                        help="queue for submission to ECMWF \
                        (e.g. ecgate or cca )")

def read_ecenv(filename):
    '''Reads the file into a dictionary where the key values are the parameter
    names.

    filename : :obj:`string`
        Path to file where the ECMWF environment parameters are stored.

    Return
    ------
    envs : :obj:`dictionary`
        Contains the environment parameter ecuid, ecgid, gateway
        and destination for ECMWF server environments.
    '''
    envs= {}

    with open(filename, 'r') as f:
        for line in f:
            data = line.strip().split()
            envs[str(data[0])] = str(data[1])

    return envs

def clean_up(c):
    '''Remove all files from intermediate directory (inputdir).

    Return
    ------

    '''

    print("clean_up")

    cleanlist = glob.glob(c.inputdir + "/*")
    for clist in cleanlist:
        if c.prefix not in clist:
            silent_remove(clist)
        if c.ecapi is False and (c.ectrans == '1' or c.ecstorage == '1'):

    print("Done")

    return

def my_error(users, message='ERROR'):
    '''Prints a specified error message which can be passed to the function
    before exiting the program.

    user : :obj:`list` of :obj:`string`
        Contains all email addresses which should be notified.
        It might also contain just the ecmwf user name which wil trigger
        mailing to the associated email address for this user.

    message : :obj:`string`, optional
        Error message. Default value is "ERROR".

    Return
    ------

    '''

    print(message)

    # comment if user does not want email notification directly from python
    for user in users:
        if '${USER}' in user:
            user = os.getenv('USER')
        try:
            p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ERROR',
                                  os.path.expandvars(user)],
            trace = '\n'.join(traceback.format_stack())
            pout = p.communicate(input=message + '\n\n' + trace)[0]
        except ValueError as e:
            print('ERROR: ', e)
            sys.exit('Email could not be sent!')
        else:
            print('Email sent to ' + os.path.expandvars(user) + ' ' +
                  pout.decode())

    sys.exit(1)

    return

def normal_exit(users, message='Done!'):
    '''Prints a specific exit message which can be passed to the function.

        Message for exiting program. Default value is "Done!".

    Return
    ------

    '''

    # comment if user does not want notification directly from python
        try:
            p = subprocess.Popen(['mail', '-s flex_extract_v7.1 normal exit',
            pout = p.communicate(input=message+'\n\n')[0]
            print('Email could not be sent!')
        else:

    return

def product(*args, **kwds):
    '''This method combines the single characters of the passed arguments
    with each other. So that each character of each argument value
    will be combined with each character of the other arguments as a tuple.

    Note
    ----
    This method is taken from an example at the ECMWF wiki website.
    https://software.ecmwf.int/wiki/display/GRIB/index.py; 2018-03-16

    Example
    -------
    product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy

    product(range(2), repeat = 3) --> 000 001 010 011 100 101 110 111

    \*args : :obj:`tuple`
        Positional arguments (arbitrary number).

    \*\*kwds : :obj:`dictionary`
        Contains all the keyword arguments from \*args.

    Return
    ------
    prod : :obj:`tuple`
        Return will be done with "yield". A tuple of combined arguments.
        See example in description above.
    '''
    pools = map(tuple, args) * kwds.get('repeat', 1)
    result = [[]]
    for pool in pools:
        result = [x + [y] for x in result for y in pool]
    for prod in result:
        yield tuple(prod)

    return

def silent_remove(filename):
    '''Remove file if it exists.
    The function does not fail if the file does not exist.

        The name of the file to be removed without notification.

    Return
    ------

    '''
    try:
        os.remove(filename)
    except OSError as e:
        if e.errno != errno.ENOENT:
            # errno.ENOENT  =  no such file or directory
            raise  # re-raise exception if a different error occured

    return

def init128(filepath):
    '''Opens and reads the grib file with table 128 information.

    filepath : :obj:`string`
        Path to file of ECMWF grib table number 128.

    Return
    ------
    table128 : :obj:`dictionary`
        Contains the ECMWF grib table 128 information.
        The key is the parameter number and the value is the
        short name of the parameter.
    '''
    table128 = dict()
    with open(filepath) as f:
        fdata = f.read().split('\n')
    for data in fdata:
        if data[0] != '!':
            table128[data[0:3]] = data[59:64].strip()

    return table128

def to_param_id(pars, table):
    '''Transform parameter names to parameter ids with ECMWF grib table 128.

    pars : :obj:`string`
        Addpar argument from CONTROL file in the format of
        parameter names instead of ids. The parameter short
        names are sepearted with "/" and they are passed as
        one single string.

    table : :obj:`dictionary`

    Return
    ------
    ipar : :obj:`list` of :obj:`integer`
        List of addpar parameters from CONTROL file transformed to
        parameter ids in the format of integer.
    '''
    cpar = pars.upper().split('/')
    ipar = []
    for par in cpar:
        for k, v in table.iteritems():
            if par == k or par == v:
                ipar.append(int(k))
                break
        else:
            print('Warning: par ' + par + ' not found in table 128')

    return ipar

def get_list_as_string(list_obj, concatenate_sign=', '):
    '''Converts a list of arbitrary content into a single string.

    list_obj : :obj:`list`
        A list with arbitrary content.

    concatenate_sign : :obj:`string`, optional
        A string which is used to concatenate the single
        list elements. Default value is ", ".

    Return
    ------
    str_of_list : :obj:`string`
        The content of the list as a single string.
    '''

    str_of_list = concatenate_sign.join(str(l) for l in list_obj)

    return str_of_list

def make_dir(directory):
    '''Creates a directory and gives a warning if the directory
    already exists. The program stops only if there is another problem.

    directory : :obj:`string`
        The directory name including the path which should be created.

    Return
    ------

    '''
    try:
        os.makedirs(directory)
        if e.errno != errno.EEXIST:
            # errno.EEXIST = directory already exists
            raise # re-raise exception if a different error occured
        else:
            print('WARNING: Directory {0} already exists!'.format(directory))

    return

def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
    '''Uses the ecaccess-file-put command to send a file to the ECMWF servers.

    Note
    ----
    The return value is just for testing reasons. It does not have
    to be used from the calling function since the whole error handling
    is done in here.

    ecd : :obj:`string`
        The path were the file is stored.

        The name of the file to send to the ECMWF server.

        The target queue where the file should be sent to.

    Return
    ------
    rcode : :obj:`string`
        Resulting code of command execution. If successful the string
        will be empty.
    '''

    try:
        rcode = subprocess.check_output(['ecaccess-file-put',
                                          ecd + '/' + filename,
                                          target + ':/home/ms/' +
                                          ecgid + '/' + ecuid +
                                          '/' + filename],
                                         stderr=subprocess.STDOUT)

        print('\n... Do you have a valid ecaccess certification key?')
        sys.exit('... ECACCESS-FILE-PUT FAILED!')

    return rcode

def submit_job_to_ecserver(target, jobname):
    '''Uses ecaccess-job-submit command to submit a job to the ECMWF server.

    Note
    ----

        The target where the file should be sent to, e.g. the queue.

    jobname : :obj:`string`
        The name of the jobfile to be submitted to the ECMWF server.

    Return
    ------
        will contain an integer number, representing the id of the job
        at the ecmwf server.
    '''

    try:
        rcode = subprocess.check_output(['ecaccess-job-submit',
                                         '-queueName', target,
                                         jobname])

        sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')