#!/usr/bin/env python
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Leopold Haimberger (University of Vienna)
#
# @Date: November 2015
#
# @Change History:
#
# February 2018 - Anne Philipp (University of Vienna):
# - applied PEP8 style guide
# - added documentation
# - applied some minor modifications in programming style/structure
# - changed name of class Control to ControlFile for more
# self-explanation naming
# - outsource of class ControlFile
# - initialisation of class attributes ( to avoid high number of
# conditional statements and set default values )
# - divided assignment of attributes and the check of conditions
# - outsourced the commandline argument assignments to control attributes
#
# @License:
# (C) Copyright 2015-2018.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# @Class Description:
# The CONTROL file is the steering part of the FLEXPART extraction
# software. All necessary parameters needed to retrieve the data fields
# from the MARS archive for driving FLEXPART are set in a CONTROL file.
# Some specific parameters like the start and end dates can be overwritten
# by the command line parameters, but in generel all parameters needed
# for a complete set of fields for FLEXPART can be set in the CONTROL file.
#
# @Class Content:
# - __init__
# - __read_controlfile__
# - __str__
# - assign_args_to_control
# - assign_envs_to_control
# - check_conditions
# - check_install_conditions
# - to_list
#
# @Class Attributes:
#
#
#*******************************************************************************
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
import os
import re
import sys
import inspect
# software specific classes and modules from flex_extract
sys.path.append('../')
import _config
from mods.tools import my_error, silent_remove
# ------------------------------------------------------------------------------
# CLASS
# ------------------------------------------------------------------------------
[docs]class ControlFile(object):
'''
Contains the information which are stored in the CONTROL files.
'''
def __init__(self, filename):
'''Initialises the instance of ControlFile class and defines
all class attributes with default values. Afterwards calls
function __read_controlfile__ to read parameter from Control file.
Parameters
----------
filename : :obj:`string`
Name of CONTROL file.
Return
------
'''
# list of all possible class attributes and their default values
self.controlfile = filename
self.start_date = None
self.end_date = None
self.date_chunk = 3
self.dtime = None
self.basetime = None
self.maxstep = None
self.type = None
self.time = None
self.step = None
self.acctype = None
self.acctime = None
self.accmaxstep = None
self.marsclass = None
self.dataset = None
self.stream = None
self.number = 'OFF'
self.expver = '1'
self.gaussian = ''
self.grid = None
self.area = ''
self.left = None
self.lower = None
self.upper = None
self.right = None
self.level = None
self.levelist = None
self.resol = None
self.gauss = 0
self.accuracy = 24
self.omega = 0
self.omegadiff = 0
self.eta = 0
self.etadiff = 0
self.etapar = 77
self.dpdeta = 1
self.smooth = 0
self.format = 'GRIB1'
self.addpar = None
self.prefix = 'EN'
self.cwc = 0
self.wrf = 0
self.ecfsdir = 'ectmp:/${USER}/econdemand/'
self.mailfail = ['${USER}']
self.mailops = ['${USER}']
self.grib2flexpart = 0
self.ecstorage = 0
self.ectrans = 0
self.inputdir = _config.PATH_INPUT_DIR
self.outputdir = None
self.ecmwfdatadir = _config.PATH_FLEXEXTRACT_DIR
self.exedir = _config.PATH_FORTRAN_SRC
self.flexpart_root_scripts = None
self.makefile = 'Makefile.gfortran'
self.destination = None
self.gateway = None
self.ecuid = None
self.ecgid = None
self.install_target = None
self.debug = 0
self.request = 0
self.public = 0
self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
'ectrans', 'debug', 'request', 'public']
self.__read_controlfile__()
return
def __read_controlfile__(self):
'''Read CONTROL file and assign all CONTROL file variables.
Parameters
----------
Return
------
'''
try:
cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
with open(cfile) as f:
fdata = f.read().split('\n')
except IOError:
print('Could not read CONTROL file "' + cfile + '"')
print('Either it does not exist or its syntax is wrong.')
print('Try "' + sys.argv[0].split('/')[-1] + \
' -h" to print usage information')
sys.exit(1)
# go through every line and store parameter
for ldata in fdata:
data = ldata.split()
if len(data) > 1:
if 'm_' in data[0].lower():
data[0] = data[0][2:]
if data[0].lower() == 'class':
data[0] = 'marsclass'
if data[0].lower() == 'day1':
data[0] = 'start_date'
if data[0].lower() == 'day2':
data[0] = 'end_date'
if data[0].lower() == 'addpar':
if '/' in data[1]:
# remove leading '/' sign from addpar content
if data[1][0] == '/':
data[1] = data[1][1:]
dd = data[1].split('/')
data = [data[0]]
for d in dd:
data.append(d)
if len(data) == 2:
if '$' in data[1]:
setattr(self, data[0].lower(), data[1])
while '$' in data[1]:
i = data[1].index('$')
j = data[1].find('{')
k = data[1].find('}')
var = os.getenv(data[1][j+1:k])
if var is not None:
data[1] = data[1][:i] + var + data[1][k+1:]
else:
my_error(self.mailfail,
'Could not find variable '
+ data[1][j+1:k] + ' while reading ' +
self.controlfile)
setattr(self, data[0].lower() + '_expanded', data[1])
else:
if data[1].lower() != 'none':
setattr(self, data[0].lower(), data[1])
else:
setattr(self, data[0].lower(), None)
elif len(data) > 2:
setattr(self, data[0].lower(), (data[1:]))
else:
pass
return
def __str__(self):
'''Prepares a string which have all the ControlFile class attributes
with its associated values. Each attribute is printed in one line and
in alphabetical order.
Example
-------
'age': 10
'color': 'Spotted'
'kids': 0
'legs': 2
'name': 'Dog'
'smell': 'Alot'
Parameters
----------
Return
------
string
Single string of concatenated ControlFile class attributes
with their values
'''
import collections
attrs = vars(self).copy()
attrs = collections.OrderedDict(sorted(attrs.items()))
return '\n'.join("%s: %s" % item for item in attrs.items())
[docs] def assign_args_to_control(self, args):
'''Overwrites the existing ControlFile instance attributes with
the command line arguments.
Parameters
----------
args : :obj:`Namespace`
Contains the commandline arguments from script/program call.
Return
------
'''
# get dictionary of command line parameters and eliminate all
# parameters which are None (were not specified)
args_dict = vars(args)
arguments = {k : args_dict[k] for k in args_dict
if args_dict[k] != None}
# assign all passed command line arguments to ControlFile instance
for k, v in arguments.iteritems():
setattr(self, str(k), v)
return
[docs] def assign_envs_to_control(self, envs):
'''Assigns the ECMWF environment parameter.
Parameters
----------
envs : :obj:`dictionary` of :obj:`strings`
Contains the ECMWF environment parameternames "ECUID", "ECGID",
"DESTINATION" and "GATEWAY" with its corresponding values.
They were read from the file "ECMWF_ENV".
Return
------
'''
for k, v in envs.iteritems():
setattr(self, str(k).lower(), str(v))
return
[docs] def check_conditions(self, queue):
'''Checks a couple of necessary attributes and conditions,
such as if they exist and contain values.
Otherwise set default values.
Parameters
----------
queue : :obj:`string`
Name of the queue if submitted to the ECMWF servers.
Used to check if ecuid, ecgid, gateway and destination
are set correctly and are not empty.
Return
------
'''
from mods.tools import my_error
import numpy as np
# check for having at least a starting date
# otherwise program is not allowed to run
if not self.start_date:
print('start_date specified neither in command line nor \
in CONTROL file ' + self.controlfile)
print('Try "' + sys.argv[0].split('/')[-1] +
' -h" to print usage information')
sys.exit(1)
# retrieve just one day if end_date isn't set
if not self.end_date:
self.end_date = self.start_date
# basetime has only two possible values
if self.basetime:
if int(self.basetime) != 0 and int(self.basetime) != 12:
print('Basetime has an invalid value!')
print('Basetime = ' + str(self.basetime))
sys.exit(1)
# assure consistency of levelist and level
# up-to-date available maximum level numbers at ECMWF, 05.10.2018
max_level_list = [16, 19, 31, 40, 50, 60, 62, 91, 137]
if not self.levelist and not self.level:
print('Warning: neither levelist nor level \
specified in CONTROL file')
sys.exit(1)
elif not self.levelist and self.level:
self.levelist = '1/to/' + self.level
elif (self.levelist and not self.level) or \
(self.levelist[-1] != self.level[-1]):
self.level = self.levelist.split('/')[-1]
else:
pass
# check if max level is a valid level
if int(self.level) not in max_level_list:
print('ERROR: ')
print('LEVEL must be the maximum level of a specified '
'level list from ECMWF, e.g.')
print('[16, 19, 31, 40, 50, 60, 62, 91 or 137]')
print('Check parameter "LEVEL" or the max level of "LEVELIST"!')
sys.exit(1)
# if area was provided (only from commandline)
# decompose area into its 4 components
if self.area:
components = self.area.split('/')
# convert float to integer coordinates
if '.' in self.area:
components = [str(int(float(item) * 1000))
for i, item in enumerate(components)]
self.upper, self.left, self.lower, self.right = components
# prepare step list if "/" signs are found
if '/' in self.step:
steps = self.step.split('/')
if 'to' in self.step.lower() and 'by' in self.step.lower():
ilist = np.arange(int(steps[0]),
int(steps[2]) + 1,
int(steps[4]))
self.step = ['{:0>3}'.format(i) for i in ilist]
elif 'to' in self.step.lower() and 'by' not in self.step.lower():
my_error(self.mailfail, self.step + ':\n' +
'if "to" is used in steps parameter, \
please use "by" as well')
else:
self.step = steps
# if maxstep wasn't provided
# search for it in the "step" parameter
if not self.maxstep:
self.maxstep = 0
for s in self.step:
if int(s) > self.maxstep:
self.maxstep = int(s)
else:
self.maxstep = int(self.maxstep)
# set root scripts since it is needed later on
if not self.flexpart_root_scripts:
self.flexpart_root_scripts = self.ecmwfdatadir
if not self.outputdir:
self.outputdir = self.inputdir
if not isinstance(self.mailfail, list):
if ',' in self.mailfail:
self.mailfail = self.mailfail.split(',')
elif ' ' in self.mailfail:
self.mailfail = self.mailfail.split()
else:
self.mailfail = [self.mailfail]
if not isinstance(self.mailops, list):
if ',' in self.mailops:
self.mailops = self.mailops.split(',')
elif ' ' in self.mailops:
self.mailops = self.mailops.split()
else:
self.mailops = [self.mailops]
if queue in _config.QUEUES_LIST and \
not self.gateway or not self.destination or \
not self.ecuid or not self.ecgid:
print('\nEnvironment variables GATEWAY, DESTINATION, ECUID and \
ECGID were not set properly!')
print('Please check for existence of file "ECMWF_ENV" in the \
python directory!')
sys.exit(1)
if self.request != 0:
marsfile = os.path.join(self.inputdir,
_config.FILE_MARS_REQUESTS)
if os.path.isfile(marsfile):
silent_remove(marsfile)
# check all logical variables for data type
# if its a string change to integer
for var in self.logicals:
if not isinstance(getattr(self, var), int):
setattr(self, var, int(getattr(self, var)))
if self.public and not self.dataset:
print('ERROR: ')
print('If public mars data wants to be retrieved, '
'the "dataset"-parameter has to be set in the control file!')
sys.exit(1)
if not isinstance(self.type, list):
self.type = [self.type]
for i, val in enumerate(self.type):
if self.type[i] == 'AN' and int(self.step[i]) != 0:
print('Analysis retrievals must have STEP = 0 (is set to 0)')
self.type[i] = 0
if not isinstance(self.time, list):
self.time = [self.time]
if not isinstance(self.step, list):
self.step = [self.step]
if not self.acctype:
print('... Control paramter ACCTYPE was not defined.')
try:
if len(self.type) > 1 and self.type[1] != 'AN':
print('Use old setting by using TYPE[1] for flux forecast!')
self.acctype = self.type[1]
except:
print('Use default value "FC" for flux forecast!')
self.acctype='FC'
if not self.acctime:
print('... Control paramter ACCTIME was not defined.')
print('Use default value "00/12" for flux forecast!')
self.acctime='00/12'
if not self.accmaxstep:
print('... Control paramter ACCMAXSTEP was not defined.')
print('Use default value "12" for flux forecast!')
self.accmaxstep='12'
return
[docs] def check_install_conditions(self):
'''Checks a couple of necessary attributes and conditions
for the installation such as if they exist and contain values.
Otherwise set default values.
Parameters
----------
Return
------
'''
if self.install_target and \
self.install_target not in ['local', 'ecgate', 'cca']:
print('ERROR: unknown or missing installation target ')
print('target: ', self.install_target)
print('please specify correct installation target ' +
'(local | ecgate | cca)')
print('use -h or --help for help')
sys.exit(1)
if self.install_target and self.install_target != 'local':
if not self.ecgid or not self.ecuid or \
not self.gateway or not self.destination:
print('Please enter your ECMWF user id and group id as well ' +
'as the \nname of the local gateway and the ectrans ' +
'destination ')
print('with command line options --ecuid --ecgid \
--gateway --destination')
print('Try "' + sys.argv[0].split('/')[-1] + \
' -h" to print usage information')
print('Please consult ecaccess documentation or ECMWF user \
support for further details')
sys.exit(1)
if not self.flexpart_root_scripts:
self.flexpart_root_scripts = '${HOME}'
else:
self.flexpart_root_scripts = self.flexpart_root_scripts
else: # local
if not self.flexpart_root_scripts:
self.flexpart_root_scripts = _config.PATH_FLEXEXTRACT_DIR
return
[docs] def to_list(self):
'''Just generates a list of strings containing the attributes and
assigned values except the attributes "_expanded", "exedir",
"ecmwfdatadir" and "flexpart_root_scripts".
Parameters
----------
Return
------
l : :obj:`list`
A sorted list of the all ControlFile class attributes with
their values except the attributes "_expanded", "exedir",
"ecmwfdatadir" and "flexpart_root_scripts".
'''
import collections
attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
l = list()
for item in attrs.items():
if '_expanded' in item[0]:
pass
elif 'exedir' in item[0]:
pass
elif 'flexpart_root_scripts' in item[0]:
pass
elif 'ecmwfdatadir' in item[0]:
pass
else:
if isinstance(item[1], list):
stot = ''
for s in item[1]:
stot += s + ' '
l.append("%s %s" % (item[0], stot))
else:
l.append("%s %s" % item)
return sorted(l)