#!/usr/bin/env python
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Leopold Haimberger (University of Vienna)
#
# @Date: November 2015
#
# @Change History:
#
# February 2018 - Anne Philipp (University of Vienna):
# - applied PEP8 style guide
# - added documentation
# - applied some minor modifications in programming style/structure
# - changed name of class Control to ControlFile for more
# self-explanation naming
# - outsource of class ControlFile
# - initialisation of class attributes ( to avoid high number of
# conditional statements and set default values )
# - divided assignment of attributes and the check of conditions
# - outsourced the commandline argument assignments to control attributes
#
# @License:
# (C) Copyright 2014-2019.
# Anne Philipp, Leopold Haimberger
#
# This work is licensed under the Creative Commons Attribution 4.0
# International License. To view a copy of this license, visit
# http://creativecommons.org/licenses/by/4.0/ or send a letter to
# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
#
# @Class Methods:
# __init__
# _read_controlfile
# __str__
# assign_args_to_control
# assign_envs_to_control
# check_conditions
# check_install_conditions
# to_list
#*******************************************************************************
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
import os
import re
import sys
import inspect
import numpy as np
# software specific classes and modules from flex_extract
sys.path.append('../')
import _config
from mods.tools import my_error, silent_remove
from mods.checks import (check_grid, check_area, check_levels, check_purefc,
check_step, check_mail, check_queue, check_pathes,
check_dates, check_maxstep, check_type, check_request,
check_basetime, check_public, check_acctype,
check_acctime, check_accmaxstep, check_time,
check_logicals_type, check_len_type_time_step,
check_addpar, check_job_chunk)
# ------------------------------------------------------------------------------
# CLASS
# ------------------------------------------------------------------------------
[docs]class ControlFile(object):
'''
Contains the information which are stored in the CONTROL files.
The CONTROL file is the steering part of the FLEXPART extraction
software. All necessary parameters needed to retrieve the data fields
from the MARS archive for driving FLEXPART are set in a CONTROL file.
Some specific parameters like the start and end dates can be overwritten
by the command line parameters, but in generel all parameters needed
for a complete set of fields for FLEXPART can be set in the CONTROL file.
Attributes
----------
controlfile : str
The name of the control file to be processed. Default value is the
filename passed to the init function when initialised.
start_date : str
The first day of the retrieval period. Default value is None.
end_date :str
The last day of the retrieval period. Default value is None.
date_chunk : int
Length of period for a single mars retrieval. Default value is 3.
dtime :str
The time step in hours. Default value is None.
basetime : str
The time for a half day retrieval. The 12 hours upfront are to be
retrieved. Default value is None.
maxstep : int
The maximum forecast step for non flux data. Default value is None.
type : list of str
List of field type per retrieving hour. Default value is None.
time : list of str
List of retrieving times in hours. Default valuer is None.
step : list of str or str
List of forecast time steps in hours for non flux data.
Default value is None.
acctype : str
The field type for the accumulated forecast fields.
Default value is None.
acctime : str
The starting time of the accumulated forecasts. Default value is None.
accmaxstep : int
The maximum forecast step for the accumulated forecast fields
(flux data). Default value is None.
marsclass : str
Characterisation of dataset. Default value is None.
dataset : str
For public datasets there is the specific naming and parameter
dataset which has to be used to characterize the type of
data. Default value is None.
stream : str
Identifies the forecasting system used to generate the data.
Default value is None.
number : str
Selects the member in ensemble forecast run. Default value is 'OFF'.
expver : str
The version number of the dataset. Default value is '1'.
gaussian : str
This parameter is deprecated and should no longer be used.
Specifies the desired type of Gaussian grid for the output.
Default value is an empty string ''.
grid : str
Specifies the output grid which can be either a Gaussian grid
or a Latitude/Longitude grid. Default value is None.
area : str
Specifies the desired sub-area of data to be extracted.
Default value is None.
left : str
The western most longitude of the area to be extracted.
Default value is None.
lower : str
The southern most latitude of the area to be extracted.
Default value is None.
upper : str
The northern most latitued of the area to be extracted.
Default value is None.
right : str
The eastern most longitude of the area to be extracted.
Default value is None.
level : str
Specifies the maximum level. Default value is None.
levelist : str
Specifies the required level list. Default value is None.
resol : str
Specifies the desired triangular truncation of retrieved data,
before carrying out any other selected post-processing.
Default value is None.
gauss : int
Switch to select gaussian fields (1) or regular lat/lon (0).
Default value is 0.
accuracy : int
Specifies the number of bits per value to be used in the
generated GRIB coded fields. Default value is 24.
omega : int
Switch to select omega retrieval (1) or not (0). Default value is 0.
omegadiff : int
Switch to decide to calculate Omega and Dps/Dt from continuity
equation for diagnostic purposes (1) or not (0). Default value is 0.
eta : int
Switch to select direct retrieval of etadot from MARS (1) or
wether it has to be calculated (0). Then Default value is 0.
etadiff : int
Switch to select calculation of etadot and Dps/Dt from continuity
equation for diagnostic purposes (1) or not (0). Default value is 0.
etapar : int
GRIB parameter Id for etadot fields. Default value is 77.
dpdeta : int
Switch to select multiplication of etadot with dpdeta.
Default value is 1.
smooth : int
Spectral truncation of ETADOT after calculation on Gaussian grid.
Default value is 0.
format : str
The format of the GRIB data. Default value is 'GRIB1'.
addpar : str
List of additional surface level ECMWF parameter to be retrieved.
Default value is None.
prefix : str
Prefix string for the final FLEXPART/FLEXTRA ready input files.
Default value is 'EN'.
cwc : int
Switch to select wether the sum of cloud liquid water content and
cloud ice water content should be retrieved. Default value is 0.
wrf : int
Switch to select further parameters for retrievment to support
WRF simulations. Default value is 0.
ecfsdir : str
Path to the ECMWF storage 'ectmp:/${USER}/econdemand/'
mailfail : list of str
Email list for sending error log files from ECMWF servers.
The email addresses should be seperated by a comma.
Default value is ['${USER}'].
mailops : list of str
Email list for sending operational log files from ECMWF servers.
The email addresses should be seperated by a comma.
Default value is ['${USER}'].
grib2flexpart : int 0
Switch to select generation of preprocessed FLEXPART files ".fp".
If it is selected, the program grib2flexpart will try
to convert the flex_extract output files into ".fp" format.
ecstorage : int
Switch to select storage of FLEXPART ready output files
in the ECFS file system. Default value is 0.
ectrans : int
Switch to select the transfer of FLEXPART ready output files
to the gateway server. Default value is 0.
inputdir : str
Path to the temporary directory for the retrieval grib files and
other processing files. Default value is _config.PATH_INPUT_DIR.
outputdir : str
Path to the final directory where the final FLEXPART ready input
files are stored. Default value is None.
flexextractdir : str
Path to the flex_extract root directory. Default value is
_config.PATH_FLEXEXTRACT_DIR.
exedir : str
Path to the FORTRAN executable file. Default value is
_config.PATH_FORTRAN_SRC.
flexpartdir : str
Path to a FLEXPART root directory. Default value is None.
makefile : str
Name of the makefile to be used for the Fortran program.
Default value is 'Makefile.gfortran'.
destination : str
The remote destination which is used to transfer files
from ECMWF server to local gateway server. Default value is None.
gateway : str
The gateway server the user is using. Default value is None.
ecuid : str
The user id on ECMWF server. Default value is None.
ecgid : str
The group id on ECMWF server. Default value is None.
install_target : str
Defines the location where the installation is to be done.
Default value is None.
debug : int
Switch to keep temporary files at the end of postprocessing (1) or
to delete all temporary files except the final output files (0).
Default value is 0.
request : int
Switch to select between just retrieving the data (0), writing the mars
parameter values to a csv file (1) or doing both (2).
Default value is 0.
public : int
Switch to select kind of ECMWF Web Api access and the
possible data sets. Public data sets (1) and Memberstate data sets (0).
Default value is 0.
ecapi : boolean
Tells wether the ECMWF Web APi was able to load or not.
Default value is None.
purefc : int
Switch to decide wether the job is a pure forecast retrieval or
coupled with analysis data. Default value is 0.
rrint: int
Switch to select between old precipitation disaggregation method (0)
or the new IA3 disaggegration method (1). Default value is 0.
logicals : list of str
List of the names of logical switches which controls the flow
of the program. Default list is ['gauss', 'omega', 'omegadiff', 'eta',
'etadiff', 'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
'ectrans', 'debug', 'request', 'public', 'purefc', 'rrint']
'''
def __init__(self, filename):
'''Initialises the instance of ControlFile class and defines
all class attributes with default values. Afterwards calls
function __read_controlfile__ to read parameter from Control file.
Parameters
----------
filename : str
Name of CONTROL file.
Return
------
'''
# list of all possible class attributes and their default values
self.controlfile = filename
self.start_date = None
self.end_date = None
self.date_chunk = 3
self.job_chunk = None
self.dtime = None
self.basetime = None
self.maxstep = None
self.type = None
self.time = None
self.step = None
self.acctype = None
self.acctime = None
self.accmaxstep = None
self.marsclass = None
self.dataset = None
self.stream = None
self.number = 'OFF'
self.expver = '1'
self.gaussian = ''
self.grid = None
self.area = ''
self.left = None
self.lower = None
self.upper = None
self.right = None
self.level = None
self.levelist = None
self.resol = None
self.gauss = 0
self.accuracy = 24
self.omega = 0
self.omegadiff = 0
self.eta = 0
self.etadiff = 0
self.etapar = 77
self.dpdeta = 1
self.smooth = 0
self.format = 'GRIB1'
self.addpar = None
self.prefix = 'EN'
self.cwc = 0
self.wrf = 0
self.ecfsdir = 'ectmp:/${USER}/econdemand/'
self.mailfail = ['${USER}']
self.mailops = ['${USER}']
self.grib2flexpart = 0
self.ecstorage = 0
self.ectrans = 0
self.inputdir = _config.PATH_INPUT_DIR
self.outputdir = None
self.flexextractdir = _config.PATH_FLEXEXTRACT_DIR
self.exedir = _config.PATH_FORTRAN_SRC
self.flexpartdir = None
self.makefile = 'Makefile.gfortran'
self.destination = None
self.gateway = None
self.ecuid = None
self.ecgid = None
self.install_target = None
self.debug = 0
self.request = 0
self.public = 0
self.ecapi = None
self.purefc = 0
self.rrint = 0
self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
'ectrans', 'debug', 'request', 'public', 'purefc',
'rrint']
self._read_controlfile()
return
def _read_controlfile(self):
'''Read CONTROL file and assign all CONTROL file variables.
Parameters
----------
Return
------
'''
try:
cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
with open(cfile) as f:
fdata = f.read().split('\n')
except IOError:
print('Could not read CONTROL file "' + cfile + '"')
print('Either it does not exist or its syntax is wrong.')
print('Try "' + sys.argv[0].split('/')[-1] + \
' -h" to print usage information')
sys.exit(1)
# go through every line and store parameter
for ldata in fdata:
if ldata and ldata[0] == '#':
# ignore comment line in control file
continue
if '#' in ldata:
# cut off comment
ldata = ldata.split('#')[0]
data = ldata.split()
if len(data) > 1:
if 'm_' in data[0].lower():
data[0] = data[0][2:]
if data[0].lower() == 'class':
data[0] = 'marsclass'
if data[0].lower() == 'day1':
data[0] = 'start_date'
if data[0].lower() == 'day2':
data[0] = 'end_date'
if len(data) == 2:
if '$' in data[1]:
setattr(self, data[0].lower(), data[1])
while '$' in data[1]:
i = data[1].index('$')
j = data[1].find('{')
k = data[1].find('}')
var = os.getenv(data[1][j+1:k])
if var is not None:
data[1] = data[1][:i] + var + data[1][k+1:]
else:
my_error(self.mailfail,
'Could not find variable '
+ data[1][j+1:k] + ' while reading ' +
self.controlfile)
setattr(self, data[0].lower() + '_expanded', data[1])
else:
if data[1].lower() != 'none':
setattr(self, data[0].lower(), data[1])
else:
setattr(self, data[0].lower(), None)
elif len(data) > 2:
setattr(self, data[0].lower(), (data[1:]))
else:
pass
return
def __str__(self):
'''Prepares a string which have all the ControlFile class attributes
with its associated values. Each attribute is printed in one line and
in alphabetical order.
Example
-------
'age': 10
'color': 'Spotted'
'kids': 0
'legs': 2
'name': 'Dog'
'smell': 'Alot'
Parameters
----------
Return
------
string
Single string of concatenated ControlFile class attributes
with their values
'''
import collections
attrs = vars(self).copy()
attrs = collections.OrderedDict(sorted(attrs.items()))
return '\n'.join("%s: %s" % item for item in attrs.items())
[docs] def assign_args_to_control(self, args):
'''Overwrites the existing ControlFile instance attributes with
the command line arguments.
Parameters
----------
args : Namespace
Contains the commandline arguments from script/program call.
Return
------
'''
# get dictionary of command line parameters and eliminate all
# parameters which are None (were not specified)
args_dict = vars(args)
arguments = {k : args_dict[k] for k in args_dict
if args_dict[k] != None}
# assign all passed command line arguments to ControlFile instance
for k, v in arguments.iteritems():
setattr(self, str(k), v)
return
[docs] def assign_envs_to_control(self, envs):
'''Assigns the ECMWF environment parameter.
Parameters
----------
envs : dict of str
Contains the ECMWF environment parameternames "ECUID", "ECGID",
"DESTINATION" and "GATEWAY" with its corresponding values.
They were read from the file "ECMWF_ENV".
Return
------
'''
for k, v in envs.iteritems():
setattr(self, str(k).lower(), str(v))
return
[docs] def check_conditions(self, queue):
'''Checks a couple of necessary attributes and conditions,
such as if they exist and contain values.
Otherwise set default values.
Parameters
----------
queue : str
Name of the queue if submitted to the ECMWF servers.
Used to check if ecuid, ecgid, gateway and destination
are set correctly and are not empty.
Return
------
'''
check_logicals_type(self, self.logicals)
self.mailfail = check_mail(self.mailfail)
self.mailops = check_mail(self.mailops)
check_queue(queue, self.gateway, self.destination,
self.ecuid, self.ecgid)
self.outputdir, self.flexpartdir = check_pathes(self.inputdir,
self.outputdir, self.flexpartdir, self.flexextractdir)
self.start_date, self.end_date = check_dates(self.start_date,
self.end_date)
check_basetime(self.basetime)
self.levelist, self.level = check_levels(self.levelist, self.level)
self.step = check_step(self.step, self.mailfail)
self.maxstep = check_maxstep(self.maxstep, self.step)
check_request(self.request,
os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS))
check_public(self.public, self.dataset)
self.type = check_type(self.type, self.step)
self.time = check_time(self.time)
self.type, self.time, self.step = check_len_type_time_step(self.type,
self.time,
self.step,
self.maxstep,
self.purefc)
self.acctype = check_acctype(self.acctype, self.type)
self.acctime = check_acctime(self.acctime, self.acctype, self.purefc)
self.accmaxstep = check_accmaxstep(self.accmaxstep, self.acctype,
self.purefc, self.maxstep)
self.purefc = check_purefc(self.type)
self.grid = check_grid(self.grid)
self.area = check_area(self.grid, self.area, self.upper, self.lower,
self.left, self.right)
self.addpar = check_addpar(self.addpar)
self.job_chunk = check_job_chunk(self.job_chunk)
return
[docs] def to_list(self):
'''Just generates a list of strings containing the attributes and
assigned values except the attributes "_expanded", "exedir",
"flexextractdir" and "flexpartdir".
Parameters
----------
Return
------
l : list of *
A sorted list of the all ControlFile class attributes with
their values except the attributes "_expanded", "exedir",
"flexextractdir" and "flexpartdir".
'''
import collections
attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
l = list()
for item in attrs.items():
if '_expanded' in item[0]:
pass
elif 'exedir' in item[0]:
pass
elif 'flexpartdir' in item[0]:
pass
elif 'flexextractdir' in item[0]:
pass
else:
if isinstance(item[1], list):
stot = ''
for s in item[1]:
stot += s + ' '
l.append("%s %s\n" % (item[0], stot))
else:
l.append("%s %s\n" % item)
return sorted(l)