Source code for ControlFile

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Leopold Haimberger (University of Vienna)
#
# @Date: November 2015
#
# @Change History:
#
#   February 2018 - Anne Philipp (University of Vienna):
#        - applied PEP8 style guide
#        - added documentation
#        - applied some minor modifications in programming style/structure
#        - changed name of class Control to ControlFile for more
#          self-explanation naming
#        - outsource of class ControlFile
#        - initialisation of class attributes ( to avoid high number of
#          conditional statements and set default values )
#        - divided assignment of attributes and the check of conditions
#        - outsourced the commandline argument assignments to control attributes
#
# @License:
#    (C) Copyright 2014-2019.
#    Anne Philipp, Leopold Haimberger
#
#    This work is licensed under the Creative Commons Attribution 4.0
#    International License. To view a copy of this license, visit
#    http://creativecommons.org/licenses/by/4.0/ or send a letter to
#    Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
#
# @Class Methods:
#    __init__
#    _read_controlfile
#    __str__
#    assign_args_to_control
#    assign_envs_to_control
#    check_conditions
#    check_install_conditions
#    to_list
#*******************************************************************************

# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
import os
import re
import sys
import inspect
import numpy as np

# software specific classes and modules from flex_extract
sys.path.append('../')
import _config
from mods.tools import my_error, silent_remove
from mods.checks import (check_grid, check_area, check_levels, check_purefc,
                         check_step, check_mail, check_queue, check_pathes,
                         check_dates, check_maxstep, check_type, check_request,
                         check_basetime, check_public, check_acctype,
                         check_acctime, check_accmaxstep, check_time,
                         check_logicals_type, check_len_type_time_step,
                         check_addpar, check_job_chunk)

# ------------------------------------------------------------------------------
# CLASS
# ------------------------------------------------------------------------------
[docs]class ControlFile(object): ''' Contains the information which are stored in the CONTROL files. The CONTROL file is the steering part of the FLEXPART extraction software. All necessary parameters needed to retrieve the data fields from the MARS archive for driving FLEXPART are set in a CONTROL file. Some specific parameters like the start and end dates can be overwritten by the command line parameters, but in generel all parameters needed for a complete set of fields for FLEXPART can be set in the CONTROL file. Attributes ---------- controlfile : str The name of the control file to be processed. Default value is the filename passed to the init function when initialised. start_date : str The first day of the retrieval period. Default value is None. end_date :str The last day of the retrieval period. Default value is None. date_chunk : int Length of period for a single mars retrieval. Default value is 3. dtime :str The time step in hours. Default value is None. basetime : str The time for a half day retrieval. The 12 hours upfront are to be retrieved. Default value is None. maxstep : int The maximum forecast step for non flux data. Default value is None. type : list of str List of field type per retrieving hour. Default value is None. time : list of str List of retrieving times in hours. Default valuer is None. step : list of str or str List of forecast time steps in hours for non flux data. Default value is None. acctype : str The field type for the accumulated forecast fields. Default value is None. acctime : str The starting time of the accumulated forecasts. Default value is None. accmaxstep : int The maximum forecast step for the accumulated forecast fields (flux data). Default value is None. marsclass : str Characterisation of dataset. Default value is None. dataset : str For public datasets there is the specific naming and parameter dataset which has to be used to characterize the type of data. Default value is None. stream : str Identifies the forecasting system used to generate the data. Default value is None. number : str Selects the member in ensemble forecast run. Default value is 'OFF'. expver : str The version number of the dataset. Default value is '1'. gaussian : str This parameter is deprecated and should no longer be used. Specifies the desired type of Gaussian grid for the output. Default value is an empty string ''. grid : str Specifies the output grid which can be either a Gaussian grid or a Latitude/Longitude grid. Default value is None. area : str Specifies the desired sub-area of data to be extracted. Default value is None. left : str The western most longitude of the area to be extracted. Default value is None. lower : str The southern most latitude of the area to be extracted. Default value is None. upper : str The northern most latitued of the area to be extracted. Default value is None. right : str The eastern most longitude of the area to be extracted. Default value is None. level : str Specifies the maximum level. Default value is None. levelist : str Specifies the required level list. Default value is None. resol : str Specifies the desired triangular truncation of retrieved data, before carrying out any other selected post-processing. Default value is None. gauss : int Switch to select gaussian fields (1) or regular lat/lon (0). Default value is 0. accuracy : int Specifies the number of bits per value to be used in the generated GRIB coded fields. Default value is 24. omega : int Switch to select omega retrieval (1) or not (0). Default value is 0. omegadiff : int Switch to decide to calculate Omega and Dps/Dt from continuity equation for diagnostic purposes (1) or not (0). Default value is 0. eta : int Switch to select direct retrieval of etadot from MARS (1) or wether it has to be calculated (0). Then Default value is 0. etadiff : int Switch to select calculation of etadot and Dps/Dt from continuity equation for diagnostic purposes (1) or not (0). Default value is 0. etapar : int GRIB parameter Id for etadot fields. Default value is 77. dpdeta : int Switch to select multiplication of etadot with dpdeta. Default value is 1. smooth : int Spectral truncation of ETADOT after calculation on Gaussian grid. Default value is 0. format : str The format of the GRIB data. Default value is 'GRIB1'. addpar : str List of additional surface level ECMWF parameter to be retrieved. Default value is None. prefix : str Prefix string for the final FLEXPART/FLEXTRA ready input files. Default value is 'EN'. cwc : int Switch to select wether the sum of cloud liquid water content and cloud ice water content should be retrieved. Default value is 0. wrf : int Switch to select further parameters for retrievment to support WRF simulations. Default value is 0. ecfsdir : str Path to the ECMWF storage 'ectmp:/${USER}/econdemand/' mailfail : list of str Email list for sending error log files from ECMWF servers. The email addresses should be seperated by a comma. Default value is ['${USER}']. mailops : list of str Email list for sending operational log files from ECMWF servers. The email addresses should be seperated by a comma. Default value is ['${USER}']. grib2flexpart : int 0 Switch to select generation of preprocessed FLEXPART files ".fp". If it is selected, the program grib2flexpart will try to convert the flex_extract output files into ".fp" format. ecstorage : int Switch to select storage of FLEXPART ready output files in the ECFS file system. Default value is 0. ectrans : int Switch to select the transfer of FLEXPART ready output files to the gateway server. Default value is 0. inputdir : str Path to the temporary directory for the retrieval grib files and other processing files. Default value is _config.PATH_INPUT_DIR. outputdir : str Path to the final directory where the final FLEXPART ready input files are stored. Default value is None. flexextractdir : str Path to the flex_extract root directory. Default value is _config.PATH_FLEXEXTRACT_DIR. exedir : str Path to the FORTRAN executable file. Default value is _config.PATH_FORTRAN_SRC. flexpartdir : str Path to a FLEXPART root directory. Default value is None. makefile : str Name of the makefile to be used for the Fortran program. Default value is 'Makefile.gfortran'. destination : str The remote destination which is used to transfer files from ECMWF server to local gateway server. Default value is None. gateway : str The gateway server the user is using. Default value is None. ecuid : str The user id on ECMWF server. Default value is None. ecgid : str The group id on ECMWF server. Default value is None. install_target : str Defines the location where the installation is to be done. Default value is None. debug : int Switch to keep temporary files at the end of postprocessing (1) or to delete all temporary files except the final output files (0). Default value is 0. request : int Switch to select between just retrieving the data (0), writing the mars parameter values to a csv file (1) or doing both (2). Default value is 0. public : int Switch to select kind of ECMWF Web Api access and the possible data sets. Public data sets (1) and Memberstate data sets (0). Default value is 0. ecapi : boolean Tells wether the ECMWF Web APi was able to load or not. Default value is None. purefc : int Switch to decide wether the job is a pure forecast retrieval or coupled with analysis data. Default value is 0. rrint: int Switch to select between old precipitation disaggregation method (0) or the new IA3 disaggegration method (1). Default value is 0. logicals : list of str List of the names of logical switches which controls the flow of the program. Default list is ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff', 'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage', 'ectrans', 'debug', 'request', 'public', 'purefc', 'rrint'] ''' def __init__(self, filename): '''Initialises the instance of ControlFile class and defines all class attributes with default values. Afterwards calls function __read_controlfile__ to read parameter from Control file. Parameters ---------- filename : str Name of CONTROL file. Return ------ ''' # list of all possible class attributes and their default values self.controlfile = filename self.start_date = None self.end_date = None self.date_chunk = 3 self.job_chunk = None self.dtime = None self.basetime = None self.maxstep = None self.type = None self.time = None self.step = None self.acctype = None self.acctime = None self.accmaxstep = None self.marsclass = None self.dataset = None self.stream = None self.number = 'OFF' self.expver = '1' self.gaussian = '' self.grid = None self.area = '' self.left = None self.lower = None self.upper = None self.right = None self.level = None self.levelist = None self.resol = None self.gauss = 0 self.accuracy = 24 self.omega = 0 self.omegadiff = 0 self.eta = 0 self.etadiff = 0 self.etapar = 77 self.dpdeta = 1 self.smooth = 0 self.format = 'GRIB1' self.addpar = None self.prefix = 'EN' self.cwc = 0 self.wrf = 0 self.ecfsdir = 'ectmp:/${USER}/econdemand/' self.mailfail = ['${USER}'] self.mailops = ['${USER}'] self.grib2flexpart = 0 self.ecstorage = 0 self.ectrans = 0 self.inputdir = _config.PATH_INPUT_DIR self.outputdir = None self.flexextractdir = _config.PATH_FLEXEXTRACT_DIR self.exedir = _config.PATH_FORTRAN_SRC self.flexpartdir = None self.makefile = 'Makefile.gfortran' self.destination = None self.gateway = None self.ecuid = None self.ecgid = None self.install_target = None self.debug = 0 self.request = 0 self.public = 0 self.ecapi = None self.purefc = 0 self.rrint = 0 self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff', 'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage', 'ectrans', 'debug', 'request', 'public', 'purefc', 'rrint'] self._read_controlfile() return def _read_controlfile(self): '''Read CONTROL file and assign all CONTROL file variables. Parameters ---------- Return ------ ''' try: cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile) with open(cfile) as f: fdata = f.read().split('\n') except IOError: print('Could not read CONTROL file "' + cfile + '"') print('Either it does not exist or its syntax is wrong.') print('Try "' + sys.argv[0].split('/')[-1] + \ ' -h" to print usage information') sys.exit(1) # go through every line and store parameter for ldata in fdata: if ldata and ldata[0] == '#': # ignore comment line in control file continue if '#' in ldata: # cut off comment ldata = ldata.split('#')[0] data = ldata.split() if len(data) > 1: if 'm_' in data[0].lower(): data[0] = data[0][2:] if data[0].lower() == 'class': data[0] = 'marsclass' if data[0].lower() == 'day1': data[0] = 'start_date' if data[0].lower() == 'day2': data[0] = 'end_date' if len(data) == 2: if '$' in data[1]: setattr(self, data[0].lower(), data[1]) while '$' in data[1]: i = data[1].index('$') j = data[1].find('{') k = data[1].find('}') var = os.getenv(data[1][j+1:k]) if var is not None: data[1] = data[1][:i] + var + data[1][k+1:] else: my_error(self.mailfail, 'Could not find variable ' + data[1][j+1:k] + ' while reading ' + self.controlfile) setattr(self, data[0].lower() + '_expanded', data[1]) else: if data[1].lower() != 'none': setattr(self, data[0].lower(), data[1]) else: setattr(self, data[0].lower(), None) elif len(data) > 2: setattr(self, data[0].lower(), (data[1:])) else: pass return def __str__(self): '''Prepares a string which have all the ControlFile class attributes with its associated values. Each attribute is printed in one line and in alphabetical order. Example ------- 'age': 10 'color': 'Spotted' 'kids': 0 'legs': 2 'name': 'Dog' 'smell': 'Alot' Parameters ---------- Return ------ string Single string of concatenated ControlFile class attributes with their values ''' import collections attrs = vars(self).copy() attrs = collections.OrderedDict(sorted(attrs.items())) return '\n'.join("%s: %s" % item for item in attrs.items())
[docs] def assign_args_to_control(self, args): '''Overwrites the existing ControlFile instance attributes with the command line arguments. Parameters ---------- args : Namespace Contains the commandline arguments from script/program call. Return ------ ''' # get dictionary of command line parameters and eliminate all # parameters which are None (were not specified) args_dict = vars(args) arguments = {k : args_dict[k] for k in args_dict if args_dict[k] != None} # assign all passed command line arguments to ControlFile instance for k, v in arguments.iteritems(): setattr(self, str(k), v) return
[docs] def assign_envs_to_control(self, envs): '''Assigns the ECMWF environment parameter. Parameters ---------- envs : dict of str Contains the ECMWF environment parameternames "ECUID", "ECGID", "DESTINATION" and "GATEWAY" with its corresponding values. They were read from the file "ECMWF_ENV". Return ------ ''' for k, v in envs.iteritems(): setattr(self, str(k).lower(), str(v)) return
[docs] def check_conditions(self, queue): '''Checks a couple of necessary attributes and conditions, such as if they exist and contain values. Otherwise set default values. Parameters ---------- queue : str Name of the queue if submitted to the ECMWF servers. Used to check if ecuid, ecgid, gateway and destination are set correctly and are not empty. Return ------ ''' check_logicals_type(self, self.logicals) self.mailfail = check_mail(self.mailfail) self.mailops = check_mail(self.mailops) check_queue(queue, self.gateway, self.destination, self.ecuid, self.ecgid) self.outputdir, self.flexpartdir = check_pathes(self.inputdir, self.outputdir, self.flexpartdir, self.flexextractdir) self.start_date, self.end_date = check_dates(self.start_date, self.end_date) check_basetime(self.basetime) self.levelist, self.level = check_levels(self.levelist, self.level) self.step = check_step(self.step, self.mailfail) self.maxstep = check_maxstep(self.maxstep, self.step) check_request(self.request, os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS)) check_public(self.public, self.dataset) self.type = check_type(self.type, self.step) self.time = check_time(self.time) self.type, self.time, self.step = check_len_type_time_step(self.type, self.time, self.step, self.maxstep, self.purefc) self.acctype = check_acctype(self.acctype, self.type) self.acctime = check_acctime(self.acctime, self.acctype, self.purefc) self.accmaxstep = check_accmaxstep(self.accmaxstep, self.acctype, self.purefc, self.maxstep) self.purefc = check_purefc(self.type) self.grid = check_grid(self.grid) self.area = check_area(self.grid, self.area, self.upper, self.lower, self.left, self.right) self.addpar = check_addpar(self.addpar) self.job_chunk = check_job_chunk(self.job_chunk) return
[docs] def to_list(self): '''Just generates a list of strings containing the attributes and assigned values except the attributes "_expanded", "exedir", "flexextractdir" and "flexpartdir". Parameters ---------- Return ------ l : list of * A sorted list of the all ControlFile class attributes with their values except the attributes "_expanded", "exedir", "flexextractdir" and "flexpartdir". ''' import collections attrs = collections.OrderedDict(sorted(vars(self).copy().items())) l = list() for item in attrs.items(): if '_expanded' in item[0]: pass elif 'exedir' in item[0]: pass elif 'flexpartdir' in item[0]: pass elif 'flexextractdir' in item[0]: pass else: if isinstance(item[1], list): stot = '' for s in item[1]: stot += s + ' ' l.append("%s %s\n" % (item[0], stot)) else: l.append("%s %s\n" % item) return sorted(l)