source: flex_extract.git/source/python/classes/ControlFile.py @ a32e042

ctbtodev
Last change on this file since a32e042 was 90a1ca0, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

added switch to select for calculation of extra synthesized ensembles from ELDA stream or not

  • Property mode set to 100644
File size: 22.9 KB
Line 
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Leopold Haimberger (University of Vienna)
5#
6# @Date: November 2015
7#
8# @Change History:
9#
10#   February 2018 - Anne Philipp (University of Vienna):
11#        - applied PEP8 style guide
12#        - added documentation
13#        - applied some minor modifications in programming style/structure
14#        - changed name of class Control to ControlFile for more
15#          self-explanation naming
16#        - outsource of class ControlFile
17#        - initialisation of class attributes ( to avoid high number of
18#          conditional statements and set default values )
19#        - divided assignment of attributes and the check of conditions
20#        - outsourced the commandline argument assignments to control attributes
21#
22# @License:
23#    (C) Copyright 2014-2019.
24#    Anne Philipp, Leopold Haimberger
25#
26#    This work is licensed under the Creative Commons Attribution 4.0
27#    International License. To view a copy of this license, visit
28#    http://creativecommons.org/licenses/by/4.0/ or send a letter to
29#    Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
30#
31# @Class Methods:
32#    __init__
33#    _read_controlfile
34#    __str__
35#    assign_args_to_control
36#    assign_envs_to_control
37#    check_conditions
38#    check_install_conditions
39#    to_list
40#*******************************************************************************
41
42# ------------------------------------------------------------------------------
43# MODULES
44# ------------------------------------------------------------------------------
45import os
46import re
47import sys
48import inspect
49import numpy as np
50
51# software specific classes and modules from flex_extract
52sys.path.append('../')
53import _config
54from mods.tools import my_error, silent_remove
55from mods.checks import (check_grid, check_area, check_levels, check_purefc,
56                         check_step, check_mail, check_queue, check_pathes,
57                         check_dates, check_maxstep, check_type, check_request,
58                         check_basetime, check_public, check_acctype,
59                         check_acctime, check_accmaxstep, check_time,
60                         check_logicals_type, check_len_type_time_step,
61                         check_addpar, check_job_chunk, check_number)
62
63# ------------------------------------------------------------------------------
64# CLASS
65# ------------------------------------------------------------------------------
66class ControlFile(object):
67    '''
68    Contains the information which are stored in the CONTROL files.
69
70    The CONTROL file is the steering part of the FLEXPART extraction
71    software. All necessary parameters needed to retrieve the data fields
72    from the MARS archive for driving FLEXPART are set in a CONTROL file.
73    Some specific parameters like the start and end dates can be overwritten
74    by the command line parameters, but in generel all parameters needed
75    for a complete set of fields for FLEXPART can be set in the CONTROL file.
76
77    Attributes
78    ----------
79    controlfile : str
80        The name of the control file to be processed. Default value is the
81        filename passed to the init function when initialised.
82
83    start_date : str
84        The first day of the retrieval period. Default value is None.
85
86    end_date :str
87        The last day of the retrieval period. Default value is None.
88
89    date_chunk : int
90        Length of period for a single mars retrieval. Default value is 3.
91
92    dtime :str
93        The time step in hours. Default value is None.
94
95    basetime : int
96        The time for a half day retrieval. The 12 hours upfront are to be
97        retrieved. Default value is None.
98
99    maxstep : int
100        The maximum forecast step for non flux data. Default value is None.
101
102    type : list of str
103        List of field type per retrieving hour. Default value is None.
104
105    time : list of str
106        List of retrieving times in hours. Default valuer is None.
107
108    step : list of str or str
109        List of forecast time steps in hours for non flux data.
110        Default value is None.
111
112    acctype : str
113        The field type for the accumulated forecast fields.
114        Default value is None.
115
116    acctime : str
117        The starting time of the accumulated forecasts. Default value is None.
118
119    accmaxstep : int
120        The maximum forecast step for the accumulated forecast fields
121        (flux data). Default value is None.
122
123    marsclass : str
124        Characterisation of dataset. Default value is None.
125
126    dataset : str
127        For public datasets there is the specific naming and parameter
128        dataset which has to be used to characterize the type of
129        data. Default value is None.
130
131    stream : str
132        Identifies the forecasting system used to generate the data.
133        Default value is None.
134
135    number : str
136        Selects the member in ensemble forecast run. Default value is 'OFF'.
137
138    expver : str
139        The version number of the dataset. Default value is '1'.
140
141    gaussian : str
142        This parameter is deprecated and should no longer be used.
143        Specifies the desired type of Gaussian grid for the output.
144        Default value is an empty string ''.
145
146    grid : str
147        Specifies the output grid which can be either a Gaussian grid
148        or a Latitude/Longitude grid. Default value is None.
149
150    area : str
151        Specifies the desired sub-area of data to be extracted.
152        Default value is None.
153
154    left : str
155        The western most longitude of the area to be extracted.
156        Default value is None.
157
158    lower : str
159        The southern most latitude of the area to be extracted.
160        Default value is None.
161
162    upper : str
163        The northern most latitued of the area to be extracted.
164        Default value is None.
165
166    right : str
167        The eastern most longitude of the area to be extracted.
168        Default value is None.
169
170    level : str
171        Specifies the maximum level. Default value is None.
172
173    levelist : str
174        Specifies the required level list. Default value is None.
175
176    resol : str
177        Specifies the desired triangular truncation of retrieved data,
178        before carrying out any other selected post-processing.
179        Default value is None.
180
181    gauss : int
182        Switch to select gaussian fields (1) or regular lat/lon (0).
183        Default value is 0.
184
185    accuracy : int
186        Specifies the number of bits per value to be used in the
187        generated GRIB coded fields. Default value is 24.
188
189    omega : int
190       Switch to select omega retrieval (1) or not (0). Default value is 0.
191
192    omegadiff : int
193        Switch to decide to calculate Omega and Dps/Dt from continuity
194        equation for diagnostic purposes (1) or not (0). Default value is 0.
195
196    eta : int
197        Switch to select direct retrieval of etadot from MARS (1) or
198        wether it has to be calculated (0). Then Default value is 0.
199
200    etadiff : int
201        Switch to select calculation of etadot and Dps/Dt from continuity
202        equation for diagnostic purposes (1) or not (0). Default value is 0.
203
204    etapar : int
205        GRIB parameter Id for etadot fields. Default value is 77.
206
207    dpdeta : int
208        Switch to select multiplication of etadot with dpdeta.
209        Default value is 1.
210
211    smooth : int
212        Spectral truncation of ETADOT after calculation on Gaussian grid.
213        Default value is 0.
214
215    format : str
216        The format of the GRIB data. Default value is 'GRIB1'.
217
218    addpar : str
219        List of additional surface level ECMWF parameter to be retrieved.
220        Default value is None.
221
222    prefix : str
223        Prefix string for the final FLEXPART/FLEXTRA ready input files.
224        Default value is 'EN'.
225
226    cwc : int
227        Switch to select wether the sum of cloud liquid water content and
228        cloud ice water content should be retrieved. Default value is 0.
229
230    wrf : int
231        Switch to select further parameters for retrievment to support
232        WRF simulations. Default value is 0.
233
234    ecfsdir : str
235        Path to the ECMWF storage  'ectmp:/${USER}/econdemand/'
236
237    mailfail : list of str
238        Email list for sending error log files from ECMWF servers.
239        The email addresses should be seperated by a comma.
240        Default value is ['${USER}'].
241
242    mailops : list of str
243        Email list for sending operational log files from ECMWF servers.
244        The email addresses should be seperated by a comma.
245        Default value is ['${USER}'].
246
247    grib2flexpart : int 0
248        Switch to select generation of preprocessed FLEXPART files ".fp".
249        If it is selected, the program grib2flexpart will try
250        to convert the flex_extract output files into ".fp" format.
251
252    ecstorage : int
253        Switch to select storage of FLEXPART ready output files
254        in the ECFS file system. Default value is 0.
255
256    ectrans : int
257        Switch to select the transfer of FLEXPART ready output files
258        to the gateway server. Default value is 0.
259
260    inputdir : str
261        Path to the temporary directory for the retrieval grib files and
262        other processing files. Default value is _config.PATH_INPUT_DIR.
263
264    outputdir : str
265        Path to the final directory where the final FLEXPART ready input
266        files are stored. Default value is None.
267
268    flexextractdir : str
269        Path to the flex_extract root directory. Default value is
270        _config.PATH_FLEXEXTRACT_DIR.
271
272    exedir : str
273        Path to the FORTRAN executable file. Default value is
274        _config.PATH_FORTRAN_SRC.
275
276    flexpartdir : str
277        Path to a FLEXPART root directory. Default value is None.
278
279    makefile : str
280        Name of the makefile to be used for the Fortran program.
281        Default value is 'Makefile.gfortran'.
282
283    destination : str
284        The remote destination which is used to transfer files
285        from ECMWF server to local gateway server. Default value is None.
286
287    gateway : str
288        The gateway server the user is using. Default value is None.
289
290    ecuid : str
291        The user id on ECMWF server. Default value is None.
292
293    ecgid : str
294        The group id on ECMWF server. Default value is None.
295
296    install_target : str
297        Defines the location where the installation is to be done.
298        Default value is None.
299
300    debug : int
301        Switch to keep temporary files at the end of postprocessing (1) or
302        to delete all temporary files except the final output files (0).
303        Default value is 0.
304
305    oper : int
306        Switch to prepare the operational job script. Start date, end date and
307        basetime will be prepared with environment variables.
308        Default value is 0.
309
310    request : int
311        Switch to select between just retrieving the data (0), writing the mars
312        parameter values to a csv file (1) or doing both (2).
313        Default value is 0.
314
315    public : int
316        Switch to select kind of ECMWF Web Api access and the
317        possible data sets. Public data sets (1) and Memberstate data sets (0).
318        Default value is 0.
319
320    ec_api : boolean
321        Tells wether the ECMWF Web API was able to load or not.
322        Default value is None.
323
324    cds_api : boolean
325        Tells wether the CDS API was able to load or not.
326        Default value is None.
327
328    purefc : int
329        Switch to decide wether the job is a pure forecast retrieval or
330        coupled with analysis data. Default value is 0.
331
332    rrint : int
333        Switch to select between old precipitation disaggregation method (0)
334        or the new IA3 disaggegration method (1). Default value is 0.
335
336    doubleelda : int
337        Switch to select the calculation of extra ensemble members for the
338        ELDA stream. It doubles the amount of retrieved ensemble members.
339
340    logicals : list of str
341        List of the names of logical switches which controls the flow
342        of the program. Default list is ['gauss', 'omega', 'omegadiff', 'eta',
343        'etadiff', 'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
344        'ectrans', 'debug', 'request', 'public', 'purefc', 'rrint', 'doubleelda']
345    '''
346
347    def __init__(self, filename):
348        '''Initialises the instance of ControlFile class and defines
349        all class attributes with default values. Afterwards calls
350        function __read_controlfile__ to read parameter from Control file.
351
352        Parameters
353        ----------
354        filename : str
355            Name of CONTROL file.
356
357        Return
358        ------
359
360        '''
361
362        # list of all possible class attributes and their default values
363        self.controlfile = filename
364        self.start_date = None
365        self.end_date = None
366        self.date_chunk = 3
367        self.job_chunk = None
368        self.dtime = None
369        self.basetime = None
370        self.maxstep = None
371        self.type = None
372        self.time = None
373        self.step = None
374        self.acctype = None
375        self.acctime = None
376        self.accmaxstep = None
377        self.marsclass = None
378        self.dataset = None
379        self.stream = None
380        self.number = 'OFF'
381        self.expver = '1'
382        self.gaussian = ''
383        self.grid = None
384        self.area = ''
385        self.left = None
386        self.lower = None
387        self.upper = None
388        self.right = None
389        self.level = None
390        self.levelist = None
391        self.resol = None
392        self.gauss = 0
393        self.accuracy = 24
394        self.omega = 0
395        self.omegadiff = 0
396        self.eta = 0
397        self.etadiff = 0
398        self.etapar = 77
399        self.dpdeta = 1
400        self.smooth = 0
401        self.format = 'GRIB1'
402        self.addpar = None
403        self.prefix = 'EN'
404        self.cwc = 0
405        self.wrf = 0
406        self.ecfsdir = 'ectmp:/${USER}/econdemand/'
407        self.mailfail = ['${USER}']
408        self.mailops = ['${USER}']
409        self.grib2flexpart = 0
410        self.ecstorage = 0
411        self.ectrans = 0
412        self.inputdir = _config.PATH_INPUT_DIR
413        self.outputdir = None
414        self.flexextractdir = _config.PATH_FLEXEXTRACT_DIR
415        self.exedir = _config.PATH_FORTRAN_SRC
416        self.flexpartdir = None
417        self.makefile = 'Makefile.gfortran'
418        self.destination = None
419        self.gateway = None
420        self.ecuid = None
421        self.ecgid = None
422        self.install_target = None
423        self.debug = 0
424        self.oper = 0
425        self.request = 0
426        self.public = 0
427        self.ec_api = None
428        self.cds_api = None
429        self.purefc = 0
430        self.rrint = 0
431        self.doubleelda = 0
432
433        self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
434                         'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
435                         'ectrans', 'debug', 'oper', 'request', 'public',
436                         'purefc', 'rrint', 'doubleelda']
437
438        self._read_controlfile()
439
440        return
441
442    def _read_controlfile(self):
443        '''Read CONTROL file and assign all CONTROL file variables.
444
445        Parameters
446        ----------
447
448        Return
449        ------
450
451        '''
452
453        try:
454            cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
455            with open(cfile) as f:
456                fdata = f.read().split('\n')
457        except IOError:
458            print('Could not read CONTROL file "' + cfile + '"')
459            print('Either it does not exist or its syntax is wrong.')
460            print('Try "' + sys.argv[0].split('/')[-1] + \
461                      ' -h" to print usage information')
462            sys.exit(1)
463
464        # go through every line and store parameter
465        for ldata in fdata:
466            if ldata and ldata[0] == '#':
467                # ignore comment line in control file
468                continue
469            if '#' in ldata:
470                # cut off comment
471                ldata = ldata.split('#')[0]
472            data = ldata.split()
473            if len(data) > 1:
474                if 'm_' in data[0].lower():
475                    data[0] = data[0][2:]
476                if data[0].lower() == 'class':
477                    data[0] = 'marsclass'
478                if data[0].lower() == 'day1':
479                    data[0] = 'start_date'
480                if data[0].lower() == 'day2':
481                    data[0] = 'end_date'
482                if len(data) == 2:
483                    if '$' in data[1]:
484                        setattr(self, data[0].lower(), data[1])
485                        while '$' in data[1]:
486                            i = data[1].index('$')
487                            j = data[1].find('{')
488                            k = data[1].find('}')
489                            var = os.getenv(data[1][j+1:k])
490                            if var is not None:
491                                data[1] = data[1][:i] + var + data[1][k+1:]
492                            else:
493                                my_error('Could not find variable '
494                                         + data[1][j+1:k] + ' while reading ' +
495                                         self.controlfile)
496                        setattr(self, data[0].lower() + '_expanded', data[1])
497                    else:
498                        if data[1].lower() != 'none':
499                            setattr(self, data[0].lower(), data[1])
500                        else:
501                            setattr(self, data[0].lower(), None)
502                elif len(data) > 2:
503                    setattr(self, data[0].lower(), (data[1:]))
504            else:
505                pass
506
507        return
508
509    def __str__(self):
510        '''Prepares a string which have all the ControlFile class attributes
511        with its associated values. Each attribute is printed in one line and
512        in alphabetical order.
513
514        Example
515        -------
516        'age': 10
517        'color': 'Spotted'
518        'kids': 0
519        'legs': 2
520        'name': 'Dog'
521        'smell': 'Alot'
522
523        Parameters
524        ----------
525
526        Return
527        ------
528        string
529            Single string of concatenated ControlFile class attributes
530            with their values
531        '''
532        import collections
533
534        attrs = vars(self).copy()
535        attrs = collections.OrderedDict(sorted(attrs.items()))
536
537        return '\n'.join("%s: %s" % item for item in attrs.items())
538
539    def assign_args_to_control(self, args):
540        '''Overwrites the existing ControlFile instance attributes with
541        the command line arguments.
542
543        Parameters
544        ----------
545        args : Namespace
546            Contains the commandline arguments from script/program call.
547
548        Return
549        ------
550
551        '''
552
553        # get dictionary of command line parameters and eliminate all
554        # parameters which are None (were not specified)
555        args_dict = vars(args)
556        arguments = {k : args_dict[k] for k in args_dict
557                     if args_dict[k] != None}
558
559        # assign all passed command line arguments to ControlFile instance
560        for k, v in arguments.items():
561            setattr(self, str(k), v)
562
563        return
564
565    def assign_envs_to_control(self, envs):
566        '''Assigns the ECMWF environment parameter.
567
568        Parameters
569        ----------
570        envs : dict of str
571            Contains the ECMWF environment parameternames "ECUID", "ECGID",
572            "DESTINATION" and "GATEWAY" with its corresponding values.
573            They were read from the file "ECMWF_ENV".
574
575        Return
576        ------
577
578        '''
579
580        for k, v in envs.items():
581            setattr(self, str(k).lower(), str(v))
582
583        return
584
585    def check_conditions(self, queue):
586        '''Checks a couple of necessary attributes and conditions,
587        such as if they exist and contain values.
588        Otherwise set default values.
589
590        Parameters
591        ----------
592        queue : str
593            Name of the queue if submitted to the ECMWF servers.
594            Used to check if ecuid, ecgid, gateway and destination
595            are set correctly and are not empty.
596
597        Return
598        ------
599
600        '''
601        check_logicals_type(self, self.logicals)
602
603        self.mailfail = check_mail(self.mailfail)
604
605        self.mailops = check_mail(self.mailops)
606
607        check_queue(queue, self.gateway, self.destination,
608                    self.ecuid, self.ecgid)
609
610        self.outputdir, self.flexpartdir = check_pathes(self.inputdir,
611             self.outputdir, self.flexpartdir, self.flexextractdir)
612
613        self.start_date, self.end_date = check_dates(self.start_date,
614                                                     self.end_date)
615
616        self.basetime = check_basetime(self.basetime)
617
618        self.levelist, self.level = check_levels(self.levelist, self.level)
619
620        self.step = check_step(self.step, self.mailfail)
621
622        self.maxstep = check_maxstep(self.maxstep, self.step)
623
624        check_request(self.request,
625                      os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS))
626
627        check_public(self.public, self.dataset)
628
629        self.type = check_type(self.type, self.step)
630
631        self.time = check_time(self.time)
632
633        self.type, self.time, self.step = check_len_type_time_step(self.type,
634                                                                   self.time,
635                                                                   self.step,
636                                                                   self.maxstep,
637                                                                   self.purefc)
638
639        self.acctype = check_acctype(self.acctype, self.type)
640
641        self.acctime = check_acctime(self.acctime, self.marsclass, self.purefc)
642
643        self.accmaxstep = check_accmaxstep(self.accmaxstep, self.marsclass,
644                                           self.purefc, self.maxstep)
645
646        self.purefc = check_purefc(self.type)
647
648        self.grid = check_grid(self.grid)
649
650        self.area = check_area(self.grid, self.area, self.upper, self.lower,
651                               self.left, self.right)
652
653        self.addpar = check_addpar(self.addpar)
654
655        self.job_chunk = check_job_chunk(self.job_chunk)
656
657        self.number = check_number(self.number, self.mailfail)
658
659        return
660
661    def to_list(self):
662        '''Just generates a list of strings containing the attributes and
663        assigned values except the attributes "_expanded", "exedir",
664        "flexextractdir" and "flexpartdir".
665
666        Parameters
667        ----------
668
669        Return
670        ------
671        l : list of *
672            A sorted list of the all ControlFile class attributes with
673            their values except the attributes "_expanded", "exedir",
674            "flexextractdir" and "flexpartdir".
675        '''
676
677        import collections
678
679        attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
680
681        l = list()
682
683        for item in attrs.items():
684            if '_expanded' in item[0]:
685                pass
686            elif 'exedir' in item[0]:
687                pass
688            elif 'flexpartdir' in item[0]:
689                pass
690            elif 'flexextractdir' in item[0]:
691                pass
692            else:
693                if isinstance(item[1], list):
694                    stot = ''
695                    for s in item[1]:
696                        stot += s + ' '
697
698                    l.append("%s %s\n" % (item[0], stot))
699                else:
700                    l.append("%s %s\n" % item)
701
702        return sorted(l)
703
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG