source: flex_extract.git/source/python/classes/ControlFile.py @ d727af2

ctbtodev
Last change on this file since d727af2 was 79729d5, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

switched from python2 to python3

  • Property mode set to 100644
File size: 22.7 KB
Line 
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Leopold Haimberger (University of Vienna)
5#
6# @Date: November 2015
7#
8# @Change History:
9#
10#   February 2018 - Anne Philipp (University of Vienna):
11#        - applied PEP8 style guide
12#        - added documentation
13#        - applied some minor modifications in programming style/structure
14#        - changed name of class Control to ControlFile for more
15#          self-explanation naming
16#        - outsource of class ControlFile
17#        - initialisation of class attributes ( to avoid high number of
18#          conditional statements and set default values )
19#        - divided assignment of attributes and the check of conditions
20#        - outsourced the commandline argument assignments to control attributes
21#
22# @License:
23#    (C) Copyright 2014-2019.
24#    Anne Philipp, Leopold Haimberger
25#
26#    This work is licensed under the Creative Commons Attribution 4.0
27#    International License. To view a copy of this license, visit
28#    http://creativecommons.org/licenses/by/4.0/ or send a letter to
29#    Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
30#
31# @Class Methods:
32#    __init__
33#    _read_controlfile
34#    __str__
35#    assign_args_to_control
36#    assign_envs_to_control
37#    check_conditions
38#    check_install_conditions
39#    to_list
40#*******************************************************************************
41
42# ------------------------------------------------------------------------------
43# MODULES
44# ------------------------------------------------------------------------------
45import os
46import re
47import sys
48import inspect
49import numpy as np
50
51# software specific classes and modules from flex_extract
52sys.path.append('../')
53import _config
54from mods.tools import my_error, silent_remove
55from mods.checks import (check_grid, check_area, check_levels, check_purefc,
56                         check_step, check_mail, check_queue, check_pathes,
57                         check_dates, check_maxstep, check_type, check_request,
58                         check_basetime, check_public, check_acctype,
59                         check_acctime, check_accmaxstep, check_time,
60                         check_logicals_type, check_len_type_time_step,
61                         check_addpar, check_job_chunk, check_number)
62
63# ------------------------------------------------------------------------------
64# CLASS
65# ------------------------------------------------------------------------------
66class ControlFile(object):
67    '''
68    Contains the information which are stored in the CONTROL files.
69
70    The CONTROL file is the steering part of the FLEXPART extraction
71    software. All necessary parameters needed to retrieve the data fields
72    from the MARS archive for driving FLEXPART are set in a CONTROL file.
73    Some specific parameters like the start and end dates can be overwritten
74    by the command line parameters, but in generel all parameters needed
75    for a complete set of fields for FLEXPART can be set in the CONTROL file.
76
77    Attributes
78    ----------
79    controlfile : str
80        The name of the control file to be processed. Default value is the
81        filename passed to the init function when initialised.
82
83    start_date : str
84        The first day of the retrieval period. Default value is None.
85
86    end_date :str
87        The last day of the retrieval period. Default value is None.
88
89    date_chunk : int
90        Length of period for a single mars retrieval. Default value is 3.
91
92    dtime :str
93        The time step in hours. Default value is None.
94
95    basetime : int
96        The time for a half day retrieval. The 12 hours upfront are to be
97        retrieved. Default value is None.
98
99    maxstep : int
100        The maximum forecast step for non flux data. Default value is None.
101
102    type : list of str
103        List of field type per retrieving hour. Default value is None.
104
105    time : list of str
106        List of retrieving times in hours. Default valuer is None.
107
108    step : list of str or str
109        List of forecast time steps in hours for non flux data.
110        Default value is None.
111
112    acctype : str
113        The field type for the accumulated forecast fields.
114        Default value is None.
115
116    acctime : str
117        The starting time of the accumulated forecasts. Default value is None.
118
119    accmaxstep : int
120        The maximum forecast step for the accumulated forecast fields
121        (flux data). Default value is None.
122
123    marsclass : str
124        Characterisation of dataset. Default value is None.
125
126    dataset : str
127        For public datasets there is the specific naming and parameter
128        dataset which has to be used to characterize the type of
129        data. Default value is None.
130
131    stream : str
132        Identifies the forecasting system used to generate the data.
133        Default value is None.
134
135    number : str
136        Selects the member in ensemble forecast run. Default value is 'OFF'.
137
138    expver : str
139        The version number of the dataset. Default value is '1'.
140
141    gaussian : str
142        This parameter is deprecated and should no longer be used.
143        Specifies the desired type of Gaussian grid for the output.
144        Default value is an empty string ''.
145
146    grid : str
147        Specifies the output grid which can be either a Gaussian grid
148        or a Latitude/Longitude grid. Default value is None.
149
150    area : str
151        Specifies the desired sub-area of data to be extracted.
152        Default value is None.
153
154    left : str
155        The western most longitude of the area to be extracted.
156        Default value is None.
157
158    lower : str
159        The southern most latitude of the area to be extracted.
160        Default value is None.
161
162    upper : str
163        The northern most latitued of the area to be extracted.
164        Default value is None.
165
166    right : str
167        The eastern most longitude of the area to be extracted.
168        Default value is None.
169
170    level : str
171        Specifies the maximum level. Default value is None.
172
173    levelist : str
174        Specifies the required level list. Default value is None.
175
176    resol : str
177        Specifies the desired triangular truncation of retrieved data,
178        before carrying out any other selected post-processing.
179        Default value is None.
180
181    gauss : int
182        Switch to select gaussian fields (1) or regular lat/lon (0).
183        Default value is 0.
184
185    accuracy : int
186        Specifies the number of bits per value to be used in the
187        generated GRIB coded fields. Default value is 24.
188
189    omega : int
190       Switch to select omega retrieval (1) or not (0). Default value is 0.
191
192    omegadiff : int
193        Switch to decide to calculate Omega and Dps/Dt from continuity
194        equation for diagnostic purposes (1) or not (0). Default value is 0.
195
196    eta : int
197        Switch to select direct retrieval of etadot from MARS (1) or
198        wether it has to be calculated (0). Then Default value is 0.
199
200    etadiff : int
201        Switch to select calculation of etadot and Dps/Dt from continuity
202        equation for diagnostic purposes (1) or not (0). Default value is 0.
203
204    etapar : int
205        GRIB parameter Id for etadot fields. Default value is 77.
206
207    dpdeta : int
208        Switch to select multiplication of etadot with dpdeta.
209        Default value is 1.
210
211    smooth : int
212        Spectral truncation of ETADOT after calculation on Gaussian grid.
213        Default value is 0.
214
215    format : str
216        The format of the GRIB data. Default value is 'GRIB1'.
217
218    addpar : str
219        List of additional surface level ECMWF parameter to be retrieved.
220        Default value is None.
221
222    prefix : str
223        Prefix string for the final FLEXPART/FLEXTRA ready input files.
224        Default value is 'EN'.
225
226    cwc : int
227        Switch to select wether the sum of cloud liquid water content and
228        cloud ice water content should be retrieved. Default value is 0.
229
230    wrf : int
231        Switch to select further parameters for retrievment to support
232        WRF simulations. Default value is 0.
233
234    ecfsdir : str
235        Path to the ECMWF storage  'ectmp:/${USER}/econdemand/'
236
237    mailfail : list of str
238        Email list for sending error log files from ECMWF servers.
239        The email addresses should be seperated by a comma.
240        Default value is ['${USER}'].
241
242    mailops : list of str
243        Email list for sending operational log files from ECMWF servers.
244        The email addresses should be seperated by a comma.
245        Default value is ['${USER}'].
246
247    grib2flexpart : int 0
248        Switch to select generation of preprocessed FLEXPART files ".fp".
249        If it is selected, the program grib2flexpart will try
250        to convert the flex_extract output files into ".fp" format.
251
252    ecstorage : int
253        Switch to select storage of FLEXPART ready output files
254        in the ECFS file system. Default value is 0.
255
256    ectrans : int
257        Switch to select the transfer of FLEXPART ready output files
258        to the gateway server. Default value is 0.
259
260    inputdir : str
261        Path to the temporary directory for the retrieval grib files and
262        other processing files. Default value is _config.PATH_INPUT_DIR.
263
264    outputdir : str
265        Path to the final directory where the final FLEXPART ready input
266        files are stored. Default value is None.
267
268    flexextractdir : str
269        Path to the flex_extract root directory. Default value is
270        _config.PATH_FLEXEXTRACT_DIR.
271
272    exedir : str
273        Path to the FORTRAN executable file. Default value is
274        _config.PATH_FORTRAN_SRC.
275
276    flexpartdir : str
277        Path to a FLEXPART root directory. Default value is None.
278
279    makefile : str
280        Name of the makefile to be used for the Fortran program.
281        Default value is 'Makefile.gfortran'.
282
283    destination : str
284        The remote destination which is used to transfer files
285        from ECMWF server to local gateway server. Default value is None.
286
287    gateway : str
288        The gateway server the user is using. Default value is None.
289
290    ecuid : str
291        The user id on ECMWF server. Default value is None.
292
293    ecgid : str
294        The group id on ECMWF server. Default value is None.
295
296    install_target : str
297        Defines the location where the installation is to be done.
298        Default value is None.
299
300    debug : int
301        Switch to keep temporary files at the end of postprocessing (1) or
302        to delete all temporary files except the final output files (0).
303        Default value is 0.
304
305    oper : int
306        Switch to prepare the operational job script. Start date, end date and
307        basetime will be prepared with environment variables.
308        Default value is 0.
309
310    request : int
311        Switch to select between just retrieving the data (0), writing the mars
312        parameter values to a csv file (1) or doing both (2).
313        Default value is 0.
314
315    public : int
316        Switch to select kind of ECMWF Web Api access and the
317        possible data sets. Public data sets (1) and Memberstate data sets (0).
318        Default value is 0.
319
320    ec_api : boolean
321        Tells wether the ECMWF Web API was able to load or not.
322        Default value is None.
323
324    cds_api : boolean
325        Tells wether the CDS API was able to load or not.
326        Default value is None.
327
328    purefc : int
329        Switch to decide wether the job is a pure forecast retrieval or
330        coupled with analysis data. Default value is 0.
331
332    rrint: int
333        Switch to select between old precipitation disaggregation method (0)
334        or the new IA3 disaggegration method (1). Default value is 0.
335
336    logicals : list of str
337        List of the names of logical switches which controls the flow
338        of the program. Default list is ['gauss', 'omega', 'omegadiff', 'eta',
339        'etadiff', 'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
340        'ectrans', 'debug', 'request', 'public', 'purefc', 'rrint']
341    '''
342
343    def __init__(self, filename):
344        '''Initialises the instance of ControlFile class and defines
345        all class attributes with default values. Afterwards calls
346        function __read_controlfile__ to read parameter from Control file.
347
348        Parameters
349        ----------
350        filename : str
351            Name of CONTROL file.
352
353        Return
354        ------
355
356        '''
357
358        # list of all possible class attributes and their default values
359        self.controlfile = filename
360        self.start_date = None
361        self.end_date = None
362        self.date_chunk = 3
363        self.job_chunk = None
364        self.dtime = None
365        self.basetime = None
366        self.maxstep = None
367        self.type = None
368        self.time = None
369        self.step = None
370        self.acctype = None
371        self.acctime = None
372        self.accmaxstep = None
373        self.marsclass = None
374        self.dataset = None
375        self.stream = None
376        self.number = 'OFF'
377        self.expver = '1'
378        self.gaussian = ''
379        self.grid = None
380        self.area = ''
381        self.left = None
382        self.lower = None
383        self.upper = None
384        self.right = None
385        self.level = None
386        self.levelist = None
387        self.resol = None
388        self.gauss = 0
389        self.accuracy = 24
390        self.omega = 0
391        self.omegadiff = 0
392        self.eta = 0
393        self.etadiff = 0
394        self.etapar = 77
395        self.dpdeta = 1
396        self.smooth = 0
397        self.format = 'GRIB1'
398        self.addpar = None
399        self.prefix = 'EN'
400        self.cwc = 0
401        self.wrf = 0
402        self.ecfsdir = 'ectmp:/${USER}/econdemand/'
403        self.mailfail = ['${USER}']
404        self.mailops = ['${USER}']
405        self.grib2flexpart = 0
406        self.ecstorage = 0
407        self.ectrans = 0
408        self.inputdir = _config.PATH_INPUT_DIR
409        self.outputdir = None
410        self.flexextractdir = _config.PATH_FLEXEXTRACT_DIR
411        self.exedir = _config.PATH_FORTRAN_SRC
412        self.flexpartdir = None
413        self.makefile = 'Makefile.gfortran'
414        self.destination = None
415        self.gateway = None
416        self.ecuid = None
417        self.ecgid = None
418        self.install_target = None
419        self.debug = 0
420        self.oper = 0
421        self.request = 0
422        self.public = 0
423        self.ec_api = None
424        self.cds_api = None
425        self.purefc = 0
426        self.rrint = 0
427
428        self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
429                         'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
430                         'ectrans', 'debug', 'oper', 'request', 'public',
431                         'purefc', 'rrint']
432
433        self._read_controlfile()
434
435        return
436
437    def _read_controlfile(self):
438        '''Read CONTROL file and assign all CONTROL file variables.
439
440        Parameters
441        ----------
442
443        Return
444        ------
445
446        '''
447
448        try:
449            cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
450            with open(cfile) as f:
451                fdata = f.read().split('\n')
452        except IOError:
453            print('Could not read CONTROL file "' + cfile + '"')
454            print('Either it does not exist or its syntax is wrong.')
455            print('Try "' + sys.argv[0].split('/')[-1] + \
456                      ' -h" to print usage information')
457            sys.exit(1)
458
459        # go through every line and store parameter
460        for ldata in fdata:
461            if ldata and ldata[0] == '#':
462                # ignore comment line in control file
463                continue
464            if '#' in ldata:
465                # cut off comment
466                ldata = ldata.split('#')[0]
467            data = ldata.split()
468            if len(data) > 1:
469                if 'm_' in data[0].lower():
470                    data[0] = data[0][2:]
471                if data[0].lower() == 'class':
472                    data[0] = 'marsclass'
473                if data[0].lower() == 'day1':
474                    data[0] = 'start_date'
475                if data[0].lower() == 'day2':
476                    data[0] = 'end_date'
477                if len(data) == 2:
478                    if '$' in data[1]:
479                        setattr(self, data[0].lower(), data[1])
480                        while '$' in data[1]:
481                            i = data[1].index('$')
482                            j = data[1].find('{')
483                            k = data[1].find('}')
484                            var = os.getenv(data[1][j+1:k])
485                            if var is not None:
486                                data[1] = data[1][:i] + var + data[1][k+1:]
487                            else:
488                                my_error('Could not find variable '
489                                         + data[1][j+1:k] + ' while reading ' +
490                                         self.controlfile)
491                        setattr(self, data[0].lower() + '_expanded', data[1])
492                    else:
493                        if data[1].lower() != 'none':
494                            setattr(self, data[0].lower(), data[1])
495                        else:
496                            setattr(self, data[0].lower(), None)
497                elif len(data) > 2:
498                    setattr(self, data[0].lower(), (data[1:]))
499            else:
500                pass
501
502        return
503
504    def __str__(self):
505        '''Prepares a string which have all the ControlFile class attributes
506        with its associated values. Each attribute is printed in one line and
507        in alphabetical order.
508
509        Example
510        -------
511        'age': 10
512        'color': 'Spotted'
513        'kids': 0
514        'legs': 2
515        'name': 'Dog'
516        'smell': 'Alot'
517
518        Parameters
519        ----------
520
521        Return
522        ------
523        string
524            Single string of concatenated ControlFile class attributes
525            with their values
526        '''
527        import collections
528
529        attrs = vars(self).copy()
530        attrs = collections.OrderedDict(sorted(attrs.items()))
531
532        return '\n'.join("%s: %s" % item for item in attrs.items())
533
534    def assign_args_to_control(self, args):
535        '''Overwrites the existing ControlFile instance attributes with
536        the command line arguments.
537
538        Parameters
539        ----------
540        args : Namespace
541            Contains the commandline arguments from script/program call.
542
543        Return
544        ------
545
546        '''
547
548        # get dictionary of command line parameters and eliminate all
549        # parameters which are None (were not specified)
550        args_dict = vars(args)
551        arguments = {k : args_dict[k] for k in args_dict
552                     if args_dict[k] != None}
553
554        # assign all passed command line arguments to ControlFile instance
555        for k, v in arguments.items():
556            setattr(self, str(k), v)
557
558        return
559
560    def assign_envs_to_control(self, envs):
561        '''Assigns the ECMWF environment parameter.
562
563        Parameters
564        ----------
565        envs : dict of str
566            Contains the ECMWF environment parameternames "ECUID", "ECGID",
567            "DESTINATION" and "GATEWAY" with its corresponding values.
568            They were read from the file "ECMWF_ENV".
569
570        Return
571        ------
572
573        '''
574
575        for k, v in envs.items():
576            setattr(self, str(k).lower(), str(v))
577
578        return
579
580    def check_conditions(self, queue):
581        '''Checks a couple of necessary attributes and conditions,
582        such as if they exist and contain values.
583        Otherwise set default values.
584
585        Parameters
586        ----------
587        queue : str
588            Name of the queue if submitted to the ECMWF servers.
589            Used to check if ecuid, ecgid, gateway and destination
590            are set correctly and are not empty.
591
592        Return
593        ------
594
595        '''
596        check_logicals_type(self, self.logicals)
597
598        self.mailfail = check_mail(self.mailfail)
599
600        self.mailops = check_mail(self.mailops)
601
602        check_queue(queue, self.gateway, self.destination,
603                    self.ecuid, self.ecgid)
604
605        self.outputdir, self.flexpartdir = check_pathes(self.inputdir,
606             self.outputdir, self.flexpartdir, self.flexextractdir)
607
608        self.start_date, self.end_date = check_dates(self.start_date,
609                                                     self.end_date)
610
611        self.basetime = check_basetime(self.basetime)
612
613        self.levelist, self.level = check_levels(self.levelist, self.level)
614
615        self.step = check_step(self.step, self.mailfail)
616
617        self.maxstep = check_maxstep(self.maxstep, self.step)
618
619        check_request(self.request,
620                      os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS))
621
622        check_public(self.public, self.dataset)
623
624        self.type = check_type(self.type, self.step)
625
626        self.time = check_time(self.time)
627
628        self.type, self.time, self.step = check_len_type_time_step(self.type,
629                                                                   self.time,
630                                                                   self.step,
631                                                                   self.maxstep,
632                                                                   self.purefc)
633
634        self.acctype = check_acctype(self.acctype, self.type)
635
636        self.acctime = check_acctime(self.acctime, self.marsclass, self.purefc)
637
638        self.accmaxstep = check_accmaxstep(self.accmaxstep, self.marsclass,
639                                           self.purefc, self.maxstep)
640
641        self.purefc = check_purefc(self.type)
642
643        self.grid = check_grid(self.grid)
644
645        self.area = check_area(self.grid, self.area, self.upper, self.lower,
646                               self.left, self.right)
647
648        self.addpar = check_addpar(self.addpar)
649
650        self.job_chunk = check_job_chunk(self.job_chunk)
651
652        self.number = check_number(self.number, self.mailfail)
653
654        return
655
656    def to_list(self):
657        '''Just generates a list of strings containing the attributes and
658        assigned values except the attributes "_expanded", "exedir",
659        "flexextractdir" and "flexpartdir".
660
661        Parameters
662        ----------
663
664        Return
665        ------
666        l : list of *
667            A sorted list of the all ControlFile class attributes with
668            their values except the attributes "_expanded", "exedir",
669            "flexextractdir" and "flexpartdir".
670        '''
671
672        import collections
673
674        attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
675
676        l = list()
677
678        for item in attrs.items():
679            if '_expanded' in item[0]:
680                pass
681            elif 'exedir' in item[0]:
682                pass
683            elif 'flexpartdir' in item[0]:
684                pass
685            elif 'flexextractdir' in item[0]:
686                pass
687            else:
688                if isinstance(item[1], list):
689                    stot = ''
690                    for s in item[1]:
691                        stot += s + ' '
692
693                    l.append("%s %s\n" % (item[0], stot))
694                else:
695                    l.append("%s %s\n" % item)
696
697        return sorted(l)
698
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG