source: flex_extract.git/source/python/classes/MarsRetrieval.py @ 6f951ca

ctbtodev
Last change on this file since 6f951ca was 6f951ca, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

new style of docstring params and updates in docstrings

  • Property mode set to 100644
File size: 19.9 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#   November 2015 - Leopold Haimberger (University of Vienna):
11#        - optimized display_info
12#        - optimized data_retrieve and seperate between python and shell
13#          script call
14#
15#   February 2018 - Anne Philipp (University of Vienna):
16#        - applied PEP8 style guide
17#        - added documentation
18#        - applied some minor modifications in programming style/structure
19#        - added writing of mars request attributes to a csv file
20#
21# @License:
22#    (C) Copyright 2014-2019.
23#    Anne Philipp, Leopold Haimberger
24#
25#    This work is licensed under the Creative Commons Attribution 4.0
26#    International License. To view a copy of this license, visit
27#    http://creativecommons.org/licenses/by/4.0/ or send a letter to
28#    Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
29#*******************************************************************************
30
31# ------------------------------------------------------------------------------
32# MODULES
33# ------------------------------------------------------------------------------
34import os
35import sys
36import subprocess
37
38# software specific classes and modules from flex_extract
39sys.path.append('../')
40import _config
41# ------------------------------------------------------------------------------
42# CLASS
43# ------------------------------------------------------------------------------
44class MarsRetrieval(object):
45    '''Specific syntax and content for submission of MARS retrievals.
46
47    A MARS revtrieval has a specific syntax with a selection of keywords and
48    their corresponding values. This class provides the necessary functions
49    by displaying the selected parameters and their values and the actual
50    retrievement of the data through a mars request or a Python web api
51    interface. The initialization already expects all the keyword values.
52
53    A description of MARS keywords/arguments and examples of their
54    values can be found here:
55    https://software.ecmwf.int/wiki/display/UDOC/\
56                   Identification+keywords#Identificationkeywords-class
57
58    Attributes
59    ----------
60    server : ECMWFService or ECMWFDataServer
61        This is the connection to the ECMWF data servers.
62
63    public : int
64        Decides which Web API Server version is used.
65
66    marsclass : str, optional
67        Characterisation of dataset.
68
69    dataset : str, optional
70        For public datasets there is the specific naming and parameter
71        dataset which has to be used to characterize the type of
72        data.
73
74    type : str, optional
75        Determines the type of fields to be retrieved.
76
77    levtype : str, optional
78        Denotes type of level.
79
80    levelist : str, optional
81        Specifies the required levels.
82
83    repres : str, optional
84        Selects the representation of the archived data.
85
86    date : str, optional
87        Specifies the Analysis date, the Forecast base date or
88        Observations date.
89
90    resol : str, optional
91        Specifies the desired triangular truncation of retrieved data,
92        before carrying out any other selected post-processing.
93
94    stream : str, optional
95        Identifies the forecasting system used to generate the data.
96
97    area : str, optional
98        Specifies the desired sub-area of data to be extracted.
99
100    time : str, optional
101        Specifies the time of the data in hours and minutes.
102
103    step : str, optional
104        Specifies the forecast time step from forecast base time.
105
106    expver : str, optional
107        The version of the dataset.
108
109    number : str, optional
110        Selects the member in ensemble forecast run.
111
112    accuracy : str, optional
113        Specifies the number of bits per value to be used in the
114        generated GRIB coded fields.
115
116    grid : str, optional
117        Specifies the output grid which can be either a Gaussian grid
118        or a Latitude/Longitude grid.
119
120    gaussian : str, optional
121        This parameter is deprecated and should no longer be used.
122        Specifies the desired type of Gaussian grid for the output.
123
124    target : str, optional
125        Specifies a file into which data is to be written after
126        retrieval or manipulation.
127
128    param : str, optional
129        Specifies the meteorological parameter.
130    '''
131
132    def __init__(self, server, public, marsclass="ei", dataset="", type="",
133                 levtype="", levelist="", repres="", date="", resol="",
134                 stream="", area="", time="", step="", expver="1",
135                 number="", accuracy="", grid="", gaussian="", target="",
136                 param=""):
137        '''Initialises the instance of the MarsRetrieval class and
138        defines and assigns a set of the necessary retrieval parameters
139        for the FLEXPART input data.
140        A description of MARS keywords/arguments, their dependencies
141        on each other and examples of their values can be found here:
142
143        https://software.ecmwf.int/wiki/display/UDOC/MARS+keywords
144
145        Parameters
146        ----------
147        server : ECMWFService or ECMWFDataServer
148            This is the connection to the ECMWF data servers.
149            It is needed for the pythonic access of ECMWF data.
150
151        public : int
152            Decides which Web API version is used:
153            0: member-state users and full archive access
154            1: public access and limited access to the public server and
155               datasets. Needs the parameter dataset.
156            Default is "0" and for member-state users.
157
158        marsclass : str, optional
159            Characterisation of dataset. E.g. EI (ERA-Interim),
160            E4 (ERA40), OD (Operational archive), ea (ERA5).
161            Default is the ERA-Interim dataset "ei".
162
163        dataset : str, optional
164            For public datasets there is the specific naming and parameter
165            dataset which has to be used to characterize the type of
166            data. Usually there is less data available, either in times,
167            domain or parameter.
168            Default is an empty string.
169
170        type : str, optional
171            Determines the type of fields to be retrieved.
172            Selects between observations, images or fields.
173            Examples for fields: Analysis (an), Forecast (fc),
174            Perturbed Forecast (pf), Control Forecast (cf) and so on.
175            Default is an empty string.
176
177        levtype : str, optional
178            Denotes type of level. Has a direct implication on valid
179            levelist values!
180            E.g. model level (ml), pressure level (pl), surface (sfc),
181            potential vorticity (pv), potential temperature (pt)
182            and depth (dp).
183            Default is an empty string.
184
185        levelist : str, optional
186            Specifies the required levels. It has to have a valid
187            correspondence to the selected levtype.
188            Examples: model level: 1/to/137, pressure levels: 500/to/1000
189            Default is an empty string.
190
191        repres : str, optional
192            Selects the representation of the archived data.
193            E.g. sh - spherical harmonics, gg - Gaussian grid,
194            ll - latitude/longitude, ...
195            Default is an empty string.
196
197        date : str, optional
198            Specifies the Analysis date, the Forecast base date or
199            Observations date. Valid formats are:
200            Absolute as YYYY-MM-DD or YYYYMMDD.
201            Default is an empty string.
202
203        resol : str, optional
204            Specifies the desired triangular truncation of retrieved data,
205            before carrying out any other selected post-processing.
206            The default is automatic truncation (auto), by which the lowest
207            resolution compatible with the value specified in grid is
208            automatically selected for the retrieval.
209            Users wanting to perform post-processing from full spectral
210            resolution should specify Archived Value (av).
211            The following are examples of existing resolutions found in
212            the archive: 63, 106, 159, 213, 255, 319, 399, 511, 799 or 1279.
213            This keyword has no meaning/effect if the archived data is
214            not in spherical harmonics representation.
215            The best selection can be found here:
216            https://software.ecmwf.int/wiki/display/UDOC/\
217                  Retrieve#Retrieve-Truncationbeforeinterpolation
218            Default is an empty string.
219
220        stream : str, optional
221            Identifies the forecasting system used to generate the data.
222            E.g. oper (Atmospheric model), enfo (Ensemble forecats), ...
223            Default is an empty string.
224
225        area : str, optional
226            Specifies the desired sub-area of data to be extracted.
227            Areas can be defined to wrap around the globe.
228
229            Latitude values must be given as signed numbers, with:
230                north latitudes (i.e. north of the equator)
231                    being positive (e.g: 40.5)
232                south latitutes (i.e. south of the equator)
233                    being negative (e.g: -50.5)
234            Longtitude values must be given as signed numbers, with:
235                east longitudes (i.e. east of the 0 degree meridian)
236                    being positive (e.g: 35.0)
237                west longitudes (i.e. west of the 0 degree meridian)
238                    being negative (e.g: -20.5)
239
240            E.g.: North/West/South/East
241            Default is an empty string.
242
243        time : str, optional
244            Specifies the time of the data in hours and minutes.
245            Valid values depend on the type of data: Analysis time,
246            Forecast base time or First guess verification time
247            (all usually at synoptic hours: 00, 06, 12 and 18 ).
248            Observation time (any combination in hours and minutes is valid,
249            subject to data availability in the archive).
250            The syntax is HHMM or HH:MM. If MM is omitted it defaults to 00.
251            Default is an empty string.
252
253        step : str, optional
254            Specifies the forecast time step from forecast base time.
255            Valid values are hours (HH) from forecast base time. It also
256            specifies the length of the forecast which verifies at
257            First Guess time.
258            E.g. 1/3/6-hourly
259            Default is an empty string.
260
261        expver : str, optional
262            The version of the dataset. Each experiment is assigned a
263            unique code (version). Production data is assigned 1 or 2,
264            and experimental data in Operations 11, 12 ,...
265            Research or Member State's experiments have a four letter
266            experiment identifier.
267            Default is "1".
268
269        number : str, optional
270            Selects the member in ensemble forecast run. (Only then it
271            is necessary.) It has a different meaning depending on
272            the type of data.
273            E.g. Perturbed Forecasts: specifies the Ensemble forecast member
274            Default is an empty string.
275
276        accuracy : str, optional
277            Specifies the number of bits per value to be used in the
278            generated GRIB coded fields.
279            A positive integer may be given to specify the preferred number
280            of bits per packed value. This must not be greater than the
281            number of bits normally used for a Fortran integer on the
282            processor handling the request (typically 32 or 64 bit).
283            Within a compute request the accuracy of the original fields
284            can be passed to the result field by specifying accuracy=av.
285            Default is an empty string.
286
287        grid : str, optional
288            Specifies the output grid which can be either a Gaussian grid
289            or a Latitude/Longitude grid. MARS requests specifying
290            grid=av will return the archived model grid.
291
292            Lat/Lon grid: The grid spacing needs to be an integer
293            fraction of 90 degrees e.g. grid = 0.5/0.5
294
295            Gaussian grid: specified by a letter denoting the type of
296            Gaussian grid followed by an integer (the grid number)
297            representing the number of lines between the Pole and Equator,
298            e.g.
299            grid = F160 - full (or regular) Gaussian grid with
300                   160 latitude lines between the pole and equator
301            grid = N320 - ECMWF original reduced Gaussian grid with
302                   320 latitude lines between the pole and equator,
303                   see Reduced Gaussian Grids for grid numbers used at ECMWF
304            grid = O640 - ECMWF octahedral (reduced) Gaussian grid with
305                   640 latitude lines between the pole and equator
306            Default is an empty string.
307
308        gaussian : str, optional
309            This parameter is deprecated and should no longer be used.
310            Specifies the desired type of Gaussian grid for the output.
311            Valid Gaussian grids are quasi-regular (reduced) or regular.
312            Keyword gaussian can only be specified together with
313            keyword grid. Gaussian without grid has no effect.
314            Default is an empty string.
315
316        target : str, optional
317            Specifies a file into which data is to be written after
318            retrieval or manipulation. Path names should always be
319            enclosed in double quotes. The MARS client supports automatic
320            generation of multiple target files using MARS keywords
321            enclosed in square brackets [ ].  If the environment variable
322            MARS_MULTITARGET_STRICT_FORMAT is set to 1 before calling mars,
323            the keyword values will be used in the filename as shown by
324            the ecCodes GRIB tool grib_ls -m, e.g. with
325            MARS_MULTITARGET_STRICT_FORMAT set to 1 the keywords time,
326            expver and param will be formatted as 0600, 0001 and 129.128
327            rather than 600, 1 and 129.
328            Default is an empty string.
329
330        param : str, optional
331            Specifies the meteorological parameter.
332            The list of meteorological parameters in MARS is extensive.
333            Their availability is directly related to their meteorological
334            meaning and, therefore, the rest of directives specified
335            in the MARS request.
336            Meteorological parameters can be specified by their
337            GRIB code (param=130), their mnemonic (param=t) or
338            full name (param=temperature).
339            The list of parameter should be seperated by a "/"-sign.
340            E.g. 130/131/133
341            Default is an empty string.
342
343        Return
344        ------
345
346        '''
347
348        self.server = server
349        self.public = public
350        self.marsclass = marsclass
351        self.dataset = dataset
352        self.type = type
353        self.levtype = levtype
354        self.levelist = levelist
355        self.repres = repres
356        self.date = date
357        self.resol = resol
358        self.stream = stream
359        self.area = area
360        self.time = time
361        self.step = step
362        self.expver = expver
363        self.number = number
364        self.accuracy = accuracy
365        self.grid = grid
366        self.gaussian = gaussian
367        self.target = target
368        self.param = param
369
370        return
371
372
373    def display_info(self):
374        '''Prints all class attributes and their values to the
375        standard output.
376
377        Parameters
378        ----------
379
380        Return
381        ------
382
383        '''
384        # Get all class attributes and their values as a dictionary
385        attrs = vars(self).copy()
386
387        # iterate through all attributes and print them
388        # with their corresponding values
389        for item in attrs.items():
390            if item[0] in ['server', 'public']:
391                pass
392            else:
393                print(item[0] + ': ' + str(item[1]))
394
395        return
396
397
398    def print_infodata_csv(self, inputdir, request_number):
399        '''Write all request parameter in alpabetical order into a "csv" file.
400
401        Parameters
402        ----------
403        inputdir : str
404            The path where all data from the retrievals are stored.
405
406        request_number : int
407            Number of mars requests for flux and non-flux data.
408
409        Return
410        ------
411
412        '''
413
414        # Get all class attributes and their values as a dictionary
415        attrs = vars(self).copy()
416        del attrs['server']
417        del attrs['public']
418
419        # open a file to store all requests to
420        with open(os.path.join(inputdir,
421                               _config.FILE_MARS_REQUESTS), 'a') as f:
422            f.write(str(request_number) + ', ')
423            f.write(', '.join(str(attrs[key])
424                              for key in sorted(attrs.iterkeys())))
425            f.write('\n')
426
427        return
428
429    def data_retrieve(self):
430        '''Submits a MARS retrieval. Depending on the existence of
431        ECMWF Web-API it is submitted via Python or a
432        subprocess in the Shell. The parameter for the mars retrieval
433        are taken from the defined class attributes.
434
435        Parameters
436        ----------
437
438        Return
439        ------
440
441        '''
442        # Get all class attributes and their values as a dictionary
443        attrs = vars(self).copy()
444
445        # eliminate unnecessary attributes from the dictionary attrs
446        del attrs['server']
447        del attrs['public']
448
449        # exchange parameter name for marsclass
450        mclass = attrs.get('marsclass')
451        del attrs['marsclass']
452        attrs['class'] = mclass
453
454        # prepare target variable as needed for the Web API mode
455        # within the dictionary for full access
456        # as a single variable for public access
457        target = attrs.get('target')
458        if not int(self.public):
459            del attrs['target']
460        print('target: ' + target)
461
462        # find all keys without a value and convert all other values to strings
463        empty_keys = []
464        for key, value in attrs.iteritems():
465            if value == '':
466                empty_keys.append(str(key))
467            else:
468                attrs[key] = str(value)
469
470        # delete all empty parameter from the dictionary
471        for key in empty_keys:
472            del attrs[key]
473
474        # MARS request via Python script
475        if self.server:
476            try:
477                if self.public:
478                    print('RETRIEVE PUBLIC DATA!')
479                    self.server.retrieve(attrs)
480                else:
481                    print('EXECUTE NON-PUBLIC RETRIEVAL!')
482                    self.server.execute(attrs, target)
483            except:
484                e = sys.exc_info()[0]
485                print("ERROR: ", e)
486                print('MARS Request failed!')
487                if not self.public and os.stat(target).st_size == 0:
488                    print('MARS Request returned no data - '
489                          'please check request')
490                    raise IOError
491                elif self.public and os.stat(target).st_size == 0:
492                    print('Public MARS Request returned no data - '
493                          'please check request')
494                    raise IOError
495                else:
496                    raise IOError
497        # MARS request via extra process in shell
498        else:
499            request_str = 'ret'
500            for key, value in attrs.iteritems():
501                request_str = request_str + ',' + key + '=' + str(value)
502            request_str += ',target="' + target + '"'
503            p = subprocess.Popen(['mars'],
504                                 stdin=subprocess.PIPE,
505                                 stdout=subprocess.PIPE,
506                                 stderr=subprocess.PIPE,
507                                 bufsize=1)
508            pout = p.communicate(input=request_str)[0]
509            print(pout.decode())
510
511            if 'Some errors reported' in pout.decode():
512                print('MARS Request failed - please check request')
513                raise IOError
514            elif os.stat(target).st_size == 0:
515                print('MARS Request returned no data - please check request')
516                raise IOError
517
518        return
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG