source: flex_extract.git/source/python/classes/MarsRetrieval.py @ f20af73

dev
Last change on this file since f20af73 was f20af73, checked in by Anne Philipp <anne.philipp@…>, 7 months ago

added CDS API support for ERA5 instead of ECMWFAPI / refactored setup of CONTROL parameter because for local version it wanted to use not installed ECMWF_ENV file.

  • Property mode set to 100644
File size: 20.2 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#   November 2015 - Leopold Haimberger (University of Vienna):
11#        - optimized display_info
12#        - optimized data_retrieve and seperate between python and shell
13#          script call
14#
15#   February 2018 - Anne Philipp (University of Vienna):
16#        - applied PEP8 style guide
17#        - added documentation
18#        - applied some minor modifications in programming style/structure
19#        - added writing of mars request attributes to a csv file
20#
21# @License:
22#    (C) Copyright 2014-2019.
23#    Anne Philipp, Leopold Haimberger
24#
25#    This work is licensed under the Creative Commons Attribution 4.0
26#    International License. To view a copy of this license, visit
27#    http://creativecommons.org/licenses/by/4.0/ or send a letter to
28#    Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
29#*******************************************************************************
30
31# ------------------------------------------------------------------------------
32# MODULES
33# ------------------------------------------------------------------------------
34import os
35import sys
36import subprocess
37import traceback
38
39# software specific classes and modules from flex_extract
40sys.path.append('../')
41import _config
42try:
43    ec_api = True
44    import ecmwfapi
45except ImportError:
46    ec_api = False
47
48try:
49    cds_api = True
50    import cdsapi
51except ImportError:
52    cds_api = False
53# ------------------------------------------------------------------------------
54# CLASS
55# ------------------------------------------------------------------------------
56class MarsRetrieval(object):
57    '''Specific syntax and content for submission of MARS retrievals.
58
59    A MARS revtrieval has a specific syntax with a selection of keywords and
60    their corresponding values. This class provides the necessary functions
61    by displaying the selected parameters and their values and the actual
62    retrievement of the data through a mars request or a Python web api
63    interface. The initialization already expects all the keyword values.
64
65    A description of MARS keywords/arguments and examples of their
66    values can be found here:
67    https://software.ecmwf.int/wiki/display/UDOC/\
68                   Identification+keywords#Identificationkeywords-class
69
70    Attributes
71    ----------
72    server : ECMWFService or ECMWFDataServer
73        This is the connection to the ECMWF data servers.
74
75    public : int
76        Decides which Web API Server version is used.
77
78    marsclass : str, optional
79        Characterisation of dataset.
80
81    dataset : str, optional
82        For public datasets there is the specific naming and parameter
83        dataset which has to be used to characterize the type of
84        data.
85
86    type : str, optional
87        Determines the type of fields to be retrieved.
88
89    levtype : str, optional
90        Denotes type of level.
91
92    levelist : str, optional
93        Specifies the required levels.
94
95    repres : str, optional
96        Selects the representation of the archived data.
97
98    date : str, optional
99        Specifies the Analysis date, the Forecast base date or
100        Observations date.
101
102    resol : str, optional
103        Specifies the desired triangular truncation of retrieved data,
104        before carrying out any other selected post-processing.
105
106    stream : str, optional
107        Identifies the forecasting system used to generate the data.
108
109    area : str, optional
110        Specifies the desired sub-area of data to be extracted.
111
112    time : str, optional
113        Specifies the time of the data in hours and minutes.
114
115    step : str, optional
116        Specifies the forecast time step from forecast base time.
117
118    expver : str, optional
119        The version of the dataset.
120
121    number : str, optional
122        Selects the member in ensemble forecast run.
123
124    accuracy : str, optional
125        Specifies the number of bits per value to be used in the
126        generated GRIB coded fields.
127
128    grid : str, optional
129        Specifies the output grid which can be either a Gaussian grid
130        or a Latitude/Longitude grid.
131
132    gaussian : str, optional
133        This parameter is deprecated and should no longer be used.
134        Specifies the desired type of Gaussian grid for the output.
135
136    target : str, optional
137        Specifies a file into which data is to be written after
138        retrieval or manipulation.
139
140    param : str, optional
141        Specifies the meteorological parameter.
142    '''
143
144    def __init__(self, server, public, marsclass="EA", dataset="", type="",
145                 levtype="", levelist="", repres="", date="", resol="",
146                 stream="", area="", time="", step="", expver="1",
147                 number="", accuracy="", grid="", gaussian="", target="",
148                 param=""):
149        '''Initialises the instance of the MarsRetrieval class and
150        defines and assigns a set of the necessary retrieval parameters
151        for the FLEXPART input data.
152        A description of MARS keywords/arguments, their dependencies
153        on each other and examples of their values can be found here:
154
155        https://software.ecmwf.int/wiki/display/UDOC/MARS+keywords
156
157        Parameters
158        ----------
159        server : ECMWFService or ECMWFDataServer
160            This is the connection to the ECMWF data servers.
161            It is needed for the pythonic access of ECMWF data.
162
163        public : int
164            Decides which Web API version is used:
165            0: member-state users and full archive access
166            1: public access and limited access to the public server and
167               datasets. Needs the parameter dataset.
168            Default is "0" and for member-state users.
169
170        marsclass : str, optional
171            Characterisation of dataset. E.g. EI (ERA-Interim),
172            E4 (ERA40), OD (Operational archive), EA (ERA5).
173            Default is the ERA5 dataset "EA".
174
175        dataset : str, optional
176            For public datasets there is the specific naming and parameter
177            dataset which has to be used to characterize the type of
178            data. Usually there is less data available, either in times,
179            domain or parameter.
180            Default is an empty string.
181
182        type : str, optional
183            Determines the type of fields to be retrieved.
184            Selects between observations, images or fields.
185            Examples for fields: Analysis (an), Forecast (fc),
186            Perturbed Forecast (pf), Control Forecast (cf) and so on.
187            Default is an empty string.
188
189        levtype : str, optional
190            Denotes type of level. Has a direct implication on valid
191            levelist values!
192            E.g. model level (ml), pressure level (pl), surface (sfc),
193            potential vorticity (pv), potential temperature (pt)
194            and depth (dp).
195            Default is an empty string.
196
197        levelist : str, optional
198            Specifies the required levels. It has to have a valid
199            correspondence to the selected levtype.
200            Examples: model level: 1/to/137, pressure levels: 500/to/1000
201            Default is an empty string.
202
203        repres : str, optional
204            Selects the representation of the archived data.
205            E.g. sh - spherical harmonics, gg - Gaussian grid,
206            ll - latitude/longitude, ...
207            Default is an empty string.
208
209        date : str, optional
210            Specifies the Analysis date, the Forecast base date or
211            Observations date. Valid formats are:
212            Absolute as YYYY-MM-DD or YYYYMMDD.
213            Default is an empty string.
214
215        resol : str, optional
216            Specifies the desired triangular truncation of retrieved data,
217            before carrying out any other selected post-processing.
218            The default is automatic truncation (auto), by which the lowest
219            resolution compatible with the value specified in grid is
220            automatically selected for the retrieval.
221            Users wanting to perform post-processing from full spectral
222            resolution should specify Archived Value (av).
223            The following are examples of existing resolutions found in
224            the archive: 63, 106, 159, 213, 255, 319, 399, 511, 799 or 1279.
225            This keyword has no meaning/effect if the archived data is
226            not in spherical harmonics representation.
227            The best selection can be found here:
228            https://software.ecmwf.int/wiki/display/UDOC/\
229                  Retrieve#Retrieve-Truncationbeforeinterpolation
230            Default is an empty string.
231
232        stream : str, optional
233            Identifies the forecasting system used to generate the data.
234            E.g. oper (Atmospheric model), enfo (Ensemble forecats), ...
235            Default is an empty string.
236
237        area : str, optional
238            Specifies the desired sub-area of data to be extracted.
239            Areas can be defined to wrap around the globe.
240
241            Latitude values must be given as signed numbers, with:
242                north latitudes (i.e. north of the equator)
243                    being positive (e.g: 40.5)
244                south latitutes (i.e. south of the equator)
245                    being negative (e.g: -50.5)
246            Longtitude values must be given as signed numbers, with:
247                east longitudes (i.e. east of the 0 degree meridian)
248                    being positive (e.g: 35.0)
249                west longitudes (i.e. west of the 0 degree meridian)
250                    being negative (e.g: -20.5)
251
252            E.g.: North/West/South/East
253            Default is an empty string.
254
255        time : str, optional
256            Specifies the time of the data in hours and minutes.
257            Valid values depend on the type of data: Analysis time,
258            Forecast base time or First guess verification time
259            (all usually at synoptic hours: 00, 06, 12 and 18 ).
260            Observation time (any combination in hours and minutes is valid,
261            subject to data availability in the archive).
262            The syntax is HHMM or HH:MM. If MM is omitted it defaults to 00.
263            Default is an empty string.
264
265        step : str, optional
266            Specifies the forecast time step from forecast base time.
267            Valid values are hours (HH) from forecast base time. It also
268            specifies the length of the forecast which verifies at
269            First Guess time.
270            E.g. 1/3/6-hourly
271            Default is an empty string.
272
273        expver : str, optional
274            The version of the dataset. Each experiment is assigned a
275            unique code (version). Production data is assigned 1 or 2,
276            and experimental data in Operations 11, 12 ,...
277            Research or Member State's experiments have a four letter
278            experiment identifier.
279            Default is "1".
280
281        number : str, optional
282            Selects the member in ensemble forecast run. (Only then it
283            is necessary.) It has a different meaning depending on
284            the type of data.
285            E.g. Perturbed Forecasts: specifies the Ensemble forecast member
286            Default is an empty string.
287
288        accuracy : str, optional
289            Specifies the number of bits per value to be used in the
290            generated GRIB coded fields.
291            A positive integer may be given to specify the preferred number
292            of bits per packed value. This must not be greater than the
293            number of bits normally used for a Fortran integer on the
294            processor handling the request (typically 32 or 64 bit).
295            Within a compute request the accuracy of the original fields
296            can be passed to the result field by specifying accuracy=av.
297            Default is an empty string.
298
299        grid : str, optional
300            Specifies the output grid which can be either a Gaussian grid
301            or a Latitude/Longitude grid. MARS requests specifying
302            grid=av will return the archived model grid.
303
304            Lat/Lon grid: The grid spacing needs to be an integer
305            fraction of 90 degrees e.g. grid = 0.5/0.5
306
307            Gaussian grid: specified by a letter denoting the type of
308            Gaussian grid followed by an integer (the grid number)
309            representing the number of lines between the Pole and Equator,
310            e.g.
311            grid = F160 - full (or regular) Gaussian grid with
312                   160 latitude lines between the pole and equator
313            grid = N320 - ECMWF original reduced Gaussian grid with
314                   320 latitude lines between the pole and equator,
315                   see Reduced Gaussian Grids for grid numbers used at ECMWF
316            grid = O640 - ECMWF octahedral (reduced) Gaussian grid with
317                   640 latitude lines between the pole and equator
318            Default is an empty string.
319
320        gaussian : str, optional
321            This parameter is deprecated and should no longer be used.
322            Specifies the desired type of Gaussian grid for the output.
323            Valid Gaussian grids are quasi-regular (reduced) or regular.
324            Keyword gaussian can only be specified together with
325            keyword grid. Gaussian without grid has no effect.
326            Default is an empty string.
327
328        target : str, optional
329            Specifies a file into which data is to be written after
330            retrieval or manipulation. Path names should always be
331            enclosed in double quotes. The MARS client supports automatic
332            generation of multiple target files using MARS keywords
333            enclosed in square brackets [ ].  If the environment variable
334            MARS_MULTITARGET_STRICT_FORMAT is set to 1 before calling mars,
335            the keyword values will be used in the filename as shown by
336            the ecCodes GRIB tool grib_ls -m, e.g. with
337            MARS_MULTITARGET_STRICT_FORMAT set to 1 the keywords time,
338            expver and param will be formatted as 0600, 0001 and 129.128
339            rather than 600, 1 and 129.
340            Default is an empty string.
341
342        param : str, optional
343            Specifies the meteorological parameter.
344            The list of meteorological parameters in MARS is extensive.
345            Their availability is directly related to their meteorological
346            meaning and, therefore, the rest of directives specified
347            in the MARS request.
348            Meteorological parameters can be specified by their
349            GRIB code (param=130), their mnemonic (param=t) or
350            full name (param=temperature).
351            The list of parameter should be seperated by a "/"-sign.
352            E.g. 130/131/133
353            Default is an empty string.
354
355        Return
356        ------
357
358        '''
359
360        self.server = server
361        self.public = public
362        self.marsclass = marsclass
363        self.dataset = dataset
364        self.type = type
365        self.levtype = levtype
366        self.levelist = levelist
367        self.repres = repres
368        self.date = date
369        self.resol = resol
370        self.stream = stream
371        self.area = area
372        self.time = time
373        self.step = step
374        self.expver = expver
375        self.number = number
376        self.accuracy = accuracy
377        self.grid = grid
378        self.gaussian = gaussian
379        self.target = target
380        self.param = param
381
382        return
383
384
385    def display_info(self):
386        '''Prints all class attributes and their values to the
387        standard output.
388
389        Parameters
390        ----------
391
392        Return
393        ------
394
395        '''
396        # Get all class attributes and their values as a dictionary
397        attrs = vars(self).copy()
398
399        # iterate through all attributes and print them
400        # with their corresponding values
401        for item in attrs.items():
402            if item[0] in ['server', 'public']:
403                pass
404            else:
405                print(item[0] + ': ' + str(item[1]))
406
407        return
408
409
410    def print_infodata_csv(self, inputdir, request_number):
411        '''Write all request parameter in alpabetical order into a "csv" file.
412
413        Parameters
414        ----------
415        inputdir : str
416            The path where all data from the retrievals are stored.
417
418        request_number : int
419            Number of mars requests for flux and non-flux data.
420
421        Return
422        ------
423
424        '''
425
426        # Get all class attributes and their values as a dictionary
427        attrs = vars(self).copy()
428        del attrs['server']
429        del attrs['public']
430
431        # open a file to store all requests to
432        with open(os.path.join(inputdir,
433                               _config.FILE_MARS_REQUESTS), 'a') as f:
434            f.write(str(request_number) + ', ')
435            f.write(', '.join(str(attrs[key])
436                              for key in sorted(attrs.iterkeys())))
437            f.write('\n')
438
439        return
440
441    def data_retrieve(self):
442        '''Submits a MARS retrieval. Depending on the existence of
443        ECMWF Web-API or CDS API it is submitted via Python or a
444        subprocess in the Shell. The parameter for the mars retrieval
445        are taken from the defined class attributes.
446
447        Parameters
448        ----------
449
450        Return
451        ------
452
453        '''
454        # Get all class attributes and their values as a dictionary
455        attrs = vars(self).copy()
456
457        # eliminate unnecessary attributes from the dictionary attrs
458        del attrs['server']
459        del attrs['public']
460
461        # exchange parameter name for marsclass
462        mclass = attrs.get('marsclass')
463        del attrs['marsclass']
464        attrs['class'] = mclass
465
466        # prepare target variable as needed for the Web API or CDS API mode
467        # within the dictionary for full access
468        # as a single variable for public access
469        target = attrs.get('target')
470        if not int(self.public):
471            del attrs['target']
472        print('target: ' + target)
473
474        # find all keys without a value and convert all other values to strings
475        empty_keys = []
476        for key, value in attrs.iteritems():
477            if value == '':
478                empty_keys.append(str(key))
479            else:
480                attrs[key] = str(value)
481
482        # delete all empty parameter from the dictionary
483        for key in empty_keys:
484            del attrs[key]
485
486        # MARS request via Python script
487        if self.server:
488            try:
489                if cds_api and isinstance(self.server, cdsapi.Client):
490                    print('RETRIEVE ERA5 WITH CDS API!')
491                    self.server.retrieve(_config.CDS_DATASET,
492                                         attrs, target)
493                elif ec_api and isinstance(self.server, ecmwfapi.ECMWFDataServer):
494                    print('RETRIEVE PUBLIC DATA (NOT ERA5)!')
495                    self.server.retrieve(attrs)
496                elif ec_api and isinstance(self.server, ecmwfapi.ECMWFService):
497                    print('EXECUTE NON-PUBLIC RETRIEVAL (NOT ERA5)!')
498                    self.server.execute(attrs, target)
499                else:
500                    print('ERROR:')
501                    print('No match for Web API instance!')
502                    raise IOError
503            except Exception as e:
504                print('\n\nMARS Request failed!')
505                print(e)
506                tb = sys.exc_info()[2]
507                print(traceback.format_exc())
508                sys.exit()
509
510        # MARS request via call in shell
511        else:
512            request_str = 'ret'
513            for key, value in attrs.iteritems():
514                request_str = request_str + ',' + key + '=' + str(value)
515            request_str += ',target="' + target + '"'
516            p = subprocess.Popen(['mars', '-p'],
517                                 stdin=subprocess.PIPE,
518                                 stdout=subprocess.PIPE,
519                                 stderr=subprocess.PIPE,
520                                 bufsize=1)
521            pout = p.communicate(input=request_str)[0]
522            print(pout.decode())
523
524            if 'Some errors reported' in pout.decode():
525                print('MARS Request failed - please check request')
526                raise IOError
527            elif os.stat(target).st_size == 0:
528                print('MARS Request returned no data - please check request')
529                raise IOError
530
531        return
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG