source: flex_extract.git/python/MarsRetrieval.py @ e1228f3

ctbtodev
Last change on this file since e1228f3 was ff99eae, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

completed application of pep8 style guide and pylint investigations. added documentation almost everywhere

  • Property mode set to 100644
File size: 16.0 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#   November 2015 - Leopold Haimberger (University of Vienna):
11#        - optimized display_info
12#        - optimized data_retrieve and seperate between python and shell
13#          script call
14#
15#   February 2018 - Anne Philipp (University of Vienna):
16#        - applied PEP8 style guide
17#        - added documentation
18#        - applied some minor modifications in programming style/structure
19#
20# @License:
21#    (C) Copyright 2015-2018.
22#
23#    This software is licensed under the terms of the Apache Licence Version 2.0
24#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
25#
26# @Class Description:
27#    A MARS revtrieval has a specific syntax with a selection of keywords and
28#    their corresponding values. This class provides the necessary functions
29#    by displaying the selected parameters and their values and the actual
30#    retrievement of the data through a mars request or a Python web api
31#    interface. The initialization already expects all the keyword values.
32#
33# @Class Content:
34#    - __init__
35#    - display_info
36#    - data_retrieve
37#
38# @Class Attributes:
39#    - server
40#    - marsclass
41#    - dtype
42#    - levtype
43#    - levelist
44#    - repres
45#    - date
46#    - resol
47#    - stream
48#    - area
49#    - time
50#    - step
51#    - expver
52#    - number
53#    - accuracy
54#    - grid
55#    - gaussian
56#    - target
57#    - param
58#
59#*******************************************************************************
60
61# ------------------------------------------------------------------------------
62# MODULES
63# ------------------------------------------------------------------------------
64import subprocess
65import os
66
67# ------------------------------------------------------------------------------
68# CLASS
69# ------------------------------------------------------------------------------
70class MarsRetrieval(object):
71    '''
72    Class for submitting MARS retrievals.
73
74    A description of MARS keywords/arguments and examples of their
75    values can be found here:
76    https://software.ecmwf.int/wiki/display/UDOC/\
77                   Identification+keywords#Identificationkeywords-class
78
79    '''
80
81    def __init__(self, server, marsclass="ei", dtype="", levtype="",
82                 levelist="", repres="", date="", resol="", stream="",
83                 area="", time="", step="", expver="1", number="",
84                 accuracy="", grid="", gaussian="", target="",
85                 param=""):
86        '''
87        @Description:
88            Initialises the instance of the MarsRetrieval class and
89            defines and assigns a set of the necessary retrieval parameters
90            for the FLEXPART input data.
91            A description of MARS keywords/arguments, their dependencies
92            on each other and examples of their values can be found here:
93
94            https://software.ecmwf.int/wiki/display/UDOC/MARS+keywords
95
96        @Input:
97            self: instance of MarsRetrieval
98                For description see class documentation.
99
100            server: instance of ECMWFService (from ECMWF Web-API)
101                This is the connection to the ECMWF data servers.
102                It is needed for the pythonic access of ECMWF data.
103
104            marsclass: string, optional
105                Characterisation of dataset. E.g. EI (ERA-Interim),
106                E4 (ERA40), OD (Operational archive), ea (ERA5).
107                Default is the ERA-Interim dataset "ei".
108
109            dtype: string, optional
110                Determines the type of fields to be retrieved.
111                Selects between observations, images or fields.
112                Examples for fields: Analysis (an), Forecast (fc),
113                Perturbed Forecast (pf), Control Forecast (cf) and so on.
114                Default is an empty string.
115
116            levtype: string, optional
117                Denotes type of level. Has a direct implication on valid
118                levelist values!
119                E.g. model level (ml), pressure level (pl), surface (sfc),
120                potential vorticity (pv), potential temperature (pt)
121                and depth (dp).
122                Default is an empty string.
123
124            levelist: string, optional
125                Specifies the required levels. It has to have a valid
126                correspondence to the selected levtype.
127                Examples: model level: 1/to/137, pressure levels: 500/to/1000
128                Default is an empty string.
129
130            repres: string, optional
131                Selects the representation of the archived data.
132                E.g. sh - spherical harmonics, gg - Gaussian grid,
133                ll - latitude/longitude, ...
134                Default is an empty string.
135
136            date: string, optional
137                Specifies the Analysis date, the Forecast base date or
138                Observations date. Valid formats are:
139                Absolute as YYYY-MM-DD or YYYYMMDD.
140                Default is an empty string.
141
142            resol: string, optional
143                Specifies the desired triangular truncation of retrieved data,
144                before carrying out any other selected post-processing.
145                The default is automatic truncation (auto), by which the lowest
146                resolution compatible with the value specified in grid is
147                automatically selected for the retrieval.
148                Users wanting to perform post-processing from full spectral
149                resolution should specify Archived Value (av).
150                The following are examples of existing resolutions found in
151                the archive: 63, 106, 159, 213, 255, 319, 399, 511, 799 or 1279.
152                This keyword has no meaning/effect if the archived data is
153                not in spherical harmonics representation.
154                The best selection can be found here:
155                https://software.ecmwf.int/wiki/display/UDOC/\
156                      Retrieve#Retrieve-Truncationbeforeinterpolation
157                Default is an empty string.
158
159            stream: string, optional
160                Identifies the forecasting system used to generate the data.
161                E.g. oper (Atmospheric model), enfo (Ensemble forecats), ...
162                Default is an empty string.
163
164            area: string, optional
165                Specifies the desired sub-area of data to be extracted.
166                Areas can be defined to wrap around the globe.
167
168                Latitude values must be given as signed numbers, with:
169                    north latitudes (i.e. north of the equator)
170                        being positive (e.g: 40.5)
171                    south latitutes (i.e. south of the equator)
172                        being negative (e.g: -50.5)
173                Longtitude values must be given as signed numbers, with:
174                    east longitudes (i.e. east of the 0 degree meridian)
175                        being positive (e.g: 35.0)
176                    west longitudes (i.e. west of the 0 degree meridian)
177                        being negative (e.g: -20.5)
178
179                E.g.: North/West/South/East
180                Default is an empty string.
181
182            time: string, optional
183                Specifies the time of the data in hours and minutes.
184                Valid values depend on the type of data: Analysis time,
185                Forecast base time or First guess verification time
186                (all usually at synoptic hours: 00, 06, 12 and 18 ).
187                Observation time (any combination in hours and minutes is valid,
188                subject to data availability in the archive).
189                The syntax is HHMM or HH:MM. If MM is omitted it defaults to 00.
190                Default is an empty string.
191
192            step: string, optional
193                Specifies the forecast time step from forecast base time.
194                Valid values are hours (HH) from forecast base time. It also
195                specifies the length of the forecast which verifies at
196                First Guess time.
197                E.g. 1/3/6-hourly
198                Default is an empty string.
199
200            expver: string, optional
201                The version of the dataset. Each experiment is assigned a
202                unique code (version). Production data is assigned 1 or 2,
203                and experimental data in Operations 11, 12 ,...
204                Research or Member State's experiments have a four letter
205                experiment identifier.
206                Default is "1".
207
208            number: string, optional
209                Selects the member in ensemble forecast run. (Only then it
210                is necessary.) It has a different meaning depending on
211                the type of data.
212                E.g. Perturbed Forecasts: specifies the Ensemble forecast member
213                Default is an empty string.
214
215            accuracy: string, optional
216                Specifies the number of bits per value to be used in the
217                generated GRIB coded fields.
218                A positive integer may be given to specify the preferred number
219                of bits per packed value. This must not be greater than the
220                number of bits normally used for a Fortran integer on the
221                processor handling the request (typically 32 or 64 bit).
222                Within a compute request the accuracy of the original fields
223                can be passed to the result field by specifying accuracy=av.
224                Default is an empty string.
225
226            grid: string, optional
227                Specifies the output grid which can be either a Gaussian grid
228                or a Latitude/Longitude grid. MARS requests specifying
229                grid=av will return the archived model grid.
230
231                Lat/Lon grid: The grid spacing needs to be an integer
232                fraction of 90 degrees e.g. grid = 0.5/0.5
233
234                Gaussian grid: specified by a letter denoting the type of
235                Gaussian grid followed by an integer (the grid number)
236                representing the number of lines between the Pole and Equator,
237                e.g.
238                grid = F160 - full (or regular) Gaussian grid with
239                       160 latitude lines between the pole and equator
240                grid = N320 - ECMWF original reduced Gaussian grid with
241                       320 latitude lines between the pole and equator,
242                       see Reduced Gaussian Grids for grid numbers used at ECMWF
243                grid = O640 - ECMWF octahedral (reduced) Gaussian grid with
244                       640 latitude lines between the pole and equator
245                Default is an empty string.
246
247            gaussian: string, optional
248                This parameter is deprecated and should no longer be used.
249                Specifies the desired type of Gaussian grid for the output.
250                Valid Gaussian grids are quasi-regular (reduced) or regular.
251                Keyword gaussian can only be specified together with
252                keyword grid. Gaussian without grid has no effect.
253                Default is an empty string.
254
255            target: string, optional
256                Specifies a file into which data is to be written after
257                retrieval or manipulation. Path names should always be
258                enclosed in double quotes. The MARS client supports automatic
259                generation of multiple target files using MARS keywords
260                enclosed in square brackets [ ].  If the environment variable
261                MARS_MULTITARGET_STRICT_FORMAT is set to 1 before calling mars,
262                the keyword values will be used in the filename as shown by
263                the ecCodes GRIB tool grib_ls -m, e.g. with
264                MARS_MULTITARGET_STRICT_FORMAT set to 1 the keywords time,
265                expver and param will be formatted as 0600, 0001 and 129.128
266                rather than 600, 1 and 129.
267                Default is an empty string.
268
269            param: string, optional
270                Specifies the meteorological parameter.
271                The list of meteorological parameters in MARS is extensive.
272                Their availability is directly related to their meteorological
273                meaning and, therefore, the rest of directives specified
274                in the MARS request.
275                Meteorological parameters can be specified by their
276                GRIB code (param=130), their mnemonic (param=t) or
277                full name (param=temperature).
278                The list of parameter should be seperated by a "/"-sign.
279                E.g. 130/131/133
280                Default is an empty string.
281
282        @Return:
283            <nothing>
284        '''
285
286        self.server = server
287        self.marsclass = marsclass
288        self.dtype = dtype
289        self.levtype = levtype
290        self.levelist = levelist
291        self.repres = repres
292        self.date = date
293        self.resol = resol
294        self.stream = stream
295        self.area = area
296        self.time = time
297        self.step = step
298        self.expver = expver
299        self.number = number
300        self.accuracy = accuracy
301        self.grid = grid
302        self.gaussian = gaussian
303        self.target = target
304        self.param = param
305
306        return
307
308
309    def display_info(self):
310        '''
311        @Description:
312            Prints all class attributes and their values.
313
314        @Input:
315            self: instance of MarsRetrieval
316                For description see class documentation.
317
318        @Return:
319            <nothing>
320        '''
321        # Get all class attributes and their values as a dictionary
322        attrs = vars(self)
323
324        # iterate through all attributes and print them
325        # with their corresponding values
326        for item in attrs.items():
327            if item[0] in 'server':
328                pass
329            else:
330                print item[0] + ': ' + str(item[1])
331
332        return
333
334    def data_retrieve(self):
335        '''
336        @Description:
337            Submits a MARS retrieval. Depending on the existence of
338            ECMWF Web-API it is submitted via Python or a
339            subprocess in the Shell. The parameter for the mars retrieval
340            are taken from the defined class attributes.
341
342        @Input:
343            self: instance of MarsRetrieval
344                For description see class documentation.
345
346        @Return:
347            <nothing>
348        '''
349        # Get all class attributes and their values as a dictionary
350        attrs = vars(self)
351
352        # convert the dictionary of attributes into a comma
353        # seperated list of attributes with their values
354        # needed for the retrieval call
355        s = 'ret'
356        for k, v in attrs.iteritems():
357            if k in 'server':
358                continue
359            if k == 'marsclass':
360                k = 'class'
361            if v == '':
362                continue
363            if k.lower() == 'target':
364                target = v
365            else:
366                s = s + ',' + k + '=' + str(v)
367
368        # MARS request via Python script
369        if self.server is not False:
370            try:
371                self.server.execute(s, target)
372            except:
373                print 'MARS Request failed, \
374                    have you already registered at apps.ecmwf.int?'
375                raise IOError
376            if os.stat(target).st_size == 0:
377                print 'MARS Request returned no data - please check request'
378                raise IOError
379        # MARS request via extra process in shell
380        else:
381            s += ',target = "' + target + '"'
382            p = subprocess.Popen(['mars'], stdin=subprocess.PIPE,
383                                 stdout=subprocess.PIPE,
384                                 stderr=subprocess.PIPE, bufsize=1)
385            pout = p.communicate(input=s)[0]
386            print pout.decode()
387
388            if 'Some errors reported' in pout.decode():
389                print 'MARS Request failed - please check request'
390                raise IOError
391
392            if os.stat(target).st_size == 0:
393                print 'MARS Request returned no data - please check request'
394                raise IOError
395
396        return
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG