source: flex_extract.git/source/python/classes/MarsRetrieval.py @ ca867de

dev
Last change on this file since ca867de was ca867de, checked in by Anne Philipp <anne.philipp@…>, 14 months ago

refactored functions in EcFlexpart? and did some minor changes

  • Property mode set to 100644
File size: 18.4 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#   November 2015 - Leopold Haimberger (University of Vienna):
11#        - optimized display_info
12#        - optimized data_retrieve and seperate between python and shell
13#          script call
14#
15#   February 2018 - Anne Philipp (University of Vienna):
16#        - applied PEP8 style guide
17#        - added documentation
18#        - applied some minor modifications in programming style/structure
19#
20# @License:
21#    (C) Copyright 2015-2018.
22#
23#    This software is licensed under the terms of the Apache Licence Version 2.0
24#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
25#
26# @Class Description:
27#    A MARS revtrieval has a specific syntax with a selection of keywords and
28#    their corresponding values. This class provides the necessary functions
29#    by displaying the selected parameters and their values and the actual
30#    retrievement of the data through a mars request or a Python web api
31#    interface. The initialization already expects all the keyword values.
32#
33# @Class Content:
34#    - __init__
35#    - display_info
36#    - data_retrieve
37#
38# @Class Attributes:
39#    - server
40#    - marsclass
41#    - dtype
42#    - levtype
43#    - levelist
44#    - repres
45#    - date
46#    - resol
47#    - stream
48#    - area
49#    - time
50#    - step
51#    - expver
52#    - number
53#    - accuracy
54#    - grid
55#    - gaussian
56#    - target
57#    - param
58#
59#*******************************************************************************
60
61# ------------------------------------------------------------------------------
62# MODULES
63# ------------------------------------------------------------------------------
64import os
65import sys
66import subprocess
67
68# software specific classes and modules from flex_extract
69sys.path.append('../')
70import _config
71# ------------------------------------------------------------------------------
72# CLASS
73# ------------------------------------------------------------------------------
74class MarsRetrieval(object):
75    '''
76    Class for submitting MARS retrievals.
77
78    A description of MARS keywords/arguments and examples of their
79    values can be found here:
80    https://software.ecmwf.int/wiki/display/UDOC/\
81                   Identification+keywords#Identificationkeywords-class
82
83    '''
84
85    def __init__(self, server, marsclass="ei", type="", levtype="",
86                 levelist="", repres="", date="", resol="", stream="",
87                 area="", time="", step="", expver="1", number="",
88                 accuracy="", grid="", gaussian="", target="",
89                 param=""):
90        '''
91        @Description:
92            Initialises the instance of the MarsRetrieval class and
93            defines and assigns a set of the necessary retrieval parameters
94            for the FLEXPART input data.
95            A description of MARS keywords/arguments, their dependencies
96            on each other and examples of their values can be found here:
97
98            https://software.ecmwf.int/wiki/display/UDOC/MARS+keywords
99
100        @Input:
101            self: instance of MarsRetrieval
102                For description see class documentation.
103
104            server: instance of ECMWFService (from ECMWF Web-API)
105                This is the connection to the ECMWF data servers.
106                It is needed for the pythonic access of ECMWF data.
107
108            marsclass: string, optional
109                Characterisation of dataset. E.g. EI (ERA-Interim),
110                E4 (ERA40), OD (Operational archive), ea (ERA5).
111                Default is the ERA-Interim dataset "ei".
112
113            type: string, optional
114                Determines the type of fields to be retrieved.
115                Selects between observations, images or fields.
116                Examples for fields: Analysis (an), Forecast (fc),
117                Perturbed Forecast (pf), Control Forecast (cf) and so on.
118                Default is an empty string.
119
120            levtype: string, optional
121                Denotes type of level. Has a direct implication on valid
122                levelist values!
123                E.g. model level (ml), pressure level (pl), surface (sfc),
124                potential vorticity (pv), potential temperature (pt)
125                and depth (dp).
126                Default is an empty string.
127
128            levelist: string, optional
129                Specifies the required levels. It has to have a valid
130                correspondence to the selected levtype.
131                Examples: model level: 1/to/137, pressure levels: 500/to/1000
132                Default is an empty string.
133
134            repres: string, optional
135                Selects the representation of the archived data.
136                E.g. sh - spherical harmonics, gg - Gaussian grid,
137                ll - latitude/longitude, ...
138                Default is an empty string.
139
140            date: string, optional
141                Specifies the Analysis date, the Forecast base date or
142                Observations date. Valid formats are:
143                Absolute as YYYY-MM-DD or YYYYMMDD.
144                Default is an empty string.
145
146            resol: string, optional
147                Specifies the desired triangular truncation of retrieved data,
148                before carrying out any other selected post-processing.
149                The default is automatic truncation (auto), by which the lowest
150                resolution compatible with the value specified in grid is
151                automatically selected for the retrieval.
152                Users wanting to perform post-processing from full spectral
153                resolution should specify Archived Value (av).
154                The following are examples of existing resolutions found in
155                the archive: 63, 106, 159, 213, 255, 319, 399, 511, 799 or 1279.
156                This keyword has no meaning/effect if the archived data is
157                not in spherical harmonics representation.
158                The best selection can be found here:
159                https://software.ecmwf.int/wiki/display/UDOC/\
160                      Retrieve#Retrieve-Truncationbeforeinterpolation
161                Default is an empty string.
162
163            stream: string, optional
164                Identifies the forecasting system used to generate the data.
165                E.g. oper (Atmospheric model), enfo (Ensemble forecats), ...
166                Default is an empty string.
167
168            area: string, optional
169                Specifies the desired sub-area of data to be extracted.
170                Areas can be defined to wrap around the globe.
171
172                Latitude values must be given as signed numbers, with:
173                    north latitudes (i.e. north of the equator)
174                        being positive (e.g: 40.5)
175                    south latitutes (i.e. south of the equator)
176                        being negative (e.g: -50.5)
177                Longtitude values must be given as signed numbers, with:
178                    east longitudes (i.e. east of the 0 degree meridian)
179                        being positive (e.g: 35.0)
180                    west longitudes (i.e. west of the 0 degree meridian)
181                        being negative (e.g: -20.5)
182
183                E.g.: North/West/South/East
184                Default is an empty string.
185
186            time: string, optional
187                Specifies the time of the data in hours and minutes.
188                Valid values depend on the type of data: Analysis time,
189                Forecast base time or First guess verification time
190                (all usually at synoptic hours: 00, 06, 12 and 18 ).
191                Observation time (any combination in hours and minutes is valid,
192                subject to data availability in the archive).
193                The syntax is HHMM or HH:MM. If MM is omitted it defaults to 00.
194                Default is an empty string.
195
196            step: string, optional
197                Specifies the forecast time step from forecast base time.
198                Valid values are hours (HH) from forecast base time. It also
199                specifies the length of the forecast which verifies at
200                First Guess time.
201                E.g. 1/3/6-hourly
202                Default is an empty string.
203
204            expver: string, optional
205                The version of the dataset. Each experiment is assigned a
206                unique code (version). Production data is assigned 1 or 2,
207                and experimental data in Operations 11, 12 ,...
208                Research or Member State's experiments have a four letter
209                experiment identifier.
210                Default is "1".
211
212            number: string, optional
213                Selects the member in ensemble forecast run. (Only then it
214                is necessary.) It has a different meaning depending on
215                the type of data.
216                E.g. Perturbed Forecasts: specifies the Ensemble forecast member
217                Default is an empty string.
218
219            accuracy: string, optional
220                Specifies the number of bits per value to be used in the
221                generated GRIB coded fields.
222                A positive integer may be given to specify the preferred number
223                of bits per packed value. This must not be greater than the
224                number of bits normally used for a Fortran integer on the
225                processor handling the request (typically 32 or 64 bit).
226                Within a compute request the accuracy of the original fields
227                can be passed to the result field by specifying accuracy=av.
228                Default is an empty string.
229
230            grid: string, optional
231                Specifies the output grid which can be either a Gaussian grid
232                or a Latitude/Longitude grid. MARS requests specifying
233                grid=av will return the archived model grid.
234
235                Lat/Lon grid: The grid spacing needs to be an integer
236                fraction of 90 degrees e.g. grid = 0.5/0.5
237
238                Gaussian grid: specified by a letter denoting the type of
239                Gaussian grid followed by an integer (the grid number)
240                representing the number of lines between the Pole and Equator,
241                e.g.
242                grid = F160 - full (or regular) Gaussian grid with
243                       160 latitude lines between the pole and equator
244                grid = N320 - ECMWF original reduced Gaussian grid with
245                       320 latitude lines between the pole and equator,
246                       see Reduced Gaussian Grids for grid numbers used at ECMWF
247                grid = O640 - ECMWF octahedral (reduced) Gaussian grid with
248                       640 latitude lines between the pole and equator
249                Default is an empty string.
250
251            gaussian: string, optional
252                This parameter is deprecated and should no longer be used.
253                Specifies the desired type of Gaussian grid for the output.
254                Valid Gaussian grids are quasi-regular (reduced) or regular.
255                Keyword gaussian can only be specified together with
256                keyword grid. Gaussian without grid has no effect.
257                Default is an empty string.
258
259            target: string, optional
260                Specifies a file into which data is to be written after
261                retrieval or manipulation. Path names should always be
262                enclosed in double quotes. The MARS client supports automatic
263                generation of multiple target files using MARS keywords
264                enclosed in square brackets [ ].  If the environment variable
265                MARS_MULTITARGET_STRICT_FORMAT is set to 1 before calling mars,
266                the keyword values will be used in the filename as shown by
267                the ecCodes GRIB tool grib_ls -m, e.g. with
268                MARS_MULTITARGET_STRICT_FORMAT set to 1 the keywords time,
269                expver and param will be formatted as 0600, 0001 and 129.128
270                rather than 600, 1 and 129.
271                Default is an empty string.
272
273            param: string, optional
274                Specifies the meteorological parameter.
275                The list of meteorological parameters in MARS is extensive.
276                Their availability is directly related to their meteorological
277                meaning and, therefore, the rest of directives specified
278                in the MARS request.
279                Meteorological parameters can be specified by their
280                GRIB code (param=130), their mnemonic (param=t) or
281                full name (param=temperature).
282                The list of parameter should be seperated by a "/"-sign.
283                E.g. 130/131/133
284                Default is an empty string.
285
286        @Return:
287            <nothing>
288        '''
289
290        self.server = server
291        self.marsclass = marsclass
292        self.type = type
293        self.levtype = levtype
294        self.levelist = levelist
295        self.repres = repres
296        self.date = date
297        self.resol = resol
298        self.stream = stream
299        self.area = area
300        self.time = time
301        self.step = step
302        self.expver = expver
303        self.number = number
304        self.accuracy = accuracy
305        self.grid = grid
306        self.gaussian = gaussian
307        self.target = target
308        self.param = param
309
310        return
311
312
313    def display_info(self):
314        '''
315        @Description:
316            Prints all class attributes and their values to the
317            standard output.
318
319        @Input:
320            self: instance of MarsRetrieval
321                For description see class documentation.
322
323        @Return:
324            <nothing>
325        '''
326        # Get all class attributes and their values as a dictionary
327        attrs = vars(self)
328
329        # iterate through all attributes and print them
330        # with their corresponding values
331        for item in attrs.items():
332            if item[0] in 'server':
333                pass
334            else:
335                print(item[0] + ': ' + str(item[1]))
336
337        return
338
339
340    def print_info(self, inputdir, request_number):
341        '''
342        @Description:
343            Prints all mars requests to an extra file for debugging and
344            information.
345
346        @Input:
347            self: instance of MarsRetrieval
348                For description see class documentation.
349
350            inputdir: string
351                The path where all data from the retrievals are stored.
352
353            request_number: integer
354                Number of mars requests for flux and non-flux data.
355
356            @Return:
357            <nothing>
358        '''
359        # Get all class attributes and their values as a dictionary
360        attrs = vars(self)
361
362        # open a file to store all requests to
363        with open(os.path.join(inputdir,
364                               _config.FILE_MARS_REQUESTS), 'a') as f:
365            f.write('mars_request #' + str(request_number) + '\n')
366            # iterate through all attributes and print them
367            # with their corresponding values
368            for item in attrs.items():
369                if item[0] in 'server':
370                    pass
371                else:
372                    f.write(item[0] + ': ' + str(item[1]) + '\n')
373            f.write('\n\n')
374
375        return
376
377
378    def print_infodata_csv(self, inputdir, request_number):
379        '''
380        @Description:
381            Write all request parameter in alpabetical order into a "csv" file.
382
383        @Input:
384            self: instance of MarsRetrieval
385                For description see class documentation.
386
387            inputdir: string
388                The path where all data from the retrievals are stored.
389
390            request_number: integer
391                Number of mars requests for flux and non-flux data.
392
393        @Return:
394            <nothing>
395        '''
396
397        # Get all class attributes and their values as a dictionary
398        attrs = vars(self)
399        del attrs['server']
400
401        # open a file to store all requests to
402        with open(os.path.join(inputdir,
403                                   _config.FILE_MARS_REQUESTS), 'a') as f:
404            f.write(str(request_number) + ', ')
405            f.write(', '.join(str(attrs[key])
406                              for key in sorted(attrs.iterkeys())))
407            f.write('\n')
408
409        return
410
411    def data_retrieve(self):
412        '''
413        @Description:
414            Submits a MARS retrieval. Depending on the existence of
415            ECMWF Web-API it is submitted via Python or a
416            subprocess in the Shell. The parameter for the mars retrieval
417            are taken from the defined class attributes.
418
419        @Input:
420            self: instance of MarsRetrieval
421                For description see class documentation.
422
423        @Return:
424            <nothing>
425        '''
426        # Get all class attributes and their values as a dictionary
427        attrs = vars(self)
428
429        # convert the dictionary of attributes into a comma
430        # seperated list of attributes with their values
431        # needed for the retrieval call
432        s = 'ret'
433        for k, v in attrs.iteritems():
434            if k in 'server':
435                continue
436            if k == 'marsclass':
437                k = 'class'
438            if v == '':
439                continue
440            if k.lower() == 'target':
441                target = v
442            else:
443                s = s + ',' + k + '=' + str(v)
444
445        # MARS request via Python script
446        if self.server is not False:
447            try:
448                self.server.execute(s, target)
449            except:
450                print('MARS Request failed, \
451                      have you already registered at apps.ecmwf.int?')
452                raise IOError
453            if os.stat(target).st_size == 0:
454                print('MARS Request returned no data - please check request')
455                raise IOError
456        # MARS request via extra process in shell
457        else:
458            s += ',target = "' + target + '"'
459            p = subprocess.Popen(['mars'], stdin=subprocess.PIPE,
460                                 stdout=subprocess.PIPE,
461                                 stderr=subprocess.PIPE, bufsize=1)
462            pout = p.communicate(input=s)[0]
463            print(pout.decode())
464
465            if 'Some errors reported' in pout.decode():
466                print('MARS Request failed - please check request')
467                raise IOError
468
469            if os.stat(target).st_size == 0:
470                print('MARS Request returned no data - please check request')
471                raise IOError
472
473        return
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG