source: flex_extract.git/source/python/classes/MarsRetrieval.py @ 295ff45

dev
Last change on this file since 295ff45 was 295ff45, checked in by Anne Philipp <anne.philipp@…>, 9 months ago

added request output option also for basetime requests; changed output to csv format

  • Property mode set to 100644
File size: 18.3 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#   November 2015 - Leopold Haimberger (University of Vienna):
11#        - optimized display_info
12#        - optimized data_retrieve and seperate between python and shell
13#          script call
14#
15#   February 2018 - Anne Philipp (University of Vienna):
16#        - applied PEP8 style guide
17#        - added documentation
18#        - applied some minor modifications in programming style/structure
19#
20# @License:
21#    (C) Copyright 2015-2018.
22#
23#    This software is licensed under the terms of the Apache Licence Version 2.0
24#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
25#
26# @Class Description:
27#    A MARS revtrieval has a specific syntax with a selection of keywords and
28#    their corresponding values. This class provides the necessary functions
29#    by displaying the selected parameters and their values and the actual
30#    retrievement of the data through a mars request or a Python web api
31#    interface. The initialization already expects all the keyword values.
32#
33# @Class Content:
34#    - __init__
35#    - display_info
36#    - data_retrieve
37#
38# @Class Attributes:
39#    - server
40#    - marsclass
41#    - dtype
42#    - levtype
43#    - levelist
44#    - repres
45#    - date
46#    - resol
47#    - stream
48#    - area
49#    - time
50#    - step
51#    - expver
52#    - number
53#    - accuracy
54#    - grid
55#    - gaussian
56#    - target
57#    - param
58#
59#*******************************************************************************
60
61# ------------------------------------------------------------------------------
62# MODULES
63# ------------------------------------------------------------------------------
64import subprocess
65import os
66
67import _config
68# ------------------------------------------------------------------------------
69# CLASS
70# ------------------------------------------------------------------------------
71class MarsRetrieval(object):
72    '''
73    Class for submitting MARS retrievals.
74
75    A description of MARS keywords/arguments and examples of their
76    values can be found here:
77    https://software.ecmwf.int/wiki/display/UDOC/\
78                   Identification+keywords#Identificationkeywords-class
79
80    '''
81
82    def __init__(self, server, marsclass="ei", type="", levtype="",
83                 levelist="", repres="", date="", resol="", stream="",
84                 area="", time="", step="", expver="1", number="",
85                 accuracy="", grid="", gaussian="", target="",
86                 param=""):
87        '''
88        @Description:
89            Initialises the instance of the MarsRetrieval class and
90            defines and assigns a set of the necessary retrieval parameters
91            for the FLEXPART input data.
92            A description of MARS keywords/arguments, their dependencies
93            on each other and examples of their values can be found here:
94
95            https://software.ecmwf.int/wiki/display/UDOC/MARS+keywords
96
97        @Input:
98            self: instance of MarsRetrieval
99                For description see class documentation.
100
101            server: instance of ECMWFService (from ECMWF Web-API)
102                This is the connection to the ECMWF data servers.
103                It is needed for the pythonic access of ECMWF data.
104
105            marsclass: string, optional
106                Characterisation of dataset. E.g. EI (ERA-Interim),
107                E4 (ERA40), OD (Operational archive), ea (ERA5).
108                Default is the ERA-Interim dataset "ei".
109
110            type: string, optional
111                Determines the type of fields to be retrieved.
112                Selects between observations, images or fields.
113                Examples for fields: Analysis (an), Forecast (fc),
114                Perturbed Forecast (pf), Control Forecast (cf) and so on.
115                Default is an empty string.
116
117            levtype: string, optional
118                Denotes type of level. Has a direct implication on valid
119                levelist values!
120                E.g. model level (ml), pressure level (pl), surface (sfc),
121                potential vorticity (pv), potential temperature (pt)
122                and depth (dp).
123                Default is an empty string.
124
125            levelist: string, optional
126                Specifies the required levels. It has to have a valid
127                correspondence to the selected levtype.
128                Examples: model level: 1/to/137, pressure levels: 500/to/1000
129                Default is an empty string.
130
131            repres: string, optional
132                Selects the representation of the archived data.
133                E.g. sh - spherical harmonics, gg - Gaussian grid,
134                ll - latitude/longitude, ...
135                Default is an empty string.
136
137            date: string, optional
138                Specifies the Analysis date, the Forecast base date or
139                Observations date. Valid formats are:
140                Absolute as YYYY-MM-DD or YYYYMMDD.
141                Default is an empty string.
142
143            resol: string, optional
144                Specifies the desired triangular truncation of retrieved data,
145                before carrying out any other selected post-processing.
146                The default is automatic truncation (auto), by which the lowest
147                resolution compatible with the value specified in grid is
148                automatically selected for the retrieval.
149                Users wanting to perform post-processing from full spectral
150                resolution should specify Archived Value (av).
151                The following are examples of existing resolutions found in
152                the archive: 63, 106, 159, 213, 255, 319, 399, 511, 799 or 1279.
153                This keyword has no meaning/effect if the archived data is
154                not in spherical harmonics representation.
155                The best selection can be found here:
156                https://software.ecmwf.int/wiki/display/UDOC/\
157                      Retrieve#Retrieve-Truncationbeforeinterpolation
158                Default is an empty string.
159
160            stream: string, optional
161                Identifies the forecasting system used to generate the data.
162                E.g. oper (Atmospheric model), enfo (Ensemble forecats), ...
163                Default is an empty string.
164
165            area: string, optional
166                Specifies the desired sub-area of data to be extracted.
167                Areas can be defined to wrap around the globe.
168
169                Latitude values must be given as signed numbers, with:
170                    north latitudes (i.e. north of the equator)
171                        being positive (e.g: 40.5)
172                    south latitutes (i.e. south of the equator)
173                        being negative (e.g: -50.5)
174                Longtitude values must be given as signed numbers, with:
175                    east longitudes (i.e. east of the 0 degree meridian)
176                        being positive (e.g: 35.0)
177                    west longitudes (i.e. west of the 0 degree meridian)
178                        being negative (e.g: -20.5)
179
180                E.g.: North/West/South/East
181                Default is an empty string.
182
183            time: string, optional
184                Specifies the time of the data in hours and minutes.
185                Valid values depend on the type of data: Analysis time,
186                Forecast base time or First guess verification time
187                (all usually at synoptic hours: 00, 06, 12 and 18 ).
188                Observation time (any combination in hours and minutes is valid,
189                subject to data availability in the archive).
190                The syntax is HHMM or HH:MM. If MM is omitted it defaults to 00.
191                Default is an empty string.
192
193            step: string, optional
194                Specifies the forecast time step from forecast base time.
195                Valid values are hours (HH) from forecast base time. It also
196                specifies the length of the forecast which verifies at
197                First Guess time.
198                E.g. 1/3/6-hourly
199                Default is an empty string.
200
201            expver: string, optional
202                The version of the dataset. Each experiment is assigned a
203                unique code (version). Production data is assigned 1 or 2,
204                and experimental data in Operations 11, 12 ,...
205                Research or Member State's experiments have a four letter
206                experiment identifier.
207                Default is "1".
208
209            number: string, optional
210                Selects the member in ensemble forecast run. (Only then it
211                is necessary.) It has a different meaning depending on
212                the type of data.
213                E.g. Perturbed Forecasts: specifies the Ensemble forecast member
214                Default is an empty string.
215
216            accuracy: string, optional
217                Specifies the number of bits per value to be used in the
218                generated GRIB coded fields.
219                A positive integer may be given to specify the preferred number
220                of bits per packed value. This must not be greater than the
221                number of bits normally used for a Fortran integer on the
222                processor handling the request (typically 32 or 64 bit).
223                Within a compute request the accuracy of the original fields
224                can be passed to the result field by specifying accuracy=av.
225                Default is an empty string.
226
227            grid: string, optional
228                Specifies the output grid which can be either a Gaussian grid
229                or a Latitude/Longitude grid. MARS requests specifying
230                grid=av will return the archived model grid.
231
232                Lat/Lon grid: The grid spacing needs to be an integer
233                fraction of 90 degrees e.g. grid = 0.5/0.5
234
235                Gaussian grid: specified by a letter denoting the type of
236                Gaussian grid followed by an integer (the grid number)
237                representing the number of lines between the Pole and Equator,
238                e.g.
239                grid = F160 - full (or regular) Gaussian grid with
240                       160 latitude lines between the pole and equator
241                grid = N320 - ECMWF original reduced Gaussian grid with
242                       320 latitude lines between the pole and equator,
243                       see Reduced Gaussian Grids for grid numbers used at ECMWF
244                grid = O640 - ECMWF octahedral (reduced) Gaussian grid with
245                       640 latitude lines between the pole and equator
246                Default is an empty string.
247
248            gaussian: string, optional
249                This parameter is deprecated and should no longer be used.
250                Specifies the desired type of Gaussian grid for the output.
251                Valid Gaussian grids are quasi-regular (reduced) or regular.
252                Keyword gaussian can only be specified together with
253                keyword grid. Gaussian without grid has no effect.
254                Default is an empty string.
255
256            target: string, optional
257                Specifies a file into which data is to be written after
258                retrieval or manipulation. Path names should always be
259                enclosed in double quotes. The MARS client supports automatic
260                generation of multiple target files using MARS keywords
261                enclosed in square brackets [ ].  If the environment variable
262                MARS_MULTITARGET_STRICT_FORMAT is set to 1 before calling mars,
263                the keyword values will be used in the filename as shown by
264                the ecCodes GRIB tool grib_ls -m, e.g. with
265                MARS_MULTITARGET_STRICT_FORMAT set to 1 the keywords time,
266                expver and param will be formatted as 0600, 0001 and 129.128
267                rather than 600, 1 and 129.
268                Default is an empty string.
269
270            param: string, optional
271                Specifies the meteorological parameter.
272                The list of meteorological parameters in MARS is extensive.
273                Their availability is directly related to their meteorological
274                meaning and, therefore, the rest of directives specified
275                in the MARS request.
276                Meteorological parameters can be specified by their
277                GRIB code (param=130), their mnemonic (param=t) or
278                full name (param=temperature).
279                The list of parameter should be seperated by a "/"-sign.
280                E.g. 130/131/133
281                Default is an empty string.
282
283        @Return:
284            <nothing>
285        '''
286
287        self.server = server
288        self.marsclass = marsclass
289        self.type = type
290        self.levtype = levtype
291        self.levelist = levelist
292        self.repres = repres
293        self.date = date
294        self.resol = resol
295        self.stream = stream
296        self.area = area
297        self.time = time
298        self.step = step
299        self.expver = expver
300        self.number = number
301        self.accuracy = accuracy
302        self.grid = grid
303        self.gaussian = gaussian
304        self.target = target
305        self.param = param
306
307        return
308
309
310    def display_info(self):
311        '''
312        @Description:
313            Prints all class attributes and their values to the
314            standard output.
315
316        @Input:
317            self: instance of MarsRetrieval
318                For description see class documentation.
319
320        @Return:
321            <nothing>
322        '''
323        # Get all class attributes and their values as a dictionary
324        attrs = vars(self)
325
326        # iterate through all attributes and print them
327        # with their corresponding values
328        for item in attrs.items():
329            if item[0] in 'server':
330                pass
331            else:
332                print(item[0] + ': ' + str(item[1]))
333
334        return
335
336
337    def print_info(self, inputdir, request_number):
338        '''
339        @Description:
340            Prints all mars requests to an extra file for debugging and
341            information.
342
343        @Input:
344            self: instance of MarsRetrieval
345                For description see class documentation.
346
347            inputdir: string
348                The path where all data from the retrievals are stored.
349
350            request_number: integer
351                Number of mars requests for flux and non-flux data.
352
353            @Return:
354            <nothing>
355        '''
356        # Get all class attributes and their values as a dictionary
357        attrs = vars(self)
358
359        # open a file to store all requests to
360        with open(os.path.join(inputdir,
361                               _config.FILE_MARS_REQUESTS), 'a') as f:
362            f.write('mars_request #' + str(request_number) + '\n')
363            # iterate through all attributes and print them
364            # with their corresponding values
365            for item in attrs.items():
366                if item[0] in 'server':
367                    pass
368                else:
369                    f.write(item[0] + ': ' + str(item[1]) + '\n')
370            f.write('\n\n')
371
372        return
373
374
375    def print_infodata_csv(self, inputdir, request_number):
376        '''
377        @Description:
378            Write all request parameter in alpabetical order into a "csv" file.
379
380        @Input:
381            self: instance of MarsRetrieval
382                For description see class documentation.
383
384            inputdir: string
385                The path where all data from the retrievals are stored.
386
387            request_number: integer
388                Number of mars requests for flux and non-flux data.
389
390        @Return:
391            <nothing>
392        '''
393
394        # Get all class attributes and their values as a dictionary
395        attrs = vars(self)
396        del attrs['server']
397
398        # open a file to store all requests to
399        with open(os.path.join(inputdir,
400                                   _config.FILE_MARS_REQUESTS), 'a') as f:
401            f.write(str(request_number) + ', ')
402            f.write(', '.join(str(attrs[key])
403                              for key in sorted(attrs.iterkeys())))
404            f.write('\n')
405
406        return
407
408    def data_retrieve(self):
409        '''
410        @Description:
411            Submits a MARS retrieval. Depending on the existence of
412            ECMWF Web-API it is submitted via Python or a
413            subprocess in the Shell. The parameter for the mars retrieval
414            are taken from the defined class attributes.
415
416        @Input:
417            self: instance of MarsRetrieval
418                For description see class documentation.
419
420        @Return:
421            <nothing>
422        '''
423        # Get all class attributes and their values as a dictionary
424        attrs = vars(self)
425
426        # convert the dictionary of attributes into a comma
427        # seperated list of attributes with their values
428        # needed for the retrieval call
429        s = 'ret'
430        for k, v in attrs.iteritems():
431            if k in 'server':
432                continue
433            if k == 'marsclass':
434                k = 'class'
435            if v == '':
436                continue
437            if k.lower() == 'target':
438                target = v
439            else:
440                s = s + ',' + k + '=' + str(v)
441
442        # MARS request via Python script
443        if self.server is not False:
444            try:
445                self.server.execute(s, target)
446            except:
447                print('MARS Request failed, \
448                      have you already registered at apps.ecmwf.int?')
449                raise IOError
450            if os.stat(target).st_size == 0:
451                print('MARS Request returned no data - please check request')
452                raise IOError
453        # MARS request via extra process in shell
454        else:
455            s += ',target = "' + target + '"'
456            p = subprocess.Popen(['mars'], stdin=subprocess.PIPE,
457                                 stdout=subprocess.PIPE,
458                                 stderr=subprocess.PIPE, bufsize=1)
459            pout = p.communicate(input=s)[0]
460            print(pout.decode())
461
462            if 'Some errors reported' in pout.decode():
463                print('MARS Request failed - please check request')
464                raise IOError
465
466            if os.stat(target).st_size == 0:
467                print('MARS Request returned no data - please check request')
468                raise IOError
469
470        return
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG