source: flex_extract.git/source/python/classes/MarsRetrieval.py @ 5bad6ec

ctbtodev
Last change on this file since 5bad6ec was 5bad6ec, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

added possibility to extract public datasets via an logical public parameter

  • Property mode set to 100644
File size: 20.4 KB
RevLine 
[efdb01a]1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
[991df6a]3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#   November 2015 - Leopold Haimberger (University of Vienna):
[ff99eae]11#        - optimized display_info
12#        - optimized data_retrieve and seperate between python and shell
[991df6a]13#          script call
14#
15#   February 2018 - Anne Philipp (University of Vienna):
16#        - applied PEP8 style guide
17#        - added documentation
18#        - applied some minor modifications in programming style/structure
19#
20# @License:
21#    (C) Copyright 2015-2018.
22#
23#    This software is licensed under the terms of the Apache Licence Version 2.0
24#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
25#
26# @Class Description:
27#    A MARS revtrieval has a specific syntax with a selection of keywords and
28#    their corresponding values. This class provides the necessary functions
29#    by displaying the selected parameters and their values and the actual
30#    retrievement of the data through a mars request or a Python web api
31#    interface. The initialization already expects all the keyword values.
32#
33# @Class Content:
34#    - __init__
[ff99eae]35#    - display_info
36#    - data_retrieve
37#
38# @Class Attributes:
39#    - server
40#    - marsclass
41#    - dtype
42#    - levtype
43#    - levelist
44#    - repres
45#    - date
46#    - resol
47#    - stream
48#    - area
49#    - time
50#    - step
51#    - expver
52#    - number
53#    - accuracy
54#    - grid
55#    - gaussian
56#    - target
57#    - param
[991df6a]58#
59#*******************************************************************************
60
61# ------------------------------------------------------------------------------
[efdb01a]62# MODULES
63# ------------------------------------------------------------------------------
64import os
[ca867de]65import sys
66import subprocess
[efdb01a]67
[ca867de]68# software specific classes and modules from flex_extract
69sys.path.append('../')
[2fb99de]70import _config
[efdb01a]71# ------------------------------------------------------------------------------
72# CLASS
73# ------------------------------------------------------------------------------
[ff99eae]74class MarsRetrieval(object):
[efdb01a]75    '''
76    Class for submitting MARS retrievals.
77
78    A description of MARS keywords/arguments and examples of their
79    values can be found here:
80    https://software.ecmwf.int/wiki/display/UDOC/\
81                   Identification+keywords#Identificationkeywords-class
82
83    '''
84
[5bad6ec]85    def __init__(self, server, public, marsclass="ei", dataset="", type="",
86                 levtype="", levelist="", repres="", date="", resol="",
87                 stream="", area="", time="", step="", expver="1",
88                 number="", accuracy="", grid="", gaussian="", target="",
[ff99eae]89                 param=""):
[efdb01a]90        '''
91        @Description:
[ff99eae]92            Initialises the instance of the MarsRetrieval class and
[efdb01a]93            defines and assigns a set of the necessary retrieval parameters
94            for the FLEXPART input data.
95            A description of MARS keywords/arguments, their dependencies
96            on each other and examples of their values can be found here:
97
98            https://software.ecmwf.int/wiki/display/UDOC/MARS+keywords
99
100        @Input:
[ff99eae]101            self: instance of MarsRetrieval
[efdb01a]102                For description see class documentation.
103
104            server: instance of ECMWFService (from ECMWF Web-API)
105                This is the connection to the ECMWF data servers.
106                It is needed for the pythonic access of ECMWF data.
107
[5bad6ec]108            public: integer
109                Decides which Web API version is used:
110                0: member-state users and full archive access
111                1: public access and limited access to the public server and
112                   datasets. Needs the parameter dataset.
113                Default is "0" and for member-state users.
114
[efdb01a]115            marsclass: string, optional
116                Characterisation of dataset. E.g. EI (ERA-Interim),
117                E4 (ERA40), OD (Operational archive), ea (ERA5).
118                Default is the ERA-Interim dataset "ei".
119
[5bad6ec]120            dataset: string, optional
121                For public datasets there is the specific naming and parameter
122                dataset which has to be used to characterize the type of
123                data. Usually there is less data available, either in times,
124                domain or parameter.
125                Default is an empty string.
126
[54a8a01]127            type: string, optional
[efdb01a]128                Determines the type of fields to be retrieved.
129                Selects between observations, images or fields.
130                Examples for fields: Analysis (an), Forecast (fc),
131                Perturbed Forecast (pf), Control Forecast (cf) and so on.
132                Default is an empty string.
133
134            levtype: string, optional
135                Denotes type of level. Has a direct implication on valid
136                levelist values!
137                E.g. model level (ml), pressure level (pl), surface (sfc),
138                potential vorticity (pv), potential temperature (pt)
139                and depth (dp).
140                Default is an empty string.
141
142            levelist: string, optional
143                Specifies the required levels. It has to have a valid
144                correspondence to the selected levtype.
145                Examples: model level: 1/to/137, pressure levels: 500/to/1000
146                Default is an empty string.
147
148            repres: string, optional
149                Selects the representation of the archived data.
150                E.g. sh - spherical harmonics, gg - Gaussian grid,
151                ll - latitude/longitude, ...
152                Default is an empty string.
153
154            date: string, optional
155                Specifies the Analysis date, the Forecast base date or
156                Observations date. Valid formats are:
157                Absolute as YYYY-MM-DD or YYYYMMDD.
158                Default is an empty string.
159
160            resol: string, optional
161                Specifies the desired triangular truncation of retrieved data,
162                before carrying out any other selected post-processing.
163                The default is automatic truncation (auto), by which the lowest
164                resolution compatible with the value specified in grid is
165                automatically selected for the retrieval.
166                Users wanting to perform post-processing from full spectral
167                resolution should specify Archived Value (av).
168                The following are examples of existing resolutions found in
169                the archive: 63, 106, 159, 213, 255, 319, 399, 511, 799 or 1279.
170                This keyword has no meaning/effect if the archived data is
171                not in spherical harmonics representation.
172                The best selection can be found here:
173                https://software.ecmwf.int/wiki/display/UDOC/\
174                      Retrieve#Retrieve-Truncationbeforeinterpolation
175                Default is an empty string.
176
177            stream: string, optional
178                Identifies the forecasting system used to generate the data.
179                E.g. oper (Atmospheric model), enfo (Ensemble forecats), ...
180                Default is an empty string.
181
182            area: string, optional
183                Specifies the desired sub-area of data to be extracted.
184                Areas can be defined to wrap around the globe.
185
186                Latitude values must be given as signed numbers, with:
187                    north latitudes (i.e. north of the equator)
188                        being positive (e.g: 40.5)
189                    south latitutes (i.e. south of the equator)
190                        being negative (e.g: -50.5)
191                Longtitude values must be given as signed numbers, with:
192                    east longitudes (i.e. east of the 0 degree meridian)
193                        being positive (e.g: 35.0)
194                    west longitudes (i.e. west of the 0 degree meridian)
195                        being negative (e.g: -20.5)
196
197                E.g.: North/West/South/East
198                Default is an empty string.
199
200            time: string, optional
201                Specifies the time of the data in hours and minutes.
202                Valid values depend on the type of data: Analysis time,
203                Forecast base time or First guess verification time
204                (all usually at synoptic hours: 00, 06, 12 and 18 ).
205                Observation time (any combination in hours and minutes is valid,
206                subject to data availability in the archive).
207                The syntax is HHMM or HH:MM. If MM is omitted it defaults to 00.
208                Default is an empty string.
209
210            step: string, optional
211                Specifies the forecast time step from forecast base time.
212                Valid values are hours (HH) from forecast base time. It also
213                specifies the length of the forecast which verifies at
214                First Guess time.
215                E.g. 1/3/6-hourly
216                Default is an empty string.
217
218            expver: string, optional
219                The version of the dataset. Each experiment is assigned a
220                unique code (version). Production data is assigned 1 or 2,
221                and experimental data in Operations 11, 12 ,...
222                Research or Member State's experiments have a four letter
223                experiment identifier.
224                Default is "1".
225
226            number: string, optional
227                Selects the member in ensemble forecast run. (Only then it
228                is necessary.) It has a different meaning depending on
229                the type of data.
230                E.g. Perturbed Forecasts: specifies the Ensemble forecast member
231                Default is an empty string.
232
233            accuracy: string, optional
234                Specifies the number of bits per value to be used in the
235                generated GRIB coded fields.
236                A positive integer may be given to specify the preferred number
237                of bits per packed value. This must not be greater than the
238                number of bits normally used for a Fortran integer on the
239                processor handling the request (typically 32 or 64 bit).
240                Within a compute request the accuracy of the original fields
241                can be passed to the result field by specifying accuracy=av.
242                Default is an empty string.
243
244            grid: string, optional
245                Specifies the output grid which can be either a Gaussian grid
246                or a Latitude/Longitude grid. MARS requests specifying
247                grid=av will return the archived model grid.
248
249                Lat/Lon grid: The grid spacing needs to be an integer
250                fraction of 90 degrees e.g. grid = 0.5/0.5
251
252                Gaussian grid: specified by a letter denoting the type of
253                Gaussian grid followed by an integer (the grid number)
254                representing the number of lines between the Pole and Equator,
255                e.g.
256                grid = F160 - full (or regular) Gaussian grid with
257                       160 latitude lines between the pole and equator
258                grid = N320 - ECMWF original reduced Gaussian grid with
259                       320 latitude lines between the pole and equator,
260                       see Reduced Gaussian Grids for grid numbers used at ECMWF
261                grid = O640 - ECMWF octahedral (reduced) Gaussian grid with
262                       640 latitude lines between the pole and equator
263                Default is an empty string.
264
265            gaussian: string, optional
266                This parameter is deprecated and should no longer be used.
267                Specifies the desired type of Gaussian grid for the output.
268                Valid Gaussian grids are quasi-regular (reduced) or regular.
269                Keyword gaussian can only be specified together with
270                keyword grid. Gaussian without grid has no effect.
271                Default is an empty string.
272
273            target: string, optional
274                Specifies a file into which data is to be written after
275                retrieval or manipulation. Path names should always be
276                enclosed in double quotes. The MARS client supports automatic
277                generation of multiple target files using MARS keywords
278                enclosed in square brackets [ ].  If the environment variable
279                MARS_MULTITARGET_STRICT_FORMAT is set to 1 before calling mars,
280                the keyword values will be used in the filename as shown by
281                the ecCodes GRIB tool grib_ls -m, e.g. with
282                MARS_MULTITARGET_STRICT_FORMAT set to 1 the keywords time,
283                expver and param will be formatted as 0600, 0001 and 129.128
284                rather than 600, 1 and 129.
285                Default is an empty string.
286
287            param: string, optional
288                Specifies the meteorological parameter.
289                The list of meteorological parameters in MARS is extensive.
290                Their availability is directly related to their meteorological
291                meaning and, therefore, the rest of directives specified
292                in the MARS request.
293                Meteorological parameters can be specified by their
294                GRIB code (param=130), their mnemonic (param=t) or
295                full name (param=temperature).
296                The list of parameter should be seperated by a "/"-sign.
297                E.g. 130/131/133
298                Default is an empty string.
299
300        @Return:
301            <nothing>
302        '''
303
304        self.server = server
[5bad6ec]305        self.public = public
[efdb01a]306        self.marsclass = marsclass
[5bad6ec]307        self.dataset = dataset
[54a8a01]308        self.type = type
[efdb01a]309        self.levtype = levtype
310        self.levelist = levelist
311        self.repres = repres
312        self.date = date
313        self.resol = resol
314        self.stream = stream
315        self.area = area
316        self.time = time
317        self.step = step
318        self.expver = expver
319        self.number = number
320        self.accuracy = accuracy
321        self.grid = grid
322        self.gaussian = gaussian
323        self.target = target
324        self.param = param
325
326        return
327
328
[ff99eae]329    def display_info(self):
[efdb01a]330        '''
331        @Description:
[2fb99de]332            Prints all class attributes and their values to the
333            standard output.
[efdb01a]334
335        @Input:
[ff99eae]336            self: instance of MarsRetrieval
[efdb01a]337                For description see class documentation.
338
339        @Return:
340            <nothing>
341        '''
342        # Get all class attributes and their values as a dictionary
[5bad6ec]343        attrs = vars(self).copy()
[efdb01a]344
345        # iterate through all attributes and print them
346        # with their corresponding values
347        for item in attrs.items():
[5bad6ec]348            if item[0] in ['server', 'public']:
[efdb01a]349                pass
350            else:
[2fb99de]351                print(item[0] + ': ' + str(item[1]))
352
353        return
354
355
[295ff45]356    def print_info(self, inputdir, request_number):
[2fb99de]357        '''
358        @Description:
359            Prints all mars requests to an extra file for debugging and
360            information.
361
362        @Input:
363            self: instance of MarsRetrieval
364                For description see class documentation.
365
[25b14be]366            inputdir: string
367                The path where all data from the retrievals are stored.
368
[295ff45]369            request_number: integer
370                Number of mars requests for flux and non-flux data.
371
372            @Return:
[2fb99de]373            <nothing>
374        '''
375        # Get all class attributes and their values as a dictionary
[5bad6ec]376        attrs = vars(self).copy()
[2fb99de]377
378        # open a file to store all requests to
[25b14be]379        with open(os.path.join(inputdir,
[2fb99de]380                               _config.FILE_MARS_REQUESTS), 'a') as f:
[295ff45]381            f.write('mars_request #' + str(request_number) + '\n')
[2fb99de]382            # iterate through all attributes and print them
383            # with their corresponding values
384            for item in attrs.items():
[5bad6ec]385                if item[0] in ['server', 'public']:
[2fb99de]386                    pass
387                else:
388                    f.write(item[0] + ': ' + str(item[1]) + '\n')
389            f.write('\n\n')
[efdb01a]390
391        return
392
[295ff45]393
394    def print_infodata_csv(self, inputdir, request_number):
395        '''
396        @Description:
397            Write all request parameter in alpabetical order into a "csv" file.
398
399        @Input:
400            self: instance of MarsRetrieval
401                For description see class documentation.
402
403            inputdir: string
404                The path where all data from the retrievals are stored.
405
406            request_number: integer
407                Number of mars requests for flux and non-flux data.
408
409        @Return:
410            <nothing>
411        '''
412
413        # Get all class attributes and their values as a dictionary
[5bad6ec]414        attrs = vars(self).copy()
[295ff45]415        del attrs['server']
[5bad6ec]416        del attrs['public']
[295ff45]417
418        # open a file to store all requests to
419        with open(os.path.join(inputdir,
420                                   _config.FILE_MARS_REQUESTS), 'a') as f:
421            f.write(str(request_number) + ', ')
422            f.write(', '.join(str(attrs[key])
423                              for key in sorted(attrs.iterkeys())))
424            f.write('\n')
425
426        return
427
[ff99eae]428    def data_retrieve(self):
[efdb01a]429        '''
430        @Description:
431            Submits a MARS retrieval. Depending on the existence of
432            ECMWF Web-API it is submitted via Python or a
433            subprocess in the Shell. The parameter for the mars retrieval
434            are taken from the defined class attributes.
435
436        @Input:
[ff99eae]437            self: instance of MarsRetrieval
[efdb01a]438                For description see class documentation.
439
440        @Return:
441            <nothing>
442        '''
443        # Get all class attributes and their values as a dictionary
[5bad6ec]444        attrs = vars(self).copy()
445
446        # eliminate unnecessary attributes from the dictionary attrs
447        del attrs['server']
448        del attrs['public']
449
450        # exchange parameter name for marsclass
451        mclass = attrs.get('marsclass')
452        del attrs['marsclass']
453        attrs['class'] = mclass
454
455        # prepare target variable as needed for the Web API mode
456        # within the dictionary for full access
457        # as a single variable for public access
458        target = attrs.get('target')
459        if not int(self.public):
460            del attrs['target']
461        print('target: ' + target)
462
463        # find all keys without a value and convert all other values to strings
464        empty_keys = []
465        for key, value in attrs.itteritems():
466            if value == '':
467                empty_keys.append(str(key))
[efdb01a]468            else:
[5bad6ec]469                attrs[key] = str(value)
470
471        # delete all empty parameter from the dictionary
472        for key in empty_keys:
473            del attrs[key]
[efdb01a]474
475        # MARS request via Python script
[5bad6ec]476        if self.server:
[efdb01a]477            try:
[5bad6ec]478                if self.public:
479                    print('RETRIEVE PUBLIC DATA!')
480                    self.server.retrieve(attrs)
481                else:
482                    print('EXECUTE NON-PUBLIC RETRIEVAL!')
483                    self.server.execute(attrs, target)
[efdb01a]484            except:
[5bad6ec]485                e = sys.exc_info()[0]
486                print("ERROR: ", e)
487                print('MARS Request failed!')
488            if not self.public and os.stat(target).st_size == 0:
[2fb99de]489                print('MARS Request returned no data - please check request')
[efdb01a]490                raise IOError
[5bad6ec]491            elif self.public and os.stat(target).st_size == 0:
492                print('Public MARS Request returned no data - '
493                      'please check request')
494                raise IOError
495            else:
496                raise IOError
[efdb01a]497        # MARS request via extra process in shell
498        else:
[5bad6ec]499            request_str = 'ret'
500            for key, value in attrs.iteritems():
501                request_str = request_str + ',' + key + '=' + str(value)
502            request_str += ',target="' + target + '"'
503            p = subprocess.Popen(['mars'],
504                                 stdin=subprocess.PIPE,
[efdb01a]505                                 stdout=subprocess.PIPE,
[5bad6ec]506                                 stderr=subprocess.PIPE,
507                                 bufsize=1)
508            pout = p.communicate(input=request_str)[0]
[2fb99de]509            print(pout.decode())
[efdb01a]510
511            if 'Some errors reported' in pout.decode():
[2fb99de]512                print('MARS Request failed - please check request')
[efdb01a]513                raise IOError
[5bad6ec]514            elif os.stat(target).st_size == 0:
[2fb99de]515                print('MARS Request returned no data - please check request')
[efdb01a]516                raise IOError
[5bad6ec]517            else:
518                raise
[efdb01a]519
520        return
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG