source: flex_extract.git/python/pythontest/TestInstallTar/flex_extract_v7.1/python/MarsRetrieval.py @ 2fb99de

ctbtodev
Last change on this file since 2fb99de was 2fb99de, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

introduced config with path definitions and changed py files accordingly; Installation works; some tests were added for tarball making; Problems in submission to ecgate

  • Property mode set to 100644
File size: 16.7 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#   November 2015 - Leopold Haimberger (University of Vienna):
11#        - optimized display_info
12#        - optimized data_retrieve and seperate between python and shell
13#          script call
14#
15#   February 2018 - Anne Philipp (University of Vienna):
16#        - applied PEP8 style guide
17#        - added documentation
18#        - applied some minor modifications in programming style/structure
19#
20# @License:
21#    (C) Copyright 2015-2018.
22#
23#    This software is licensed under the terms of the Apache Licence Version 2.0
24#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
25#
26# @Class Description:
27#    A MARS revtrieval has a specific syntax with a selection of keywords and
28#    their corresponding values. This class provides the necessary functions
29#    by displaying the selected parameters and their values and the actual
30#    retrievement of the data through a mars request or a Python web api
31#    interface. The initialization already expects all the keyword values.
32#
33# @Class Content:
34#    - __init__
35#    - display_info
36#    - data_retrieve
37#
38# @Class Attributes:
39#    - server
40#    - marsclass
41#    - dtype
42#    - levtype
43#    - levelist
44#    - repres
45#    - date
46#    - resol
47#    - stream
48#    - area
49#    - time
50#    - step
51#    - expver
52#    - number
53#    - accuracy
54#    - grid
55#    - gaussian
56#    - target
57#    - param
58#
59#*******************************************************************************
60
61# ------------------------------------------------------------------------------
62# MODULES
63# ------------------------------------------------------------------------------
64import subprocess
65import os
66
67import _config
68# ------------------------------------------------------------------------------
69# CLASS
70# ------------------------------------------------------------------------------
71class MarsRetrieval(object):
72    '''
73    Class for submitting MARS retrievals.
74
75    A description of MARS keywords/arguments and examples of their
76    values can be found here:
77    https://software.ecmwf.int/wiki/display/UDOC/\
78                   Identification+keywords#Identificationkeywords-class
79
80    '''
81
82    def __init__(self, server, marsclass="ei", type="", levtype="",
83                 levelist="", repres="", date="", resol="", stream="",
84                 area="", time="", step="", expver="1", number="",
85                 accuracy="", grid="", gaussian="", target="",
86                 param=""):
87        '''
88        @Description:
89            Initialises the instance of the MarsRetrieval class and
90            defines and assigns a set of the necessary retrieval parameters
91            for the FLEXPART input data.
92            A description of MARS keywords/arguments, their dependencies
93            on each other and examples of their values can be found here:
94
95            https://software.ecmwf.int/wiki/display/UDOC/MARS+keywords
96
97        @Input:
98            self: instance of MarsRetrieval
99                For description see class documentation.
100
101            server: instance of ECMWFService (from ECMWF Web-API)
102                This is the connection to the ECMWF data servers.
103                It is needed for the pythonic access of ECMWF data.
104
105            marsclass: string, optional
106                Characterisation of dataset. E.g. EI (ERA-Interim),
107                E4 (ERA40), OD (Operational archive), ea (ERA5).
108                Default is the ERA-Interim dataset "ei".
109
110            type: string, optional
111                Determines the type of fields to be retrieved.
112                Selects between observations, images or fields.
113                Examples for fields: Analysis (an), Forecast (fc),
114                Perturbed Forecast (pf), Control Forecast (cf) and so on.
115                Default is an empty string.
116
117            levtype: string, optional
118                Denotes type of level. Has a direct implication on valid
119                levelist values!
120                E.g. model level (ml), pressure level (pl), surface (sfc),
121                potential vorticity (pv), potential temperature (pt)
122                and depth (dp).
123                Default is an empty string.
124
125            levelist: string, optional
126                Specifies the required levels. It has to have a valid
127                correspondence to the selected levtype.
128                Examples: model level: 1/to/137, pressure levels: 500/to/1000
129                Default is an empty string.
130
131            repres: string, optional
132                Selects the representation of the archived data.
133                E.g. sh - spherical harmonics, gg - Gaussian grid,
134                ll - latitude/longitude, ...
135                Default is an empty string.
136
137            date: string, optional
138                Specifies the Analysis date, the Forecast base date or
139                Observations date. Valid formats are:
140                Absolute as YYYY-MM-DD or YYYYMMDD.
141                Default is an empty string.
142
143            resol: string, optional
144                Specifies the desired triangular truncation of retrieved data,
145                before carrying out any other selected post-processing.
146                The default is automatic truncation (auto), by which the lowest
147                resolution compatible with the value specified in grid is
148                automatically selected for the retrieval.
149                Users wanting to perform post-processing from full spectral
150                resolution should specify Archived Value (av).
151                The following are examples of existing resolutions found in
152                the archive: 63, 106, 159, 213, 255, 319, 399, 511, 799 or 1279.
153                This keyword has no meaning/effect if the archived data is
154                not in spherical harmonics representation.
155                The best selection can be found here:
156                https://software.ecmwf.int/wiki/display/UDOC/\
157                      Retrieve#Retrieve-Truncationbeforeinterpolation
158                Default is an empty string.
159
160            stream: string, optional
161                Identifies the forecasting system used to generate the data.
162                E.g. oper (Atmospheric model), enfo (Ensemble forecats), ...
163                Default is an empty string.
164
165            area: string, optional
166                Specifies the desired sub-area of data to be extracted.
167                Areas can be defined to wrap around the globe.
168
169                Latitude values must be given as signed numbers, with:
170                    north latitudes (i.e. north of the equator)
171                        being positive (e.g: 40.5)
172                    south latitutes (i.e. south of the equator)
173                        being negative (e.g: -50.5)
174                Longtitude values must be given as signed numbers, with:
175                    east longitudes (i.e. east of the 0 degree meridian)
176                        being positive (e.g: 35.0)
177                    west longitudes (i.e. west of the 0 degree meridian)
178                        being negative (e.g: -20.5)
179
180                E.g.: North/West/South/East
181                Default is an empty string.
182
183            time: string, optional
184                Specifies the time of the data in hours and minutes.
185                Valid values depend on the type of data: Analysis time,
186                Forecast base time or First guess verification time
187                (all usually at synoptic hours: 00, 06, 12 and 18 ).
188                Observation time (any combination in hours and minutes is valid,
189                subject to data availability in the archive).
190                The syntax is HHMM or HH:MM. If MM is omitted it defaults to 00.
191                Default is an empty string.
192
193            step: string, optional
194                Specifies the forecast time step from forecast base time.
195                Valid values are hours (HH) from forecast base time. It also
196                specifies the length of the forecast which verifies at
197                First Guess time.
198                E.g. 1/3/6-hourly
199                Default is an empty string.
200
201            expver: string, optional
202                The version of the dataset. Each experiment is assigned a
203                unique code (version). Production data is assigned 1 or 2,
204                and experimental data in Operations 11, 12 ,...
205                Research or Member State's experiments have a four letter
206                experiment identifier.
207                Default is "1".
208
209            number: string, optional
210                Selects the member in ensemble forecast run. (Only then it
211                is necessary.) It has a different meaning depending on
212                the type of data.
213                E.g. Perturbed Forecasts: specifies the Ensemble forecast member
214                Default is an empty string.
215
216            accuracy: string, optional
217                Specifies the number of bits per value to be used in the
218                generated GRIB coded fields.
219                A positive integer may be given to specify the preferred number
220                of bits per packed value. This must not be greater than the
221                number of bits normally used for a Fortran integer on the
222                processor handling the request (typically 32 or 64 bit).
223                Within a compute request the accuracy of the original fields
224                can be passed to the result field by specifying accuracy=av.
225                Default is an empty string.
226
227            grid: string, optional
228                Specifies the output grid which can be either a Gaussian grid
229                or a Latitude/Longitude grid. MARS requests specifying
230                grid=av will return the archived model grid.
231
232                Lat/Lon grid: The grid spacing needs to be an integer
233                fraction of 90 degrees e.g. grid = 0.5/0.5
234
235                Gaussian grid: specified by a letter denoting the type of
236                Gaussian grid followed by an integer (the grid number)
237                representing the number of lines between the Pole and Equator,
238                e.g.
239                grid = F160 - full (or regular) Gaussian grid with
240                       160 latitude lines between the pole and equator
241                grid = N320 - ECMWF original reduced Gaussian grid with
242                       320 latitude lines between the pole and equator,
243                       see Reduced Gaussian Grids for grid numbers used at ECMWF
244                grid = O640 - ECMWF octahedral (reduced) Gaussian grid with
245                       640 latitude lines between the pole and equator
246                Default is an empty string.
247
248            gaussian: string, optional
249                This parameter is deprecated and should no longer be used.
250                Specifies the desired type of Gaussian grid for the output.
251                Valid Gaussian grids are quasi-regular (reduced) or regular.
252                Keyword gaussian can only be specified together with
253                keyword grid. Gaussian without grid has no effect.
254                Default is an empty string.
255
256            target: string, optional
257                Specifies a file into which data is to be written after
258                retrieval or manipulation. Path names should always be
259                enclosed in double quotes. The MARS client supports automatic
260                generation of multiple target files using MARS keywords
261                enclosed in square brackets [ ].  If the environment variable
262                MARS_MULTITARGET_STRICT_FORMAT is set to 1 before calling mars,
263                the keyword values will be used in the filename as shown by
264                the ecCodes GRIB tool grib_ls -m, e.g. with
265                MARS_MULTITARGET_STRICT_FORMAT set to 1 the keywords time,
266                expver and param will be formatted as 0600, 0001 and 129.128
267                rather than 600, 1 and 129.
268                Default is an empty string.
269
270            param: string, optional
271                Specifies the meteorological parameter.
272                The list of meteorological parameters in MARS is extensive.
273                Their availability is directly related to their meteorological
274                meaning and, therefore, the rest of directives specified
275                in the MARS request.
276                Meteorological parameters can be specified by their
277                GRIB code (param=130), their mnemonic (param=t) or
278                full name (param=temperature).
279                The list of parameter should be seperated by a "/"-sign.
280                E.g. 130/131/133
281                Default is an empty string.
282
283        @Return:
284            <nothing>
285        '''
286
287        self.server = server
288        self.marsclass = marsclass
289        self.type = type
290        self.levtype = levtype
291        self.levelist = levelist
292        self.repres = repres
293        self.date = date
294        self.resol = resol
295        self.stream = stream
296        self.area = area
297        self.time = time
298        self.step = step
299        self.expver = expver
300        self.number = number
301        self.accuracy = accuracy
302        self.grid = grid
303        self.gaussian = gaussian
304        self.target = target
305        self.param = param
306
307        return
308
309
310    def display_info(self):
311        '''
312        @Description:
313            Prints all class attributes and their values.
314
315        @Input:
316            self: instance of MarsRetrieval
317                For description see class documentation.
318
319        @Return:
320            <nothing>
321        '''
322        # Get all class attributes and their values as a dictionary
323        attrs = vars(self)
324
325        # iterate through all attributes and print them
326        # with their corresponding values
327        for item in attrs.items():
328            if item[0] in 'server':
329                pass
330            else:
331                print item[0] + ': ' + str(item[1])
332
333        return
334
335
336    def print_info(self):
337        '''
338        '''
339        # Get all class attributes and their values as a dictionary
340        attrs = vars(self)
341
342        # open a file to store all requests to
343        with open(os.path.join(_config.PATH_RUN_DIR + os.path.sep +
344                               _config.FILE_MARS_REQUESTS), 'a') as f:
345            f.write('mars\n')
346            # iterate through all attributes and print them
347            # with their corresponding values
348            for item in attrs.items():
349                if item[0] in 'server':
350                    pass
351                else:
352                    f.write(item[0] + ': ' + str(item[1]) + '\n')
353            f.write('\n\n')
354
355        return
356
357    def data_retrieve(self):
358        '''
359        @Description:
360            Submits a MARS retrieval. Depending on the existence of
361            ECMWF Web-API it is submitted via Python or a
362            subprocess in the Shell. The parameter for the mars retrieval
363            are taken from the defined class attributes.
364
365        @Input:
366            self: instance of MarsRetrieval
367                For description see class documentation.
368
369        @Return:
370            <nothing>
371        '''
372        # Get all class attributes and their values as a dictionary
373        attrs = vars(self)
374
375        # convert the dictionary of attributes into a comma
376        # seperated list of attributes with their values
377        # needed for the retrieval call
378        s = 'ret'
379        for k, v in attrs.iteritems():
380            if k in 'server':
381                continue
382            if k == 'marsclass':
383                k = 'class'
384            if v == '':
385                continue
386            if k.lower() == 'target':
387                target = v
388            else:
389                s = s + ',' + k + '=' + str(v)
390
391        # MARS request via Python script
392        if self.server is not False:
393            try:
394                self.server.execute(s, target)
395            except:
396                print('MARS Request failed, '
397                      'have you already registered at apps.ecmwf.int?')
398                raise IOError
399            if os.stat(target).st_size == 0:
400                print('MARS Request returned no data - please check request')
401                raise IOError
402        # MARS request via extra process in shell
403        else:
404            s += ',target = "' + target + '"'
405            p = subprocess.Popen(['mars'], stdin=subprocess.PIPE,
406                                 stdout=subprocess.PIPE,
407                                 stderr=subprocess.PIPE, bufsize=1)
408            pout = p.communicate(input=s)[0]
409            print pout.decode()
410
411            if 'Some errors reported' in pout.decode():
412                print('MARS Request failed - please check request')
413                raise IOError
414
415            if os.stat(target).st_size == 0:
416                print('MARS Request returned no data - please check request')
417                raise IOError
418
419        return
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG