source: flex_extract.git/source/python/mods/get_mars_data.py @ f20af73

dev
Last change on this file since f20af73 was f20af73, checked in by Anne Philipp <anne.philipp@…>, 3 months ago

added CDS API support for ERA5 instead of ECMWFAPI / refactored setup of CONTROL parameter because for local version it wanted to use not installed ECMWF_ENV file.

  • Property mode set to 100755
File size: 10.8 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - moved the getEIdata program into a function "get_mars_data"
12#        - moved the AgurmentParser into a seperate function
13#        - adatpted the function for the use in flex_extract
14#        - renamed file to get_mars_data
15#
16#    February 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added structured documentation
19#        - minor changes in programming style for consistence
20#        - added function main and moved function calls vom __main__ there
21#          (necessary for better documentation with docstrings for later
22#          online documentation)
23#        - use of UIFiles class for file selection and deletion
24#        - seperated get_mars_data function into several smaller pieces:
25#          write_reqheader, mk_server, mk_dates, remove_old, do_retrievment
26#
27# @License:
28#    (C) Copyright 2014-2019.
29#    Anne Philipp, Leopold Haimberger
30#
31#    This work is licensed under the Creative Commons Attribution 4.0
32#    International License. To view a copy of this license, visit
33#    http://creativecommons.org/licenses/by/4.0/ or send a letter to
34#    Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
35#*******************************************************************************
36'''This script extracts MARS data from ECMWF servers.
37
38At first, the necessary parameters from command line and CONTROL files are
39extracted. They define the data set to be extracted from MARS.
40
41This file can also be imported as a module and contains the following
42functions:
43
44    * main - the main function of the script
45    * get_mars_data - overall control of ECMWF data retrievment
46    * write_reqheader - writes the header into the mars_request file
47    * mk_server - creates the server connection to ECMWF servers
48    * mk_dates - defines the start and end date
49    * remove_old - deletes old retrieved grib files
50    * do_retrievement - creates individual retrievals
51
52Type: get_mars_data.py --help
53to get information about command line parameters.
54Read the documentation for usage instructions.
55'''
56# ------------------------------------------------------------------------------
57# MODULES
58# ------------------------------------------------------------------------------
59import os
60import sys
61import inspect
62from datetime import datetime, timedelta
63
64# software specific classes and modules from flex_extract
65# add path to local main python path for flex_extract to get full access
66sys.path.append(os.path.dirname(os.path.abspath(
67    inspect.getfile(inspect.currentframe()))) + '/../')
68import _config
69from .tools import (setup_controldata, my_error, normal_exit, get_cmdline_args,
70                   read_ecenv, make_dir)
71from classes.EcFlexpart import EcFlexpart
72from classes.UioFiles import UioFiles
73from classes.MarsRetrieval import MarsRetrieval
74
75try:
76    ec_api = True
77    import ecmwfapi
78except ImportError:
79    ec_api = False
80
81try:
82    cds_api = True
83    import cdsapi
84except ImportError:
85    cds_api = False
86# ------------------------------------------------------------------------------
87# FUNCTION
88# ------------------------------------------------------------------------------
89def main():
90    '''Controls the program to get data out of mars.
91
92    This is done if it is called directly from command line.
93    Then it also takes program call arguments and control file input.
94
95    Parameters
96    ----------
97
98    Return
99    ------
100
101    '''
102
103    c, _, _, _ = setup_controldata()
104    get_mars_data(c)
105    normal_exit('Retrieving MARS data: Done!')
106
107    return
108
109def get_mars_data(c):
110    '''Retrieves the EC data needed for a FLEXPART simulation.
111
112    Start and end dates for retrieval period is set. Retrievals
113    are divided into smaller periods if necessary and datechunk parameter
114    is set.
115
116    Parameters
117    ----------
118    c : ControlFile
119        Contains all the parameters of CONTROL file and
120        command line.
121
122    Return
123    ------
124
125    '''
126    c.ec_api = ec_api
127    c.cds_api = cds_api
128
129    if not os.path.exists(c.inputdir):
130        make_dir(c.inputdir)
131
132    if c.request == 0:
133        print("Retrieving EC data!")
134    else:
135        if c.request == 1:
136            print("Printing mars requests!")
137        elif c.request == 2:
138            print("Retrieving EC data and printing mars request!")
139        write_reqheader(os.path.join(c.inputdir, _config.FILE_MARS_REQUESTS))
140
141    print("start date %s " % (c.start_date))
142    print("end date %s " % (c.end_date))
143
144    server = mk_server(c)
145
146    # if data are to be retrieved, clean up any old grib files
147    if c.request == 0 or c.request == 2:
148        remove_old('*grb', c.inputdir)
149
150    # --------------  flux data ------------------------------------------------
151    start, end, datechunk = mk_dates(c, fluxes=True)
152    do_retrievement(c, server, start, end, datechunk, fluxes=True)
153
154    # --------------  non flux data --------------------------------------------
155    start, end, datechunk = mk_dates(c, fluxes=False)
156    do_retrievement(c, server, start, end, datechunk, fluxes=False)
157
158    return
159
160def write_reqheader(marsfile):
161    '''Writes header with column names into mars request file.
162
163    Parameters
164    ----------
165    marsfile : str
166        Path to the mars request file.
167
168    Return
169    ------
170
171    '''
172    MR = MarsRetrieval(None, None)
173    attrs = vars(MR).copy()
174    del attrs['server']
175    del attrs['public']
176    with open(marsfile, 'w') as f:
177        f.write('request_number' + ', ')
178        f.write(', '.join(str(key) for key in sorted(attrs.iterkeys())))
179        f.write('\n')
180
181    return
182
183def mk_server(c):
184    '''Creates a server connection with available python API.
185
186    Which API is used depends on availability and the dataset to be retrieved.
187    The CDS API is used for ERA5 dataset no matter if the user is a member or
188    a public user. ECMWF WebAPI is used for all other available datasets.
189
190    Parameters
191    ----------
192    c : ControlFile
193        Contains all the parameters of CONTROL file and
194        command line.
195
196    Return
197    ------
198    server : ECMWFDataServer, ECMWFService or Client
199        Connection to ECMWF server via python interface ECMWF WebAPI or CDS API.
200
201    '''
202    if cds_api and (c.marsclass.upper() == 'EA'):
203        server = cdsapi.Client()
204        c.ec_api = False
205    elif c.ec_api:
206        if c.public:
207            server = ecmwfapi.ECMWFDataServer()
208        else:
209            server = ecmwfapi.ECMWFService("mars")
210        c.cds_api = False
211    else:
212        server = False
213
214    print('Using ECMWF WebAPI: ' + str(c.ec_api))
215    print('Using CDS API: ' + str(c.cds_api))
216
217    return server
218
219
220def check_dates_for_nonflux_fc_times(types, times):
221    '''
222    '''
223    for ty, ti in zip(types,times):
224        if ty.upper() == 'FC' and int(ti) == 18:
225            return True
226    return False
227
228
229def mk_dates(c, fluxes):
230    '''Prepares start and end date depending on flux or non flux data.
231
232    If forecast for maximum one day (upto 24h) are to be retrieved, then
233    collect accumulation data (flux data) with additional days in the
234    beginning and at the end (used for complete disaggregation of
235    original period)
236
237    If forecast data longer than 24h are to be retrieved, then
238    collect accumulation data (flux data) with the exact start and end date
239    (disaggregation will be done for the exact time period with
240    boundary conditions)
241
242    Since for basetime the extraction contains the 12 hours upfront,
243    if basetime is 0, the starting date has to be the day before and
244
245    Parameters
246    ----------
247    c : ControlFile
248        Contains all the parameters of CONTROL file and
249        command line.
250
251    fluxes : boolean, optional
252        Decides if the flux parameter settings are stored or
253        the rest of the parameter list.
254        Default value is False.
255
256    Return
257    ------
258    start : datetime
259        The start date of the retrieving data set.
260
261    end : datetime
262        The end date of the retrieving data set.
263
264    chunk : datetime
265        Time period in days for one single mars retrieval.
266
267    '''
268    start = datetime.strptime(c.start_date, '%Y%m%d')
269    end = datetime.strptime(c.end_date, '%Y%m%d')
270    chunk = timedelta(days=int(c.date_chunk))
271
272    if c.basetime == 0:
273        start = start - timedelta(days=1)
274
275    if not c.purefc and fluxes and not c.basetime == 0:
276        start = start - timedelta(days=1)
277        end = end + timedelta(days=1)
278
279    # if we have non-flux forecast data starting at 18 UTC
280    # we need to start retrieving data one day in advance
281    if not fluxes and check_dates_for_nonflux_fc_times(c.type, c.time):
282        start = start - timedelta(days=1)
283
284    return start, end, chunk
285
286def remove_old(pattern, inputdir):
287    '''Deletes old retrieval files from current input directory
288    matching the pattern.
289
290    Parameters
291    ----------
292    pattern : str
293        The sub string pattern which identifies the files to be deleted.
294
295    inputdir : str, optional
296        Path to the directory where the retrieved data is stored.
297
298    Return
299    ------
300
301    '''
302    print('... removing old content of ' + inputdir)
303
304    tobecleaned = UioFiles(inputdir, pattern)
305    tobecleaned.delete_files()
306
307    return
308
309
310def do_retrievement(c, server, start, end, delta_t, fluxes=False):
311    '''Divides the complete retrieval period in smaller chunks and
312    retrieves the data from MARS.
313
314    Parameters
315    ----------
316    c : ControlFile
317        Contains all the parameters of CONTROL file and
318        command line.
319
320    server : ECMWFService or ECMWFDataServer
321            The server connection to ECMWF.
322
323    start : datetime
324        The start date of the retrieval.
325
326    end : datetime
327        The end date of the retrieval.
328
329    delta_t : datetime
330        Delta_t + 1 is the maximal time period of a single
331        retrieval.
332
333    fluxes : boolean, optional
334        Decides if the flux parameters are to be retrieved or
335        the rest of the parameter list.
336        Default value is False.
337
338    Return
339    ------
340
341    '''
342
343    # since actual day also counts as one day,
344    # we only need to add datechunk - 1 days to retrieval
345    # for a period
346    delta_t_m1 = delta_t - timedelta(days=1)
347
348    day = start
349    while day <= end:
350        flexpart = EcFlexpart(c, fluxes)
351        tmpday = day + delta_t_m1
352        if tmpday < end:
353            dates = day.strftime("%Y%m%d") + "/to/" + \
354                    tmpday.strftime("%Y%m%d")
355        else:
356            dates = day.strftime("%Y%m%d") + "/to/" + \
357                    end.strftime("%Y%m%d")
358
359        print("... retrieve " + dates + " in dir " + c.inputdir)
360
361        try:
362            flexpart.retrieve(server, dates, c.public, c.request, c.inputdir)
363        except IOError:
364            my_error('MARS request failed')
365
366        day += delta_t
367
368    return
369
370if __name__ == "__main__":
371    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG