source: flex_extract.git/source/python/mods/get_mars_data.py @ 45b99e6

dev
Last change on this file since 45b99e6 was 45b99e6, checked in by Anne Philipp <anne.philipp@…>, 5 months ago

added possibility to extract ensemble members with ELDA stream (and ENFO)

  • Property mode set to 100755
File size: 10.5 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - moved the getEIdata program into a function "get_mars_data"
12#        - moved the AgurmentParser into a seperate function
13#        - adatpted the function for the use in flex_extract
14#        - renamed file to get_mars_data
15#
16#    February 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added structured documentation
19#        - minor changes in programming style for consistence
20#        - added function main and moved function calls vom __main__ there
21#          (necessary for better documentation with docstrings for later
22#          online documentation)
23#        - use of UIFiles class for file selection and deletion
24#        - seperated get_mars_data function into several smaller pieces:
25#          write_reqheader, mk_server, mk_dates, remove_old, do_retrievment
26#
27# @License:
28#    (C) Copyright 2014-2019.
29#    Anne Philipp, Leopold Haimberger
30#
31#    This work is licensed under the Creative Commons Attribution 4.0
32#    International License. To view a copy of this license, visit
33#    http://creativecommons.org/licenses/by/4.0/ or send a letter to
34#    Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
35#*******************************************************************************
36'''This script extracts MARS data from ECMWF servers.
37
38At first, the necessary parameters from command line and CONTROL files are
39extracted. They define the data set to be extracted from MARS.
40
41This file can also be imported as a module and contains the following
42functions:
43
44    * main - the main function of the script
45    * get_mars_data - overall control of ECMWF data retrievment
46    * write_reqheader - writes the header into the mars_request file
47    * mk_server - creates the server connection to ECMWF servers
48    * mk_dates - defines the start and end date
49    * remove_old - deletes old retrieved grib files
50    * do_retrievement - creates individual retrievals
51
52Type: get_mars_data.py --help
53to get information about command line parameters.
54Read the documentation for usage instructions.
55'''
56# ------------------------------------------------------------------------------
57# MODULES
58# ------------------------------------------------------------------------------
59import os
60import sys
61import inspect
62from datetime import datetime, timedelta
63
64# software specific classes and modules from flex_extract
65# add path to local main python path for flex_extract to get full access
66sys.path.append(os.path.dirname(os.path.abspath(
67    inspect.getfile(inspect.currentframe()))) + '/../')
68import _config
69from tools import (my_error, normal_exit, get_cmdline_args,
70                   read_ecenv, make_dir)
71from classes.EcFlexpart import EcFlexpart
72from classes.UioFiles import UioFiles
73from classes.MarsRetrieval import MarsRetrieval
74
75try:
76    ecapi = True
77    import ecmwfapi
78except ImportError:
79    ecapi = False
80# ------------------------------------------------------------------------------
81# FUNCTION
82# ------------------------------------------------------------------------------
83def main():
84    '''Controls the program to get data out of mars.
85
86    This is done if it is called directly from command line.
87    Then it also takes program call arguments and control file input.
88
89    Parameters
90    ----------
91
92    Return
93    ------
94
95    '''
96
97    args = get_cmdline_args()
98    c = ControlFile(args.controlfile)
99
100    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
101    c.assign_args_to_control(args)
102    c.assign_envs_to_control(env_parameter)
103    c.check_conditions(args.queue)
104
105    get_mars_data(c)
106    normal_exit(c.mailops, c.queue, 'Done!')
107
108    return
109
110def get_mars_data(c):
111    '''Retrieves the EC data needed for a FLEXPART simulation.
112
113    Start and end dates for retrieval period is set. Retrievals
114    are divided into smaller periods if necessary and datechunk parameter
115    is set.
116
117    Parameters
118    ----------
119    c : ControlFile
120        Contains all the parameters of CONTROL file and
121        command line.
122
123    Return
124    ------
125
126    '''
127    c.ecapi = ecapi
128
129    if not os.path.exists(c.inputdir):
130        make_dir(c.inputdir)
131
132    if c.request == 0:
133        print("Retrieving EC data!")
134    else:
135        if c.request == 1:
136            print("Printing mars requests!")
137        elif c.request == 2:
138            print("Retrieving EC data and printing mars request!")
139        write_reqheader(os.path.join(c.inputdir, _config.FILE_MARS_REQUESTS))
140
141    print("start date %s " % (c.start_date))
142    print("end date %s " % (c.end_date))
143
144    server = mk_server(c)
145
146    # if data are to be retrieved, clean up any old grib files
147    if c.request == 0 or c.request == 2:
148        remove_old('*grb', c.inputdir)
149
150    # --------------  flux data ------------------------------------------------
151    start, end, datechunk = mk_dates(c, fluxes=True)
152    do_retrievement(c, server, start, end, datechunk, fluxes=True)
153
154    # --------------  non flux data --------------------------------------------
155    start, end, datechunk = mk_dates(c, fluxes=False)
156    do_retrievement(c, server, start, end, datechunk, fluxes=False)
157
158    return
159
160def write_reqheader(marsfile):
161    '''Writes header with column names into mars request file.
162
163    Parameters
164    ----------
165    marsfile : str
166        Path to the mars request file.
167
168    Return
169    ------
170
171    '''
172    MR = MarsRetrieval(None, None)
173    attrs = vars(MR).copy()
174    del attrs['server']
175    del attrs['public']
176    with open(marsfile, 'w') as f:
177        f.write('request_number' + ', ')
178        f.write(', '.join(str(key) for key in sorted(attrs.iterkeys())))
179        f.write('\n')
180
181    return
182
183def mk_server(c):
184    '''Creates server connection if ECMWF WebAPI is available.
185
186    Parameters
187    ----------
188    c : ControlFile
189        Contains all the parameters of CONTROL file and
190        command line.
191
192    Return
193    ------
194    server : ECMWFDataServer or ECMWFService
195        Connection to ECMWF server via python interface ECMWF WebAPI.
196
197    '''
198    if c.ecapi:
199        if c.public:
200            server = ecmwfapi.ECMWFDataServer()
201        else:
202            server = ecmwfapi.ECMWFService("mars")
203    else:
204        server = False
205
206    print('Using ECMWF WebAPI: ' + str(c.ecapi))
207
208    return server
209
210
211def check_dates_for_nonflux_fc_times(types, times):
212    '''
213    '''
214    for ty, ti in zip(types,times):
215        if ty.upper() == 'FC' and int(ti) == 18:
216            return True
217    return False
218
219
220def mk_dates(c, fluxes):
221    '''Prepares start and end date depending on flux or non flux data.
222
223    If forecast for maximum one day (upto 24h) are to be retrieved, then
224    collect accumulation data (flux data) with additional days in the
225    beginning and at the end (used for complete disaggregation of
226    original period)
227
228    If forecast data longer than 24h are to be retrieved, then
229    collect accumulation data (flux data) with the exact start and end date
230    (disaggregation will be done for the exact time period with
231    boundary conditions)
232
233    Since for basetime the extraction contains the 12 hours upfront,
234    if basetime is 0, the starting date has to be the day before and
235
236    Parameters
237    ----------
238    c : ControlFile
239        Contains all the parameters of CONTROL file and
240        command line.
241
242    fluxes : boolean, optional
243        Decides if the flux parameter settings are stored or
244        the rest of the parameter list.
245        Default value is False.
246
247    Return
248    ------
249    start : datetime
250        The start date of the retrieving data set.
251
252    end : datetime
253        The end date of the retrieving data set.
254
255    chunk : datetime
256        Time period in days for one single mars retrieval.
257
258    '''
259    start = datetime.strptime(c.start_date, '%Y%m%d')
260    end = datetime.strptime(c.end_date, '%Y%m%d')
261    chunk = timedelta(days=int(c.date_chunk))
262
263    if c.basetime == 0:
264        start = start - timedelta(days=1)
265
266    if not c.purefc and fluxes and not c.basetime == 0:
267        start = start - timedelta(days=1)
268        end = end + timedelta(days=1)
269
270    # if we have non-flux forecast data starting at 18 UTC
271    # we need to start retrieving data one day in advance
272    if not fluxes and check_dates_for_nonflux_fc_times(c.type, c.time):
273        start = start - timedelta(days=1)
274
275    return start, end, chunk
276
277def remove_old(pattern, inputdir):
278    '''Deletes old retrieval files from current input directory
279    matching the pattern.
280
281    Parameters
282    ----------
283    pattern : str
284        The sub string pattern which identifies the files to be deleted.
285
286    inputdir : str, optional
287        Path to the directory where the retrieved data is stored.
288
289    Return
290    ------
291
292    '''
293    print('... removing old content of ' + inputdir)
294
295    tobecleaned = UioFiles(inputdir, pattern)
296    tobecleaned.delete_files()
297
298    return
299
300
301def do_retrievement(c, server, start, end, delta_t, fluxes=False):
302    '''Divides the complete retrieval period in smaller chunks and
303    retrieves the data from MARS.
304
305    Parameters
306    ----------
307    c : ControlFile
308        Contains all the parameters of CONTROL file and
309        command line.
310
311    server : ECMWFService or ECMWFDataServer
312            The server connection to ECMWF.
313
314    start : datetime
315        The start date of the retrieval.
316
317    end : datetime
318        The end date of the retrieval.
319
320    delta_t : datetime
321        Delta_t + 1 is the maximal time period of a single
322        retrieval.
323
324    fluxes : boolean, optional
325        Decides if the flux parameters are to be retrieved or
326        the rest of the parameter list.
327        Default value is False.
328
329    Return
330    ------
331
332    '''
333
334    # since actual day also counts as one day,
335    # we only need to add datechunk - 1 days to retrieval
336    # for a period
337    delta_t_m1 = delta_t - timedelta(days=1)
338
339    day = start
340    while day <= end:
341        flexpart = EcFlexpart(c, fluxes)
342        tmpday = day + delta_t_m1
343        if tmpday < end:
344            dates = day.strftime("%Y%m%d") + "/to/" + \
345                    tmpday.strftime("%Y%m%d")
346        else:
347            dates = day.strftime("%Y%m%d") + "/to/" + \
348                    end.strftime("%Y%m%d")
349
350        print("... retrieve " + dates + " in dir " + c.inputdir)
351
352        try:
353            flexpart.retrieve(server, dates, c.public, c.request, c.inputdir)
354        except IOError:
355            my_error(c.mailfail, 'MARS request failed')
356
357        day += delta_t
358
359    return
360
361if __name__ == "__main__":
362    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG