source: flex_extract.git/source/python/mods/get_mars_data.py @ d4696e0

ctbtodev
Last change on this file since d4696e0 was d4696e0, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

bugfix retrievement with basetime parameter

  • Property mode set to 100755
File size: 10.1 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - moved the getEIdata program into a function "get_mars_data"
12#        - moved the AgurmentParser into a seperate function
13#        - adatpted the function for the use in flex_extract
14#        - renamed file to get_mars_data
15#
16#    February 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added structured documentation
19#        - minor changes in programming style for consistence
20#        - added function main and moved function calls vom __main__ there
21#          (necessary for better documentation with docstrings for later
22#          online documentation)
23#        - use of UIFiles class for file selection and deletion
24#        - seperated get_mars_data function into several smaller pieces:
25#          write_reqheader, mk_server, mk_dates, remove_old, do_retrievment
26#
27# @License:
28#    (C) Copyright 2014-2019.
29#    Anne Philipp, Leopold Haimberger
30#
31#    This work is licensed under the Creative Commons Attribution 4.0
32#    International License. To view a copy of this license, visit
33#    http://creativecommons.org/licenses/by/4.0/ or send a letter to
34#    Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
35#*******************************************************************************
36'''This script extracts MARS data from ECMWF servers.
37
38At first, the necessary parameters from command line and CONTROL files are
39extracted. They define the data set to be extracted from MARS.
40
41This file can also be imported as a module and contains the following
42functions:
43
44    * main - the main function of the script
45    * get_mars_data - overall control of ECMWF data retrievment
46    * write_reqheader - writes the header into the mars_request file
47    * mk_server - creates the server connection to ECMWF servers
48    * mk_dates - defines the start and end date
49    * remove_old - deletes old retrieved grib files
50    * do_retrievement - creates individual retrievals
51
52Type: get_mars_data.py --help
53to get information about command line parameters.
54Read the documentation for usage instructions.
55'''
56# ------------------------------------------------------------------------------
57# MODULES
58# ------------------------------------------------------------------------------
59import os
60import sys
61import inspect
62from datetime import datetime, timedelta
63
64# software specific classes and modules from flex_extract
65# add path to local main python path for flex_extract to get full access
66sys.path.append(os.path.dirname(os.path.abspath(
67    inspect.getfile(inspect.currentframe()))) + '/../')
68import _config
69from tools import (my_error, normal_exit, get_cmdline_args,
70                   read_ecenv, make_dir)
71from classes.EcFlexpart import EcFlexpart
72from classes.UioFiles import UioFiles
73from classes.MarsRetrieval import MarsRetrieval
74
75try:
76    ecapi = True
77    import ecmwfapi
78except ImportError:
79    ecapi = False
80# ------------------------------------------------------------------------------
81# FUNCTION
82# ------------------------------------------------------------------------------
83def main():
84    '''Controls the program to get data out of mars.
85
86    This is done if it is called directly from command line.
87    Then it also takes program call arguments and control file input.
88
89    Parameters
90    ----------
91
92    Return
93    ------
94
95    '''
96
97    args = get_cmdline_args()
98    c = ControlFile(args.controlfile)
99
100    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
101    c.assign_args_to_control(args)
102    c.assign_envs_to_control(env_parameter)
103    c.check_conditions(args.queue)
104
105    get_mars_data(c)
106    normal_exit(c.mailops, c.queue, 'Done!')
107
108    return
109
110def get_mars_data(c):
111    '''Retrieves the EC data needed for a FLEXPART simulation.
112
113    Start and end dates for retrieval period is set. Retrievals
114    are divided into smaller periods if necessary and datechunk parameter
115    is set.
116
117    Parameters
118    ----------
119    c : ControlFile
120        Contains all the parameters of CONTROL file and
121        command line.
122
123    Return
124    ------
125
126    '''
127    c.ecapi = ecapi
128
129    if not os.path.exists(c.inputdir):
130        make_dir(c.inputdir)
131
132    if c.request == 0:
133        print("Retrieving EC data!")
134    else:
135        if c.request == 1:
136            print("Printing mars requests!")
137        elif c.request == 2:
138            print("Retrieving EC data and printing mars request!")
139        write_reqheader(os.path.join(c.inputdir, _config.FILE_MARS_REQUESTS))
140
141    print("start date %s " % (c.start_date))
142    print("end date %s " % (c.end_date))
143
144    server = mk_server(c)
145
146    # if data are to be retrieved, clean up any old grib files
147    if c.request == 0 or c.request == 2:
148        remove_old('*grb', c.inputdir)
149
150    # --------------  flux data ------------------------------------------------
151    start, end, datechunk = mk_dates(c, fluxes=True)
152    do_retrievement(c, server, start, end, datechunk, fluxes=True)
153
154    # --------------  non flux data --------------------------------------------
155    start, end, datechunk = mk_dates(c, fluxes=False)
156    do_retrievement(c, server, start, end, datechunk, fluxes=False)
157
158    return
159
160def write_reqheader(marsfile):
161    '''Writes header with column names into mars request file.
162
163    Parameters
164    ----------
165    marsfile : str
166        Path to the mars request file.
167
168    Return
169    ------
170
171    '''
172    MR = MarsRetrieval(None, None)
173    attrs = vars(MR).copy()
174    del attrs['server']
175    del attrs['public']
176    with open(marsfile, 'w') as f:
177        f.write('request_number' + ', ')
178        f.write(', '.join(str(key) for key in sorted(attrs.iterkeys())))
179        f.write('\n')
180
181    return
182
183def mk_server(c):
184    '''Creates server connection if ECMWF WebAPI is available.
185
186    Parameters
187    ----------
188    c : ControlFile
189        Contains all the parameters of CONTROL file and
190        command line.
191
192    Return
193    ------
194    server : ECMWFDataServer or ECMWFService
195        Connection to ECMWF server via python interface ECMWF WebAPI.
196
197    '''
198    if c.ecapi:
199        if c.public:
200            server = ecmwfapi.ECMWFDataServer()
201        else:
202            server = ecmwfapi.ECMWFService("mars")
203    else:
204        server = False
205
206    print('Using ECMWF WebAPI: ' + str(c.ecapi))
207
208    return server
209
210
211def mk_dates(c, fluxes):
212    '''Prepares start and end date depending on flux or non flux data.
213
214    If forecast for maximum one day (upto 24h) are to be retrieved, then
215    collect accumulation data (flux data) with additional days in the
216    beginning and at the end (used for complete disaggregation of
217    original period)
218
219    If forecast data longer than 24h are to be retrieved, then
220    collect accumulation data (flux data) with the exact start and end date
221    (disaggregation will be done for the exact time period with
222    boundary conditions)
223
224    Since for basetime the extraction contains the 12 hours upfront,
225    if basetime is 0, the starting date has to be the day before and
226
227    Parameters
228    ----------
229    c : ControlFile
230        Contains all the parameters of CONTROL file and
231        command line.
232
233    fluxes : boolean, optional
234        Decides if the flux parameter settings are stored or
235        the rest of the parameter list.
236        Default value is False.
237
238    Return
239    ------
240    start : datetime
241        The start date of the retrieving data set.
242
243    end : datetime
244        The end date of the retrieving data set.
245
246    chunk : datetime
247        Time period in days for one single mars retrieval.
248
249    '''
250    start = datetime.strptime(c.start_date, '%Y%m%d')
251    end = datetime.strptime(c.end_date, '%Y%m%d')
252    chunk = timedelta(days=int(c.date_chunk))
253
254    if c.basetime == 0:
255        start = start - timedelta(days=1)
256
257    if not c.purefc and fluxes and not c.basetime == 0:
258        start = start - timedelta(days=1)
259        end = end + timedelta(days=1)
260
261    return start, end, chunk
262
263def remove_old(pattern, inputdir):
264    '''Deletes old retrieval files from current input directory
265    matching the pattern.
266
267    Parameters
268    ----------
269    pattern : str
270        The sub string pattern which identifies the files to be deleted.
271
272    inputdir : str, optional
273        Path to the directory where the retrieved data is stored.
274
275    Return
276    ------
277
278    '''
279    print('... removing old content of ' + inputdir)
280
281    tobecleaned = UioFiles(inputdir, pattern)
282    tobecleaned.delete_files()
283
284    return
285
286
287def do_retrievement(c, server, start, end, delta_t, fluxes=False):
288    '''Divides the complete retrieval period in smaller chunks and
289    retrieves the data from MARS.
290
291    Parameters
292    ----------
293    c : ControlFile
294        Contains all the parameters of CONTROL file and
295        command line.
296
297    server : ECMWFService or ECMWFDataServer
298            The server connection to ECMWF.
299
300    start : datetime
301        The start date of the retrieval.
302
303    end : datetime
304        The end date of the retrieval.
305
306    delta_t : datetime
307        Delta_t + 1 is the maximal time period of a single
308        retrieval.
309
310    fluxes : boolean, optional
311        Decides if the flux parameters are to be retrieved or
312        the rest of the parameter list.
313        Default value is False.
314
315    Return
316    ------
317
318    '''
319
320    # since actual day also counts as one day,
321    # we only need to add datechunk - 1 days to retrieval
322    # for a period
323    delta_t_m1 = delta_t - timedelta(days=1)
324
325    day = start
326    while day <= end:
327        flexpart = EcFlexpart(c, fluxes)
328        tmpday = day + delta_t_m1
329        if tmpday < end:
330            dates = day.strftime("%Y%m%d") + "/to/" + \
331                    tmpday.strftime("%Y%m%d")
332        else:
333            dates = day.strftime("%Y%m%d") + "/to/" + \
334                    end.strftime("%Y%m%d")
335
336        print("... retrieve " + dates + " in dir " + c.inputdir)
337
338        try:
339            flexpart.retrieve(server, dates, c.public, c.request, c.inputdir)
340        except IOError:
341            my_error(c.mailfail, 'MARS request failed')
342
343        day += delta_t
344
345    return
346
347if __name__ == "__main__":
348    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG