source: flex_extract.git/source/python/mods/get_mars_data.py @ 403cbf1

ctbtodev
Last change on this file since 403cbf1 was 403cbf1, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

added header writing to marsrequest file and corrected bug in normalexit call in submit

  • Property mode set to 100755
File size: 9.1 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - moved the getEIdata program into a function "get_mars_data"
12#        - moved the AgurmentParser into a seperate function
13#        - adatpted the function for the use in flex_extract
14#        - renamed file to get_mars_data
15#
16#    February 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added structured documentation
19#        - minor changes in programming style for consistence
20#        - added function main and moved function calls vom __main__ there
21#          (necessary for better documentation with docstrings for later
22#          online documentation)
23#        - use of UIFiles class for file selection and deletion
24#
25#
26# @License:
27#    (C) Copyright 2014-2018.
28#
29#    This software is licensed under the terms of the Apache Licence Version 2.0
30#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
31#
32# @Program Functionality:
33#    This program can be used as a module in the whole flex_extract process
34#    or can be run by itself to just extract MARS data from ECMWF. To do so,
35#    a couple of necessary parameters has to be passed with the program call.
36#    See documentation for more details.
37#
38# @Program Content:
39#    - main
40#    - get_mars_data
41#    - do_retrievement
42#
43#*******************************************************************************
44"""ToDo: Name of litte program
45
46ToDo: Add desccription
47
48ToDo: Add Conditions
49
50This script requires that `...` be installed within the Python
51environment you are running this script in.
52
53This file can also be imported as a module and contains the following
54functions:
55
56    * get_mars_data -
57    * do_retrievement -
58    * main - the main function of the script
59"""
60# ------------------------------------------------------------------------------
61# MODULES
62# ------------------------------------------------------------------------------
63import os
64import sys
65import inspect
66from datetime import datetime, timedelta
67
68# software specific classes and modules from flex_extract
69sys.path.append(os.path.dirname(os.path.abspath(
70    inspect.getfile(inspect.currentframe()))) + '/../')
71import _config
72from tools import (my_error, normal_exit, get_cmdline_arguments,
73                   read_ecenv, make_dir)
74from classes.EcFlexpart import EcFlexpart
75from classes.UioFiles import UioFiles
76from classes.MarsRetrieval import MarsRetrieval
77
78try:
79    ecapi = True
80    import ecmwfapi
81except ImportError:
82    ecapi = False
83# ------------------------------------------------------------------------------
84# FUNCTION
85# ------------------------------------------------------------------------------
86def main():
87    '''Controls the program to get data out of mars.
88
89    This is done if it is called directly from command line.
90    Then it also takes program call arguments and control file input.
91
92    Parameters
93    ----------
94
95    Return
96    ------
97
98    '''
99
100    args = get_cmdline_arguments()
101    c = ControlFile(args.controlfile)
102
103    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
104    c.assign_args_to_control(args)
105    c.assign_envs_to_control(env_parameter)
106    c.check_conditions(args.queue)
107
108    get_mars_data(c)
109    normal_exit(c.mailops, c.queue, 'Done!')
110
111    return
112
113def get_mars_data(c):
114    '''Retrieves the EC data needed for a FLEXPART simulation.
115
116    Start and end dates for retrieval period is set. Retrievals
117    are divided into smaller periods if necessary and datechunk parameter
118    is set.
119
120    Parameters
121    ----------
122    c : :obj:`ControlFile`
123        Contains all the parameters of CONTROL file and
124        command line.
125
126    Return
127    ------
128
129    '''
130
131    if not os.path.exists(c.inputdir):
132        make_dir(c.inputdir)
133
134    if c.request == 0:
135        print("Retrieving EC data!")
136    else:
137        if c.request == 1:
138            print("Printing mars requests!")
139        elif c.request == 2:
140            print("Retrieving EC data and printing mars request!")
141        # first, write header with the mars parameter to file
142        # create a dummy MarsRetrieval to get parameter
143        MR = MarsRetrieval(None, None)
144        attrs = vars(MR).copy()
145        del attrs['server']
146        del attrs['public']
147        marsfile = os.path.join(c.inputdir, _config.FILE_MARS_REQUESTS)
148        with open(marsfile, 'w') as f:
149            f.write('request_number' + ', ')
150            f.write(', '.join(str(key) for key in sorted(attrs.iterkeys())))
151            f.write('\n')
152
153    print("start date %s " % (c.start_date))
154    print("end date %s " % (c.end_date))
155
156    if ecapi:
157        if c.public:
158            server = ecmwfapi.ECMWFDataServer()
159        else:
160            server = ecmwfapi.ECMWFService("mars")
161    else:
162        server = False
163
164    c.ecapi = ecapi
165    print('Using ECMWF WebAPI: ' + str(c.ecapi))
166
167    # basetime geht rückwärts
168
169    # if basetime 00
170    # dann wird von 12 am vortag bis 00 am start tag geholt
171    # aber ohne 12 selbst sondern 12 + step
172
173    # if basetime 12
174    # dann wird von 00 + step bis 12 am start tag geholt
175
176    # purer forecast wird vorwärts bestimmt.
177    # purer forecast mode ist dann wenn  größer 24 stunden
178    # wie kann das noch festgestellt werden ????
179    # nur FC und steps mehr als 24 ?
180    # die einzige problematik beim reinen forecast ist die benennung der files!
181    # also sobald es Tagesüberschneidungen gibt
182    # allerdings ist das relevant und ersichtlich an den NICHT FLUSS DATEN
183
184    start = datetime.strptime(c.start_date, '%Y%m%d')
185    end = datetime.strptime(c.end_date, '%Y%m%d')
186    # time period for one single retrieval
187    datechunk = timedelta(days=int(c.date_chunk))
188
189    if c.basetime == '00':
190        start = start - timedelta(days=1)
191
192    if c.maxstep <= 24:
193        startm1 = start - timedelta(days=1)
194
195    if c.basetime == '00' or c.basetime == '12':
196        # endp1 = end + timedelta(days=1)
197        endp1 = end
198    else:
199        # endp1 = end + timedelta(days=2)
200        endp1 = end + timedelta(days=1)
201
202    # --------------  flux data ------------------------------------------------
203    if c.request == 0 or c.request == 2:
204        print('... removing old flux content of ' + c.inputdir)
205        tobecleaned = UioFiles(c.inputdir,
206                               '*_acc_*.' + str(os.getppid()) + '.*.grb')
207        tobecleaned.delete_files()
208
209    # if forecast for maximum one day (upto 24h) are to be retrieved,
210    # collect accumulation data (flux data)
211    # with additional days in the beginning and at the end
212    # (used for complete disaggregation of original period)
213    if c.maxstep <= 24:
214        do_retrievement(c, server, startm1, endp1, datechunk, fluxes=True)
215
216    # if forecast data longer than 24h are to be retrieved,
217    # collect accumulation data (flux data)
218    # with the exact start and end date
219    # (disaggregation will be done for the
220    # exact time period with boundary conditions)
221    else:
222        do_retrievement(c, server, start, end, datechunk, fluxes=True)
223
224    # --------------  non flux data --------------------------------------------
225    if c.request == 0 or c.request == 2:
226        print('... removing old non flux content of ' + c.inputdir)
227        tobecleaned = UioFiles(c.inputdir,
228                               '*__*.' + str(os.getppid()) + '.*.grb')
229        tobecleaned.delete_files()
230
231    do_retrievement(c, server, start, end, datechunk, fluxes=False)
232
233    return
234
235def do_retrievement(c, server, start, end, delta_t, fluxes=False):
236    '''Divides the complete retrieval period in smaller chunks and
237    retrieves the data from MARS.
238
239    Parameters
240    ----------
241    c : :obj:`ControlFile`
242        Contains all the parameters of CONTROL file and
243        command line.
244
245    server : :obj:`ECMWFService`
246            The server connection to ECMWF.
247
248    start : :obj:`datetime`
249        The start date of the retrieval.
250
251    end : :obj:`datetime`
252        The end date of the retrieval.
253
254    delta_t : :obj:`datetime`
255        Delta_t + 1 is the maximal time period of a single
256        retrieval.
257
258    fluxes : :obj:`boolean`, optional
259        Decides if the flux parameters are to be retrieved or
260        the rest of the parameter list.
261        Default value is False.
262
263    Return
264    ------
265
266    '''
267
268    # since actual day also counts as one day,
269    # we only need to add datechunk - 1 days to retrieval
270    # for a period
271    delta_t_m1 = delta_t - timedelta(days=1)
272
273    day = start
274    while day <= end:
275        flexpart = EcFlexpart(c, fluxes)
276        tmpday = day + delta_t_m1
277        if tmpday < end:
278            dates = day.strftime("%Y%m%d") + "/to/" + \
279                    tmpday.strftime("%Y%m%d")
280        else:
281            dates = day.strftime("%Y%m%d") + "/to/" + \
282                    end.strftime("%Y%m%d")
283
284        print("... retrieve " + dates + " in dir " + c.inputdir)
285
286        try:
287            flexpart.retrieve(server, dates, c.public, c.request, c.inputdir)
288        except IOError:
289            my_error(c.mailfail, 'MARS request failed')
290
291        day += delta_t
292
293    return
294
295if __name__ == "__main__":
296    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG