source: flex_extract.git/source/python/mods/get_mars_data.py @ 2e62398

ctbtodev
Last change on this file since 2e62398 was 96e1533, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

redefined test data dir and completed unittests for tools module

  • Property mode set to 100755
File size: 8.5 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - moved the getEIdata program into a function "get_mars_data"
12#        - moved the AgurmentParser into a seperate function
13#        - adatpted the function for the use in flex_extract
14#        - renamed file to get_mars_data
15#
16#    February 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added structured documentation
19#        - minor changes in programming style for consistence
20#        - added function main and moved function calls vom __main__ there
21#          (necessary for better documentation with docstrings for later
22#          online documentation)
23#        - use of UIFiles class for file selection and deletion
24#
25#
26# @License:
27#    (C) Copyright 2014-2018.
28#
29#    This software is licensed under the terms of the Apache Licence Version 2.0
30#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
31#
32# @Program Functionality:
33#    This program can be used as a module in the whole flex_extract process
34#    or can be run by itself to just extract MARS data from ECMWF. To do so,
35#    a couple of necessary parameters has to be passed with the program call.
36#    See documentation for more details.
37#
38# @Program Content:
39#    - main
40#    - get_mars_data
41#    - do_retrievement
42#
43#*******************************************************************************
44"""ToDo: Name of litte program
45
46ToDo: Add desccription
47
48ToDo: Add Conditions
49
50This script requires that `...` be installed within the Python
51environment you are running this script in.
52
53This file can also be imported as a module and contains the following
54functions:
55
56    * get_mars_data -
57    * do_retrievement -
58    * main - the main function of the script
59"""
60# ------------------------------------------------------------------------------
61# MODULES
62# ------------------------------------------------------------------------------
63import os
64import sys
65import inspect
66from datetime import datetime, timedelta
67
68# software specific classes and modules from flex_extract
69sys.path.append(os.path.dirname(os.path.abspath(
70    inspect.getfile(inspect.currentframe()))) + '/../')
71import _config
72from tools import (my_error, normal_exit, get_cmdline_arguments,
73                   read_ecenv, make_dir)
74from classes.EcFlexpart import EcFlexpart
75from classes.UioFiles import UioFiles
76
77try:
78    ecapi = True
79    import ecmwfapi
80except ImportError:
81    ecapi = False
82# ------------------------------------------------------------------------------
83# FUNCTION
84# ------------------------------------------------------------------------------
85def main():
86    '''Controls the program to get data out of mars.
87
88    This is done if it is called directly from command line.
89    Then it also takes program call arguments and control file input.
90
91    Parameters
92    ----------
93
94    Return
95    ------
96
97    '''
98
99    args = get_cmdline_arguments()
100    c = ControlFile(args.controlfile)
101
102    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
103    c.assign_args_to_control(args)
104    c.assign_envs_to_control(env_parameter)
105    c.check_conditions(args.queue)
106
107    get_mars_data(c)
108    normal_exit(c.mailops, c.queue, 'Done!')
109
110    return
111
112def get_mars_data(c):
113    '''Retrieves the EC data needed for a FLEXPART simulation.
114
115    Start and end dates for retrieval period is set. Retrievals
116    are divided into smaller periods if necessary and datechunk parameter
117    is set.
118
119    Parameters
120    ----------
121    c : :obj:`ControlFile`
122        Contains all the parameters of CONTROL file and
123        command line.
124
125    Return
126    ------
127
128    '''
129
130    if not os.path.exists(c.inputdir):
131        make_dir(c.inputdir)
132
133    if c.request == 0 or c.request == 2:
134        print("Retrieving EC data!")
135    elif c.request == 1:
136        print("Printing mars requests!")
137
138    print("start date %s " % (c.start_date))
139    print("end date %s " % (c.end_date))
140
141    if ecapi:
142        if c.public:
143            server = ecmwfapi.ECMWFDataServer()
144        else:
145            server = ecmwfapi.ECMWFService("mars")
146    else:
147        server = False
148
149    c.ecapi = ecapi
150    print('Using ECMWF WebAPI: ' + str(c.ecapi))
151
152    # basetime geht rückwärts
153
154    # if basetime 00
155    # dann wird von 12 am vortag bis 00 am start tag geholt
156    # aber ohne 12 selbst sondern 12 + step
157
158    # if basetime 12
159    # dann wird von 00 + step bis 12 am start tag geholt
160
161    # purer forecast wird vorwärts bestimmt.
162    # purer forecast mode ist dann wenn  größer 24 stunden
163    # wie kann das noch festgestellt werden ????
164    # nur FC und steps mehr als 24 ?
165    # die einzige problematik beim reinen forecast ist die benennung der files!
166    # also sobald es Tagesüberschneidungen gibt
167    # allerdings ist das relevant und ersichtlich an den NICHT FLUSS DATEN
168
169    start = datetime.strptime(c.start_date, '%Y%m%d')
170    end = datetime.strptime(c.end_date, '%Y%m%d')
171    # time period for one single retrieval
172    datechunk = timedelta(days=int(c.date_chunk))
173
174    if c.basetime == '00':
175        start = start - timedelta(days=1)
176
177    if c.maxstep <= 24:
178        startm1 = start - timedelta(days=1)
179
180    if c.basetime == '00' or c.basetime == '12':
181        # endp1 = end + timedelta(days=1)
182        endp1 = end
183    else:
184        # endp1 = end + timedelta(days=2)
185        endp1 = end + timedelta(days=1)
186
187    # --------------  flux data ------------------------------------------------
188    if c.request == 0 or c.request == 2:
189        print('... removing old flux content of ' + c.inputdir)
190        tobecleaned = UioFiles(c.inputdir,
191                               '*_acc_*.' + str(os.getppid()) + '.*.grb')
192        tobecleaned.delete_files()
193
194    # if forecast for maximum one day (upto 24h) are to be retrieved,
195    # collect accumulation data (flux data)
196    # with additional days in the beginning and at the end
197    # (used for complete disaggregation of original period)
198    if c.maxstep <= 24:
199        do_retrievement(c, server, startm1, endp1, datechunk, fluxes=True)
200
201    # if forecast data longer than 24h are to be retrieved,
202    # collect accumulation data (flux data)
203    # with the exact start and end date
204    # (disaggregation will be done for the
205    # exact time period with boundary conditions)
206    else:
207        do_retrievement(c, server, start, end, datechunk, fluxes=True)
208
209    # --------------  non flux data --------------------------------------------
210    if c.request == 0 or c.request == 2:
211        print('... removing old non flux content of ' + c.inputdir)
212        tobecleaned = UioFiles(c.inputdir,
213                               '*__*.' + str(os.getppid()) + '.*.grb')
214        tobecleaned.delete_files()
215
216    do_retrievement(c, server, start, end, datechunk, fluxes=False)
217
218    return
219
220def do_retrievement(c, server, start, end, delta_t, fluxes=False):
221    '''Divides the complete retrieval period in smaller chunks and
222    retrieves the data from MARS.
223
224    Parameters
225    ----------
226    c : :obj:`ControlFile`
227        Contains all the parameters of CONTROL file and
228        command line.
229
230    server : :obj:`ECMWFService`
231            The server connection to ECMWF.
232
233    start : :obj:`datetime`
234        The start date of the retrieval.
235
236    end : :obj:`datetime`
237        The end date of the retrieval.
238
239    delta_t : :obj:`datetime`
240        Delta_t + 1 is the maximal time period of a single
241        retrieval.
242
243    fluxes : :obj:`boolean`, optional
244        Decides if the flux parameters are to be retrieved or
245        the rest of the parameter list.
246        Default value is False.
247
248    Return
249    ------
250
251    '''
252
253    # since actual day also counts as one day,
254    # we only need to add datechunk - 1 days to retrieval
255    # for a period
256    delta_t_m1 = delta_t - timedelta(days=1)
257
258    day = start
259    while day <= end:
260        flexpart = EcFlexpart(c, fluxes)
261        tmpday = day + delta_t_m1
262        if tmpday < end:
263            dates = day.strftime("%Y%m%d") + "/to/" + \
264                    tmpday.strftime("%Y%m%d")
265        else:
266            dates = day.strftime("%Y%m%d") + "/to/" + \
267                    end.strftime("%Y%m%d")
268
269        print("... retrieve " + dates + " in dir " + c.inputdir)
270
271        try:
272            flexpart.retrieve(server, dates, c.public, c.request, c.inputdir)
273        except IOError:
274            my_error(c.mailfail, 'MARS request failed')
275
276        day += delta_t
277
278    return
279
280if __name__ == "__main__":
281    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG