source: flex_extract.git/python/get_mars_data.py @ 70a0bec

ctbtodev
Last change on this file since 70a0bec was 54a8a01, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

restructuring, documentations and bug fixes

  • Property mode set to 100755
File size: 9.8 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - moved the getEIdata program into a function "get_mars_data"
12#        - moved the AgurmentParser into a seperate function
13#        - adatpted the function for the use in flex_extract
14#        - renamed file to get_mars_data
15#
16#    February 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added structured documentation
19#        - minor changes in programming style for consistence
20#        - added function main and moved function calls vom __main__ there
21#          (necessary for better documentation with docstrings for later
22#          online documentation)
23#        - use of UIFiles class for file selection and deletion
24#
25#
26# @License:
27#    (C) Copyright 2014-2018.
28#
29#    This software is licensed under the terms of the Apache Licence Version 2.0
30#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
31#
32# @Program Functionality:
33#    This program can be used as a module in the whole flex_extract process
34#    or can be run by itself to just extract MARS data from ECMWF. To do so,
35#    a couple of necessary parameters has to be passed with the program call.
36#    See documentation for more details.
37#
38# @Program Content:
39#    - main
40#    - get_mars_data
41#    - do_retrievement
42#
43#*******************************************************************************
44
45# ------------------------------------------------------------------------------
46# MODULES
47# ------------------------------------------------------------------------------
48import os
49import sys
50import datetime
51import inspect
52try:
53    ecapi = True
54    import ecmwfapi
55except ImportError:
56    ecapi = False
57
58# software specific classes and modules from flex_extract
59from tools import my_error, normal_exit, get_cmdline_arguments, read_ecenv
60from EcFlexpart import EcFlexpart
61from UioFiles import UioFiles
62
63# add path to pythonpath so that python finds its buddies
64LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
65    inspect.getfile(inspect.currentframe())))
66if LOCAL_PYTHON_PATH not in sys.path:
67    sys.path.append(LOCAL_PYTHON_PATH)
68
69# ------------------------------------------------------------------------------
70# FUNCTION
71# ------------------------------------------------------------------------------
72def main():
73    '''
74    @Description:
75        If get_mars_data is called from command line, this function controls
76        the program flow and calls the argumentparser function and
77        the get_mars_data function for retrieving EC data.
78
79    @Input:
80        <nothing>
81
82    @Return:
83        <nothing>
84    '''
85
86    args = get_cmdline_arguments()
87
88    try:
89        c = ControlFile(args.controlfile)
90    except IOError:
91        try:
92            c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile)
93        except IOError:
94            print 'Could not read CONTROL file "' + args.controlfile + '"'
95            print 'Either it does not exist or its syntax is wrong.'
96            print 'Try "' + sys.argv[0].split('/')[-1] + \
97                  ' -h" to print usage information'
98            sys.exit(1)
99
100    env_parameter = read_ecenv(c.ecmwfdatadir + 'python/ECMWF_ENV')
101    c.assign_args_to_control(args, env_parameter)
102    c.assign_envs_to_control(env_parameter)
103    c.check_conditions()
104
105    get_mars_data(c)
106    normal_exit(c.mailfail, 'Done!')
107
108    return
109
110def get_mars_data(c):
111    '''
112    @Description:
113        Retrieves the EC data needed for a FLEXPART simulation.
114        Start and end dates for retrieval period is set. Retrievals
115        are divided into smaller periods if necessary and datechunk parameter
116        is set.
117
118    @Input:
119        c: instance of class ControlFile
120            Contains all the parameters of CONTROL file, which are e.g.:
121            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
122            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
123            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
124            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
125            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
126            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
127            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
128
129            For more information about format and content of the parameter
130            see documentation.
131
132    @Return:
133        <nothing>
134    '''
135
136    if not os.path.exists(c.inputdir):
137        os.makedirs(c.inputdir)
138
139    print "Retrieving EC data!"
140    print "start date %s " % (c.start_date)
141    print "end date %s " % (c.end_date)
142
143    if ecapi:
144        server = ecmwfapi.ECMWFService("mars")
145    else:
146        server = False
147
148    c.ecapi = ecapi
149    print 'ecapi: ', c.ecapi
150
151    # basetime geht rückwärts
152
153    # if basetime 00
154    # dann wird von 12 am vortag bis 00 am start tag geholt
155    # aber ohne 12 selbst sondern 12 + step
156
157    # if basetime 12
158    # dann wird von 00 + step bis 12 am start tag geholt
159
160    # purer forecast wird vorwärts bestimmt.
161    # purer forecast mode ist dann wenn  größer 24 stunden
162    # wie kann das noch festgestellt werden ????
163    # nur FC und steps mehr als 24 ?
164    # die einzige problematik beim reinen forecast ist die benennung der files!
165    # also sobald es Tagesüberschneidungen gibt
166    # allerdings ist das relevant und ersichtlich an den NICHT FLUSS DATEN
167
168
169    # set start date of retrieval period
170    start = datetime.date(year=int(c.start_date[:4]),
171                          month=int(c.start_date[4:6]),
172                          day=int(c.start_date[6:]))
173    startm1 = start - datetime.timedelta(days=1)
174
175    # set end date of retrieval period
176    end = datetime.date(year=int(c.end_date[:4]),
177                        month=int(c.end_date[4:6]),
178                        day=int(c.end_date[6:]))
179
180    # set time period for one single retrieval
181    datechunk = datetime.timedelta(days=int(c.date_chunk))
182
183    if c.basetime == '00':
184        start = startm1
185
186    if c.basetime == '00' or c.basetime == '12':
187        # endp1 = end + datetime.timedelta(days=1)
188        endp1 = end
189    else:
190        # endp1 = end + datetime.timedelta(days=2)
191        endp1 = end + datetime.timedelta(days=1)
192
193    # --------------  flux data ------------------------------------------------
194    print 'removing old flux content of ' + c.inputdir
195    tobecleaned = UioFiles(c.inputdir,
196                           '*_acc_*.' + str(os.getppid()) + '.*.grb')
197    tobecleaned.delete_files()
198
199    # if forecast for maximum one day (upto 24h) are to be retrieved,
200    # collect accumulation data (flux data)
201    # with additional days in the beginning and at the end
202    # (used for complete disaggregation of original period)
203    if c.maxstep <= 24:
204        do_retrievement(c, server, startm1, endp1, datechunk, fluxes=True)
205
206    # if forecast data longer than 24h are to be retrieved,
207    # collect accumulation data (flux data)
208    # with the exact start and end date
209    # (disaggregation will be done for the
210    # exact time period with boundary conditions)
211    else:
212        do_retrievement(c, server, start, end, datechunk, fluxes=True)
213
214    # --------------  non flux data --------------------------------------------
215    print 'removing old non flux content of ' + c.inputdir
216    tobecleaned = UioFiles(c.inputdir,
217                           '*__*.' + str(os.getppid()) + '.*.grb')
218    tobecleaned.delete_files()
219
220    do_retrievement(c, server, start, end, datechunk, fluxes=False)
221
222    return
223
224def do_retrievement(c, server, start, end, delta_t, fluxes=False):
225    '''
226    @Description:
227        Divides the complete retrieval period in smaller chunks and
228        retrieves the data from MARS.
229
230    @Input:
231        c: instance of ControlFile
232            Contains all the parameters of CONTROL file, which are e.g.:
233            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
234            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
235            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
236            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
237            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
238            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
239            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
240
241            For more information about format and content of the parameter
242            see documentation.
243
244        server: instance of ECMWFService
245            The server connection to ECMWF
246
247        start: instance of datetime
248            The start date of the retrieval.
249
250        end: instance of datetime
251            The end date of the retrieval.
252
253        delta_t: instance of datetime
254            Delta_t +1 is the maximal time period of a single
255            retrieval.
256
257        fluxes: boolean, optional
258            Decides if the flux parameters are to be retrieved or
259            the rest of the parameter list.
260            Default value is False.
261
262    @Return:
263        <nothing>
264    '''
265
266    # since actual day also counts as one day,
267    # we only need to add datechunk - 1 days to retrieval
268    # for a period
269    delta_t_m1 = delta_t - datetime.timedelta(days=1)
270
271    day = start
272    while day <= end:
273        flexpart = EcFlexpart(c, fluxes)
274        tmpday = day + delta_t_m1
275        if tmpday < end:
276            dates = day.strftime("%Y%m%d") + "/to/" + \
277                    tmpday.strftime("%Y%m%d")
278        else:
279            dates = day.strftime("%Y%m%d") + "/to/" + \
280                    end.strftime("%Y%m%d")
281
282        print "retrieve " + dates + " in dir " + c.inputdir
283
284        try:
285            flexpart.retrieve(server, dates, c.inputdir)
286        except IOError:
287            my_error(c.mailfail, 'MARS request failed')
288
289        day += delta_t
290
291    return
292
293if __name__ == "__main__":
294    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG