source: flex_extract.git/python/pythontest/TestInstallTar/flex_extract_v7.1/python/get_mars_data.py @ 2fb99de

ctbtodev
Last change on this file since 2fb99de was 2fb99de, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

introduced config with path definitions and changed py files accordingly; Installation works; some tests were added for tarball making; Problems in submission to ecgate
  • Property mode set to 100755
File size: 10.1 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - moved the getEIdata program into a function "get_mars_data"
12#        - moved the AgurmentParser into a seperate function
13#        - adatpted the function for the use in flex_extract
14#        - renamed file to get_mars_data
15#
16#    February 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added structured documentation
19#        - minor changes in programming style for consistence
20#        - added function main and moved function calls vom __main__ there
21#          (necessary for better documentation with docstrings for later
22#          online documentation)
23#        - use of UIFiles class for file selection and deletion
24#
25#
26# @License:
27#    (C) Copyright 2014-2018.
28#
29#    This software is licensed under the terms of the Apache Licence Version 2.0
30#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
31#
32# @Program Functionality:
33#    This program can be used as a module in the whole flex_extract process
34#    or can be run by itself to just extract MARS data from ECMWF. To do so,
35#    a couple of necessary parameters has to be passed with the program call.
36#    See documentation for more details.
37#
38# @Program Content:
39#    - main
40#    - get_mars_data
41#    - do_retrievement
42#
43#*******************************************************************************
44
45# ------------------------------------------------------------------------------
46# MODULES
47# ------------------------------------------------------------------------------
48import os
49import sys
50import datetime
51import inspect
52try:
53    ecapi = True
54    import ecmwfapi
55except ImportError:
56    ecapi = False
57
58# software specific classes and modules from flex_extract
59from tools import my_error, normal_exit, get_cmdline_arguments, read_ecenv
60from EcFlexpart import EcFlexpart
61from UioFiles import UioFiles
62
63# add path to pythonpath so that python finds its buddies
64LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
65    inspect.getfile(inspect.currentframe())))
66if LOCAL_PYTHON_PATH not in sys.path:
67    sys.path.append(LOCAL_PYTHON_PATH)
68
69# ------------------------------------------------------------------------------
70# FUNCTION
71# ------------------------------------------------------------------------------
72def main():
73    '''
74    @Description:
75        If get_mars_data is called from command line, this function controls
76        the program flow and calls the argumentparser function and
77        the get_mars_data function for retrieving EC data.
78
79    @Input:
80        <nothing>
81
82    @Return:
83        <nothing>
84    '''
85
86    args = get_cmdline_arguments()
87
88    try:
89        c = ControlFile(args.controlfile)
90    except IOError:
91        try:
92            c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile)
93        except IOError:
94            print 'Could not read CONTROL file "' + args.controlfile + '"'
95            print 'Either it does not exist or its syntax is wrong.'
96            print 'Try "' + sys.argv[0].split('/')[-1] + \
97                  ' -h" to print usage information'
98            sys.exit(1)
99
100    env_parameter = read_ecenv(c.ecmwfdatadir + 'python/ECMWF_ENV')
101    c.assign_args_to_control(args, env_parameter)
102    c.assign_envs_to_control(env_parameter)
103    c.check_conditions()
104
105    get_mars_data(c)
106    normal_exit(c.mailfail, 'Done!')
107
108    return
109
110def get_mars_data(c):
111    '''
112    @Description:
113        Retrieves the EC data needed for a FLEXPART simulation.
114        Start and end dates for retrieval period is set. Retrievals
115        are divided into smaller periods if necessary and datechunk parameter
116        is set.
117
118    @Input:
119        c: instance of class ControlFile
120            Contains all the parameters of CONTROL file, which are e.g.:
121            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
122            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
123            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
124            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
125            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
126            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
127            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
128
129            For more information about format and content of the parameter
130            see documentation.
131
132    @Return:
133        <nothing>
134    '''
135
136    if not os.path.exists(c.inputdir):
137        os.makedirs(c.inputdir)
138
139    if c.request == 0 or c.request == 2:
140        print("Retrieving EC data!")
141    elif c.request == 1:
142        print("Printing mars requests!")
143
144    print("start date %s " % (c.start_date))
145    print("end date %s " % (c.end_date))
146
147    if ecapi:
148        server = ecmwfapi.ECMWFService("mars")
149    else:
150        server = False
151
152    c.ecapi = ecapi
153    print('Using ECMWF WebAPI: ' + str(c.ecapi))
154
155    # basetime geht rückwärts
156
157    # if basetime 00
158    # dann wird von 12 am vortag bis 00 am start tag geholt
159    # aber ohne 12 selbst sondern 12 + step
160
161    # if basetime 12
162    # dann wird von 00 + step bis 12 am start tag geholt
163
164    # purer forecast wird vorwärts bestimmt.
165    # purer forecast mode ist dann wenn  größer 24 stunden
166    # wie kann das noch festgestellt werden ????
167    # nur FC und steps mehr als 24 ?
168    # die einzige problematik beim reinen forecast ist die benennung der files!
169    # also sobald es Tagesüberschneidungen gibt
170    # allerdings ist das relevant und ersichtlich an den NICHT FLUSS DATEN
171
172
173    # set start date of retrieval period
174    start = datetime.date(year=int(c.start_date[:4]),
175                          month=int(c.start_date[4:6]),
176                          day=int(c.start_date[6:]))
177    startm1 = start - datetime.timedelta(days=1)
178
179    # set end date of retrieval period
180    end = datetime.date(year=int(c.end_date[:4]),
181                        month=int(c.end_date[4:6]),
182                        day=int(c.end_date[6:]))
183
184    # set time period for one single retrieval
185    datechunk = datetime.timedelta(days=int(c.date_chunk))
186
187    if c.basetime == '00':
188        start = startm1
189
190    if c.basetime == '00' or c.basetime == '12':
191        # endp1 = end + datetime.timedelta(days=1)
192        endp1 = end
193    else:
194        # endp1 = end + datetime.timedelta(days=2)
195        endp1 = end + datetime.timedelta(days=1)
196
197    # --------------  flux data ------------------------------------------------
198    if c.request == 0 or c.request == 2:
199        print('... removing old flux content of ' + c.inputdir)
200        tobecleaned = UioFiles(c.inputdir,
201                               '*_acc_*.' + str(os.getppid()) + '.*.grb')
202        tobecleaned.delete_files()
203
204    # if forecast for maximum one day (upto 24h) are to be retrieved,
205    # collect accumulation data (flux data)
206    # with additional days in the beginning and at the end
207    # (used for complete disaggregation of original period)
208    if c.maxstep <= 24:
209        do_retrievement(c, server, startm1, endp1, datechunk, fluxes=True)
210
211    # if forecast data longer than 24h are to be retrieved,
212    # collect accumulation data (flux data)
213    # with the exact start and end date
214    # (disaggregation will be done for the
215    # exact time period with boundary conditions)
216    else:
217        do_retrievement(c, server, start, end, datechunk, fluxes=True)
218
219    # --------------  non flux data --------------------------------------------
220    if c.request == 0 or c.request == 2:
221        print('... removing old non flux content of ' + c.inputdir)
222        tobecleaned = UioFiles(c.inputdir,
223                               '*__*.' + str(os.getppid()) + '.*.grb')
224        tobecleaned.delete_files()
225
226    do_retrievement(c, server, start, end, datechunk, fluxes=False)
227
228    return
229
230def do_retrievement(c, server, start, end, delta_t, fluxes=False):
231    '''
232    @Description:
233        Divides the complete retrieval period in smaller chunks and
234        retrieves the data from MARS.
235
236    @Input:
237        c: instance of ControlFile
238            Contains all the parameters of CONTROL file, which are e.g.:
239            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
240            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
241            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
242            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
243            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
244            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
245            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
246
247            For more information about format and content of the parameter
248            see documentation.
249
250        server: instance of ECMWFService
251            The server connection to ECMWF
252
253        start: instance of datetime
254            The start date of the retrieval.
255
256        end: instance of datetime
257            The end date of the retrieval.
258
259        delta_t: instance of datetime
260            Delta_t +1 is the maximal time period of a single
261            retrieval.
262
263        fluxes: boolean, optional
264            Decides if the flux parameters are to be retrieved or
265            the rest of the parameter list.
266            Default value is False.
267
268    @Return:
269        <nothing>
270    '''
271
272    # since actual day also counts as one day,
273    # we only need to add datechunk - 1 days to retrieval
274    # for a period
275    delta_t_m1 = delta_t - datetime.timedelta(days=1)
276
277    day = start
278    while day <= end:
279        flexpart = EcFlexpart(c, fluxes)
280        tmpday = day + delta_t_m1
281        if tmpday < end:
282            dates = day.strftime("%Y%m%d") + "/to/" + \
283                    tmpday.strftime("%Y%m%d")
284        else:
285            dates = day.strftime("%Y%m%d") + "/to/" + \
286                    end.strftime("%Y%m%d")
287
288
289        print("... retrieve " + dates + " in dir " + c.inputdir)
290
291        try:
292            flexpart.retrieve(server, dates, c.request, c.inputdir)
293        except IOError:
294            my_error(c.mailfail, 'MARS request failed')
295
296        day += delta_t
297
298    return
299
300if __name__ == "__main__":
301    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG