source: flex_extract.git/source/python/mods/get_mars_data.py @ 4971f63

dev
Last change on this file since 4971f63 was 4971f63, checked in by Anne Philipp <anne.philipp@…>, 11 months ago

eliminated some redundancy and exchanged CONTROL2 with the config-filename

  • Property mode set to 100755
File size: 9.5 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - moved the getEIdata program into a function "get_mars_data"
12#        - moved the AgurmentParser into a seperate function
13#        - adatpted the function for the use in flex_extract
14#        - renamed file to get_mars_data
15#
16#    February 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added structured documentation
19#        - minor changes in programming style for consistence
20#        - added function main and moved function calls vom __main__ there
21#          (necessary for better documentation with docstrings for later
22#          online documentation)
23#        - use of UIFiles class for file selection and deletion
24#
25#
26# @License:
27#    (C) Copyright 2014-2018.
28#
29#    This software is licensed under the terms of the Apache Licence Version 2.0
30#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
31#
32# @Program Functionality:
33#    This program can be used as a module in the whole flex_extract process
34#    or can be run by itself to just extract MARS data from ECMWF. To do so,
35#    a couple of necessary parameters has to be passed with the program call.
36#    See documentation for more details.
37#
38# @Program Content:
39#    - main
40#    - get_mars_data
41#    - do_retrievement
42#
43#*******************************************************************************
44
45# ------------------------------------------------------------------------------
46# MODULES
47# ------------------------------------------------------------------------------
48import os
49import sys
50import datetime
51import inspect
52try:
53    ecapi = True
54    import ecmwfapi
55except ImportError:
56    ecapi = False
57
58# software specific classes and modules from flex_extract
59import _config
60from tools import my_error, normal_exit, get_cmdline_arguments, read_ecenv
61from classes.EcFlexpart import EcFlexpart
62from classes.UioFiles import UioFiles
63# ------------------------------------------------------------------------------
64# FUNCTION
65# ------------------------------------------------------------------------------
66def main():
67    '''
68    @Description:
69        If get_mars_data is called from command line, this function controls
70        the program flow and calls the argumentparser function and
71        the get_mars_data function for retrieving EC data.
72
73    @Input:
74        <nothing>
75
76    @Return:
77        <nothing>
78    '''
79
80    args = get_cmdline_arguments()
81    c = ControlFile(args.controlfile)
82
83    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
84    c.assign_args_to_control(args, env_parameter)
85    c.assign_envs_to_control(env_parameter)
86    c.check_conditions(args.queue)
87
88    get_mars_data(c)
89    normal_exit(c.mailfail, 'Done!')
90
91    return
92
93def get_mars_data(c):
94    '''
95    @Description:
96        Retrieves the EC data needed for a FLEXPART simulation.
97        Start and end dates for retrieval period is set. Retrievals
98        are divided into smaller periods if necessary and datechunk parameter
99        is set.
100
101    @Input:
102        c: instance of class ControlFile
103            Contains all the parameters of CONTROL file, which are e.g.:
104            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
105            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
106            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
107            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
108            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
109            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
110            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
111
112            For more information about format and content of the parameter
113            see documentation.
114
115    @Return:
116        <nothing>
117    '''
118
119    if not os.path.exists(c.inputdir):
120        os.makedirs(c.inputdir)
121
122    if c.request == 0 or c.request == 2:
123        print("Retrieving EC data!")
124    elif c.request == 1:
125        print("Printing mars requests!")
126
127    print("start date %s " % (c.start_date))
128    print("end date %s " % (c.end_date))
129
130    if ecapi:
131        server = ecmwfapi.ECMWFService("mars")
132    else:
133        server = False
134
135    c.ecapi = ecapi
136    print('Using ECMWF WebAPI: ' + str(c.ecapi))
137
138    # basetime geht rückwärts
139
140    # if basetime 00
141    # dann wird von 12 am vortag bis 00 am start tag geholt
142    # aber ohne 12 selbst sondern 12 + step
143
144    # if basetime 12
145    # dann wird von 00 + step bis 12 am start tag geholt
146
147    # purer forecast wird vorwärts bestimmt.
148    # purer forecast mode ist dann wenn  größer 24 stunden
149    # wie kann das noch festgestellt werden ????
150    # nur FC und steps mehr als 24 ?
151    # die einzige problematik beim reinen forecast ist die benennung der files!
152    # also sobald es Tagesüberschneidungen gibt
153    # allerdings ist das relevant und ersichtlich an den NICHT FLUSS DATEN
154
155
156    # set start date of retrieval period
157    start = datetime.date(year=int(c.start_date[:4]),
158                          month=int(c.start_date[4:6]),
159                          day=int(c.start_date[6:]))
160    startm1 = start - datetime.timedelta(days=1)
161
162    # set end date of retrieval period
163    end = datetime.date(year=int(c.end_date[:4]),
164                        month=int(c.end_date[4:6]),
165                        day=int(c.end_date[6:]))
166
167    # set time period for one single retrieval
168    datechunk = datetime.timedelta(days=int(c.date_chunk))
169
170    if c.basetime == '00':
171        start = startm1
172
173    if c.basetime == '00' or c.basetime == '12':
174        # endp1 = end + datetime.timedelta(days=1)
175        endp1 = end
176    else:
177        # endp1 = end + datetime.timedelta(days=2)
178        endp1 = end + datetime.timedelta(days=1)
179
180    # --------------  flux data ------------------------------------------------
181    if c.request == 0 or c.request == 2:
182        print('... removing old flux content of ' + c.inputdir)
183        tobecleaned = UioFiles(c.inputdir,
184                               '*_acc_*.' + str(os.getppid()) + '.*.grb')
185        tobecleaned.delete_files()
186
187    # if forecast for maximum one day (upto 24h) are to be retrieved,
188    # collect accumulation data (flux data)
189    # with additional days in the beginning and at the end
190    # (used for complete disaggregation of original period)
191    if c.maxstep <= 24:
192        do_retrievement(c, server, startm1, endp1, datechunk, fluxes=True)
193
194    # if forecast data longer than 24h are to be retrieved,
195    # collect accumulation data (flux data)
196    # with the exact start and end date
197    # (disaggregation will be done for the
198    # exact time period with boundary conditions)
199    else:
200        do_retrievement(c, server, start, end, datechunk, fluxes=True)
201
202    # --------------  non flux data --------------------------------------------
203    if c.request == 0 or c.request == 2:
204        print('... removing old non flux content of ' + c.inputdir)
205        tobecleaned = UioFiles(c.inputdir,
206                               '*__*.' + str(os.getppid()) + '.*.grb')
207        tobecleaned.delete_files()
208
209    do_retrievement(c, server, start, end, datechunk, fluxes=False)
210
211    return
212
213def do_retrievement(c, server, start, end, delta_t, fluxes=False):
214    '''
215    @Description:
216        Divides the complete retrieval period in smaller chunks and
217        retrieves the data from MARS.
218
219    @Input:
220        c: instance of ControlFile
221            Contains all the parameters of CONTROL file, which are e.g.:
222            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
223            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
224            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
225            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
226            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
227            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
228            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
229
230            For more information about format and content of the parameter
231            see documentation.
232
233        server: instance of ECMWFService
234            The server connection to ECMWF
235
236        start: instance of datetime
237            The start date of the retrieval.
238
239        end: instance of datetime
240            The end date of the retrieval.
241
242        delta_t: instance of datetime
243            Delta_t +1 is the maximal time period of a single
244            retrieval.
245
246        fluxes: boolean, optional
247            Decides if the flux parameters are to be retrieved or
248            the rest of the parameter list.
249            Default value is False.
250
251    @Return:
252        <nothing>
253    '''
254
255    # since actual day also counts as one day,
256    # we only need to add datechunk - 1 days to retrieval
257    # for a period
258    delta_t_m1 = delta_t - datetime.timedelta(days=1)
259
260    day = start
261    while day <= end:
262        flexpart = EcFlexpart(c, fluxes)
263        tmpday = day + delta_t_m1
264        if tmpday < end:
265            dates = day.strftime("%Y%m%d") + "/to/" + \
266                    tmpday.strftime("%Y%m%d")
267        else:
268            dates = day.strftime("%Y%m%d") + "/to/" + \
269                    end.strftime("%Y%m%d")
270
271        print("... retrieve " + dates + " in dir " + c.inputdir)
272
273        try:
274            flexpart.retrieve(server, dates, c.request, c.inputdir)
275        except IOError:
276            my_error(c.mailfail, 'MARS request failed')
277
278        day += delta_t
279
280    return
281
282if __name__ == "__main__":
283    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG