source: flex_extract.git/source/python/mods/get_mars_data.py @ 295ff45

dev
Last change on this file since 295ff45 was 295ff45, checked in by Anne Philipp <anne.philipp@…>, 9 months ago

added request output option also for basetime requests; changed output to csv format

  • Property mode set to 100755
File size: 9.5 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - moved the getEIdata program into a function "get_mars_data"
12#        - moved the AgurmentParser into a seperate function
13#        - adatpted the function for the use in flex_extract
14#        - renamed file to get_mars_data
15#
16#    February 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added structured documentation
19#        - minor changes in programming style for consistence
20#        - added function main and moved function calls vom __main__ there
21#          (necessary for better documentation with docstrings for later
22#          online documentation)
23#        - use of UIFiles class for file selection and deletion
24#
25#
26# @License:
27#    (C) Copyright 2014-2018.
28#
29#    This software is licensed under the terms of the Apache Licence Version 2.0
30#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
31#
32# @Program Functionality:
33#    This program can be used as a module in the whole flex_extract process
34#    or can be run by itself to just extract MARS data from ECMWF. To do so,
35#    a couple of necessary parameters has to be passed with the program call.
36#    See documentation for more details.
37#
38# @Program Content:
39#    - main
40#    - get_mars_data
41#    - do_retrievement
42#
43#*******************************************************************************
44
45# ------------------------------------------------------------------------------
46# MODULES
47# ------------------------------------------------------------------------------
48import os
49import sys
50import datetime
51import inspect
52try:
53    ecapi = True
54    import ecmwfapi
55except ImportError:
56    ecapi = False
57
58# software specific classes and modules from flex_extract
59import _config
60from tools import my_error, normal_exit, get_cmdline_arguments, read_ecenv
61from classes.EcFlexpart import EcFlexpart
62from classes.UioFiles import UioFiles
63# ------------------------------------------------------------------------------
64# FUNCTION
65# ------------------------------------------------------------------------------
66def main():
67    '''
68    @Description:
69        If get_mars_data is called directly from command line,
70
71        the program flow and calls the argumentparser function and
72        the get_mars_data function for retrieving EC data.
73
74    @Input:
75        <nothing>
76
77    @Return:
78        <nothing>
79    '''
80
81    args = get_cmdline_arguments()
82    c = ControlFile(args.controlfile)
83
84    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
85    c.assign_args_to_control(args)
86    c.assign_envs_to_control(env_parameter)
87    c.check_conditions(args.queue)
88
89    get_mars_data(c)
90    normal_exit(c.mailfail, 'Done!')
91
92    return
93
94def get_mars_data(c):
95    '''
96    @Description:
97        Retrieves the EC data needed for a FLEXPART simulation.
98        Start and end dates for retrieval period is set. Retrievals
99        are divided into smaller periods if necessary and datechunk parameter
100        is set.
101
102    @Input:
103        c: instance of class ControlFile
104            Contains all the parameters of CONTROL file, which are e.g.:
105            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
106            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
107            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
108            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
109            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
110            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
111            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
112
113            For more information about format and content of the parameter
114            see documentation.
115
116    @Return:
117        <nothing>
118    '''
119
120    if not os.path.exists(c.inputdir):
121        os.makedirs(c.inputdir)
122
123    if c.request == 0 or c.request == 2:
124        print("Retrieving EC data!")
125    elif c.request == 1:
126        print("Printing mars requests!")
127
128    print("start date %s " % (c.start_date))
129    print("end date %s " % (c.end_date))
130
131    if ecapi:
132        server = ecmwfapi.ECMWFService("mars")
133    else:
134        server = False
135
136    c.ecapi = ecapi
137    print('Using ECMWF WebAPI: ' + str(c.ecapi))
138
139    # basetime geht rückwärts
140
141    # if basetime 00
142    # dann wird von 12 am vortag bis 00 am start tag geholt
143    # aber ohne 12 selbst sondern 12 + step
144
145    # if basetime 12
146    # dann wird von 00 + step bis 12 am start tag geholt
147
148    # purer forecast wird vorwärts bestimmt.
149    # purer forecast mode ist dann wenn  größer 24 stunden
150    # wie kann das noch festgestellt werden ????
151    # nur FC und steps mehr als 24 ?
152    # die einzige problematik beim reinen forecast ist die benennung der files!
153    # also sobald es Tagesüberschneidungen gibt
154    # allerdings ist das relevant und ersichtlich an den NICHT FLUSS DATEN
155
156
157    # set start date of retrieval period
158    start = datetime.date(year=int(c.start_date[:4]),
159                          month=int(c.start_date[4:6]),
160                          day=int(c.start_date[6:]))
161    startm1 = start - datetime.timedelta(days=1)
162
163    # set end date of retrieval period
164    end = datetime.date(year=int(c.end_date[:4]),
165                        month=int(c.end_date[4:6]),
166                        day=int(c.end_date[6:]))
167
168    # set time period for one single retrieval
169    datechunk = datetime.timedelta(days=int(c.date_chunk))
170
171    if c.basetime == '00':
172        start = startm1
173
174    if c.basetime == '00' or c.basetime == '12':
175        # endp1 = end + datetime.timedelta(days=1)
176        endp1 = end
177    else:
178        # endp1 = end + datetime.timedelta(days=2)
179        endp1 = end + datetime.timedelta(days=1)
180
181    # --------------  flux data ------------------------------------------------
182    if c.request == 0 or c.request == 2:
183        print('... removing old flux content of ' + c.inputdir)
184        tobecleaned = UioFiles(c.inputdir,
185                               '*_acc_*.' + str(os.getppid()) + '.*.grb')
186        tobecleaned.delete_files()
187
188    # if forecast for maximum one day (upto 24h) are to be retrieved,
189    # collect accumulation data (flux data)
190    # with additional days in the beginning and at the end
191    # (used for complete disaggregation of original period)
192    if c.maxstep <= 24:
193        do_retrievement(c, server, startm1, endp1, datechunk, fluxes=True)
194
195    # if forecast data longer than 24h are to be retrieved,
196    # collect accumulation data (flux data)
197    # with the exact start and end date
198    # (disaggregation will be done for the
199    # exact time period with boundary conditions)
200    else:
201        do_retrievement(c, server, start, end, datechunk, fluxes=True)
202
203    # --------------  non flux data --------------------------------------------
204    if c.request == 0 or c.request == 2:
205        print('... removing old non flux content of ' + c.inputdir)
206        tobecleaned = UioFiles(c.inputdir,
207                               '*__*.' + str(os.getppid()) + '.*.grb')
208        tobecleaned.delete_files()
209
210    do_retrievement(c, server, start, end, datechunk, fluxes=False)
211
212    return
213
214def do_retrievement(c, server, start, end, delta_t, fluxes=False):
215    '''
216    @Description:
217        Divides the complete retrieval period in smaller chunks and
218        retrieves the data from MARS.
219
220    @Input:
221        c: instance of ControlFile
222            Contains all the parameters of CONTROL file, which are e.g.:
223            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
224            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
225            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
226            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
227            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
228            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
229            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
230
231            For more information about format and content of the parameter
232            see documentation.
233
234        server: instance of ECMWFService
235            The server connection to ECMWF
236
237        start: instance of datetime
238            The start date of the retrieval.
239
240        end: instance of datetime
241            The end date of the retrieval.
242
243        delta_t: instance of datetime
244            Delta_t +1 is the maximal time period of a single
245            retrieval.
246
247        fluxes: boolean, optional
248            Decides if the flux parameters are to be retrieved or
249            the rest of the parameter list.
250            Default value is False.
251
252    @Return:
253        <nothing>
254    '''
255
256    # since actual day also counts as one day,
257    # we only need to add datechunk - 1 days to retrieval
258    # for a period
259    delta_t_m1 = delta_t - datetime.timedelta(days=1)
260
261    day = start
262    while day <= end:
263        flexpart = EcFlexpart(c, fluxes)
264        tmpday = day + delta_t_m1
265        if tmpday < end:
266            dates = day.strftime("%Y%m%d") + "/to/" + \
267                    tmpday.strftime("%Y%m%d")
268        else:
269            dates = day.strftime("%Y%m%d") + "/to/" + \
270                    end.strftime("%Y%m%d")
271
272        print("... retrieve " + dates + " in dir " + c.inputdir)
273
274        try:
275            flexpart.retrieve(server, dates, c.request, c.inputdir)
276        except IOError:
277            my_error(c.mailfail, 'MARS request failed')
278
279        day += delta_t
280
281    return
282
283if __name__ == "__main__":
284    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG