source: flex_extract.git/source/python/mods/get_mars_data.py @ ca867de

ctbtodev
Last change on this file since ca867de was ca867de, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

refactored functions in EcFlexpart? and did some minor changes

  • Property mode set to 100755
File size: 8.3 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - moved the getEIdata program into a function "get_mars_data"
12#        - moved the AgurmentParser into a seperate function
13#        - adatpted the function for the use in flex_extract
14#        - renamed file to get_mars_data
15#
16#    February 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added structured documentation
19#        - minor changes in programming style for consistence
20#        - added function main and moved function calls vom __main__ there
21#          (necessary for better documentation with docstrings for later
22#          online documentation)
23#        - use of UIFiles class for file selection and deletion
24#
25#
26# @License:
27#    (C) Copyright 2014-2018.
28#
29#    This software is licensed under the terms of the Apache Licence Version 2.0
30#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
31#
32# @Program Functionality:
33#    This program can be used as a module in the whole flex_extract process
34#    or can be run by itself to just extract MARS data from ECMWF. To do so,
35#    a couple of necessary parameters has to be passed with the program call.
36#    See documentation for more details.
37#
38# @Program Content:
39#    - main
40#    - get_mars_data
41#    - do_retrievement
42#
43#*******************************************************************************
44
45# ------------------------------------------------------------------------------
46# MODULES
47# ------------------------------------------------------------------------------
48import os
49import sys
50import inspect
51from datetime import datetime, timedelta
52
53# software specific classes and modules from flex_extract
54sys.path.append('../')
55import _config
56from tools import my_error, normal_exit, get_cmdline_arguments, read_ecenv
57from classes.EcFlexpart import EcFlexpart
58from classes.UioFiles import UioFiles
59
60try:
61    ecapi = True
62    import ecmwfapi
63except ImportError:
64    ecapi = False
65# ------------------------------------------------------------------------------
66# FUNCTION
67# ------------------------------------------------------------------------------
68def main():
69    '''
70    @Description:
71        If get_mars_data is called directly from command line,
72
73        the program flow and calls the argumentparser function and
74        the get_mars_data function for retrieving EC data.
75
76    @Input:
77        <nothing>
78
79    @Return:
80        <nothing>
81    '''
82
83    args = get_cmdline_arguments()
84    c = ControlFile(args.controlfile)
85
86    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
87    c.assign_args_to_control(args)
88    c.assign_envs_to_control(env_parameter)
89    c.check_conditions(args.queue)
90
91    get_mars_data(c)
92    normal_exit(c.mailfail, 'Done!')
93
94    return
95
96def get_mars_data(c):
97    '''
98    @Description:
99        Retrieves the EC data needed for a FLEXPART simulation.
100        Start and end dates for retrieval period is set. Retrievals
101        are divided into smaller periods if necessary and datechunk parameter
102        is set.
103
104    @Input:
105        c: instance of class ControlFile
106            Contains all the parameters of CONTROL file and
107            command line.
108            For more information about format and content of the parameter
109            see documentation.
110
111    @Return:
112        <nothing>
113    '''
114
115    if not os.path.exists(c.inputdir):
116        os.makedirs(c.inputdir)
117
118    if c.request == 0 or c.request == 2:
119        print("Retrieving EC data!")
120    elif c.request == 1:
121        print("Printing mars requests!")
122
123    print("start date %s " % (c.start_date))
124    print("end date %s " % (c.end_date))
125
126    if ecapi:
127        server = ecmwfapi.ECMWFService("mars")
128    else:
129        server = False
130
131    c.ecapi = ecapi
132    print('Using ECMWF WebAPI: ' + str(c.ecapi))
133
134    # basetime geht rückwärts
135
136    # if basetime 00
137    # dann wird von 12 am vortag bis 00 am start tag geholt
138    # aber ohne 12 selbst sondern 12 + step
139
140    # if basetime 12
141    # dann wird von 00 + step bis 12 am start tag geholt
142
143    # purer forecast wird vorwärts bestimmt.
144    # purer forecast mode ist dann wenn  größer 24 stunden
145    # wie kann das noch festgestellt werden ????
146    # nur FC und steps mehr als 24 ?
147    # die einzige problematik beim reinen forecast ist die benennung der files!
148    # also sobald es Tagesüberschneidungen gibt
149    # allerdings ist das relevant und ersichtlich an den NICHT FLUSS DATEN
150
151    start = datetime.strptime(c.start_date, '%Y%m%d')
152    end = datetime.strptime(c.end_date, '%Y%m%d')
153    # time period for one single retrieval
154    datechunk = timedelta(days=int(c.date_chunk))
155
156    if c.basetime == '00':
157        start = start - timedelta(days=1)
158
159    if c.maxstep <= 24:
160        startm1 = start - timedelta(days=1)
161
162    if c.basetime == '00' or c.basetime == '12':
163        # endp1 = end + timedelta(days=1)
164        endp1 = end
165    else:
166        # endp1 = end + timedelta(days=2)
167        endp1 = end + timedelta(days=1)
168
169    # --------------  flux data ------------------------------------------------
170    if c.request == 0 or c.request == 2:
171        print('... removing old flux content of ' + c.inputdir)
172        tobecleaned = UioFiles(c.inputdir,
173                               '*_acc_*.' + str(os.getppid()) + '.*.grb')
174        tobecleaned.delete_files()
175
176    # if forecast for maximum one day (upto 24h) are to be retrieved,
177    # collect accumulation data (flux data)
178    # with additional days in the beginning and at the end
179    # (used for complete disaggregation of original period)
180    if c.maxstep <= 24:
181        do_retrievement(c, server, startm1, endp1, datechunk, fluxes=True)
182
183    # if forecast data longer than 24h are to be retrieved,
184    # collect accumulation data (flux data)
185    # with the exact start and end date
186    # (disaggregation will be done for the
187    # exact time period with boundary conditions)
188    else:
189        do_retrievement(c, server, start, end, datechunk, fluxes=True)
190
191    # --------------  non flux data --------------------------------------------
192    if c.request == 0 or c.request == 2:
193        print('... removing old non flux content of ' + c.inputdir)
194        tobecleaned = UioFiles(c.inputdir,
195                               '*__*.' + str(os.getppid()) + '.*.grb')
196        tobecleaned.delete_files()
197
198    do_retrievement(c, server, start, end, datechunk, fluxes=False)
199
200    return
201
202def do_retrievement(c, server, start, end, delta_t, fluxes=False):
203    '''
204    @Description:
205        Divides the complete retrieval period in smaller chunks and
206        retrieves the data from MARS.
207
208    @Input:
209        c: instance of class ControlFile
210            Contains all the parameters of CONTROL file and
211            command line.
212            For more information about format and content of the parameter
213            see documentation.
214
215        server: instance of ECMWFService
216            The server connection to ECMWF
217
218        start: instance of datetime
219            The start date of the retrieval.
220
221        end: instance of datetime
222            The end date of the retrieval.
223
224        delta_t: instance of datetime
225            Delta_t +1 is the maximal time period of a single
226            retrieval.
227
228        fluxes: boolean, optional
229            Decides if the flux parameters are to be retrieved or
230            the rest of the parameter list.
231            Default value is False.
232
233    @Return:
234        <nothing>
235    '''
236
237    # since actual day also counts as one day,
238    # we only need to add datechunk - 1 days to retrieval
239    # for a period
240    delta_t_m1 = delta_t - timedelta(days=1)
241
242    day = start
243    while day <= end:
244        flexpart = EcFlexpart(c, fluxes)
245        tmpday = day + delta_t_m1
246        if tmpday < end:
247            dates = day.strftime("%Y%m%d") + "/to/" + \
248                    tmpday.strftime("%Y%m%d")
249        else:
250            dates = day.strftime("%Y%m%d") + "/to/" + \
251                    end.strftime("%Y%m%d")
252
253        print("... retrieve " + dates + " in dir " + c.inputdir)
254
255        try:
256            flexpart.retrieve(server, dates, c.request, c.inputdir)
257        except IOError:
258            my_error(c.mailfail, 'MARS request failed')
259
260        day += delta_t
261
262    return
263
264if __name__ == "__main__":
265    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG