source: flex_extract.git/python/get_mars_data.py @ 2fb99de

devfeature/makefilesorigin/task/language-editingtask/language-editing
Last change on this file since 2fb99de was 2fb99de, checked in by Anne Philipp <anne.philipp@…>, 23 months ago

introduced config with path definitions and changed py files accordingly; Installation works; some tests were added for tarball making; Problems in submission to ecgate

  • Property mode set to 100755
File size: 9.8 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - moved the getEIdata program into a function "get_mars_data"
12#        - moved the AgurmentParser into a seperate function
13#        - adatpted the function for the use in flex_extract
14#        - renamed file to get_mars_data
15#
16#    February 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added structured documentation
19#        - minor changes in programming style for consistence
20#        - added function main and moved function calls vom __main__ there
21#          (necessary for better documentation with docstrings for later
22#          online documentation)
23#        - use of UIFiles class for file selection and deletion
24#
25#
26# @License:
27#    (C) Copyright 2014-2018.
28#
29#    This software is licensed under the terms of the Apache Licence Version 2.0
30#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
31#
32# @Program Functionality:
33#    This program can be used as a module in the whole flex_extract process
34#    or can be run by itself to just extract MARS data from ECMWF. To do so,
35#    a couple of necessary parameters has to be passed with the program call.
36#    See documentation for more details.
37#
38# @Program Content:
39#    - main
40#    - get_mars_data
41#    - do_retrievement
42#
43#*******************************************************************************
44
45# ------------------------------------------------------------------------------
46# MODULES
47# ------------------------------------------------------------------------------
48import os
49import sys
50import datetime
51import inspect
52try:
53    ecapi = True
54    import ecmwfapi
55except ImportError:
56    ecapi = False
57
58# software specific classes and modules from flex_extract
59import _config
60from tools import my_error, normal_exit, get_cmdline_arguments, read_ecenv
61from EcFlexpart import EcFlexpart
62from UioFiles import UioFiles
63# ------------------------------------------------------------------------------
64# FUNCTION
65# ------------------------------------------------------------------------------
66def main():
67    '''
68    @Description:
69        If get_mars_data is called from command line, this function controls
70        the program flow and calls the argumentparser function and
71        the get_mars_data function for retrieving EC data.
72
73    @Input:
74        <nothing>
75
76    @Return:
77        <nothing>
78    '''
79
80    args = get_cmdline_arguments()
81
82    try:
83        c = ControlFile(args.controlfile)
84    except IOError:
85        print('Could not read CONTROL file "' + args.controlfile + '"')
86        print('Either it does not exist or its syntax is wrong.')
87        print('Try "' + sys.argv[0].split('/')[-1] + \
88              ' -h" to print usage information')
89        sys.exit(1)
90
91    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
92    c.assign_args_to_control(args, env_parameter)
93    c.assign_envs_to_control(env_parameter)
94    c.check_conditions(args.queue)
95
96    get_mars_data(c)
97    normal_exit(c.mailfail, 'Done!')
98
99    return
100
101def get_mars_data(c):
102    '''
103    @Description:
104        Retrieves the EC data needed for a FLEXPART simulation.
105        Start and end dates for retrieval period is set. Retrievals
106        are divided into smaller periods if necessary and datechunk parameter
107        is set.
108
109    @Input:
110        c: instance of class ControlFile
111            Contains all the parameters of CONTROL file, which are e.g.:
112            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
113            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
114            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
115            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
116            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
117            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
118            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
119
120            For more information about format and content of the parameter
121            see documentation.
122
123    @Return:
124        <nothing>
125    '''
126
127    if not os.path.exists(c.inputdir):
128        os.makedirs(c.inputdir)
129
130    if c.request == 0 or c.request == 2:
131        print("Retrieving EC data!")
132    elif c.request == 1:
133        print("Printing mars requests!")
134
135    print("start date %s " % (c.start_date))
136    print("end date %s " % (c.end_date))
137
138    if ecapi:
139        server = ecmwfapi.ECMWFService("mars")
140    else:
141        server = False
142
143    c.ecapi = ecapi
144    print('Using ECMWF WebAPI: ' + str(c.ecapi))
145
146    # basetime geht rückwärts
147
148    # if basetime 00
149    # dann wird von 12 am vortag bis 00 am start tag geholt
150    # aber ohne 12 selbst sondern 12 + step
151
152    # if basetime 12
153    # dann wird von 00 + step bis 12 am start tag geholt
154
155    # purer forecast wird vorwärts bestimmt.
156    # purer forecast mode ist dann wenn  größer 24 stunden
157    # wie kann das noch festgestellt werden ????
158    # nur FC und steps mehr als 24 ?
159    # die einzige problematik beim reinen forecast ist die benennung der files!
160    # also sobald es Tagesüberschneidungen gibt
161    # allerdings ist das relevant und ersichtlich an den NICHT FLUSS DATEN
162
163
164    # set start date of retrieval period
165    start = datetime.date(year=int(c.start_date[:4]),
166                          month=int(c.start_date[4:6]),
167                          day=int(c.start_date[6:]))
168    startm1 = start - datetime.timedelta(days=1)
169
170    # set end date of retrieval period
171    end = datetime.date(year=int(c.end_date[:4]),
172                        month=int(c.end_date[4:6]),
173                        day=int(c.end_date[6:]))
174
175    # set time period for one single retrieval
176    datechunk = datetime.timedelta(days=int(c.date_chunk))
177
178    if c.basetime == '00':
179        start = startm1
180
181    if c.basetime == '00' or c.basetime == '12':
182        # endp1 = end + datetime.timedelta(days=1)
183        endp1 = end
184    else:
185        # endp1 = end + datetime.timedelta(days=2)
186        endp1 = end + datetime.timedelta(days=1)
187
188    # --------------  flux data ------------------------------------------------
189    if c.request == 0 or c.request == 2:
190        print('... removing old flux content of ' + c.inputdir)
191        tobecleaned = UioFiles(c.inputdir,
192                               '*_acc_*.' + str(os.getppid()) + '.*.grb')
193        tobecleaned.delete_files()
194
195    # if forecast for maximum one day (upto 24h) are to be retrieved,
196    # collect accumulation data (flux data)
197    # with additional days in the beginning and at the end
198    # (used for complete disaggregation of original period)
199    if c.maxstep <= 24:
200        do_retrievement(c, server, startm1, endp1, datechunk, fluxes=True)
201
202    # if forecast data longer than 24h are to be retrieved,
203    # collect accumulation data (flux data)
204    # with the exact start and end date
205    # (disaggregation will be done for the
206    # exact time period with boundary conditions)
207    else:
208        do_retrievement(c, server, start, end, datechunk, fluxes=True)
209
210    # --------------  non flux data --------------------------------------------
211    if c.request == 0 or c.request == 2:
212        print('... removing old non flux content of ' + c.inputdir)
213        tobecleaned = UioFiles(c.inputdir,
214                               '*__*.' + str(os.getppid()) + '.*.grb')
215        tobecleaned.delete_files()
216
217    do_retrievement(c, server, start, end, datechunk, fluxes=False)
218
219    return
220
221def do_retrievement(c, server, start, end, delta_t, fluxes=False):
222    '''
223    @Description:
224        Divides the complete retrieval period in smaller chunks and
225        retrieves the data from MARS.
226
227    @Input:
228        c: instance of ControlFile
229            Contains all the parameters of CONTROL file, which are e.g.:
230            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
231            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
232            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
233            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
234            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
235            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
236            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
237
238            For more information about format and content of the parameter
239            see documentation.
240
241        server: instance of ECMWFService
242            The server connection to ECMWF
243
244        start: instance of datetime
245            The start date of the retrieval.
246
247        end: instance of datetime
248            The end date of the retrieval.
249
250        delta_t: instance of datetime
251            Delta_t +1 is the maximal time period of a single
252            retrieval.
253
254        fluxes: boolean, optional
255            Decides if the flux parameters are to be retrieved or
256            the rest of the parameter list.
257            Default value is False.
258
259    @Return:
260        <nothing>
261    '''
262
263    # since actual day also counts as one day,
264    # we only need to add datechunk - 1 days to retrieval
265    # for a period
266    delta_t_m1 = delta_t - datetime.timedelta(days=1)
267
268    day = start
269    while day <= end:
270        flexpart = EcFlexpart(c, fluxes)
271        tmpday = day + delta_t_m1
272        if tmpday < end:
273            dates = day.strftime("%Y%m%d") + "/to/" + \
274                    tmpday.strftime("%Y%m%d")
275        else:
276            dates = day.strftime("%Y%m%d") + "/to/" + \
277                    end.strftime("%Y%m%d")
278
279        print("... retrieve " + dates + " in dir " + c.inputdir)
280
281        try:
282            flexpart.retrieve(server, dates, c.request, c.inputdir)
283        except IOError:
284            my_error(c.mailfail, 'MARS request failed')
285
286        day += delta_t
287
288    return
289
290if __name__ == "__main__":
291    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG