source: flex_extract.git/python/get_mars_data.py @ ff99eae

ctbtodev
Last change on this file since ff99eae was ff99eae, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

completed application of pep8 style guide and pylint investigations. added documentation almost everywhere

  • Property mode set to 100755
File size: 8.1 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - moved the getEIdata program into a function "get_mars_data"
12#        - moved the AgurmentParser into a seperate function
13#        - adatpted the function for the use in flex_extract
14#        - renamed file to get_mars_data
15#
16#    February 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added structured documentation
19#        - minor changes in programming style for consistence
20#        - added function main and moved function calls vom __main__ there
21#          (necessary for better documentation with docstrings for later
22#          online documentation)
23#        - use of UIFiles class for file selection and deletion
24
25#
26# @License:
27#    (C) Copyright 2014-2018.
28#
29#    This software is licensed under the terms of the Apache Licence Version 2.0
30#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
31#
32# @Program Functionality:
33#    This program can be used as a module in the whole flex_extract process
34#    or can be run by itself to just extract MARS data from ECMWF. To do so,
35#    a couple of necessary parameters has to be passed with the program call.
36#    See documentation for more details.
37#
38# @Program Content:
39#    - main
40#    - get_mars_data
41#
42#*******************************************************************************
43
44# ------------------------------------------------------------------------------
45# MODULES
46# ------------------------------------------------------------------------------
47import os
48import sys
49import datetime
50import inspect
51try:
52    ecapi = True
53    import ecmwfapi
54except ImportError:
55    ecapi = False
56
57# software specific classes and modules from flex_extract
58from tools import my_error, normal_exit, interpret_args_and_control
59from EcFlexpart import EcFlexpart
60from UioFiles import UioFiles
61
62# add path to pythonpath so that python finds its buddies
63LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
64    inspect.getfile(inspect.currentframe())))
65if LOCAL_PYTHON_PATH not in sys.path:
66    sys.path.append(LOCAL_PYTHON_PATH)
67
68# ------------------------------------------------------------------------------
69# FUNCTION
70# ------------------------------------------------------------------------------
71def main():
72    '''
73    @Description:
74        If get_mars_data is called from command line, this function controls
75        the program flow and calls the argumentparser function and
76        the get_mars_data function for retrieving EC data.
77
78    @Input:
79        <nothing>
80
81    @Return:
82        <nothing>
83    '''
84    args, c = interpret_args_and_control()
85    get_mars_data(c)
86    normal_exit(c)
87
88    return
89
90def get_mars_data(c):
91    '''
92    @Description:
93        Retrieves the EC data needed for a FLEXPART simulation.
94        Start and end dates for retrieval period is set. Retrievals
95        are divided into smaller periods if necessary and datechunk parameter
96        is set.
97
98    @Input:
99        c: instance of class ControlFile
100            Contains all the parameters of CONTROL file, which are e.g.:
101            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
102            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
103            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
104            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
105            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
106            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
107            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
108
109            For more information about format and content of the parameter
110            see documentation.
111
112    @Return:
113        <nothing>
114    '''
115
116    if not os.path.exists(c.inputdir):
117        os.makedirs(c.inputdir)
118
119    print "Retrieving EC data!"
120    print "start date %s " % (c.start_date)
121    print "end date %s " % (c.end_date)
122
123    if ecapi:
124        server = ecmwfapi.ECMWFService("mars")
125    else:
126        server = False
127
128    c.ecapi = ecapi
129    print 'ecapi: ', c.ecapi
130
131    # set start date of retrieval period
132    start = datetime.date(year=int(c.start_date[:4]),
133                          month=int(c.start_date[4:6]),
134                          day=int(c.start_date[6:]))
135    startm1 = start - datetime.timedelta(days=1)
136    if c.basetime == '00':
137        start = startm1
138
139    # set end date of retrieval period
140    end = datetime.date(year=int(c.end_date[:4]),
141                        month=int(c.end_date[4:6]),
142                        day=int(c.end_date[6:]))
143    if c.basetime == '00' or c.basetime == '12':
144        endp1 = end + datetime.timedelta(days=1)
145    else:
146        endp1 = end + datetime.timedelta(days=2)
147
148    # set time period of one single retrieval
149    datechunk = datetime.timedelta(days=int(c.date_chunk))
150
151    # --------------  flux data ------------------------------------------------
152    print 'removing old flux content of ' + c.inputdir
153    tobecleaned = UioFiles('*_acc_*.' + str(os.getppid()) + '.*.grb')
154    tobecleaned.list_files(c.inputdir)
155    tobecleaned.delete_files()
156
157    # if forecast for maximum one day (upto 23h) are to be retrieved,
158    # collect accumulation data (flux data)
159    # with additional days in the beginning and at the end
160    # (used for complete disaggregation of original period)
161    if c.maxstep < 24:
162        day = startm1
163        while day < endp1:
164            # retrieve MARS data for the whole period
165            flexpart = EcFlexpart(c, fluxes=True)
166            tmpday = day + datechunk - datetime.timedelta(days=1)
167            if tmpday < endp1:
168                dates = day.strftime("%Y%m%d") + "/to/" + \
169                        tmpday.strftime("%Y%m%d")
170            else:
171                dates = day.strftime("%Y%m%d") + "/to/" + \
172                        end.strftime("%Y%m%d")
173
174            print "retrieve " + dates + " in dir " + c.inputdir
175
176            try:
177                flexpart.retrieve(server, dates, c.inputdir)
178            except IOError:
179                my_error(c, 'MARS request failed')
180
181            day += datechunk
182
183    # if forecast data longer than 24h are to be retrieved,
184    # collect accumulation data (flux data)
185    # with the exact start and end date
186    # (disaggregation will be done for the
187    # exact time period with boundary conditions)
188    else:
189        day = start
190        while day <= end:
191            # retrieve MARS data for the whole period
192            flexpart = EcFlexpart(c, fluxes=True)
193            tmpday = day + datechunk - datetime.timedelta(days=1)
194            if tmpday < end:
195                dates = day.strftime("%Y%m%d") + "/to/" + \
196                        tmpday.trftime("%Y%m%d")
197            else:
198                dates = day.strftime("%Y%m%d") + "/to/" + \
199                        end.strftime("%Y%m%d")
200
201            print "retrieve " + dates + " in dir " + c.inputdir
202
203            try:
204                flexpart.retrieve(server, dates, c.inputdir)
205            except IOError:
206                my_error(c, 'MARS request failed')
207
208            day += datechunk
209
210    # --------------  non flux data --------------------------------------------
211    print 'removing old non flux content of ' + c.inputdir
212    tobecleaned = UioFiles('*__*.' + str(os.getppid()) + '.*.grb')
213    tobecleaned.list_files(c.inputdir)
214    tobecleaned.delete_files()
215
216    day = start
217    while day <= end:
218        # retrieve all non flux MARS data for the whole period
219        flexpart = EcFlexpart(c, fluxes=False)
220        tmpday = day + datechunk - datetime.timedelta(days=1)
221        if tmpday < end:
222            dates = day.strftime("%Y%m%d") + "/to/" + \
223                    tmpday.strftime("%Y%m%d")
224        else:
225            dates = day.strftime("%Y%m%d") + "/to/" + \
226                    end.strftime("%Y%m%d")
227
228        print "retrieve " + dates + " in dir " + c.inputdir
229
230        try:
231            flexpart.retrieve(server, dates, c.inputdir)
232        except IOError:
233            my_error(c, 'MARS request failed')
234
235        day += datechunk
236
237    return
238
239if __name__ == "__main__":
240    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG