source: flex_extract.git/python/get_mars_data.py @ 54a8a01

ctbtodev
Last change on this file since 54a8a01 was 54a8a01, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

restructuring, documentations and bug fixes

  • Property mode set to 100755
File size: 9.8 KB
RevLine 
[d69b677]1#!/usr/bin/env python
[efdb01a]2# -*- coding: utf-8 -*-
[991df6a]3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
[ff99eae]11#        - moved the getEIdata program into a function "get_mars_data"
[991df6a]12#        - moved the AgurmentParser into a seperate function
13#        - adatpted the function for the use in flex_extract
[ff99eae]14#        - renamed file to get_mars_data
[991df6a]15#
16#    February 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added structured documentation
19#        - minor changes in programming style for consistence
20#        - added function main and moved function calls vom __main__ there
21#          (necessary for better documentation with docstrings for later
22#          online documentation)
23#        - use of UIFiles class for file selection and deletion
[54a8a01]24#
[991df6a]25#
26# @License:
27#    (C) Copyright 2014-2018.
28#
29#    This software is licensed under the terms of the Apache Licence Version 2.0
30#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
31#
32# @Program Functionality:
33#    This program can be used as a module in the whole flex_extract process
34#    or can be run by itself to just extract MARS data from ECMWF. To do so,
35#    a couple of necessary parameters has to be passed with the program call.
36#    See documentation for more details.
37#
38# @Program Content:
39#    - main
[ff99eae]40#    - get_mars_data
[54a8a01]41#    - do_retrievement
[991df6a]42#
43#*******************************************************************************
44
[efdb01a]45# ------------------------------------------------------------------------------
46# MODULES
47# ------------------------------------------------------------------------------
[991df6a]48import os
49import sys
50import datetime
51import inspect
[d69b677]52try:
[ff99eae]53    ecapi = True
[d69b677]54    import ecmwfapi
55except ImportError:
[ff99eae]56    ecapi = False
57
58# software specific classes and modules from flex_extract
[54a8a01]59from tools import my_error, normal_exit, get_cmdline_arguments, read_ecenv
[ff99eae]60from EcFlexpart import EcFlexpart
61from UioFiles import UioFiles
[d69b677]62
[991df6a]63# add path to pythonpath so that python finds its buddies
[ff99eae]64LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
[991df6a]65    inspect.getfile(inspect.currentframe())))
[ff99eae]66if LOCAL_PYTHON_PATH not in sys.path:
67    sys.path.append(LOCAL_PYTHON_PATH)
[64cf353]68
[efdb01a]69# ------------------------------------------------------------------------------
70# FUNCTION
71# ------------------------------------------------------------------------------
[991df6a]72def main():
73    '''
74    @Description:
[ff99eae]75        If get_mars_data is called from command line, this function controls
[991df6a]76        the program flow and calls the argumentparser function and
[ff99eae]77        the get_mars_data function for retrieving EC data.
[991df6a]78
79    @Input:
80        <nothing>
81
82    @Return:
83        <nothing>
84    '''
[54a8a01]85
86    args = get_cmdline_arguments()
87
88    try:
89        c = ControlFile(args.controlfile)
90    except IOError:
91        try:
92            c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile)
93        except IOError:
94            print 'Could not read CONTROL file "' + args.controlfile + '"'
95            print 'Either it does not exist or its syntax is wrong.'
96            print 'Try "' + sys.argv[0].split('/')[-1] + \
97                  ' -h" to print usage information'
98            sys.exit(1)
99
100    env_parameter = read_ecenv(c.ecmwfdatadir + 'python/ECMWF_ENV')
101    c.assign_args_to_control(args, env_parameter)
102    c.assign_envs_to_control(env_parameter)
103    c.check_conditions()
104
[ff99eae]105    get_mars_data(c)
[54a8a01]106    normal_exit(c.mailfail, 'Done!')
[d69b677]107
[991df6a]108    return
109
[ff99eae]110def get_mars_data(c):
[991df6a]111    '''
112    @Description:
113        Retrieves the EC data needed for a FLEXPART simulation.
114        Start and end dates for retrieval period is set. Retrievals
115        are divided into smaller periods if necessary and datechunk parameter
116        is set.
117
118    @Input:
119        c: instance of class ControlFile
120            Contains all the parameters of CONTROL file, which are e.g.:
121            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
122            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
123            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
124            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
125            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
126            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
127            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
128
129            For more information about format and content of the parameter
130            see documentation.
131
132    @Return:
133        <nothing>
134    '''
[d69b677]135
136    if not os.path.exists(c.inputdir):
137        os.makedirs(c.inputdir)
[991df6a]138
[ff99eae]139    print "Retrieving EC data!"
140    print "start date %s " % (c.start_date)
141    print "end date %s " % (c.end_date)
[d69b677]142
143    if ecapi:
144        server = ecmwfapi.ECMWFService("mars")
145    else:
146        server = False
147
[64cf353]148    c.ecapi = ecapi
[991df6a]149    print 'ecapi: ', c.ecapi
150
[54a8a01]151    # basetime geht rückwärts
152
153    # if basetime 00
154    # dann wird von 12 am vortag bis 00 am start tag geholt
155    # aber ohne 12 selbst sondern 12 + step
156
157    # if basetime 12
158    # dann wird von 00 + step bis 12 am start tag geholt
159
160    # purer forecast wird vorwärts bestimmt.
161    # purer forecast mode ist dann wenn  größer 24 stunden
162    # wie kann das noch festgestellt werden ????
163    # nur FC und steps mehr als 24 ?
164    # die einzige problematik beim reinen forecast ist die benennung der files!
165    # also sobald es Tagesüberschneidungen gibt
166    # allerdings ist das relevant und ersichtlich an den NICHT FLUSS DATEN
167
168
[991df6a]169    # set start date of retrieval period
170    start = datetime.date(year=int(c.start_date[:4]),
171                          month=int(c.start_date[4:6]),
172                          day=int(c.start_date[6:]))
[64cf353]173    startm1 = start - datetime.timedelta(days=1)
[efdb01a]174
[991df6a]175    # set end date of retrieval period
176    end = datetime.date(year=int(c.end_date[:4]),
177                        month=int(c.end_date[4:6]),
178                        day=int(c.end_date[6:]))
[64cf353]179
[54a8a01]180    # set time period for one single retrieval
[64cf353]181    datechunk = datetime.timedelta(days=int(c.date_chunk))
[efdb01a]182
[54a8a01]183    if c.basetime == '00':
184        start = startm1
185
186    if c.basetime == '00' or c.basetime == '12':
187        # endp1 = end + datetime.timedelta(days=1)
188        endp1 = end
189    else:
190        # endp1 = end + datetime.timedelta(days=2)
191        endp1 = end + datetime.timedelta(days=1)
192
[991df6a]193    # --------------  flux data ------------------------------------------------
194    print 'removing old flux content of ' + c.inputdir
[54a8a01]195    tobecleaned = UioFiles(c.inputdir,
196                           '*_acc_*.' + str(os.getppid()) + '.*.grb')
[ff99eae]197    tobecleaned.delete_files()
[64cf353]198
[54a8a01]199    # if forecast for maximum one day (upto 24h) are to be retrieved,
[efdb01a]200    # collect accumulation data (flux data)
201    # with additional days in the beginning and at the end
202    # (used for complete disaggregation of original period)
[54a8a01]203    if c.maxstep <= 24:
204        do_retrievement(c, server, startm1, endp1, datechunk, fluxes=True)
[efdb01a]205
206    # if forecast data longer than 24h are to be retrieved,
207    # collect accumulation data (flux data)
208    # with the exact start and end date
209    # (disaggregation will be done for the
210    # exact time period with boundary conditions)
[d69b677]211    else:
[54a8a01]212        do_retrievement(c, server, start, end, datechunk, fluxes=True)
[991df6a]213
214    # --------------  non flux data --------------------------------------------
215    print 'removing old non flux content of ' + c.inputdir
[54a8a01]216    tobecleaned = UioFiles(c.inputdir,
217                           '*__*.' + str(os.getppid()) + '.*.grb')
[ff99eae]218    tobecleaned.delete_files()
[991df6a]219
[54a8a01]220    do_retrievement(c, server, start, end, datechunk, fluxes=False)
221
222    return
223
224def do_retrievement(c, server, start, end, delta_t, fluxes=False):
225    '''
226    @Description:
227        Divides the complete retrieval period in smaller chunks and
228        retrieves the data from MARS.
229
230    @Input:
231        c: instance of ControlFile
232            Contains all the parameters of CONTROL file, which are e.g.:
233            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
234            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
235            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
236            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
237            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
238            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
239            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
240
241            For more information about format and content of the parameter
242            see documentation.
243
244        server: instance of ECMWFService
245            The server connection to ECMWF
246
247        start: instance of datetime
248            The start date of the retrieval.
249
250        end: instance of datetime
251            The end date of the retrieval.
252
253        delta_t: instance of datetime
254            Delta_t +1 is the maximal time period of a single
255            retrieval.
256
257        fluxes: boolean, optional
258            Decides if the flux parameters are to be retrieved or
259            the rest of the parameter list.
260            Default value is False.
261
262    @Return:
263        <nothing>
264    '''
265
266    # since actual day also counts as one day,
267    # we only need to add datechunk - 1 days to retrieval
268    # for a period
269    delta_t_m1 = delta_t - datetime.timedelta(days=1)
270
[efdb01a]271    day = start
272    while day <= end:
[54a8a01]273        flexpart = EcFlexpart(c, fluxes)
274        tmpday = day + delta_t_m1
[ff99eae]275        if tmpday < end:
276            dates = day.strftime("%Y%m%d") + "/to/" + \
277                    tmpday.strftime("%Y%m%d")
278        else:
279            dates = day.strftime("%Y%m%d") + "/to/" + \
280                    end.strftime("%Y%m%d")
[efdb01a]281
[ff99eae]282        print "retrieve " + dates + " in dir " + c.inputdir
[64cf353]283
[ff99eae]284        try:
285            flexpart.retrieve(server, dates, c.inputdir)
286        except IOError:
[54a8a01]287            my_error(c.mailfail, 'MARS request failed')
[efdb01a]288
[54a8a01]289        day += delta_t
[64cf353]290
[efdb01a]291    return
[d69b677]292
293if __name__ == "__main__":
[991df6a]294    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG