source: flex_extract.git/python/getMARSdata.py @ efdb01a

ctbtodev
Last change on this file since efdb01a was efdb01a, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

whole bunch of modifications due to new structure of ECMWFDATA, added basics of documentation, minor programming corrections

  • Property mode set to 100755
File size: 7.0 KB
RevLine 
[d69b677]1#!/usr/bin/env python
[efdb01a]2# -*- coding: utf-8 -*-
3#************************************************************************
4# TODO AP
5# - Change History ist nicht angepasst ans File!
6# - add file description
7#************************************************************************
8"""
9@Author: Anne Fouilloux (University of Oslo)
10
11@Date: October 2014
12
13@ChangeHistory:
14    November 2015 - Leopold Haimberger (University of Vienna):
15        - using the WebAPI also for general MARS retrievals
16        - job submission on ecgate and cca
17        - job templates suitable for twice daily operational dissemination
18        - dividing retrievals of longer periods into digestable chunks
19        - retrieve also longer term forecasts, not only analyses and
20          short term forecast data
21        - conversion into GRIB2
22        - conversion into .fp format for faster execution of FLEXPART
23
24    February 2018 - Anne Philipp (University of Vienna):
25        - applied PEP8 style guide
26        - added documentation
27        - minor changes in programming style for consistence
28
29@License:
30    (C) Copyright 2014-2018.
31
32    This software is licensed under the terms of the Apache Licence Version 2.0
33    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
34
35@Requirements:
36    - A standard python 2.6 or 2.7 installation
37    - dateutils
38    - ECMWF specific packages, all available from https://software.ecmwf.int/
39        ECMWF WebMARS, gribAPI with python enabled, emoslib and
40        ecaccess web toolkit
41
42@Description:
43    Further documentation may be obtained from www.flexpart.eu.
44
45"""
46# ------------------------------------------------------------------------------
47# MODULES
48# ------------------------------------------------------------------------------
[d69b677]49try:
50    ecapi=True
51    import ecmwfapi
52except ImportError:
53    ecapi=False
54
55import calendar
56import shutil
57import datetime
58import time
[efdb01a]59import os
60import glob
61import sys
62import inspect
[d69b677]63# add path to submit.py to pythonpath so that python finds its buddies
64localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
65if localpythonpath not in sys.path:
66    sys.path.append(localpythonpath)
67
[efdb01a]68from Control import Control
69from Tools import myerror, normalexit, \
[64cf353]70                          interpret_args_and_control
[efdb01a]71from ECFlexpart import ECFlexpart
[64cf353]72
[efdb01a]73# ------------------------------------------------------------------------------
74# FUNCTION
75# ------------------------------------------------------------------------------
[64cf353]76def getMARSdata(args, c):
[d69b677]77
78
79    if not os.path.exists(c.inputdir):
80        os.makedirs(c.inputdir)
[64cf353]81    print("start date %s " % (c.start_date))
82    print("end date %s " % (c.end_date))
[d69b677]83
84    if ecapi:
85        server = ecmwfapi.ECMWFService("mars")
86    else:
87        server = False
88
[64cf353]89    c.ecapi = ecapi
90    print 'ecapi:', c.ecapi
[d69b677]91
[efdb01a]92# Retrieve EC data for running flexpart
[64cf353]93#AP change this variant to correct format conversion with datetime
94#AP import datetime and timedelta explicitly
95    syear = int(c.start_date[:4])
96    smonth = int(c.start_date[4:6])
97    sday = int(c.start_date[6:])
98    start = datetime.date(year=syear, month=smonth, day=sday)
99    startm1 = start - datetime.timedelta(days=1)
100    if c.basetime == '00':
101        start = startm1
[efdb01a]102
[64cf353]103    eyear = int(c.end_date[:4])
104    emonth = int(c.end_date[4:6])
105    eday = int(c.end_date[6:])
106    end = datetime.date(year=eyear, month=emonth, day=eday)
107    if c.basetime == '00' or c.basetime == '12':
108        endp1 = end + datetime.timedelta(days=1)
[d69b677]109    else:
[64cf353]110        endp1 = end + datetime.timedelta(days=2)
111
112    datechunk = datetime.timedelta(days=int(c.date_chunk))
[efdb01a]113
114    # retrieving of accumulated data fields (flux data), (maximum one month)
115
116    # remove old files
[64cf353]117    print 'removing content of ' + c.inputdir
[efdb01a]118    tobecleaned = glob.glob(c.inputdir + '/*_acc_*.' + \
119                            str(os.getppid()) + '.*.grb')
[d69b677]120    for f in tobecleaned:
121        os.remove(f)
[64cf353]122
[efdb01a]123    times = None
124    # if forecast for maximum one day (upto 23h) are to be retrieved,
125    # collect accumulation data (flux data)
126    # with additional days in the beginning and at the end
127    # (used for complete disaggregation of original period)
128    if c.maxstep < 24:
129        day = startm1
130        while day < endp1:
131                # retrieve MARS data for the whole period
132                flexpart = ECFlexpart(c, fluxes=True)
133                tmpday = day + datechunk - datetime.timedelta(days=1)
134                if tmpday < endp1:
135                    dates = day.strftime("%Y%m%d") + "/to/" + \
136                            tmpday.strftime("%Y%m%d")
[d69b677]137                else:
[efdb01a]138                    dates = day.strftime("%Y%m%d") + "/to/" + \
139                            end.strftime("%Y%m%d")
[64cf353]140
[d69b677]141                print "retrieve " + dates + " in dir " + c.inputdir
[efdb01a]142
[d69b677]143                try:
144                    flexpart.retrieve(server, dates, times, c.inputdir)
145                except IOError:
146                    myerror(c,'MARS request failed')
[64cf353]147
[efdb01a]148                day += datechunk
149
150    # if forecast data longer than 24h are to be retrieved,
151    # collect accumulation data (flux data)
152    # with the exact start and end date
153    # (disaggregation will be done for the
154    # exact time period with boundary conditions)
[d69b677]155    else:
[efdb01a]156        day = start
157        while day <= end:
158                # retrieve MARS data for the whole period
159                flexpart = ECFlexpart(c, fluxes=True)
160                tmpday = day + datechunk - datetime.timedelta(days=1)
161                if tmpday < end:
162                    dates = day.strftime("%Y%m%d") + "/to/" + \
163                            tmpday.trftime("%Y%m%d")
[d69b677]164                else:
[efdb01a]165                    dates = day.strftime("%Y%m%d") + "/to/" + \
166                            end.strftime("%Y%m%d")
[64cf353]167
[d69b677]168                print "retrieve " + dates + " in dir " + c.inputdir
[64cf353]169
[efdb01a]170                try:
171                    flexpart.retrieve(server, dates, times, c.inputdir)
172                except IOError:
173                    myerror(c, 'MARS request failed')
174
175                day += datechunk
[d69b677]176
[efdb01a]177    # retrieving of normal data fields (non flux data), (maximum one month)
[d69b677]178
[efdb01a]179    # remove old *__* files
180    tobecleaned = glob.glob(c.inputdir + '/*__*.' +
181                            str(os.getppid()) + '.*.grb')
[d69b677]182    for f in tobecleaned:
183        os.remove(f)
[efdb01a]184    day = start
185    times = None
186    while day <= end:
187            # retrieve MARS data for the whole period
188            flexpart = ECFlexpart(c, fluxes=False)
189            tmpday = day+datechunk-datetime.timedelta(days=1)
190            if tmpday < end:
191                dates = day.strftime("%Y%m%d") + "/to/" + \
192                        tmpday.strftime("%Y%m%d")
[d69b677]193            else:
[efdb01a]194                dates = day.strftime("%Y%m%d") + "/to/" + \
195                        end.strftime("%Y%m%d")
196
[d69b677]197            print "retrieve " + dates + " in dir " + c.inputdir
[64cf353]198
[efdb01a]199            try:
200                flexpart.retrieve(server, dates, times, c.inputdir)
201            except IOError:
202                myerror(c, 'MARS request failed')
203
204            day += datechunk
[64cf353]205
[efdb01a]206    return
[d69b677]207
208if __name__ == "__main__":
[64cf353]209
[efdb01a]210    args, c = interpret_args_and_control()
211    getMARSdata(args, c)
[d69b677]212    normalexit(c)
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG