Index: python/getMARSdata.py
===================================================================
--- python/getMARSdata.py	(revision e18f4b5f53f063ab746e1307bdd3162d31cb3e8a)
+++ python/getMARSdata.py	(revision e18f4b5f53f063ab746e1307bdd3162d31cb3e8a)
@@ -0,0 +1,155 @@
+#!/usr/bin/env python
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Functionality provided: Prepare input 3D-wind fields in hybrid coordinates + surface fields for FLEXPART runs
+#
+# Creation: October  2014 - Anne Fouilloux - University of Oslo
+# Extension November 2015 - Leopold Haimberger - University of Vienna for:
+# - using the WebAPI also for general MARS retrievals
+# - job submission on ecgate and cca
+# - job templates suitable for twice daily operational dissemination
+# - dividing retrievals of longer periods into digestable chunks
+# - retrieve also longer term forecasts, not only analyses and short term forecast data
+# - conversion into GRIB2
+# - conversion into .fp format for faster execution of FLEXPART
+#
+#
+# Further documentation may be obtained from www.flexpart.eu
+#
+# Requirements:
+# in addition to a standard python 2.6 or 2.7 installation the following packages need to be installed
+# ECMWF WebMARS, gribAPI with python enabled, emoslib, ecaccess web toolkit, all available from https://software.ecmwf.int/
+# dateutils
+# matplotlib (optional, for debugging)
+#
+# Get MARS GRIB fields from ECMWF for FLEXPART
+#
+
+#import socket
+
+#hostname=socket.gethostname()
+#ecapi= 'ecmwf' not in hostname
+try:
+    ecapi=True
+    import ecmwfapi
+except ImportError:
+    ecapi=False
+
+import calendar
+import shutil
+import datetime
+import time
+import os,glob,sys,inspect
+#from string import strip
+from argparse import ArgumentParser,ArgumentDefaultsHelpFormatter
+# add path to submit.py to pythonpath so that python finds its buddies
+localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+if localpythonpath not in sys.path:
+    sys.path.append(localpythonpath)
+
+from FlexpartTools import MARSretrieval, EIFlexpart, silentremove, \
+     Control,myerror,normalexit, interpret_args_and_control
+
+
+def getMARSdata(args,c):
+
+    if not os.path.exists(c.inputdir):
+        os.makedirs(c.inputdir)
+    print "start date %s "%(c.start_date)
+    print "end date %s "%(c.end_date)
+
+    if ecapi:
+        if int(c.public):
+            server = ecmwfapi.ECMWFDataServer()
+        else:
+            server = ecmwfapi.ECMWFService("mars")
+    else:
+        server = False
+
+    c.ecapi=ecapi
+    print 'ecapi:',c.ecapi
+# Retrieve ERA interim data for running flexpart
+
+    syear=int(c.start_date[:4])
+    smonth=int(c.start_date[4:6])
+    sday=int(c.start_date[6:])
+    start = datetime.date( year = syear, month = smonth, day = sday )
+    startm1=start- datetime.timedelta(days=1)
+    if c.basetime=='00':
+        start=startm1
+    eyear=int(c.end_date[:4])
+    emonth=int(c.end_date[4:6])
+    eday=int(c.end_date[6:])
+
+    end = datetime.date( year = eyear, month = emonth, day = eday )
+    if c.basetime=='00' or c.basetime=='12':
+        endp1=end+ datetime.timedelta(days=1)
+    else:
+        endp1=end+ datetime.timedelta(days=2)
+
+    datechunk=datetime.timedelta(days=int(c.date_chunk))
+    print 'removing content of '+c.inputdir
+    tobecleaned=glob.glob(c.inputdir+'/*_acc_*.'+str(os.getppid())+'.*.grb')
+    for f in tobecleaned:
+        os.remove(f)
+
+    times=None
+    if c.maxstep<=24:
+        day=startm1
+        while day<endp1:
+            # we need to retrieve MARS data for this period (maximum one month)
+            flexpart = EIFlexpart(c,fluxes=True)
+            if day+datechunk-datetime.timedelta(days=1)<endp1:
+                dates= day.strftime("%Y%m%d") + "/to/" + (day+datechunk-datetime.timedelta(days=1)).strftime("%Y%m%d")
+            else:
+                dates= day.strftime("%Y%m%d") + "/to/" + end.strftime("%Y%m%d")
+
+            print "retrieve " + dates + " in dir " + c.inputdir
+            try:
+                flexpart.retrieve(server, c.public, dates, times, c.inputdir)
+            except IOError:
+                myerror(c,'MARS request failed')
+
+            day+=datechunk
+    else:
+        day=start
+        while day<=end:
+            # we need to retrieve MARS data for this period (maximum one month)
+            flexpart = EIFlexpart(c,fluxes=True)
+            if day+datechunk-datetime.timedelta(days=1)<end:
+                dates= day.strftime("%Y%m%d") + "/to/" + (day+datechunk-datetime.timedelta(days=1)).strftime("%Y%m%d")
+            else:
+                dates= day.strftime("%Y%m%d") + "/to/" + end.strftime("%Y%m%d")
+
+            print "retrieve " + dates + " in dir " + c.inputdir
+            flexpart.retrieve(server, c.public, dates, times, c.inputdir)
+            day+=datechunk
+
+
+
+    tobecleaned=glob.glob(c.inputdir+'/*__*.'+str(os.getppid())+'.*.grb')
+    for f in tobecleaned:
+        os.remove(f)
+    day=start
+    times=None
+    while day<=end:
+
+            # we need to retrieve MARS data for this period (maximum one month)
+        flexpart = EIFlexpart(c)
+        if day+datechunk-datetime.timedelta(days=1)<end:
+            dates= day.strftime("%Y%m%d") + "/to/" + (day+datechunk-datetime.timedelta(days=1)).strftime("%Y%m%d")
+        else:
+            dates= day.strftime("%Y%m%d") + "/to/" + end.strftime("%Y%m%d")
+        print "retrieve " + dates + " in dir " + c.inputdir
+
+        flexpart.retrieve(server, c.public, dates, times, c.inputdir)
+        day+=datechunk
+
+
+if __name__ == "__main__":
+
+    args,c=interpret_args_and_control()
+    getMARSdata(args,c)
+    normalexit(c)
