Changeset efdb01a in flex_extract.git for python/getMARSdata.py

Timestamp:

May 9, 2018, 12:15:00 PM (6 years ago)

Author:

Anne Philipp <anne.philipp@…>

Branches:

master, ctbto, dev

Children:

991df6a

Parents:

02c8c50

Message:

whole bunch of modifications due to new structure of ECMWFDATA, added basics of documentation, minor programming corrections

File:

• python/getMARSdata.py (modified) (7 diffs, 1 prop)

python/getMARSdata.py

@@ -1 +1 @@
 #!/usr/bin/env python
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# Functionality provided: Prepare input 3D-wind fields in hybrid coordinates + surface fields for FLEXPART runs
-#
-# Creation: October  2014 - Anne Fouilloux - University of Oslo
-# Extension November 2015 - Leopold Haimberger - University of Vienna for:
-# - using the WebAPI also for general MARS retrievals
-# - job submission on ecgate and cca
-# - job templates suitable for twice daily operational dissemination
-# - dividing retrievals of longer periods into digestable chunks
-# - retrieve also longer term forecasts, not only analyses and short term forecast data
-# - conversion into GRIB2
-# - conversion into .fp format for faster execution of FLEXPART
-#
-#
-# Further documentation may be obtained from www.flexpart.eu
-#
-# Requirements:
-# in addition to a standard python 2.6 or 2.7 installation the following packages need to be installed
-# ECMWF WebMARS, gribAPI with python enabled, emoslib, ecaccess web toolkit, all available from https://software.ecmwf.int/
-# dateutils
-# matplotlib (optional, for debugging)
-#
-# Get MARS GRIB fields from ECMWF for FLEXPART
-#
-
-#import socket
-
-#hostname=socket.gethostname()
-#ecapi= 'ecmwf' not in hostname
+# -*- coding: utf-8 -*-
+#************************************************************************
+# TODO AP
+# - Change History ist nicht angepasst ans File!
+# - add file description
+#************************************************************************
+"""
+@Author: Anne Fouilloux (University of Oslo)
+
+@Date: October 2014
+
+@ChangeHistory:
+    November 2015 - Leopold Haimberger (University of Vienna):
+        - using the WebAPI also for general MARS retrievals
+        - job submission on ecgate and cca
+        - job templates suitable for twice daily operational dissemination
+        - dividing retrievals of longer periods into digestable chunks
+        - retrieve also longer term forecasts, not only analyses and
+          short term forecast data
+        - conversion into GRIB2
+        - conversion into .fp format for faster execution of FLEXPART
+
+    February 2018 - Anne Philipp (University of Vienna):
+        - applied PEP8 style guide
+        - added documentation
+        - minor changes in programming style for consistence
+
+@License:
+    (C) Copyright 2014-2018.
+
+    This software is licensed under the terms of the Apache Licence Version 2.0
+    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+@Requirements:
+    - A standard python 2.6 or 2.7 installation
+    - dateutils
+    - ECMWF specific packages, all available from https://software.ecmwf.int/
+        ECMWF WebMARS, gribAPI with python enabled, emoslib and
+        ecaccess web toolkit
+
+@Description:
+    Further documentation may be obtained from www.flexpart.eu.
+
+"""
+# ------------------------------------------------------------------------------
+# MODULES
+# ------------------------------------------------------------------------------
 try:
     ecapi=True
@@ -42 +57 @@
 import datetime
 import time
-import os,glob,sys,inspect
-#from string import strip
-from argparse import ArgumentParser,ArgumentDefaultsHelpFormatter
+import os
+import glob
+import sys
+import inspect
 # add path to submit.py to pythonpath so that python finds its buddies
 localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
@@ -50 +66 @@
     sys.path.append(localpythonpath)
 
-from FlexpartTools import ECFlexpart,  \
-                          Control, myerror, normalexit, \
+from Control import Control
+from Tools import myerror, normalexit, \
                           interpret_args_and_control
-
-
+from ECFlexpart import ECFlexpart
+
+# ------------------------------------------------------------------------------
+# FUNCTION
+# ------------------------------------------------------------------------------
 def getMARSdata(args, c):
 
@@ -71 +90 @@
     print 'ecapi:', c.ecapi
 
-# Retrieve ERA interim data for running flexpart
+# Retrieve EC data for running flexpart
 #AP change this variant to correct format conversion with datetime
 #AP import datetime and timedelta explicitly
@@ -81 +100 @@
     if c.basetime == '00':
         start = startm1
+
     eyear = int(c.end_date[:4])
     emonth = int(c.end_date[4:6])
@@ -91 +111 @@
 
     datechunk = datetime.timedelta(days=int(c.date_chunk))
+
+    # retrieving of accumulated data fields (flux data), (maximum one month)
+
+    # remove old files
     print 'removing content of ' + c.inputdir
-    tobecleaned = glob.glob(c.inputdir + '/*_acc_*.' + str(os.getppid()) + '.*.grb')
+    tobecleaned = glob.glob(c.inputdir + '/*_acc_*.' + \
+                            str(os.getppid()) + '.*.grb')
     for f in tobecleaned:
         os.remove(f)
 
-    times=None
-    if c.maxstep<24:
-        day=startm1
-        while day<endp1:
-                # we need to retrieve MARS data for this period (maximum one month)
-                flexpart = ECFlexpart(c,fluxes=True)
-                if day+datechunk-datetime.timedelta(days=1)<endp1:
-                    dates= day.strftime("%Y%m%d") + "/to/" + (day+datechunk-datetime.timedelta(days=1)).strftime("%Y%m%d")
+    times = None
+    # if forecast for maximum one day (upto 23h) are to be retrieved,
+    # collect accumulation data (flux data)
+    # with additional days in the beginning and at the end
+    # (used for complete disaggregation of original period)
+    if c.maxstep < 24:
+        day = startm1
+        while day < endp1:
+                # retrieve MARS data for the whole period
+                flexpart = ECFlexpart(c, fluxes=True)
+                tmpday = day + datechunk - datetime.timedelta(days=1)
+                if tmpday < endp1:
+                    dates = day.strftime("%Y%m%d") + "/to/" + \
+                            tmpday.strftime("%Y%m%d")
                 else:
-                    dates= day.strftime("%Y%m%d") + "/to/" + end.strftime("%Y%m%d")
+                    dates = day.strftime("%Y%m%d") + "/to/" + \
+                            end.strftime("%Y%m%d")
 
                 print "retrieve " + dates + " in dir " + c.inputdir
+
                 try:
                     flexpart.retrieve(server, dates, times, c.inputdir)
@@ -113 +146 @@
                     myerror(c,'MARS request failed')
 
-                day+=datechunk
+                day += datechunk
+
+    # if forecast data longer than 24h are to be retrieved,
+    # collect accumulation data (flux data)
+    # with the exact start and end date
+    # (disaggregation will be done for the
+    # exact time period with boundary conditions)
     else:
-        day=start
-        while day<=end:
-                # we need to retrieve MARS data for this period (maximum one month)
-                flexpart = ECFlexpart(c,fluxes=True)
-                if day+datechunk-datetime.timedelta(days=1)<end:
-                    dates= day.strftime("%Y%m%d") + "/to/" + (day+datechunk-datetime.timedelta(days=1)).strftime("%Y%m%d")
+        day = start
+        while day <= end:
+                # retrieve MARS data for the whole period
+                flexpart = ECFlexpart(c, fluxes=True)
+                tmpday = day + datechunk - datetime.timedelta(days=1)
+                if tmpday < end:
+                    dates = day.strftime("%Y%m%d") + "/to/" + \
+                            tmpday.trftime("%Y%m%d")
                 else:
-                    dates= day.strftime("%Y%m%d") + "/to/" + end.strftime("%Y%m%d")
+                    dates = day.strftime("%Y%m%d") + "/to/" + \
+                            end.strftime("%Y%m%d")
 
                 print "retrieve " + dates + " in dir " + c.inputdir
-                flexpart.retrieve(server, dates, times, c.inputdir)
-                day+=datechunk
-
-
-
-    tobecleaned=glob.glob(c.inputdir+'/*__*.'+str(os.getppid())+'.*.grb')
+
+                try:
+                    flexpart.retrieve(server, dates, times, c.inputdir)
+                except IOError:
+                    myerror(c, 'MARS request failed')
+
+                day += datechunk
+
+    # retrieving of normal data fields (non flux data), (maximum one month)
+
+    # remove old *__* files
+    tobecleaned = glob.glob(c.inputdir + '/*__*.' +
+                            str(os.getppid()) + '.*.grb')
     for f in tobecleaned:
         os.remove(f)
-    day=start
-    times=None
-    while day<=end:
-
-               # we need to retrieve MARS data for this period (maximum one month)
-            flexpart = ECFlexpart(c)
-            if day+datechunk-datetime.timedelta(days=1)<end:
-                dates= day.strftime("%Y%m%d") + "/to/" + (day+datechunk-datetime.timedelta(days=1)).strftime("%Y%m%d")
+    day = start
+    times = None
+    while day <= end:
+            # retrieve MARS data for the whole period
+            flexpart = ECFlexpart(c, fluxes=False)
+            tmpday = day+datechunk-datetime.timedelta(days=1)
+            if tmpday < end:
+                dates = day.strftime("%Y%m%d") + "/to/" + \
+                        tmpday.strftime("%Y%m%d")
             else:
-                dates= day.strftime("%Y%m%d") + "/to/" + end.strftime("%Y%m%d")
+                dates = day.strftime("%Y%m%d") + "/to/" + \
+                        end.strftime("%Y%m%d")
+
             print "retrieve " + dates + " in dir " + c.inputdir
 
-            flexpart.retrieve(server, dates, times, c.inputdir)
-            day+=datechunk
-
+            try:
+                flexpart.retrieve(server, dates, times, c.inputdir)
+            except IOError:
+                myerror(c, 'MARS request failed')
+
+            day += datechunk
+
+    return
 
 if __name__ == "__main__":
 
-    args,c=interpret_args_and_control()
-    getMARSdata(args,c)
+    args, c = interpret_args_and_control()
+    getMARSdata(args, c)
     normalexit(c)