Changeset f20af73 in flex_extract.git for source/python/classes/MarsRetrieval.py

Timestamp:

Mar 8, 2019, 10:05:20 AM (5 years ago)

Author:

Anne Philipp <anne.philipp@…>

Branches:

master, ctbto, dev

Children:

82c2959

Parents:

b4a4777

Message:

added CDS API support for ERA5 instead of ECMWFAPI / refactored setup of CONTROL parameter because for local version it wanted to use not installed ECMWF_ENV file.

File:

• source/python/classes/MarsRetrieval.py (modified) (7 diffs)

source/python/classes/MarsRetrieval.py

@@ -35 +35 @@
 import sys
 import subprocess
+import traceback
 
 # software specific classes and modules from flex_extract
 sys.path.append('../')
 import _config
+try:
+    ec_api = True
+    import ecmwfapi
+except ImportError:
+    ec_api = False
+
+try:
+    cds_api = True
+    import cdsapi
+except ImportError:
+    cds_api = False
 # ------------------------------------------------------------------------------
 # CLASS
@@ -130 +142 @@
     '''
 
-    def __init__(self, server, public, marsclass="ei", dataset="", type="",
+    def __init__(self, server, public, marsclass="EA", dataset="", type="",
                  levtype="", levelist="", repres="", date="", resol="",
                  stream="", area="", time="", step="", expver="1",
@@ -158 +170 @@
         marsclass : str, optional
             Characterisation of dataset. E.g. EI (ERA-Interim),
-            E4 (ERA40), OD (Operational archive), ea (ERA5).
-            Default is the ERA-Interim dataset "ei".
+            E4 (ERA40), OD (Operational archive), EA (ERA5).
+            Default is the ERA5 dataset "EA".
 
         dataset : str, optional
@@ -429 +441 @@
     def data_retrieve(self):
         '''Submits a MARS retrieval. Depending on the existence of
-        ECMWF Web-API it is submitted via Python or a
+        ECMWF Web-API or CDS API it is submitted via Python or a
         subprocess in the Shell. The parameter for the mars retrieval
         are taken from the defined class attributes.
@@ -452 +464 @@
         attrs['class'] = mclass
 
-        # prepare target variable as needed for the Web API mode
+        # prepare target variable as needed for the Web API or CDS API mode
         # within the dictionary for full access
         # as a single variable for public access
@@ -475 +487 @@
         if self.server:
             try:
-                if self.public:
-                    print('RETRIEVE PUBLIC DATA!')
+                if cds_api and isinstance(self.server, cdsapi.Client):
+                    print('RETRIEVE ERA5 WITH CDS API!')
+                    self.server.retrieve(_config.CDS_DATASET,
+                                         attrs, target)
+                elif ec_api and isinstance(self.server, ecmwfapi.ECMWFDataServer):
+                    print('RETRIEVE PUBLIC DATA (NOT ERA5)!')
                     self.server.retrieve(attrs)
+                elif ec_api and isinstance(self.server, ecmwfapi.ECMWFService):
+                    print('EXECUTE NON-PUBLIC RETRIEVAL (NOT ERA5)!')
+                    self.server.execute(attrs, target)
                 else:
-                    print('EXECUTE NON-PUBLIC RETRIEVAL!')
-                    self.server.execute(attrs, target)
-            except:
-                e = sys.exc_info()[0]
-                print("ERROR: ", e)
-                print('MARS Request failed!')
-                if not self.public and os.stat(target).st_size == 0:
-                    print('MARS Request returned no data - '
-                          'please check request')
+                    print('ERROR:')
+                    print('No match for Web API instance!')
                     raise IOError
-                elif self.public and os.stat(target).st_size == 0:
-                    print('Public MARS Request returned no data - '
-                          'please check request')
-                    raise IOError
-                else:
-                    raise IOError
-        # MARS request via extra process in shell
+            except Exception as e:
+                print('\n\nMARS Request failed!')
+                print(e)
+                tb = sys.exc_info()[2]
+                print(traceback.format_exc())
+                sys.exit()
+
+        # MARS request via call in shell
         else:
             request_str = 'ret'
@@ -501 +514 @@
                 request_str = request_str + ',' + key + '=' + str(value)
             request_str += ',target="' + target + '"'
-            p = subprocess.Popen(['mars'],
+            p = subprocess.Popen(['mars', '-p'],
                                  stdin=subprocess.PIPE,
                                  stdout=subprocess.PIPE,