Changeset 5bad6ec in flex_extract.git for source/python/classes/MarsRetrieval.py

Timestamp:

Oct 5, 2018, 5:20:48 PM (6 years ago)

Author:

Anne Philipp <anne.philipp@…>

Branches:

master, ctbto, dev

Children:

ae88f7d

Parents:

ca867de

Message:

added possibility to extract public datasets via an logical public parameter

File:

• source/python/classes/MarsRetrieval.py (modified) (9 diffs)

source/python/classes/MarsRetrieval.py

@@ -83 +83 @@
     '''
 
-    def __init__(self, server, marsclass="ei", type="", levtype="",
-                 levelist="", repres="", date="", resol="", stream="",
-                 area="", time="", step="", expver="1", number="",
-                 accuracy="", grid="", gaussian="", target="",
+    def __init__(self, server, public, marsclass="ei", dataset="", type="",
+                 levtype="", levelist="", repres="", date="", resol="",
+                 stream="", area="", time="", step="", expver="1",
+                 number="", accuracy="", grid="", gaussian="", target="",
                  param=""):
         '''
@@ -106 +106 @@
                 It is needed for the pythonic access of ECMWF data.
 
+            public: integer
+                Decides which Web API version is used:
+                0: member-state users and full archive access
+                1: public access and limited access to the public server and
+                   datasets. Needs the parameter dataset.
+                Default is "0" and for member-state users.
+
             marsclass: string, optional
                 Characterisation of dataset. E.g. EI (ERA-Interim),
                 E4 (ERA40), OD (Operational archive), ea (ERA5).
                 Default is the ERA-Interim dataset "ei".
+
+            dataset: string, optional
+                For public datasets there is the specific naming and parameter
+                dataset which has to be used to characterize the type of
+                data. Usually there is less data available, either in times,
+                domain or parameter.
+                Default is an empty string.
 
             type: string, optional
@@ -289 +303 @@
 
         self.server = server
+        self.public = public
         self.marsclass = marsclass
+        self.dataset = dataset
         self.type = type
         self.levtype = levtype
@@ -325 +341 @@
         '''
         # Get all class attributes and their values as a dictionary
-        attrs = vars(self)
+        attrs = vars(self).copy()
 
         # iterate through all attributes and print them
         # with their corresponding values
         for item in attrs.items():
-            if item[0] in 'server':
+            if item[0] in ['server', 'public']:
                 pass
             else:
@@ -358 +374 @@
         '''
         # Get all class attributes and their values as a dictionary
-        attrs = vars(self)
+        attrs = vars(self).copy()
 
         # open a file to store all requests to
@@ -367 +383 @@
             # with their corresponding values
             for item in attrs.items():
-                if item[0] in 'server':
+                if item[0] in ['server', 'public']:
                     pass
                 else:
@@ -396 +412 @@
 
         # Get all class attributes and their values as a dictionary
-        attrs = vars(self)
+        attrs = vars(self).copy()
         del attrs['server']
+        del attrs['public']
 
         # open a file to store all requests to
@@ -425 +442 @@
         '''
         # Get all class attributes and their values as a dictionary
-        attrs = vars(self)
-
-        # convert the dictionary of attributes into a comma
-        # seperated list of attributes with their values
-        # needed for the retrieval call
-        s = 'ret'
-        for k, v in attrs.iteritems():
-            if k in 'server':
-                continue
-            if k == 'marsclass':
-                k = 'class'
-            if v == '':
-                continue
-            if k.lower() == 'target':
-                target = v
+        attrs = vars(self).copy()
+
+        # eliminate unnecessary attributes from the dictionary attrs
+        del attrs['server']
+        del attrs['public']
+
+        # exchange parameter name for marsclass
+        mclass = attrs.get('marsclass')
+        del attrs['marsclass']
+        attrs['class'] = mclass
+
+        # prepare target variable as needed for the Web API mode
+        # within the dictionary for full access
+        # as a single variable for public access
+        target = attrs.get('target')
+        if not int(self.public):
+            del attrs['target']
+        print('target: ' + target)
+
+        # find all keys without a value and convert all other values to strings
+        empty_keys = []
+        for key, value in attrs.itteritems():
+            if value == '':
+                empty_keys.append(str(key))
             else:
-                s = s + ',' + k + '=' + str(v)
+                attrs[key] = str(value)
+
+        # delete all empty parameter from the dictionary
+        for key in empty_keys:
+            del attrs[key]
 
         # MARS request via Python script
-        if self.server is not False:
+        if self.server:
             try:
-                self.server.execute(s, target)
+                if self.public:
+                    print('RETRIEVE PUBLIC DATA!')
+                    self.server.retrieve(attrs)
+                else:
+                    print('EXECUTE NON-PUBLIC RETRIEVAL!')
+                    self.server.execute(attrs, target)
             except:
-                print('MARS Request failed, \
-                      have you already registered at apps.ecmwf.int?')
+                e = sys.exc_info()[0]
+                print("ERROR: ", e)
+                print('MARS Request failed!')
+            if not self.public and os.stat(target).st_size == 0:
+                print('MARS Request returned no data - please check request')
                 raise IOError
-            if os.stat(target).st_size == 0:
-                print('MARS Request returned no data - please check request')
+            elif self.public and os.stat(target).st_size == 0:
+                print('Public MARS Request returned no data - '
+                      'please check request')
+                raise IOError
+            else:
                 raise IOError
         # MARS request via extra process in shell
         else:
-            s += ',target = "' + target + '"'
-            p = subprocess.Popen(['mars'], stdin=subprocess.PIPE,
+            request_str = 'ret'
+            for key, value in attrs.iteritems():
+                request_str = request_str + ',' + key + '=' + str(value)
+            request_str += ',target="' + target + '"'
+            p = subprocess.Popen(['mars'],
+                                 stdin=subprocess.PIPE,
                                  stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE, bufsize=1)
-            pout = p.communicate(input=s)[0]
+                                 stderr=subprocess.PIPE,
+                                 bufsize=1)
+            pout = p.communicate(input=request_str)[0]
             print(pout.decode())
 
@@ -466 +512 @@
                 print('MARS Request failed - please check request')
                 raise IOError
-
-            if os.stat(target).st_size == 0:
+            elif os.stat(target).st_size == 0:
                 print('MARS Request returned no data - please check request')
                 raise IOError
+            else:
+                raise
 
         return