Changeset 0a75335 in flex_extract.git

Timestamp:

Jul 10, 2020, 8:20:31 AM (4 years ago)

Author:

Anne Philipp <anne.philipp@…>

Branches:

master, ctbto, dev

Children:

8028176

Parents:

bbbe1e1

Message:

changed cds api retrieval for single levels to retrieve from CS3 disks instead of MARS - severe performance boost

Files:

• Source/Python/Classes/ControlFile.py (modified) (1 diff)
• Source/Python/Classes/MarsRetrieval.py (modified) (3 diffs)
• Source/Python/Mods/checks.py (modified) (2 diffs)
• Source/Python/_config.py (modified) (1 diff)
• Testing/Installation/Api_testscripts/test_cdsapi_modellevel.py (modified) (1 diff)
• Testing/Installation/Api_testscripts/test_ecmwfapi_public.py (modified) (1 diff)

Source/Python/Classes/ControlFile.py

@@ -616 +616 @@
                       os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS))
 
-        check_public(self.public, self.dataset)
+        check_public(self.public, self.dataset, self.marsclass)
 
         self.type = check_type(self.type, self.step)

Source/Python/Classes/MarsRetrieval.py

@@ -446 +446 @@
 
         return
+    
+    def _convert_to_cdsera5_sfc_request(self, attrs):
+        '''
+        The keywords and values for the single level download
+        with CDS API is different from MARS. This function 
+        converts the old request keywords to the new ones.
+        
+        Example request for single level downloads in CDS API
+        
+        retrieve(
+            'reanalysis-era5-single-levels',
+            {
+                'product_type': 'reanalysis',
+                'variable': 'total_precipitation',
+                'year': '2019',
+                'month': '01',
+                'day': '01',
+                'time': '00:00',
+                'format': 'grib',
+                'grid':[1.0, 1.0],
+                'area': [
+                    45, 0, 43,
+                    12,
+                ],
+            },
+            'download.grib')
+            
+        Parameters
+        ----------
+        attrs : dict
+            Dictionary of the mars request parameters.
+
+        Return
+        ------
+
+        '''
+        from datetime import datetime, timedelta
+        newattrs = {}
+
+        if '/' in attrs['date']:
+            year = set()
+            month = set()
+            day = set()
+            start,end = attrs['date'].split('/')[::2]
+            sdate = datetime.strptime(start, '%Y%m%d')
+            edate = datetime.strptime(end, '%Y%m%d')
+            date = sdate
+            while date <= edate:
+                year.add(date.year)
+                month.add(date.month)
+                day.add(date.day)      
+                date = date + timedelta(days=1)
+            newattrs['year'] =list(year)
+            newattrs['month'] = list(month)
+            newattrs['day'] =  list(day)                       
+        else:
+            date = datetime.strptime(attrs['date'], '%Y%m%d')
+            newattrs['year'] = date.year
+            newattrs['month'] = date.month
+            newattrs['day'] =  date.day                  
+        
+        newattrs['product_type'] = 'reanalysis'
+        newattrs['area'] = attrs['area'].split('/')
+        newattrs['grid'] = list(map(float,attrs['grid'].split('/')))
+        newattrs['param'] = attrs['param'].split('/')        
+        newattrs['time'] = list(map(str,range(0,24,3)))
+        newattrs['format'] = 'grib'
+                
+        return newattrs
 
     def data_retrieve(self):
@@ -479 +548 @@
             del attrs['target']
         print('target: ' + target)
-
+        
         # find all keys without a value and convert all other values to strings
         empty_keys = []
@@ -498 +567 @@
             try:
                 if cds_api and isinstance(self.server, cdsapi.Client):
+                    # distinguish between model (ECMWF MARS access) 
+                    # and surface level (CS3 online access)
+                    if attrs['levtype'].lower() == 'ml':
+                        dataset = _config.CDS_DATASET_ML
+                    else:
+                        dataset = _config.CDS_DATASET_SFC
+                        attrs = self._convert_to_cdsera5_sfc_request(attrs)
                     print('RETRIEVE ERA5 WITH CDS API!')
-                    self.server.retrieve(_config.CDS_DATASET,
+                    self.server.retrieve(dataset,
                                          attrs, target)
                 elif ec_api and isinstance(self.server, ecmwfapi.ECMWFDataServer):

Source/Python/Mods/checks.py

@@ -651 +651 @@
     return
 
-def check_public(public, dataset):
+def check_public(public, dataset, marsclass):
     '''Check wether the dataset parameter is set to a
     public data set.
@@ -667 +667 @@
 
     '''
-    if public and not dataset:
+    if public and not dataset and not (marsclass.upper() == 'EA'):
         raise ValueError('ERROR: If public MARS data are to be retrieved, '
                          'the "dataset"-parameter has to be set, too!')

Source/Python/_config.py

@@ -46 +46 @@
 INSTALL_TARGETS = ['local', 'ecgate', 'cca', 'ccb']
 
-CDS_DATASET = 'reanalysis-era5-complete'
+CDS_DATASET_ML = 'reanalysis-era5-complete'
+CDS_DATASET_SFC = 'reanalysis-era5-single-levels'
 
 # up-to-date available maximum level numbers at ECMWF, 05.10.2018

Testing/Installation/Api_testscripts/test_cdsapi_modellevel.py

@@ -13 +13 @@
         'type'    : 'fc',
         'step'    : '3/to/12/by/3',
-        'param'   : '130.128',
+        'param'   : 't',
         'levtype' : 'ml',
         'levelist': '135/to/137',

Testing/Installation/Api_testscripts/test_ecmwfapi_public.py

@@ -17 +17 @@
         'type'      : "an",
         'class'     : "ep",
+    'number'    : "0",
         'target'    : "download_cera20c_ecmwfapi.grib"
 })