Changeset f20af73 in flex_extract.git for source/python/mods

Timestamp:

Mar 8, 2019, 10:05:20 AM (5 years ago)

Author:

Anne Philipp <anne.philipp@…>

Branches:

master, ctbto, dev

Children:

82c2959

Parents:

b4a4777

Message:

added CDS API support for ERA5 instead of ECMWFAPI / refactored setup of CONTROL parameter because for local version it wanted to use not installed ECMWF_ENV file.

Location:

source/python/mods

Files:

• checks.py (modified) (2 diffs)
• get_mars_data.py (modified) (7 diffs)
• prepare_flexpart.py (modified) (3 diffs)
• tools.py (modified) (6 diffs)

source/python/mods/checks.py

@@ -27 +27 @@
 import sys
 import _config
-import exceptions
-from tools import my_error, silent_remove
+try:
+    import exceptions
+except ImportError:
+    import builtins as exceptions
+from .tools import my_error, silent_remove
 from datetime import datetime
 import numpy as np
@@ -296 +299 @@
             step = ['{:0>3}'.format(i) for i in ilist]
         elif 'to' in step.lower() and 'by' not in step.lower():
-            my_error(mailfail, step + ':\n' +
+            my_error(step + ':\n' +
                      'if "to" is used in steps parameter, '
                      'please use "by" as well')

source/python/mods/get_mars_data.py

@@ -67 +67 @@
     inspect.getfile(inspect.currentframe()))) + '/../')
 import _config
-from tools import (my_error, normal_exit, get_cmdline_args,
+from .tools import (setup_controldata, my_error, normal_exit, get_cmdline_args,
                    read_ecenv, make_dir)
 from classes.EcFlexpart import EcFlexpart
@@ -74 +74 @@
 
 try:
-    ecapi = True
+    ec_api = True
     import ecmwfapi
 except ImportError:
-    ecapi = False
+    ec_api = False
+
+try:
+    cds_api = True
+    import cdsapi
+except ImportError:
+    cds_api = False
 # ------------------------------------------------------------------------------
 # FUNCTION
@@ -95 +101 @@
     '''
 
-    args = get_cmdline_args()
-    c = ControlFile(args.controlfile)
-
-    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
-    c.assign_args_to_control(args)
-    c.assign_envs_to_control(env_parameter)
-    c.check_conditions(args.queue)
-
+    c, _, _, _ = setup_controldata()
     get_mars_data(c)
-    normal_exit(c.mailops, c.queue, 'Done!')
+    normal_exit('Retrieving MARS data: Done!')
 
     return
@@ -125 +124 @@
 
     '''
-    c.ecapi = ecapi
+    c.ec_api = ec_api
+    c.cds_api = cds_api
 
     if not os.path.exists(c.inputdir):
@@ -182 +182 @@
 
 def mk_server(c):
-    '''Creates server connection if ECMWF WebAPI is available.
+    '''Creates a server connection with available python API.
+
+    Which API is used depends on availability and the dataset to be retrieved.
+    The CDS API is used for ERA5 dataset no matter if the user is a member or
+    a public user. ECMWF WebAPI is used for all other available datasets.
 
     Parameters
@@ -192 +196 @@
     Return
     ------
-    server : ECMWFDataServer or ECMWFService
-        Connection to ECMWF server via python interface ECMWF WebAPI.
-
-    '''
-    if c.ecapi:
+    server : ECMWFDataServer, ECMWFService or Client
+        Connection to ECMWF server via python interface ECMWF WebAPI or CDS API.
+
+    '''
+    if cds_api and (c.marsclass.upper() == 'EA'):
+        server = cdsapi.Client()
+        c.ec_api = False
+    elif c.ec_api:
         if c.public:
             server = ecmwfapi.ECMWFDataServer()
         else:
             server = ecmwfapi.ECMWFService("mars")
+        c.cds_api = False
     else:
         server = False
 
-    print('Using ECMWF WebAPI: ' + str(c.ecapi))
+    print('Using ECMWF WebAPI: ' + str(c.ec_api))
+    print('Using CDS API: ' + str(c.cds_api))
 
     return server
@@ -353 +362 @@
             flexpart.retrieve(server, dates, c.public, c.request, c.inputdir)
         except IOError:
-            my_error(c.mailfail, 'MARS request failed')
+            my_error('MARS request failed')
 
         day += delta_t

source/python/mods/prepare_flexpart.py

@@ -71 +71 @@
     inspect.getfile(inspect.currentframe()))) + '/../')
 import _config
-from checks import check_ppid
+from .checks import check_ppid
 from classes.UioFiles import UioFiles
 from classes.ControlFile import ControlFile
-from tools import clean_up, get_cmdline_args, read_ecenv, make_dir
+from .tools import (setup_controldata, clean_up, get_cmdline_args,
+                   read_ecenv, make_dir)
 from classes.EcFlexpart import EcFlexpart
-
-ecapi = 'ecmwf' not in socket.gethostname()
-try:
-    if ecapi:
-        import ecmwfapi
-except ImportError:
-    ecapi = False
 
 # ------------------------------------------------------------------------------
@@ -101 +95 @@
     '''
 
-    args = get_cmdline_args()
-    c = ControlFile(args.controlfile)
-
-    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
-    c.assign_args_to_control(args)
-    c.assign_envs_to_control(env_parameter)
-    c.check_conditions(args.queue)
-
-    prepare_flexpart(args.ppid, c)
+    c, ppid, _, _ = setup_controldata()
+    prepare_flexpart(ppid, c)
+    normal_exit('Preparing FLEXPART output files: Done!')
 
     return
@@ -136 +124 @@
     '''
     check_ppid(c, ppid)
-
-    c.ecapi = ecapi
 
     # create the start and end date

source/python/mods/tools.py

@@ -65 +65 @@
 import subprocess
 import traceback
-import exceptions
+try:
+    import exceptions
+except ImportError:
+    import builtins as exceptions
 from datetime import datetime
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+
+
 
 # ------------------------------------------------------------------------------
 # METHODS
 # ------------------------------------------------------------------------------
+
+def setup_controldata():
+    '''Collects, stores and checks controlling arguments from command line,
+    CONTROL file and ECMWF_ENV file.
+
+    Parameters
+    ----------
+
+    Return
+    ------
+    c : ControlFile
+        Contains all the parameters of CONTROL file and
+        command line.
+
+    ppid : str
+        Parent process id.
+
+    queue : str
+        Name of queue for submission to ECMWF (e.g. ecgate or cca )
+
+    job_template : str
+        Name of the job template file for submission to ECMWF server.
+    '''
+    import _config
+    from classes.ControlFile import ControlFile
+
+    args = get_cmdline_args()
+    c = ControlFile(args.controlfile)
+    c.assign_args_to_control(args)
+    if os.path.isfile(_config.PATH_ECMWF_ENV):
+        env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
+        c.assign_envs_to_control(env_parameter)
+    c.check_conditions(args.queue)
+
+    return c, args.ppid, args.queue, args.job_template
 
 def none_or_str(value):
@@ -259 +299 @@
     print("... clean inputdir!")
 
-    cleanlist = glob.glob(os.path.join(c.inputdir, "*"))
+    cleanlist = [file for file in glob.glob(os.path.join(c.inputdir, "*"))
+                 if not os.path.basename(file).startswith(c.prefix)]
 
     if cleanlist:
         for element in cleanlist:
-            if c.prefix not in element:
-                silent_remove(element)
-            if c.ecapi is False and (c.ectrans == 1 or c.ecstorage == 1):
-                silent_remove(element)
+            silent_remove(element)
         print("... done!")
     else:
@@ -274 +312 @@
 
 
-def my_error(users, message='ERROR'):
+def my_error(message='ERROR'):
     '''Prints a specified error message which can be passed to the function
     before exiting the program.
@@ -280 +318 @@
     Parameters
     ----------
-    user : list of str
-        Contains all email addresses which should be notified.
-        It might also contain just the ecmwf user name which wil trigger
-        mailing to the associated email address for this user.
-
     message : str, optional
         Error message. Default value is "ERROR".
@@ -464 +497 @@
     else:
         for data in fdata:
-            if data[0] != '!':
-                table128[data[0:3]] = data[59:64].strip()
+            if data != '' and data[0] != '!':
+                table128[data[0:3]] = data[59:65].strip()
 
     return table128
@@ -508 +541 @@
 
     return ipar
+
+def to_param_id_with_tablenumber(pars, table):
+    '''Transform parameter names to parameter ids and add table id.
+
+    Conversion with ECMWF grib table 128.
+
+    Parameters
+    ----------
+    pars : str
+        Addpar argument from CONTROL file in the format of
+        parameter names instead of ids. The parameter short
+        names are sepearted with "/" and they are passed as
+        one single string.
+
+    table : dict
+        Contains the ECMWF grib table 128 information.
+        The key is the parameter number and the value is the
+        short name of the parameter.
+
+    Return
+    ------
+    spar : str
+        List of addpar parameters from CONTROL file transformed to
+        parameter ids in the format of integer.
+    '''
+    if not pars:
+        return []
+    if not isinstance(pars, str):
+        pars=str(pars)
+
+    cpar = pars.upper().split('/')
+    spar = []
+    for par in cpar:
+        for k, v in table.iteritems():
+            if par == k or par == v:
+                spar.append(k + '.128')
+                break
+        else:
+            print('\n\n\t\tWarning: par ' + par + ' not found in table 128\n\n')
+
+    return '/'.join(spar)
 
 def get_list_as_string(list_obj, concatenate_sign=', '):