Changeset f20af73 in flex_extract.git for source/python/classes


Ignore:
Timestamp:
Mar 8, 2019, 10:05:20 AM (5 years ago)
Author:
Anne Philipp <anne.philipp@…>
Branches:
master, ctbto, dev
Children:
82c2959
Parents:
b4a4777
Message:

added CDS API support for ERA5 instead of ECMWFAPI / refactored setup of CONTROL parameter because for local version it wanted to use not installed ECMWF_ENV file.

Location:
source/python/classes
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • source/python/classes/ControlFile.py

    rae2756e rf20af73  
    318318        Default value is 0.
    319319
    320     ecapi : boolean
    321         Tells wether the ECMWF Web APi was able to load or not.
     320    ec_api : boolean
     321        Tells wether the ECMWF Web API was able to load or not.
     322        Default value is None.
     323
     324    cds_api : boolean
     325        Tells wether the CDS API was able to load or not.
    322326        Default value is None.
    323327
    … …  
    417421        self.request = 0
    418422        self.public = 0
    419         self.ecapi = None
     423        self.ec_api = None
     424        self.cds_api = None
    420425        self.purefc = 0
    421426        self.rrint = 0
    … …  
    481486                                data[1] = data[1][:i] + var + data[1][k+1:]
    482487                            else:
    483                                 my_error(self.mailfail,
    484                                          'Could not find variable '
     488                                my_error('Could not find variable '
    485489                                         + data[1][j+1:k] + ' while reading ' +
    486490                                         self.controlfile)

  • source/python/classes/EcFlexpart.py

    re446e85 rf20af73  
    7373sys.path.append('../')
    7474import _config
    75 from GribUtil import GribUtil
     75from .GribUtil import GribUtil
    7676from mods.tools import (init128, to_param_id, silent_remove, product,
    7777                        my_error, make_dir, get_informations, get_dimensions,
    78                         execute_subprocess)
    79 from MarsRetrieval import MarsRetrieval
    80 from UioFiles import UioFiles
     78                        execute_subprocess, to_param_id_with_tablenumber)
     79from .MarsRetrieval import MarsRetrieval
     80from .UioFiles import UioFiles
    8181import mods.disaggregation as disaggregation
    8282
    … …  
    403403        elif not gauss and not eta:
    404404            self.params['OG__ML'][0] += '/U/V'
    405         else:
    406             print('Warning: Collecting etadot and parameters for gaussian grid \
    407                             is a very costly parameter combination, \
    408                             use this combination only for debugging!')
    409             self.params['GG__SL'] = ['Q', 'ML', '1', \
     405        else:  # GAUSS and ETA
     406            print('Warning: Collecting etadot and parameters for gaussian grid '
     407                           'is a very costly parameter combination, '
     408                           'use this combination only for debugging!')
     409            self.params['GG__SL'] = ['Q', 'ML', '1',
    410410                                     '{}'.format((int(self.resol) + 1) / 2)]
    411             self.params['GG__ML'] = ['U/V/D/77', 'ML', self.glevelist, \
     411            self.params['GG__ML'] = ['U/V/D/ETADOT', 'ML', self.glevelist,
    412412                                     '{}'.format((int(self.resol) + 1) / 2)]
    413413
    … …  
    419419
    420420        # ADDITIONAL FIELDS FOR FLEXPART-WRF MODEL (IF QUESTIONED)
    421         #-----------------------------------------------------------------------
     421        # -----------------------------------------------------------------------
    422422        if wrf:
    423423            self.params['OG__ML'][0] += '/Z/VO'
    424424            if '/D' not in self.params['OG__ML'][0]:
    425425                self.params['OG__ML'][0] += '/D'
     426
    426427            wrf_sfc = ['SP','SKT','SST','CI','STL1','STL2', 'STL3','STL4',
    427428                       'SWVL1','SWVL2','SWVL3','SWVL4']
    … …  
    452453
    453454        '''
    454         self.params['OG_acc_SL'] = ["LSP/CP/SSHF/EWSS/NSSS/SSR", \
     455        self.params['OG_acc_SL'] = ["LSP/CP/SSHF/EWSS/NSSS/SSR",
    455456                                    'SFC', '1', self.grid]
    456457        return
    … …  
    684685                                        pk,
    685686                                        retr_param_dict['date'].split('/')[0])
    686                 retr_param_dict['param'] = pv[0]
     687                table128 = init128(_config.PATH_GRIBTABLE)
     688                ids = to_param_id_with_tablenumber(pv[0], table128)
     689                retr_param_dict['param'] = ids
    687690                retr_param_dict['levtype'] = pv[1]
    688691                retr_param_dict['levelist'] = pv[2]
    … …  
    12381241
    12391242        '''
    1240         print('... disaggregation or precipitation with new method.')
     1243        print('... disaggregation of precipitation with new method.')
    12411244        lsp_new_np = np.zeros((ni * nj, nt * 3), dtype=np.float64)
    12421245        cp_new_np = np.zeros((ni * nj, nt * 3), dtype=np.float64)
    … …  
    12681271                    filename1 = c.prefix + date.strftime('%y%m%d%H') + '_1'
    12691272                    filename2 = c.prefix + date.strftime('%y%m%d%H') + '_2'
    1270 
    1271                 # collect for final processing
    1272                 self.outputfilelist.append(os.path.basename(fluxfilename))
    1273                 self.outputfilelist.append(os.path.basename(filename1))
    1274                 self.outputfilelist.append(os.path.basename(filename2))
    12751273
    12761274                # write original time step to flux file as usual
    … …  
    14491447            cdate_hour = datetime.strftime(timestamp, '%Y%m%d%H')
    14501448
    1451             # eliminate all temporary times
     1449            # skip all temporary times
    14521450            # which are outside the retrieval period
    14531451            if timestamp < start_period or \
    … …  
    15451543            os.chdir(c.inputdir)
    15461544            if os.stat('fort.21').st_size == 0 and c.eta:
    1547                 print('Parameter 77 (etadot) is missing, most likely it is \
    1548                        not available for this type or date/time\n')
     1545                print('Parameter 77 (etadot) is missing, most likely it is '
     1546                      'not available for this type or date / time\n')
    15491547                print('Check parameters CLASS, TYPE, STREAM, START_DATE\n')
    1550                 my_error(c.mailfail, 'fort.21 is empty while parameter eta \
    1551                          is set to 1 in CONTROL file')
    1552 #============================================================================================
     1548                my_error('fort.21 is empty while parameter eta '
     1549                         'is set to 1 in CONTROL file')
     1550# ============================================================================================
    15531551            # write out all output to log file before starting fortran programm
    15541552            sys.stdout.flush()
    … …  
    15601558
    15611559            os.chdir(pwd)
    1562 #============================================================================================
     1560# ============================================================================================
    15631561            # create name of final output file, e.g. EN13040500 (ENYYMMDDHH)
    15641562            if c.purefc:
    … …  
    15771575            # collect for final processing
    15781576            self.outputfilelist.append(os.path.basename(fnout))
    1579 #============================================================================================
     1577            # get additional precipitation subgrid data if available
     1578            if c.rrint:
     1579                self.outputfilelist.append(os.path.basename(fnout + '_1'))
     1580                self.outputfilelist.append(os.path.basename(fnout + '_2'))
     1581# ============================================================================================
    15801582            # create outputfile and copy all data from intermediate files
    15811583            # to the outputfile (final GRIB input files for FLEXPART)
    … …  
    15971599                    shutil.copyfileobj(open(os.path.join(c.inputdir, 'fort.25'),
    15981600                                            'rb'), fout)
    1599 #============================================================================================
     1601# ============================================================================================
    16001602        if c.wrf:
    16011603            fwrf.close()
    … …  
    16921694        print('\n\nPostprocessing:\n Format: {}\n'.format(c.format))
    16931695
    1694         if not c.ecapi:
     1696        if _config.FLAG_ON_ECMWFSERVER:
    16951697            print('ecstorage: {}\n ecfsdir: {}\n'.
    16961698                  format(c.ecstorage, c.ecfsdir))
    … …  
    17141716                                   'FILES FAILED!')
    17151717
    1716             if c.ectrans and not c.ecapi:
     1718            if c.ectrans and _config.FLAG_ON_ECMWFSERVER:
    17171719                execute_subprocess(['ectrans', '-overwrite', '-gateway',
    17181720                                    c.gateway, '-remote', c.destination,
    … …  
    17201722                                   error_msg='TRANSFER TO LOCAL SERVER FAILED!')
    17211723
    1722             if c.ecstorage and not c.ecapi:
     1724            if c.ecstorage and _config.FLAG_ON_ECMWFSERVER:
    17231725                execute_subprocess(['ecp', '-o', ofile,
    17241726                                    os.path.expandvars(c.ecfsdir)],

  • source/python/classes/MarsRetrieval.py

    r6f951ca rf20af73  
    3535import sys
    3636import subprocess
     37import traceback
    3738
    3839# software specific classes and modules from flex_extract
    3940sys.path.append('../')
    4041import _config
     42try:
     43    ec_api = True
     44    import ecmwfapi
     45except ImportError:
     46    ec_api = False
     47
     48try:
     49    cds_api = True
     50    import cdsapi
     51except ImportError:
     52    cds_api = False
    4153# ------------------------------------------------------------------------------
    4254# CLASS
    … …  
    130142    '''
    131143
    132     def __init__(self, server, public, marsclass="ei", dataset="", type="",
     144    def __init__(self, server, public, marsclass="EA", dataset="", type="",
    133145                 levtype="", levelist="", repres="", date="", resol="",
    134146                 stream="", area="", time="", step="", expver="1",
    … …  
    158170        marsclass : str, optional
    159171            Characterisation of dataset. E.g. EI (ERA-Interim),
    160             E4 (ERA40), OD (Operational archive), ea (ERA5).
    161             Default is the ERA-Interim dataset "ei".
     172            E4 (ERA40), OD (Operational archive), EA (ERA5).
     173            Default is the ERA5 dataset "EA".
    162174
    163175        dataset : str, optional
    … …  
    429441    def data_retrieve(self):
    430442        '''Submits a MARS retrieval. Depending on the existence of
    431         ECMWF Web-API it is submitted via Python or a
     443        ECMWF Web-API or CDS API it is submitted via Python or a
    432444        subprocess in the Shell. The parameter for the mars retrieval
    433445        are taken from the defined class attributes.
    … …  
    452464        attrs['class'] = mclass
    453465
    454         # prepare target variable as needed for the Web API mode
     466        # prepare target variable as needed for the Web API or CDS API mode
    455467        # within the dictionary for full access
    456468        # as a single variable for public access
    … …  
    475487        if self.server:
    476488            try:
    477                 if self.public:
    478                     print('RETRIEVE PUBLIC DATA!')
     489                if cds_api and isinstance(self.server, cdsapi.Client):
     490                    print('RETRIEVE ERA5 WITH CDS API!')
     491                    self.server.retrieve(_config.CDS_DATASET,
     492                                         attrs, target)
     493                elif ec_api and isinstance(self.server, ecmwfapi.ECMWFDataServer):
     494                    print('RETRIEVE PUBLIC DATA (NOT ERA5)!')
    479495                    self.server.retrieve(attrs)
     496                elif ec_api and isinstance(self.server, ecmwfapi.ECMWFService):
     497                    print('EXECUTE NON-PUBLIC RETRIEVAL (NOT ERA5)!')
     498                    self.server.execute(attrs, target)
    480499                else:
    481                     print('EXECUTE NON-PUBLIC RETRIEVAL!')
    482                     self.server.execute(attrs, target)
    483             except:
    484                 e = sys.exc_info()[0]
    485                 print("ERROR: ", e)
    486                 print('MARS Request failed!')
    487                 if not self.public and os.stat(target).st_size == 0:
    488                     print('MARS Request returned no data - '
    489                           'please check request')
     500                    print('ERROR:')
     501                    print('No match for Web API instance!')
    490502                    raise IOError
    491                 elif self.public and os.stat(target).st_size == 0:
    492                     print('Public MARS Request returned no data - '
    493                           'please check request')
    494                     raise IOError
    495                 else:
    496                     raise IOError
    497         # MARS request via extra process in shell
     503            except Exception as e:
     504                print('\n\nMARS Request failed!')
     505                print(e)
     506                tb = sys.exc_info()[2]
     507                print(traceback.format_exc())
     508                sys.exit()
     509
     510        # MARS request via call in shell
    498511        else:
    499512            request_str = 'ret'
    … …  
    501514                request_str = request_str + ',' + key + '=' + str(value)
    502515            request_str += ',target="' + target + '"'
    503             p = subprocess.Popen(['mars'],
     516            p = subprocess.Popen(['mars', '-p'],
    504517                                 stdin=subprocess.PIPE,
    505518                                 stdout=subprocess.PIPE,
Note: See TracChangeset for help on using the changeset viewer.
hosted by ZAMG