Changeset 54a8a01 in flex_extract.git


Ignore:
Timestamp:
Aug 31, 2018, 7:50:37 AM (14 months ago)
Author:
Anne Philipp <anne.philipp@…>
Branches:
dev
Children:
597d4d1
Parents:
e1228f3
Message:

restructuring, documentations and bug fixes

Files:
15 added
2 deleted
25 edited

Legend:

Unmodified
Added
Removed
  • python/CONTROL.temp

    rccab809 r54a8a01  
    1 DAY1
     1DAY1 20130501
    22DAY2
    33DTIME 3
  • python/CONTROL_FC

    rd69b677 r54a8a01  
    1 DAY1 20131107
    2 DAY2 20131108
    3 DTIME 1
     1DAY1
     2DAY2
     3DTIME 3
    44M_TYPE FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC
    55M_TIME 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
    66M_STEP 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
     7ACCTYPE FC
     8ACCTIME 00
     9ACCMAXSTEP 36
    710M_MAXSTEP 36
    811M_LEVEL 137
     
    1316M_EXPVER 1
    1417M_GRID 1000 
    15 M_LEFT -179000
    16 M_LOWER -90000
    17 M_UPPER 90000
    18 M_RIGHT 180000
     18M_LEFT -25000
     19M_LOWER 10000
     20M_UPPER 60000
     21M_RIGHT 60000
    1922M_RESOL 159
    20 M_GAUSS 0
    21 M_ACCURACY 16
    22 M_OMEGA 0
    23 M_OMEGADIFF 0
    2423M_ETA 1
    25 M_ETADIFF 0
    2624M_DPDETA 1
    2725M_SMOOTH 0
    28 M_FORMAT GRIB1
    2926M_ADDPAR /186/187/188/235/139/39
    3027M_WRF 1
    3128M_CWC 1
    3229PREFIX EE
    33 ECSTORAGE 1
    34 ECTRANS 0
    35 ECFSDIR ectmp:/${USER}/econdemand/
    36 MAILOPS ${USER}
    37 MAILFAIL ${USER}
     30ECTRANS 1
    3831EOF
  • python/CONTROL_PF.temp

    rd69b677 r54a8a01  
    3434MAILOPS ${USER}
    3535MAILFAIL ${USER}
    36 FLEXPARTDIR ${HOME}/ECMWFDATA7.0/flexpart_code
     36FLEXPARTDIR ${HOME}/flex_extract_v7.1/flexpart_code
    3737GRIB2FLEXPART grib2flexpart
    3838EOF
  • python/ControlFile.py

    re1228f3 r54a8a01  
    11#!/usr/bin/env python
    22# -*- coding: utf-8 -*-
    3 #************************************************************************
    4 # ToDo AP
    5 # - write a test class
    6 #************************************************************************
    73#*******************************************************************************
    84# @Author: Leopold Haimberger (University of Vienna)
     
    1915#          self-explanation naming
    2016#        - outsource of class ControlFile
     17#        - initialisation of class attributes ( to avoid high number of
     18#          conditional statements and set default values )
     19#        - divided assignment of attributes and the check of conditions
     20#        - outsourced the commandline argument assignments to control attributes
    2121#
    2222# @License:
     
    3636# @Class Content:
    3737#    - __init__
     38#    - __read_controlfile__
    3839#    - __str__
     40#    - assign_args_to_control
     41#    - assign_envs_to_control
     42#    - check_conditions
     43#    - check_install_conditions
    3944#    - to_list
    4045#
    4146# @Class Attributes:
    42 #    - start_date
    43 #    - end_date
    44 #    - accuracy
    45 #    - omega
    46 #    - cwc
    47 #    - omegadiff
    48 #    - etadiff
    49 #    - level
    50 #    - levelist
    51 #    - step
    52 #    - maxstep
    53 #    - prefix
    54 #    - makefile
    55 #    - basetime
    56 #    - date_chunk
    57 #    - grib2flexpart
    58 #    - exedir
    59 #    - flexpart_root_scripts
    60 #    - ecmwfdatadir
     47#
    6148#
    6249#*******************************************************************************
     
    6653# ------------------------------------------------------------------------------
    6754import os
     55import sys
    6856import inspect
    6957
     
    9280        '''
    9381        @Description:
    94             Initialises the instance of ControlFile class and defines and
    95             assign all CONTROL file variables. Set default values if
    96             parameter was not in CONTROL file.
     82            Initialises the instance of ControlFile class and defines
     83            all class attributes with default values. Afterwards calls
     84            function __read_controlfile__ to read parameter from
     85            Control file.
    9786
    9887        @Input:
     
    10695            <nothing>
    10796        '''
     97
     98        # list of all possible class attributes and their default values
     99        self.controlfile = filename
     100        self.start_date = None
     101        self.end_date = None
     102        self.date_chunk = 3
     103        self.dtime = None
     104        self.basetime = None
     105        self.maxstep = None
     106        self.type = None
     107        self.time = None
     108        self.step = None
     109        self.marsclass = None
     110        self.stream = None
     111        self.number = 'OFF'
     112        self.expver = None
     113        self.grid = None
     114        self.area = ''
     115        self.left = None
     116        self.lower = None
     117        self.upper = None
     118        self.right = None
     119        self.level = None
     120        self.levelist = None
     121        self.resol = None
     122        self.gauss = 0
     123        self.accuracy = 24
     124        self.omega = 0
     125        self.omegadiff = 0
     126        self.eta = 0
     127        self.etadiff = 0
     128        self.etapar = 77
     129        self.dpdeta = 1
     130        self.smooth = 0
     131        self.format = 'GRIB1'
     132        self.addpar = None
     133        self.prefix = 'EN'
     134        self.cwc = 0
     135        self.wrf = 0
     136        self.ecfsdir = 'ectmp:/${USER}/econdemand/'
     137        self.mailfail = ['${USER}']
     138        self.mailops = ['${USER}']
     139        self.grib2flexpart = 0
     140        self.ecstorage = 0
     141        self.ectrans = 0
     142        self.inputdir = '../work'
     143        self.outputdir = self.inputdir
     144        self.ecmwfdatadir = None
     145        self.exedir = None
     146        self.flexpart_root_scripts = None
     147        self.makefile = None
     148        self.destination = None
     149        self.gateway = None
     150        self.ecuid = None
     151        self.ecgid = None
     152        self.install_target = None
     153        self.debug = 0
     154
     155        self.__read_controlfile__()
     156
     157        return
     158
     159    def __read_controlfile__(self):
     160        '''
     161        @Description:
     162            Read CONTROL file and assign all CONTROL file variables.
     163
     164        @Input:
     165            self: instance of ControlFile class
     166                Description see class documentation.
     167
     168        @Return:
     169            <nothing>
     170        '''
    108171        from tools import my_error
    109172
    110173        # read whole CONTROL file
    111         with open(filename) as f:
     174        with open(self.controlfile) as f:
    112175            fdata = f.read().split('\n')
    113176
    114177        # go through every line and store parameter
    115         # as class variable
    116178        for ldata in fdata:
    117179            data = ldata.split()
     
    145207                                data[1] = data[1][:i] + var + data[1][k+1:]
    146208                            else:
    147                                 my_error(None, 'Could not find variable ' +
    148                                          data[1][j+1:k] + ' while reading ' +
    149                                          filename)
     209                                my_error(self.mailfail,
     210                                         'Could not find variable '
     211                                         + data[1][j+1:k] + ' while reading ' +
     212                                         self.controlfile)
    150213                        setattr(self, data[0].lower() + '_expanded', data[1])
    151214                    else:
     
    159222                pass
    160223
    161         # check a couple of necessary attributes if they contain values
    162         # otherwise set default values
    163         if not hasattr(self, 'start_date'):
    164             self.start_date = None
    165         if not hasattr(self, 'end_date'):
     224        # script directory
     225        self.ecmwfdatadir = os.path.dirname(os.path.abspath(inspect.getfile(
     226            inspect.currentframe()))) + '/../'
     227
     228        # Fortran source directory
     229        self.exedir = self.ecmwfdatadir + 'src/'
     230
     231        return
     232
     233    def __str__(self):
     234        '''
     235        @Description:
     236            Prepares a string which have all the ControlFile
     237            class attributes with its associated values.
     238            Each attribute is printed in one line and in
     239            alphabetical order.
     240
     241            Example:
     242            'age': 10
     243            'color': 'Spotted'
     244            'kids': 0
     245            'legs': 2
     246            'name': 'Dog'
     247            'smell': 'Alot'
     248
     249        @Input:
     250            self: instance of ControlFile class
     251                Description see class documentation.
     252
     253        @Return:
     254            string of ControlFile class attributes with their values
     255        '''
     256        import collections
     257
     258        attrs = vars(self)
     259        attrs = collections.OrderedDict(sorted(attrs.items()))
     260
     261        return '\n'.join("%s: %s" % item for item in attrs.items())
     262
     263    def assign_args_to_control(self, args):
     264        '''
     265        @Description:
     266            Overwrites the existing ControlFile instance attributes with
     267            the command line arguments.
     268
     269        @Input:
     270            self: instance of ControlFile class
     271                Description see class documentation.
     272
     273            args: instance of ArgumentParser
     274                Contains the commandline arguments from script/program call.
     275
     276        @Return:
     277            <nothing>
     278        '''
     279
     280        # get dictionary of command line parameters and eliminate all
     281        # parameters which are None (were not specified)
     282        args_dict = vars(args)
     283        arguments = {k : args_dict[k] for k in args_dict
     284                     if args_dict[k] != None}
     285
     286        # assign all passed command line arguments to ControlFile instance
     287        for k, v in arguments.iteritems():
     288            setattr(self, str(k), v)
     289
     290        return
     291
     292    def assign_envs_to_control(self, envs):
     293        '''
     294        @Description:
     295            Assigns the ECMWF environment parameter.
     296
     297        @Input:
     298            envs: dict of strings
     299                Contains the ECMWF environment parameternames "ECUID", "ECGID",
     300                "DESTINATION" and "GATEWAY" with its corresponding values.
     301                They were read from the file "ECMWF_ENV".
     302
     303        @Return:
     304            <nothing>
     305        '''
     306
     307        for k, v in envs.iteritems():
     308            setattr(self, str(k).lower(), str(v))
     309
     310        return
     311
     312    def check_conditions(self):
     313        '''
     314        @Description:
     315            Checks a couple of necessary attributes and conditions,
     316            such as if they exist and contain values.
     317            Otherwise set default values.
     318
     319        @Input:
     320            self: instance of ControlFile class
     321                Description see class documentation.
     322
     323        @Return:
     324            <nothing>
     325        '''
     326        from tools import my_error
     327        import numpy as np
     328
     329        # check for having at least a starting date
     330        # otherwise program is not allowed to run
     331        if self.start_date is None:
     332            print 'start_date specified neither in command line nor ' + \
     333                  'in CONTROL file ' +  self.controlfile
     334            print 'Try "' + sys.argv[0].split('/')[-1] + \
     335                  ' -h" to print usage information'
     336            sys.exit(1)
     337
     338        # retrieve just one day if end_date isn't set
     339        if self.end_date is None:
    166340            self.end_date = self.start_date
    167         if not hasattr(self, 'accuracy'):
    168             self.accuracy = 24
    169         if not hasattr(self, 'omega'):
    170             self.omega = '0'
    171         if not hasattr(self, 'cwc'):
    172             self.cwc = '0'
    173         if not hasattr(self, 'omegadiff'):
    174             self.omegadiff = '0'
    175         if not hasattr(self, 'etadiff'):
    176             self.etadiff = '0'
    177         if not hasattr(self, 'levelist'):
    178             if not hasattr(self, 'level'):
    179                 print 'Warning: neither levelist nor level \
    180                        specified in CONTROL file'
     341
     342        # assure consistency of levelist and level
     343        if self.levelist is None:
     344            if self.level is None:
     345                print 'Warning: neither levelist nor level ' + \
     346                      'specified in CONTROL file'
     347                sys.exit(1)
    181348            else:
    182349                self.levelist = '1/to/' + self.level
    183350        else:
    184             if 'to' in self.levelist:
     351            if 'to' in self.levelist.lower():
    185352                self.level = self.levelist.split('/')[2]
    186353            else:
    187354                self.level = self.levelist.split('/')[-1]
    188355
    189         if not hasattr(self, 'maxstep'):
    190             # find out maximum step
     356        # if area was provided at command line
     357        # decompse area into its 4 components
     358        if self.area:
     359            afloat = '.' in self.area
     360            l = self.area.split('/')
     361            if afloat:
     362                for i, item in enumerate(l):
     363                    item = str(int(float(item) * 1000))
     364            self.upper, self.left, self.lower, self.right = l
     365
     366        # prepare step for correct usage
     367        if '/' in self.step:
     368            l = self.step.split('/')
     369            if 'to' in self.step.lower():
     370                if 'by' in self.step.lower():
     371                    ilist = np.arange(int(l[0]), int(l[2]) + 1, int(l[4]))
     372                    self.step = ['{:0>3}'.format(i) for i in ilist]
     373                else:
     374                    my_error(self.mailfail, self.step + ':\n' +
     375                             'if "to" is used, please use "by" as well')
     376            else:
     377                self.step = l
     378
     379        # if maxstep wasn't provided
     380        # search for it in the "step" parameter
     381        if self.maxstep is None:
    191382            self.maxstep = 0
    192383            for s in self.step:
     
    196387            self.maxstep = int(self.maxstep)
    197388
    198         if not hasattr(self, 'prefix'):
    199             self.prefix = 'EN'
    200         if not hasattr(self, 'makefile'):
    201             self.makefile = None
    202         if not hasattr(self, 'basetime'):
    203             self.basetime = None
    204         if not hasattr(self, 'date_chunk'):
    205             self.date_chunk = '3'
    206         if not hasattr(self, 'grib2flexpart'):
    207             self.grib2flexpart = '0'
    208 
    209         # script directory
    210         self.ecmwfdatadir = os.path.dirname(os.path.abspath(inspect.getfile(
    211             inspect.currentframe()))) + '/../'
    212         # Fortran source directory
    213         self.exedir = self.ecmwfdatadir + 'src/'
    214 
    215         # FLEXPART directory
    216         if not hasattr(self, 'flexpart_root_scripts'):
     389        # set root scripts since it is needed later on
     390        if not self.flexpart_root_scripts:
    217391            self.flexpart_root_scripts = self.ecmwfdatadir
    218392
     393        if not isinstance(self.mailfail, list):
     394            if ',' in self.mailfail:
     395                self.mailfail = self.mailfail.split(',')
     396            elif ' ' in self.mailfail:
     397                self.mailfail = self.mailfail.split()
     398            else:
     399                self.mailfail = [self.mailfail]
     400
     401        if not isinstance(self.mailops, list):
     402            if ',' in self.mailops:
     403                self.mailops = self.mailops.split(',')
     404            elif ' ' in self.mailops:
     405                self.mailops = self.mailops.split()
     406            else:
     407                self.mailops = [self.mailops]
     408
     409        if not self.gateway or not self.destination or \
     410           not self.ecuid or not self.ecgid:
     411            print '\nEnvironment variables GATWAY, DESTINATION, ECUID and ' + \
     412                  'ECGID were not set properly!'
     413            print 'Please check for excistence of file "ECMWF_ENV" in the ' + \
     414                  'python directory!'
     415            sys.exit(1)
     416
    219417        return
    220418
    221     def __str__(self):
    222         '''
    223         @Description:
    224             Prepares a single string with all the comma seperated ControlFile
    225             class attributes including their values.
    226 
    227             Example:
    228             {'kids': 0, 'name': 'Dog', 'color': 'Spotted',
    229              'age': 10, 'legs': 2, 'smell': 'Alot'}
    230 
    231         @Input:
    232             self: instance of ControlFile class
    233                 Description see class documentation.
    234 
    235         @Return:
    236             string of ControlFile class attributes with their values
    237         '''
    238 
    239         attrs = vars(self)
    240 
    241         return ', '.join("%s: %s" % item for item in attrs.items())
     419    def check_install_conditions(self):
     420        '''
     421        @Description:
     422            Checks a couple of necessary attributes and conditions
     423            for the installation such as if they exist and contain values.
     424            Otherwise set default values.
     425
     426        @Input:
     427            self: instance of ControlFile class
     428                Description see class documentation.
     429
     430        @Return:
     431            <nothing>
     432        '''
     433
     434        if self.install_target and \
     435           self.install_target not in ['local', 'ecgate', 'cca']:
     436            print 'ERROR: unknown or missing installation target '
     437            print 'target: ', self.install_target
     438            print 'please specify correct installation target \
     439                   (local | ecgate | cca)'
     440            print 'use -h or --help for help'
     441            sys.exit(1)
     442
     443        if self.install_target and self.install_target != 'local':
     444            if not self.ecgid or not self.ecuid or \
     445               not self.gateway or not self.destination:
     446                print 'Please enter your ECMWF user id and group id as well as \
     447                       the \nname of the local gateway and the ectrans \
     448                       destination '
     449                print 'with command line options --ecuid --ecgid \
     450                       --gateway --destination'
     451                print 'Try "' + sys.argv[0].split('/')[-1] + \
     452                      ' -h" to print usage information'
     453                print 'Please consult ecaccess documentation or ECMWF user \
     454                       support for further details'
     455                sys.exit(1)
     456
     457            if not self.flexpart_root_scripts:
     458                self.flexpart_root_scripts = '${HOME}'
     459            else:
     460                self.flexpart_root_scripts = self.flexpart_root_scripts
     461        else:
     462            if not self.flexpart_root_scripts:
     463                self.flexpart_root_scripts = '../'
     464
     465        if not self.makefile:
     466            self.makefile = 'Makefile.gfortran'
     467
     468        return
    242469
    243470    def to_list(self):
     
    259486        '''
    260487
    261         attrs = vars(self)
     488        import collections
     489
     490        attrs = collections.OrderedDict(sorted(vars(self).items()))
     491
    262492        l = list()
    263493
     
    283513        return sorted(l)
    284514
    285     # def to_dict(self):
    286         # '''
    287 
    288         # '''
    289         # parameters_dict = vars(self)
    290 
    291         # # remove unneeded parameter
    292         # parameters_dict.pop('_expanded', None)
    293         # parameters_dict.pop('exedir', None)
    294         # parameters_dict.pop('flexpart_root_scripts', None)
    295         # parameters_dict.pop('ecmwfdatadir', None)
    296 
    297         # parameters_dict_str = {}
    298         # for key, value in parameters_dict.iteritems():
    299             # if isinstance(value, list):
    300                 # parameters_dict_str[str(key)] = get_list_as_string(value, ' ')
    301             # else:
    302                 # parameters_dict_str[str(key)] = str(value)
    303 
    304         # return parameters_dict_str
  • python/EcFlexpart.py

    re1228f3 r54a8a01  
    11#!/usr/bin/env python
    22# -*- coding: utf-8 -*-
    3 #************************************************************************
    4 # ToDo AP
    5 # - specifiy file header documentation
    6 # - add class description in header information
    7 # - apply classtests
    8 # - add references to ECMWF specific software packages
    9 # - add describtion of deacc_fluxes
    10 # - change name of func deacc ( weil disagg )
    11 # - add desc of retrieve function
    12 #************************************************************************
    133#*******************************************************************************
    144# @Author: Anne Fouilloux (University of Oslo)
     
    151141
    152142            fluxes: boolean, optional
    153                 Decides if a the flux parameter settings are stored or
     143                Decides if the flux parameter settings are stored or
    154144                the rest of the parameter list.
    155145                Default value is False.
     
    175165        self.dtime = c.dtime
    176166        i = 0
    177         if fluxes is True and c.maxstep < 24:
     167        if fluxes and c.maxstep <= 24:
    178168            # no forecast beyond one day is needed!
    179169            # Thus, prepare flux data manually as usual
     
    192182                    btlist = [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 0]
    193183
    194                 if i % int(c.dtime) == 0 and c.maxstep > 24 or i in btlist:
     184                if i % int(c.dtime) == 0 and (i in btlist or c.maxstep > 24):
    195185
    196186                    if ty not in self.types.keys():
     
    207197                        self.types[ty]['steps'] += st
    208198                i += 1
    209 
    210199
    211200        self.marsclass = c.marsclass
     
    304293                                         '{}'.format((int(self.resol) + 1) / 2)]
    305294
    306             if c.omega == '1':
     295            if hasattr(c, 'omega') and c.omega == '1':
    307296                self.params['OG__ML'][0] += '/W'
    308297
    309298            # add cloud water content if necessary
    310             if c.cwc == '1':
     299            if hasattr(c, 'cwc') and c.cwc == '1':
    311300                self.params['OG__ML'][0] += '/CLWC/CIWC'
    312301
    313302            # add vorticity and geopotential height for WRF if necessary
    314             if c.wrf == '1':
     303            if hasattr(c, 'wrf') and c.wrf == '1':
    315304                self.params['OG__ML'][0] += '/Z/VO'
    316305                if '/D' not in self.params['OG__ML'][0]:
     
    740729                                           c.gateway, '-remote', c.destination,
    741730                                           '-source', ofile])
    742                 print('ectrans:', p)
     731                #print('ectrans:', p)
    743732
    744733        if int(c.ecstorage) == 1 and c.ecapi is False:
     
    890879
    891880        for prod in product(*index_vals):
    892             # flag for Fortran program CONVERT2, initially False
     881            # flag for Fortran program CONVERT2 and file merging
    893882            convertFlag = False
    894883            print 'current prod: ', prod
     
    906895            # prepare some date and time parameter before reading the data
    907896            if gid is not None:
    908                 # Fortran program CONVERT2 is only done if gid at this time is
    909                 # not None, therefore save information in convertFlag
     897                # Combine all temporary data files into final grib file if
     898                # gid is at least one time not None. Therefore set convertFlag
     899                # to save information. The fortran program CONVERT2 is also
     900                # only done if convertFlag is True
    910901                convertFlag = True
    911902                # remove old fort.* files and open new ones
     903                # they are just valid for a single product
    912904                for k, f in fdict.iteritems():
    913905                    silent_remove(c.inputdir + "/fort." + k)
     
    922914                                              '%Y%m%d%H')
    923915                timestamp += timedelta(hours=int(step))
    924 
    925916                cdateH = datetime.strftime(timestamp, '%Y%m%d%H')
    926917
     
    966957                levtype = grib_get(gid, 'typeOfLevel')
    967958                if paramId == 133 and gridtype == 'reduced_gg':
    968                 # Relative humidity (Q.grb) is used as a template only
     959                # Specific humidity (Q.grb) is used as a template only
    969960                # so we need the first we "meet"
    970961                    with open(c.inputdir + '/fort.18', 'w') as fout:
     
    10281019                           not available for this type or date/time\n'
    10291020                    print 'Check parameters CLASS, TYPE, STREAM, START_DATE\n'
    1030                     my_error(c, 'fort.21 is empty while parameter eta is set \
    1031                                to 1 in CONTROL file')
     1021                    my_error(c.mailfail, 'fort.21 is empty while parameter eta \
     1022                             is set to 1 in CONTROL file')
    10321023
    10331024                # create the corresponding output file fort.15
     
    10611052                with open(fnout, 'wb') as fout:
    10621053                    for f in flist:
    1063                         shutil.copyfileobj(open(c.inputdir + '/' + f, 'rb'), fout)
     1054                        shutil.copyfileobj(
     1055                            open(c.inputdir + '/' + f, 'rb'), fout)
    10641056
    10651057                if c.omega == '1':
     
    10681060                            open(c.inputdir + '/fort.25', 'rb'), fout)
    10691061
    1070         if c.wrf == '1':
     1062        if hasattr(c, 'wrf') and c.wrf == '1':
    10711063            fwrf.close()
    10721064
     
    13071299        grib_index_release(iid)
    13081300
    1309         exit()
    13101301        return
  • python/GribTools.py

    re1228f3 r54a8a01  
    309309                    iid = grib_index_new_from_file(filename, index_keys)
    310310                else:
    311                     print 'in else zweig'
    312311                    grib_index_add_file(iid, filename)
    313312
  • python/MarsRetrieval.py

    rff99eae r54a8a01  
    7979    '''
    8080
    81     def __init__(self, server, marsclass="ei", dtype="", levtype="",
     81    def __init__(self, server, marsclass="ei", type="", levtype="",
    8282                 levelist="", repres="", date="", resol="", stream="",
    8383                 area="", time="", step="", expver="1", number="",
     
    107107                Default is the ERA-Interim dataset "ei".
    108108
    109             dtype: string, optional
     109            type: string, optional
    110110                Determines the type of fields to be retrieved.
    111111                Selects between observations, images or fields.
     
    286286        self.server = server
    287287        self.marsclass = marsclass
    288         self.dtype = dtype
     288        self.type = type
    289289        self.levtype = levtype
    290290        self.levelist = levelist
  • python/UioFiles.py

    rff99eae r54a8a01  
    3434# @Class Content:
    3535#    - __init__
    36 #    - list_files
     36#    - __str__
     37#    - __list_files__
    3738#    - delete_files
    3839#
     
    5152# software specific module from flex_extract
    5253#import profiling
    53 from tools import silent_remove
     54from tools import silent_remove, get_list_as_string
    5455
    5556# ------------------------------------------------------------------------------
     
    6667    # CLASS FUNCTIONS
    6768    # --------------------------------------------------------------------------
    68     def __init__(self, pattern):
     69    def __init__(self, path, pattern):
    6970        '''
    7071        @Description:
     
    7576                Description see class documentation.
    7677
     78            path: string
     79                Directory where to list the files.
     80
    7781            pattern: string
    7882                Regular expression pattern. For example: '*.grb'
     
    8286        '''
    8387
     88        self.path = path
    8489        self.pattern = pattern
    8590        self.files = None
     91
     92        self.__list_files__(self.path)
    8693
    8794        return
    8895
    8996    #@profiling.timefn
    90     def list_files(self, path, callid=0):
     97    def __list_files__(self, path, callid=0):
    9198        '''
    9299        @Description:
     
    99106
    100107            path: string
    101                 Directory where to list the files.
     108                Path to the files.
    102109
    103110            callid: integer
     
    129136        if subdirs:
    130137            for subdir in subdirs:
    131                 self.list_files(os.path.join(path, subdir), callid=1)
     138                self.__list_files__(os.path.join(path, subdir), callid=1)
    132139
    133140        return
     141
     142    def __str__(self):
     143        '''
     144        @Description:
     145            Converts the list of files into a single string.
     146            The entries are sepereated by "," sign.
     147
     148        @Input:
     149            self: instance of UioFiles
     150                Description see class documentation.
     151
     152        @Return:
     153            files_string: string
     154                The content of the list as a single string.
     155        '''
     156
     157        filenames = [os.path.basename(f) for f in self.files]
     158        files_string = get_list_as_string(filenames, concatenate_sign=', ')
     159
     160        return files_string
    134161
    135162    def delete_files(self):
  • python/compilejob.ksh

    r991df6a r54a8a01  
    55# start with sbatch NAME_OF_THIS_FILE directly on machine
    66
    7 #SBATCH --workdir=/scratch/ms/spatlh00/lh0
     7#SBATCH --workdir=/scratch/ms/at/km4a
    88#SBATCH --qos=normal
    99#SBATCH --job-name=flex_ecmwf
     
    3333  module load grib_api/1.14.5
    3434  module load emos/437-r64
    35 export FLEXPART_ROOT_SCRIPTS=$HOME
    36 #  export ECMWFDATA=$FLEXPART_ROOT/ECMWFDATA$VERSION
    37 #  export PYTHONPATH=$ECMWFDATA/python
    38 #  export PATH=${PATH}:$ECMWFDATA/python
    39   export MAKEFILE=Makefile.gfortran
     35export FLEXPART_ROOT_SCRIPTS=${HOME}
     36export MAKEFILE=Makefile.gfortran
    4037  ;;
    4138  *cca*)
     
    4946  export GROUP=`echo $HOME | awk -F / '{print $4}'`
    5047  export SCRATCH=/scratch/ms/${GROUP}/${USER}
    51 export FLEXPART_ROOT_SCRIPTS=$HOME
    52 #  export ECMWFDATA=$FLEXPART_ROOT/ECMWFDATA$VERSION
    53 #  export PYTHONPATH=$ECMWFDATA/python
    54 #  export PATH=${PATH}:$ECMWFDATA/python
    55   export MAKEFILE=Makefile.CRAY
     48export FLEXPART_ROOT_SCRIPTS=${HOME}
     49export MAKEFILE=Makefile.gfortran
    5650  ;;
    5751esac
    5852
    59 mkdir -p $FLEXPART_ROOT_SCRIPTS/ECMWFDATA$VERSION
    60 cd $FLEXPART_ROOT_SCRIPTS/ECMWFDATA$VERSION   # if FLEXPART_ROOT is not set this means cd to the home directory
    61 tar -xvf $HOME/ECMWFDATA$VERSION.tar
     53mkdir -p $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION
     54cd $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION   # if FLEXPART_ROOT is not set this means cd to the home directory
     55tar -xvf $HOME/flex_extract_v$VERSION.tar
    6256cd src
    6357\rm *.o *.mod CONVERT2
  • python/compilejob.temp

    refdb01a r54a8a01  
    3434  module load emos/437-r64
    3535  export FLEXPART_ROOT_SCRIPTS=
    36 #  export ECMWFDATA=$FLEXPART_ROOT/ECMWFDATA$VERSION
    37 #  export PYTHONPATH=$ECMWFDATA/python
    38 #  export PATH=${PATH}:$ECMWFDATA/python
    3936  export MAKEFILE=Makefile.gfortran
    4037  ;;
     
    5047  export SCRATCH=/scratch/ms/${GROUP}/${USER}
    5148  export FLEXPART_ROOT_SCRIPTS=
    52 #  export ECMWFDATA=$FLEXPART_ROOT/ECMWFDATA$VERSION
    53 #  export PYTHONPATH=$ECMWFDATA/python
    54 #  export PATH=${PATH}:$ECMWFDATA/python
    5549  export MAKEFILE=Makefile.CRAY
    5650  ;;
    5751esac
    5852
    59 mkdir -p $FLEXPART_ROOT_SCRIPTS/ECMWFDATA$VERSION
    60 cd $FLEXPART_ROOT_SCRIPTS/ECMWFDATA$VERSION   # if FLEXPART_ROOT is not set this means cd to the home directory
    61 tar -xvf $HOME/ECMWFDATA$VERSION.tar
     53mkdir -p $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION
     54cd $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION   # if FLEXPART_ROOT is not set this means cd to the home directory
     55tar -xvf $HOME/flex_extract_v$VERSION.tar
    6256cd src
    6357\rm *.o *.mod CONVERT2
  • python/disaggregation.py

    rff99eae r54a8a01  
    11#!/usr/bin/env python
    22# -*- coding: utf-8 -*-
    3 #************************************************************************
    4 # ToDo AP
    5 # - check alist of size 4 ?
    6 # - write a test, IMPORTANT
    7 #************************************************************************
    83#*******************************************************************************
    94# @Author: Anne Philipp (University of Vienna)
     
    127#
    138# @Change History:
     9#
    1410#    November 2015 - Leopold Haimberger (University of Vienna):
    1511#        - migration of the methods dapoly and darain from Fortran
     
    2925#
    3026# @Module Description:
    31 #    disaggregationregation of deaccumulated flux data from an ECMWF model FG field.
     27#    disaggregation of deaccumulated flux data from an ECMWF model FG field.
    3228#    Initially the flux data to be concerned are:
    3329#    - large-scale precipitation
  • python/get_mars_data.py

    rff99eae r54a8a01  
    2222#          online documentation)
    2323#        - use of UIFiles class for file selection and deletion
    24 
     24#
    2525#
    2626# @License:
     
    3939#    - main
    4040#    - get_mars_data
     41#    - do_retrievement
    4142#
    4243#*******************************************************************************
     
    5657
    5758# software specific classes and modules from flex_extract
    58 from tools import my_error, normal_exit, interpret_args_and_control
     59from tools import my_error, normal_exit, get_cmdline_arguments, read_ecenv
    5960from EcFlexpart import EcFlexpart
    6061from UioFiles import UioFiles
     
    8283        <nothing>
    8384    '''
    84     args, c = interpret_args_and_control()
     85
     86    args = get_cmdline_arguments()
     87
     88    try:
     89        c = ControlFile(args.controlfile)
     90    except IOError:
     91        try:
     92            c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile)
     93        except IOError:
     94            print 'Could not read CONTROL file "' + args.controlfile + '"'
     95            print 'Either it does not exist or its syntax is wrong.'
     96            print 'Try "' + sys.argv[0].split('/')[-1] + \
     97                  ' -h" to print usage information'
     98            sys.exit(1)
     99
     100    env_parameter = read_ecenv(c.ecmwfdatadir + 'python/ECMWF_ENV')
     101    c.assign_args_to_control(args, env_parameter)
     102    c.assign_envs_to_control(env_parameter)
     103    c.check_conditions()
     104
    85105    get_mars_data(c)
    86     normal_exit(c)
     106    normal_exit(c.mailfail, 'Done!')
    87107
    88108    return
     
    129149    print 'ecapi: ', c.ecapi
    130150
     151    # basetime geht rückwärts
     152
     153    # if basetime 00
     154    # dann wird von 12 am vortag bis 00 am start tag geholt
     155    # aber ohne 12 selbst sondern 12 + step
     156
     157    # if basetime 12
     158    # dann wird von 00 + step bis 12 am start tag geholt
     159
     160    # purer forecast wird vorwärts bestimmt.
     161    # purer forecast mode ist dann wenn  größer 24 stunden
     162    # wie kann das noch festgestellt werden ????
     163    # nur FC und steps mehr als 24 ?
     164    # die einzige problematik beim reinen forecast ist die benennung der files!
     165    # also sobald es Tagesüberschneidungen gibt
     166    # allerdings ist das relevant und ersichtlich an den NICHT FLUSS DATEN
     167
     168
    131169    # set start date of retrieval period
    132170    start = datetime.date(year=int(c.start_date[:4]),
     
    134172                          day=int(c.start_date[6:]))
    135173    startm1 = start - datetime.timedelta(days=1)
    136     if c.basetime == '00':
    137         start = startm1
    138174
    139175    # set end date of retrieval period
     
    141177                        month=int(c.end_date[4:6]),
    142178                        day=int(c.end_date[6:]))
     179
     180    # set time period for one single retrieval
     181    datechunk = datetime.timedelta(days=int(c.date_chunk))
     182
     183    if c.basetime == '00':
     184        start = startm1
     185
    143186    if c.basetime == '00' or c.basetime == '12':
     187        # endp1 = end + datetime.timedelta(days=1)
     188        endp1 = end
     189    else:
     190        # endp1 = end + datetime.timedelta(days=2)
    144191        endp1 = end + datetime.timedelta(days=1)
    145     else:
    146         endp1 = end + datetime.timedelta(days=2)
    147 
    148     # set time period of one single retrieval
    149     datechunk = datetime.timedelta(days=int(c.date_chunk))
    150192
    151193    # --------------  flux data ------------------------------------------------
    152194    print 'removing old flux content of ' + c.inputdir
    153     tobecleaned = UioFiles('*_acc_*.' + str(os.getppid()) + '.*.grb')
    154     tobecleaned.list_files(c.inputdir)
     195    tobecleaned = UioFiles(c.inputdir,
     196                           '*_acc_*.' + str(os.getppid()) + '.*.grb')
    155197    tobecleaned.delete_files()
    156198
    157     # if forecast for maximum one day (upto 23h) are to be retrieved,
     199    # if forecast for maximum one day (upto 24h) are to be retrieved,
    158200    # collect accumulation data (flux data)
    159201    # with additional days in the beginning and at the end
    160202    # (used for complete disaggregation of original period)
    161     if c.maxstep < 24:
    162         day = startm1
    163         while day < endp1:
    164             # retrieve MARS data for the whole period
    165             flexpart = EcFlexpart(c, fluxes=True)
    166             tmpday = day + datechunk - datetime.timedelta(days=1)
    167             if tmpday < endp1:
    168                 dates = day.strftime("%Y%m%d") + "/to/" + \
    169                         tmpday.strftime("%Y%m%d")
    170             else:
    171                 dates = day.strftime("%Y%m%d") + "/to/" + \
    172                         end.strftime("%Y%m%d")
    173 
    174             print "retrieve " + dates + " in dir " + c.inputdir
    175 
    176             try:
    177                 flexpart.retrieve(server, dates, c.inputdir)
    178             except IOError:
    179                 my_error(c, 'MARS request failed')
    180 
    181             day += datechunk
     203    if c.maxstep <= 24:
     204        do_retrievement(c, server, startm1, endp1, datechunk, fluxes=True)
    182205
    183206    # if forecast data longer than 24h are to be retrieved,
     
    187210    # exact time period with boundary conditions)
    188211    else:
    189         day = start
    190         while day <= end:
    191             # retrieve MARS data for the whole period
    192             flexpart = EcFlexpart(c, fluxes=True)
    193             tmpday = day + datechunk - datetime.timedelta(days=1)
    194             if tmpday < end:
    195                 dates = day.strftime("%Y%m%d") + "/to/" + \
    196                         tmpday.trftime("%Y%m%d")
    197             else:
    198                 dates = day.strftime("%Y%m%d") + "/to/" + \
    199                         end.strftime("%Y%m%d")
    200 
    201             print "retrieve " + dates + " in dir " + c.inputdir
    202 
    203             try:
    204                 flexpart.retrieve(server, dates, c.inputdir)
    205             except IOError:
    206                 my_error(c, 'MARS request failed')
    207 
    208             day += datechunk
     212        do_retrievement(c, server, start, end, datechunk, fluxes=True)
    209213
    210214    # --------------  non flux data --------------------------------------------
    211215    print 'removing old non flux content of ' + c.inputdir
    212     tobecleaned = UioFiles('*__*.' + str(os.getppid()) + '.*.grb')
    213     tobecleaned.list_files(c.inputdir)
     216    tobecleaned = UioFiles(c.inputdir,
     217                           '*__*.' + str(os.getppid()) + '.*.grb')
    214218    tobecleaned.delete_files()
     219
     220    do_retrievement(c, server, start, end, datechunk, fluxes=False)
     221
     222    return
     223
     224def do_retrievement(c, server, start, end, delta_t, fluxes=False):
     225    '''
     226    @Description:
     227        Divides the complete retrieval period in smaller chunks and
     228        retrieves the data from MARS.
     229
     230    @Input:
     231        c: instance of ControlFile
     232            Contains all the parameters of CONTROL file, which are e.g.:
     233            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
     234            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
     235            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
     236            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
     237            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
     238            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
     239            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
     240
     241            For more information about format and content of the parameter
     242            see documentation.
     243
     244        server: instance of ECMWFService
     245            The server connection to ECMWF
     246
     247        start: instance of datetime
     248            The start date of the retrieval.
     249
     250        end: instance of datetime
     251            The end date of the retrieval.
     252
     253        delta_t: instance of datetime
     254            Delta_t +1 is the maximal time period of a single
     255            retrieval.
     256
     257        fluxes: boolean, optional
     258            Decides if the flux parameters are to be retrieved or
     259            the rest of the parameter list.
     260            Default value is False.
     261
     262    @Return:
     263        <nothing>
     264    '''
     265
     266    # since actual day also counts as one day,
     267    # we only need to add datechunk - 1 days to retrieval
     268    # for a period
     269    delta_t_m1 = delta_t - datetime.timedelta(days=1)
    215270
    216271    day = start
    217272    while day <= end:
    218         # retrieve all non flux MARS data for the whole period
    219         flexpart = EcFlexpart(c, fluxes=False)
    220         tmpday = day + datechunk - datetime.timedelta(days=1)
     273        flexpart = EcFlexpart(c, fluxes)
     274        tmpday = day + delta_t_m1
    221275        if tmpday < end:
    222276            dates = day.strftime("%Y%m%d") + "/to/" + \
     
    231285            flexpart.retrieve(server, dates, c.inputdir)
    232286        except IOError:
    233             my_error(c, 'MARS request failed')
    234 
    235         day += datechunk
     287            my_error(c.mailfail, 'MARS request failed')
     288
     289        day += delta_t
    236290
    237291    return
  • python/install.py

    rff99eae r54a8a01  
    11#!/usr/bin/env python
    22# -*- coding: utf-8 -*-
    3 #************************************************************************
    4 # ToDo AP
    5 # - create a class Installation and divide installation in 3 subdefs for
    6 #   ecgate, local and cca seperatly
    7 # - Change History ist nicht angepasst ans File! Original geben lassen
    8 #************************************************************************
    93#*******************************************************************************
    104# @Author: Leopold Haimberger (University of Vienna)
     
    3731# @Program Content:
    3832#    - main
    39 #    - install_args_and_control
     33#    - get_install_cmdline_arguments
    4034#    - install_via_gateway
     35#    - mk_tarball
     36#    - mk_env_vars
     37#    - mk_compilejob
     38#    - mk_job_template
     39#    - delete_convert_build
     40#    - make_convert_build
    4141#
    4242#*******************************************************************************
     
    4747import os
    4848import sys
    49 import glob
    5049import subprocess
    5150import inspect
     
    5453# software specific classes and modules from flex_extract
    5554from ControlFile import ControlFile
     55from UioFiles import UioFiles
     56from tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
    5657
    5758# add path to pythonpath so that python finds its buddies
     
    6162    sys.path.append(LOCAL_PYTHON_PATH)
    6263
     64_VERSION_STR = '7.1'
     65
    6366# ------------------------------------------------------------------------------
    6467# FUNCTIONS
     
    7881
    7982    os.chdir(LOCAL_PYTHON_PATH)
    80     args, c = install_args_and_control()
    81 
    82     if args.install_target is not None:
    83         install_via_gateway(c, args.install_target)
    84     else:
    85         print 'Please specify installation target (local|ecgate|cca)'
    86         print 'use -h or --help for help'
    87 
    88     sys.exit()
    89 
    90     return
    91 
    92 
    93 def install_args_and_control():
    94     '''
    95     @Description:
    96         Assigns the command line arguments for installation and reads
    97         CONTROL file content. Apply default values for non mentioned arguments.
     83    args = get_install_cmdline_arguments()
     84
     85    try:
     86        c = ControlFile(args.controlfile)
     87    except IOError:
     88        print 'Could not read CONTROL file "' + args.controlfile + '"'
     89        print 'Either it does not exist or its syntax is wrong.'
     90        print 'Try "' + sys.argv[0].split('/')[-1] + \
     91              ' -h" to print usage information'
     92        exit(1)
     93
     94    c.assign_args_to_control(args)
     95    c.check_install_conditions()
     96
     97    install_via_gateway(c)
     98
     99    return
     100
     101def get_install_cmdline_arguments():
     102    '''
     103    @Description:
     104        Decomposes the command line arguments and assigns them to variables.
     105        Apply default values for non mentioned arguments.
    98106
    99107    @Input:
     
    103111        args: instance of ArgumentParser
    104112            Contains the commandline arguments from script/program call.
    105 
     113    '''
     114    parser = ArgumentParser(description='Install flex_extract software locally or \
     115                            on ECMWF machines',
     116                            formatter_class=ArgumentDefaultsHelpFormatter)
     117
     118    parser.add_argument('--target', dest='install_target', default=None,
     119                        help="Valid targets: local | ecgate | cca , \
     120                        the latter two are at ECMWF")
     121    parser.add_argument("--makefile", dest="makefile", default=None,
     122                        help='Name of Makefile to use for compiling CONVERT2')
     123    parser.add_argument("--ecuid", dest="ecuid", default=None,
     124                        help='user id at ECMWF')
     125    parser.add_argument("--ecgid", dest="ecgid", default=None,
     126                        help='group id at ECMWF')
     127    parser.add_argument("--gateway", dest="gateway", default=None,
     128                        help='name of local gateway server')
     129    parser.add_argument("--destination", dest="destination", default=None,
     130                        help='ecaccess destination, e.g. leo@genericSftp')
     131
     132    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
     133                        default=None, help="FLEXPART root directory on ECMWF \
     134                        servers (to find grib2flexpart and COMMAND file)\n\
     135                        Normally flex_extract resides in the scripts directory \
     136                        of the FLEXPART distribution, thus the:")
     137
     138    # arguments for job submission to ECMWF, only needed by submit.py
     139    parser.add_argument("--job_template", dest='job_template',
     140                        default="job.temp.o",
     141                        help="job template file for submission to ECMWF")
     142
     143    parser.add_argument("--controlfile", dest="controlfile",
     144                        default='CONTROL.temp',
     145                        help="file with CONTROL parameters")
     146
     147    args = parser.parse_args()
     148
     149    return args
     150
     151
     152def install_via_gateway(c):
     153    '''
     154    @Description:
     155        Perform the actual installation on local machine or prepare data
     156        transfer to remote gate and submit a job script which will
     157        install everything on the remote gate.
     158
     159    @Input:
    106160        c: instance of class ControlFile
    107161            Contains all necessary information of a CONTROL file. The parameters
     
    113167            For more information about format and content of the parameter see
    114168            documentation.
    115     '''
    116     parser = ArgumentParser(description='Install ECMWFDATA software locally or \
    117                             on ECMWF machines',
    118                             formatter_class=ArgumentDefaultsHelpFormatter)
    119 
    120     parser.add_argument('--target', dest='install_target',
    121                         help="Valid targets: local | ecgate | cca , \
    122                         the latter two are at ECMWF")
    123     parser.add_argument("--makefile", dest="makefile",
    124                         help='Name of Makefile to use for compiling CONVERT2')
    125     parser.add_argument("--ecuid", dest="ecuid",
    126                         help='user id at ECMWF')
    127     parser.add_argument("--ecgid", dest="ecgid",
    128                         help='group id at ECMWF')
    129     parser.add_argument("--gateway", dest="gateway",
    130                         help='name of local gateway server')
    131     parser.add_argument("--destination", dest="destination",
    132                         help='ecaccess destination, e.g. leo@genericSftp')
    133 
    134     parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
    135                         help="FLEXPART root directory on ECMWF servers \
    136                         (to find grib2flexpart and COMMAND file)\n\
    137                         Normally ECMWFDATA resides in the scripts directory \
    138                         of the FLEXPART distribution, thus the:")
    139 
    140 # arguments for job submission to ECMWF, only needed by submit.py
    141     parser.add_argument("--job_template", dest='job_template',
    142                         default="job.temp.o",
    143                         help="job template file for submission to ECMWF")
    144 
    145     parser.add_argument("--controlfile", dest="controlfile",
    146                         default='CONTROL.temp',
    147                         help="file with CONTROL parameters")
    148 
    149     args = parser.parse_args()
    150 
    151     try:
    152         c = ControlFile(args.controlfile)
    153     except IOError:
    154         print 'Could not read CONTROL file "' + args.controlfile + '"'
    155         print 'Either it does not exist or its syntax is wrong.'
    156         print 'Try "' + sys.argv[0].split('/')[-1] + \
    157               ' -h" to print usage information'
    158         exit(1)
    159 
    160     if args.install_target != 'local':
    161         if args.ecgid is None or args.ecuid is None or args.gateway is None \
    162            or args.destination is None:
    163             print 'Please enter your ECMWF user id and group id as well as \
    164                    the \nname of the local gateway and the ectrans \
    165                    destination '
    166             print 'with command line options --ecuid --ecgid \
    167                    --gateway --destination'
    168             print 'Try "' + sys.argv[0].split('/')[-1] + \
    169                   ' -h" to print usage information'
    170             print 'Please consult ecaccess documentation or ECMWF user support \
    171                    for further details'
    172             sys.exit(1)
    173         else:
    174             c.ecuid = args.ecuid
    175             c.ecgid = args.ecgid
    176             c.gateway = args.gateway
    177             c.destination = args.destination
    178 
    179     if args.makefile:
    180         c.makefile = args.makefile
    181 
    182     if args.install_target == 'local':
    183         if args.flexpart_root_scripts is None:
    184             c.flexpart_root_scripts = '../'
    185         else:
    186             c.flexpart_root_scripts = args.flexpart_root_scripts
    187 
    188     if args.install_target != 'local':
    189         if args.flexpart_root_scripts is None:
    190             c.ec_flexpart_root_scripts = '${HOME}'
    191         else:
    192             c.ec_flexpart_root_scripts = args.flexpart_root_scripts
    193 
    194     return args, c
    195 
    196 
    197 def install_via_gateway(c, target):
    198     '''
    199     @Description:
    200         Perform the actual installation on local machine or prepare data
    201         transfer to remote gate and submit a job script which will
    202         install everything on the remote gate.
    203 
    204     @Input:
    205         c: instance of class ControlFile
    206             Contains all necessary information of a CONTROL file. The parameters
    207             are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
    208             NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
    209             RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
    210             SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
    211             ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR
    212             For more information about format and content of the parameter see
    213             documentation.
    214 
    215         target: string
    216             The target where the installation should be processed.
    217             E.g. "local", "ecgate" or "cca"
    218 
    219     @Return:
    220         <nothing>
    221     '''
     169
     170    @Return:
     171        <nothing>
     172    '''
     173
    222174    ecd = c.ecmwfdatadir
    223     template = ecd + 'python/compilejob.temp'
    224     job = ecd + 'python/compilejob.ksh'
    225     fo = open(job, 'w')
    226 #AP could do with open(template) as f, open(job, 'w') as fo:
    227 #AP or nested with statements
    228     with open(template) as f:
    229         fdata = f.read().split('\n')
    230         for data in fdata:
    231             if 'MAKEFILE=' in data:
    232                 if c.makefile is not None:
    233                     data = 'export MAKEFILE=' + c.makefile
    234             if 'FLEXPART_ROOT_SCRIPTS=' in data:
    235                 if c.flexpart_root_scripts != '../':
    236                     data = 'export FLEXPART_ROOT_SCRIPTS=' + \
    237                             c.flexpart_root_scripts
    238                 else:
    239                     data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
    240             if target.lower() != 'local':
    241                 if '--workdir' in data:
    242                     data = '#SBATCH --workdir=/scratch/ms/' + c.ecgid + \
    243                             '/' + c.ecuid
    244                 if '##PBS -o' in data:
    245                     data = '##PBS -o /scratch/ms/' + c.ecgid + '/' + c.ecuid + \
    246                             'flex_ecmwf.$Jobname.$Job_ID.out'
    247                 if 'FLEXPART_ROOT_SCRIPTS=' in data:
    248                     if c.ec_flexpart_root_scripts != '../':
    249                         data = 'export FLEXPART_ROOT_SCRIPTS=' + \
    250                                 c.ec_flexpart_root_scripts
    251                     else:
    252                         data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
    253             fo.write(data + '\n')
    254     f.close()
    255     fo.close()
    256 
    257     if target.lower() != 'local':
    258         template = ecd + 'python/job.temp.o'
    259 #AP hier eventuell Zeile für Zeile lesen und dann if Entscheidung
    260         with open(template) as f:
    261             fdata = f.read().split('\n')
    262         f.close()
    263         fo = open(template[:-2], 'w')
    264         for data in fdata:
    265             if '--workdir' in data:
    266                 data = '#SBATCH --workdir=/scratch/ms/' + c.ecgid + \
    267                         '/' + c.ecuid
    268             if '##PBS -o' in data:
    269                 data = '##PBS -o /scratch/ms/' + c.ecgid + '/' + \
    270                         c.ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out'
    271             if  'export PATH=${PATH}:' in data:
    272                 data += c.ec_flexpart_root_scripts + '/ECMWFDATA7.1/python'
    273             if 'cat>>' in data or 'cat >>' in data:
    274                 i = data.index('>')
    275                 fo.write(data[:i] + data[i+1:] + '\n')
    276                 fo.write('GATEWAY ' + c.gateway + '\n')
    277                 fo.write('DESTINATION ' + c.destination + '\n')
    278                 fo.write('EOF\n')
    279 
    280             fo.write(data + '\n')
    281         fo.close()
    282 
    283         job = ecd + 'python/ECMWF_ENV'
    284         with open(job, 'w') as fo:
    285             fo.write('ECUID ' + c.ecuid + '\n')
    286             fo.write('ECGID ' + c.ecgid + '\n')
    287             fo.write('GATEWAY ' + c.gateway + '\n')
    288             fo.write('DESTINATION ' + c.destination + '\n')
    289         fo.close()
    290 
    291     if target.lower() == 'local':
    292         # compile CONVERT2
    293         if c.flexpart_root_scripts is None or c.flexpart_root_scripts == '../':
    294             print 'Warning: FLEXPART_ROOT_SCRIPTS has not been specified'
    295             print 'Only CONVERT2 will be compiled in ' + ecd + '/../src'
    296         else:
     175    tarball_name = 'flex_extract_v' + _VERSION_STR + '.tar'
     176    target_dir = 'flex_extract_v' + _VERSION_STR
     177    fortran_executable = 'CONVERT2'
     178
     179    if c.install_target.lower() != 'local':
     180
     181        mk_compilejob(ecd + 'python/compilejob.temp', c.makefile,
     182                      c.install_target, c.ecuid, c.ecgid,
     183                      c.flexpart_root_scripts)
     184
     185        mk_job_template(ecd + 'python/job.temp.o', c.ecuid, c.ecgid, c.gateway,
     186                        c.destination, c.flexpart_root_scripts)
     187
     188        mk_env_vars(ecd, c.ecuid, c.ecgid, c.gateway, c.destination)
     189
     190        #os.chdir('/')
     191
     192        mk_tarball(ecd, tarball_name)
     193
     194        put_file_to_ecserver(ecd, tarball_name, c.install_target,
     195                             c.ecuid, c.ecgid)
     196
     197        submit_job_to_ecserver(ecd + '/python/', c.install_target,
     198                               'compilejob.ksh')
     199
     200        print 'job compilation script has been submitted to ecgate for ' + \
     201              'installation in ' + c.flexpart_root_scripts + \
     202               '/' + target_dir
     203        print 'You should get an email with subject flexcompile within ' + \
     204              'the next few minutes!'
     205
     206    else: #local
     207        if not c.flexpart_root_scripts or c.flexpart_root_scripts == '../':
     208            print 'WARNING: FLEXPART_ROOT_SCRIPTS has not been specified'
     209            print 'There will be only the compilation of ' + \
     210                  ' in ' + ecd + '/src'
     211            os.chdir(ecd + '/src')
     212        else: # creates the target working directory for flex_extract
    297213            c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
    298214                c.flexpart_root_scripts))
    299215            if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
    300216                os.chdir('/')
    301                 p = subprocess.check_call(['tar', '-cvf',
    302                                            ecd + '../ECMWFDATA7.1.tar',
    303                                            ecd + 'python',
    304                                            ecd + 'grib_templates',
    305                                            ecd + 'src'])
    306                 try:
    307                     os.makedirs(c.flexpart_root_scripts + '/ECMWFDATA7.1')
    308                 finally:
    309                     pass
    310                 os.chdir(c.flexpart_root_scripts + '/ECMWFDATA7.1')
    311                 p = subprocess.check_call(['tar', '-xvf',
    312                                            ecd + '../ECMWFDATA7.1.tar'])
    313                 os.chdir(c.flexpart_root_scripts + '/ECMWFDATA7.1/src')
    314 
    315         os.chdir('../src')
    316         print(('install ECMWFDATA7.1 software on ' + target + ' in directory '
    317                + os.getcwd()))
    318         if c.makefile is None:
    319             makefile = 'Makefile.local.ifort'
    320         else:
    321             makefile = c.makefile
    322         flist = glob.glob('*.mod') + glob.glob('*.o')
    323         if flist:
    324             p = subprocess.check_call(['rm'] + flist)
    325         try:
    326             print 'Using makefile: ' + makefile
    327             p = subprocess.check_call(['make', '-f', makefile])
    328             p = subprocess.check_call(['ls', '-l', 'CONVERT2'])
    329         except subprocess.CalledProcessError as e:
    330             print 'compile failed with the following error:'
    331             print e.output
    332             print 'please edit ' + makefile + \
     217                mk_tarball(ecd, tarball_name)
     218                make_dir(c.flexpart_root_scripts + '/' + target_dir)
     219                os.chdir(c.flexpart_root_scripts + '/' + target_dir)
     220                print 'Untar ...'
     221                subprocess.check_call(['tar', '-xvf',
     222                                       ecd + '../' + tarball_name])
     223                os.chdir(c.flexpart_root_scripts + '/' + target_dir + '/src')
     224
     225        # Create Fortran executable - CONVERT2
     226        print 'Install ' + target_dir + ' software on ' + \
     227              c.install_target + ' in directory ' + \
     228              os.path.abspath(os.getcwd() + '/../') + '\n'
     229
     230        delete_convert_build('')
     231        make_convert_build('', c.makefile, fortran_executable)
     232
     233    return
     234
     235def mk_tarball(ecd, tarname):
     236    '''
     237    @Description:
     238        Creates a tarball from all files which need to be sent to the
     239        installation directory.
     240        It does not matter if this is local or remote.
     241        Collects all python files, the Fortran source and makefiles,
     242        the ECMWF_ENV file, the CONTROL files as well as
     243        the korn shell and template files.
     244
     245    @Input:
     246        ecd: string
     247            The path were the file is to be stored.
     248
     249        tarname: string
     250            The name of the file to send to the ECMWF server.
     251
     252    @Return:
     253        <nothing>
     254    '''
     255
     256    print 'Create tarball ...'
     257    try:
     258        subprocess.check_call(['tar -cvf '+
     259                               ecd + '../' + tarname + ' ' +
     260                               ecd + 'python/*py ' +
     261                               ecd + 'python/CONTROL* ' +
     262                               ecd + 'python/*ksh ' +
     263                               ecd + 'python/*temp* ' +
     264                               ecd + 'python/ECMWF_ENV ' +
     265                               ecd + 'grib_templates ' +
     266                               ecd + 'src/*.f ' +
     267                               ecd + 'src/*.f90 ' +
     268                               ecd + 'src/*.h ' +
     269                               ecd + 'src/Makefile*'], shell=True)
     270    except subprocess.CalledProcessError as e:
     271        print 'ERROR:'
     272        print e.output
     273        sys.exit('could not make installation tar ball!')
     274
     275    return
     276
     277def mk_env_vars(ecd, ecuid, ecgid, gateway, destination):
     278    '''
     279    @Description:
     280        Creates a file named ECMWF_ENV which contains the
     281        necessary environmental variables at ECMWF servers.
     282
     283    @Input:
     284        ecd: string
     285            The path were the file is to be stored.
     286
     287        ecuid: string
     288            The user id on ECMWF server.
     289
     290        ecgid: string
     291            The group id on ECMWF server.
     292
     293        gateway: string
     294            The gateway server the user is using.
     295
     296        destination: string
     297            The remote destination which is used to transfer files
     298            from ECMWF server to local gateway server.
     299
     300    @Return:
     301        <nothing>
     302    '''
     303
     304    with open(ecd + 'python/ECMWF_ENV', 'w') as fo:
     305        fo.write('ECUID ' + ecuid + '\n')
     306        fo.write('ECGID ' + ecgid + '\n')
     307        fo.write('GATEWAY ' + gateway + '\n')
     308        fo.write('DESTINATION ' + destination + '\n')
     309
     310    return
     311
     312def mk_compilejob(template, makefile, target, ecuid, ecgid, fp_root):
     313    '''
     314    @Description:
     315        Modifies the original job template file so that it is specified
     316        for the user and the environment were it will be applied. Result
     317        is stored in a new file "job.temp" in the python directory.
     318
     319    @Input:
     320        template: string
     321            File which contains the original text for the job template.
     322            It must contain the complete path to the file.
     323
     324        makefile: string
     325            Name of the makefile which should be used to compile FORTRAN
     326            CONVERT2 program.
     327
     328        target: string
     329            The target where the installation should be done, e.g. the queue.
     330
     331        ecuid: string
     332            The user id on ECMWF server.
     333
     334        ecgid: string
     335            The group id on ECMWF server.
     336
     337        fp_root: string
     338           Path to the root directory of FLEXPART environment or flex_extract
     339           environment.
     340
     341    @Return:
     342        <nothing>
     343    '''
     344
     345    with open(template) as f:
     346        fdata = f.read().split('\n')
     347
     348    with open(template[:-4] + 'ksh', 'w') as fo:
     349        for data in fdata:
     350            if 'MAKEFILE=' in data:
     351                data = 'export MAKEFILE=' + makefile
     352            elif 'FLEXPART_ROOT_SCRIPTS=' in data:
     353                if fp_root != '../':
     354                    data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
     355                else:
     356                    data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
     357            elif target.lower() != 'local':
     358                if '--workdir' in data:
     359                    data = '#SBATCH --workdir=/scratch/ms/' + \
     360                            ecgid + '/' + ecuid
     361                elif '##PBS -o' in data:
     362                    data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \
     363                           'flex_ecmwf.$Jobname.$Job_ID.out'
     364                elif 'FLEXPART_ROOT_SCRIPTS=' in data:
     365                    if fp_root != '../':
     366                        data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
     367                    else:
     368                        data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
     369            fo.write(data + '\n')
     370
     371    return
     372
     373def mk_job_template(template, ecuid, ecgid, gateway, destination, fp_root):
     374    '''
     375    @Description:
     376        Modifies the original job template file so that it is specified
     377        for the user and the environment were it will be applied. Result
     378        is stored in a new file "job.temp" in the python directory.
     379
     380    @Input:
     381        template: string
     382            File which contains the original text for the job template.
     383            It must contain the complete path to the file.
     384
     385        ecuid: string
     386            The user id on ECMWF server.
     387
     388        ecgid: string
     389            The group id on ECMWF server.
     390
     391        gateway: string
     392            The gateway server the user is using.
     393
     394        destination: string
     395            The remote destination which is used to transfer files
     396            from ECMWF server to local gateway server.
     397
     398        fp_root: string
     399           Path to the root directory of FLEXPART environment or flex_extract
     400           environment.
     401
     402    @Return:
     403        <nothing>
     404    '''
     405
     406    with open(template) as f:
     407        fdata = f.read().split('\n')
     408
     409    with open(template[:-2], 'w') as fo:
     410        for data in fdata:
     411            if '--workdir' in data:
     412                data = '#SBATCH --workdir=/scratch/ms/' + ecgid + \
     413                        '/' + ecuid
     414            elif '##PBS -o' in data:
     415                data = '##PBS -o /scratch/ms/' + ecgid + '/' + \
     416                        ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out'
     417            elif  'export PATH=${PATH}:' in data:
     418                data += fp_root + '/flex_extract_v7.1/python'
     419
     420            fo.write(data + '\n')
     421    return
     422
     423def delete_convert_build(ecd):
     424    '''
     425    @Description:
     426        Clean up the Fortran source directory and remove all
     427        build files (e.g. *.o, *.mod and CONVERT2)
     428
     429    @Input:
     430        ecd: string
     431            The path to the Fortran program.
     432
     433    @Return:
     434        <nothing>
     435    '''
     436
     437    modfiles = UioFiles(ecd, '*.mod')
     438    objfiles = UioFiles(ecd, '*.o')
     439    exefile = UioFiles(ecd, 'CONVERT2')
     440
     441    modfiles.delete_files()
     442    objfiles.delete_files()
     443    exefile.delete_files()
     444
     445    return
     446
     447def make_convert_build(ecd, makefile, f_executable):
     448    '''
     449    @Description:
     450        Compiles the Fortran code and generates the executable.
     451
     452    @Input:
     453        ecd: string
     454            The path were the file is to be stored.
     455
     456        makefile: string
     457            The name of the makefile which should be used.
     458
     459        f_executable: string
     460            The name of the executable the Fortran program generates after
     461            compilation.
     462
     463    @Return:
     464        <nothing>
     465    '''
     466
     467    try:
     468        print 'Using makefile: ' + makefile
     469        p = subprocess.Popen(['make', '-f', ecd + makefile],
     470                             stdin=subprocess.PIPE,
     471                             stdout=subprocess.PIPE,
     472                             stderr=subprocess.PIPE,
     473                             bufsize=1)
     474        pout, perr = p.communicate()
     475        print pout
     476        if p.returncode != 0:
     477            print perr
     478            print 'Please edit ' + makefile + \
    333479                  ' or try another Makefile in the src directory.'
    334             print 'most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB \
    335                    and EMOSLIB must be adapted.'
     480            print 'Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB ' \
     481                  'and EMOSLIB must be adapted.'
    336482            print 'Available Makefiles:'
    337             print glob.glob('Makefile*')
    338     elif target.lower() == 'ecgate':
    339         os.chdir('/')
    340         p = subprocess.check_call(['tar', '-cvf',
    341                                    ecd + '../ECMWFDATA7.1.tar',
    342                                    ecd + 'python',
    343                                    ecd + 'grib_templates',
    344                                    ecd + 'src'])
    345         try:
    346             p = subprocess.check_call(['ecaccess-file-put',
    347                                        ecd + '../ECMWFDATA7.1.tar',
    348                                        'ecgate:/home/ms/' + c.ecgid + '/' +
    349                                        c.ecuid + '/ECMWFDATA7.1.tar'])
    350         except subprocess.CalledProcessError as e:
    351             print 'ecaccess-file-put failed! \
    352                    Probably the eccert key has expired.'
    353             exit(1)
    354 
    355         try:
    356             p = subprocess.check_call(['ecaccess-job-submit',
    357                                        '-queueName',
    358                                        target,
    359                                        ecd + 'python/compilejob.ksh'])
    360             print 'compilejob.ksh has been submitted to ecgate for  \
    361                    installation in ' + c.ec_flexpart_root_scripts + \
    362                    '/ECMWFDATA7.1'
    363             print 'You should get an email with subject flexcompile within  \
    364                    the next few minutes'
    365         except subprocess.CalledProcessError as e:
    366             print 'ecaccess-job-submit failed!'
    367             exit(1)
    368 
    369     elif target.lower() == 'cca':
    370         os.chdir('/')
    371         p = subprocess.check_call(['tar', '-cvf',
    372                                    ecd + '../ECMWFDATA7.1.tar',
    373                                    ecd + 'python',
    374                                    ecd + 'grib_templates',
    375                                    ecd + 'src'])
    376         try:
    377             p = subprocess.check_call(['ecaccess-file-put',
    378                                        ecd + '../ECMWFDATA7.1.tar',
    379                                        'cca:/home/ms/' + c.ecgid + '/' +
    380                                        c.ecuid + '/ECMWFDATA7.1.tar'])
    381         except subprocess.CalledProcessError as e:
    382             print 'ecaccess-file-put failed! \
    383                    Probably the eccert key has expired.'
    384             exit(1)
    385 
    386         try:
    387             p = subprocess.check_call(['ecaccess-job-submit',
    388                                        '-queueName',
    389                                        target,
    390                                        ecd + 'python/compilejob.ksh'])
    391             print 'compilejob.ksh has been submitted to cca for installation in ' +\
    392                   c.ec_flexpart_root_scripts + '/ECMWFDATA7.1'
    393             print 'You should get an email with subject flexcompile \
    394                    within the next few minutes'
    395         except subprocess.CalledProcessError as e:
    396             print 'ecaccess-job-submit failed!'
    397             exit(1)
    398 
     483            print UioFiles('.', 'Makefile*')
     484            sys.exit('Compilation failed!')
     485    except ValueError as e:
     486        print 'ERROR: Makefile call failed:'
     487        print e
    399488    else:
    400         print 'ERROR: unknown installation target ', target
    401         print 'Valid targets: ecgate, cca, local'
     489        subprocess.check_call(['ls', '-l', ecd + f_executable])
    402490
    403491    return
  • python/job.ksh

    rccab809 r54a8a01  
    3333  module load grib_api/1.14.5
    3434  module load emos/437-r64
    35 #  export ECMWFDATA=$HOME/ECMWFDATA$VERSION
    36 #  export PYTHONPATH=$ECMWFDATA/python
    37   export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python
     35  export PATH=${PATH}:${HOME}/flex_extract_v7.1/python
    3836  ;;
    3937  *cca*)
     
    4341  module load python
    4442  export SCRATCH=$TMPDIR
    45 #  export ECMWFDATA=$HOME/ECMWFDATA$VERSION
    46 #  export PYTHONPATH=$ECMWFDATA/python
    47   export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python
     43  export PATH=${PATH}:${HOME}/flex_extract_v7.1/python
    4844  ;;
    49 #  *)
    50 #  export ECMWFDATA=$HOME/ECMWFDATA$VERSION
    51 #  export PATH=/opt/anaconda/bin:$ECMWFDATA/python:${PATH}
    52 #  export PYTHONPATH=/opt/anaconda/lib/python2.7/site-packages/grib_api:$ECMWFDATA/python
    53 #  export SCRATCH=$ECMWFDATA/python
    54 #  which python
    55 #  ;;
    5645esac
    5746
     
    6352
    6453cat >$CONTROL<<EOF
    65 GATEWAY srvx8.img.univie.ac.at
    66 DESTINATION annep@genericSftp
    67 accuracy 16
     54accuracy 24
    6855addpar 186 187 188 235 139 39
     56area
    6957basetime None
     58controlfile CONTROL.test
    7059cwc 0
    7160date_chunk 3
    72 debug 1
     61debug 0
     62destination annep@genericSftp
    7363dpdeta 1
    7464dtime 3
    7565ecfsdir ectmp:/${USER}/econdemand/
     66ecgid at
    7667ecstorage 0
    7768ectrans 1
    78 end_date 20160809
     69ecuid km4a
     70end_date 20000101
    7971eta 0
    8072etadiff 0
     73etapar 77
    8174expver 1
    8275format GRIB1
     76gateway srvx8.img.univie.ac.at
    8377gauss 1
    8478grib2flexpart 0
    8579grid 5000
    8680inputdir ../work
     81install_target None
     82job_template job.temp
    8783left -15000
    8884level 60
    8985levelist 55/to/60
    9086lower 30000
    91 mailfail ${USER}
    92 mailops ${USER}
     87mailfail ${USER} 
     88mailops ${USER} 
    9389makefile None
    9490marsclass EI
     
    9894omegadiff 0
    9995outputdir ../work
    100 prefix EI
     96prefix EItest_
     97queue ecgate
    10198resol 63
    10299right 45000
    103100smooth 0
    104 start_date 20160809
     101start_date 20000101
    105102step 00 01 02 03 04 05 00 07 08 09 10 11 00 01 02 03 04 05 00 07 08 09 10 11
    106103stream OPER
     
    108105type AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC
    109106upper 75000
     107wrf 0
    110108EOF
    111109
  • python/job.temp

    r991df6a r54a8a01  
    3333  module load grib_api/1.14.5
    3434  module load emos/437-r64
    35 #  export ECMWFDATA=$HOME/ECMWFDATA$VERSION
    36 #  export PYTHONPATH=$ECMWFDATA/python
    37   export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python
     35  export PATH=${PATH}:${HOME}/flex_extract_v7.1/python
    3836  ;;
    3937  *cca*)
     
    4341  module load python
    4442  export SCRATCH=$TMPDIR
    45 #  export ECMWFDATA=$HOME/ECMWFDATA$VERSION
    46 #  export PYTHONPATH=$ECMWFDATA/python
    47   export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python
     43  export PATH=${PATH}:${HOME}/flex_extract_v7.1/python
    4844  ;;
    49 #  *)
    50 #  export ECMWFDATA=$HOME/ECMWFDATA$VERSION
    51 #  export PATH=/opt/anaconda/bin:$ECMWFDATA/python:${PATH}
    52 #  export PYTHONPATH=/opt/anaconda/lib/python2.7/site-packages/grib_api:$ECMWFDATA/python
    53 #  export SCRATCH=$ECMWFDATA/python
    54 #  which python
    55 #  ;;
    5645esac
    5746
     
    6352
    6453cat >$CONTROL<<EOF
    65 GATEWAY srvx8.img.univie.ac.at
    66 DESTINATION annep@genericSftp
    67 EOF
    68 cat >>$CONTROL<<EOF
    6954EOF
    7055
  • python/job.temp.o

    refdb01a r54a8a01  
    3333  module load grib_api/1.14.5
    3434  module load emos/437-r64
    35 #  export ECMWFDATA=$HOME/ECMWFDATA$VERSION
    36 #  export PYTHONPATH=$ECMWFDATA/python
    3735  export PATH=${PATH}:
    3836  ;;
     
    4341  module load python
    4442  export SCRATCH=$TMPDIR
    45 #  export ECMWFDATA=$HOME/ECMWFDATA$VERSION
    46 #  export PYTHONPATH=$ECMWFDATA/python
    4743  export PATH=${PATH}:
    4844  ;;
    49 #  *)
    50 #  export ECMWFDATA=$HOME/ECMWFDATA$VERSION
    51 #  export PATH=/opt/anaconda/bin:$ECMWFDATA/python:${PATH}
    52 #  export PYTHONPATH=/opt/anaconda/lib/python2.7/site-packages/grib_api:$ECMWFDATA/python
    53 #  export SCRATCH=$ECMWFDATA/python
    54 #  which python
    55 #  ;;
    5645esac
    5746
     
    6251export CONTROL=CONTROL
    6352
    64 cat >>$CONTROL<<EOF
     53cat >$CONTROL<<EOF
    6554EOF
    6655
  • python/joboper.ksh

    rccab809 r54a8a01  
    3333  module load grib_api/1.14.5
    3434  module load emos/437-r64
    35 #  export ECMWFDATA=$HOME/ECMWFDATA$VERSION
    36 #  export PYTHONPATH=$ECMWFDATA/python
    37   export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python
     35  export PATH=${PATH}:${HOME}/flex_extract_v7.1/python
    3836  ;;
    3937  *cca*)
     
    4341  module load python
    4442  export SCRATCH=$TMPDIR
    45 #  export ECMWFDATA=$HOME/ECMWFDATA$VERSION
    46 #  export PYTHONPATH=$ECMWFDATA/python
    47   export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python
     43  export PATH=${PATH}:${HOME}/flex_extract_v7.1/python
    4844  ;;
    49 #  *)
    50 #  export ECMWFDATA=$HOME/ECMWFDATA$VERSION
    51 #  export PATH=/opt/anaconda/bin:$ECMWFDATA/python:${PATH}
    52 #  export PYTHONPATH=/opt/anaconda/lib/python2.7/site-packages/grib_api:$ECMWFDATA/python
    53 #  export SCRATCH=$ECMWFDATA/python
    54 #  which python
    55 #  ;;
    5645esac
    5746
     
    6756accuracy 16
    6857addpar 186 187 188 235 139 39
     58area
     59base_time ${MSJ_BASETIME}
    6960basetime None
     61controlfile CONTROL.temp
    7062cwc 0
    7163date_chunk 3
    7264debug 1
     65destination None
    7366dpdeta 1
    7467dtime 3
    7568ecfsdir ectmp:/${USER}/econdemand/
     69ecgid None
    7670ecstorage 0
    7771ectrans 1
    78 start_date ${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}
     72ecuid None
     73end_date ${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}
    7974eta 0
    8075etadiff 0
     76etapar 77
    8177expver 1
    8278format GRIB1
     79gateway None
    8380gauss 1
    8481grib2flexpart 0
    8582grid 5000
    8683inputdir ../work
     84install_target None
     85job_template job.temp
    8786left -15000
    8887level 60
    8988levelist 55/to/60
    9089lower 30000
    91 mailfail ${USER}
    92 mailops ${USER}
     90mailfail ${USER} 
     91mailops ${USER} 
    9392makefile None
    9493marsclass EI
     
    9998outputdir ../work
    10099prefix EI
     100queue ecgate
    101101resol 63
    102102right 45000
     
    108108type AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC
    109109upper 75000
     110wrf 0
    110111EOF
    111112
  • python/plot_retrieved.py

    rff99eae r54a8a01  
    11#!/usr/bin/env python
    22# -*- coding: utf-8 -*-
    3 #************************************************************************
    4 # ToDo AP
    5 # - documentation der Funktionen
    6 # - docu der progam functionality
    7 # - apply pep8
    8 #************************************************************************
    93#*******************************************************************************
    104# @Author: Leopold Haimberger (University of Vienna)
     
    3428#    - main
    3529#    - get_basics
     30#    - get_files_per_date
    3631#    - plot_retrieved
    3732#    - plot_timeseries
     
    219214    c.area = np.asarray(c.area)
    220215
    221     files = UioFiles(c.prefix+'*')
    222     files.list_files(c.inputdir)
     216    files = UioFiles(c.inputdir, c.prefix+'*')
    223217    ifiles = get_files_per_date(files.files, datelist)
    224218    ifiles.sort()
     
    622616                        help="FLEXPART root directory (to find \
    623617                        'grib2flexpart and COMMAND file)\n \
    624                         Normally ECMWFDATA resides in the scripts directory \
     618                        Normally flex_extract resides in the scripts directory \
    625619                        of the FLEXPART distribution")
    626620
  • python/prepare_flexpart.py

    rff99eae r54a8a01  
    11#!/usr/bin/env python
    22# -*- coding: utf-8 -*-
    3 #************************************************************************
    4 # ToDo AP
    5 # - wieso start=startm1 wenn basetime = 0 ?  wenn die fluxes nicht mehr
    6 #   relevant sind? verstehe ich nicht
    7 #************************************************************************
    83#*******************************************************************************
    94# @Author: Anne Fouilloux (University of Oslo)
     
    6257import sys
    6358import socket
     59import _config
    6460
    6561# software specific classes and modules from flex_extract
    6662from UioFiles import UioFiles
    67 from tools import interpret_args_and_control, clean_up
     63from tools import clean_up, get_cmdline_arguments, read_ecenv
    6864from EcFlexpart import EcFlexpart
    6965
     
    8177    sys.path.append(LOCAL_PYTHON_PATH)
    8278
     79
    8380# ------------------------------------------------------------------------------
    8481# FUNCTION
     
    9794        <nothing>
    9895    '''
    99     args, c = interpret_args_and_control()
    100     prepare_flexpart(args, c)
     96
     97    args = get_cmdline_arguments()
     98
     99    try:
     100        c = ControlFile(args.controlfile)
     101    except IOError:
     102        try:
     103            c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile)
     104        except IOError:
     105            print 'Could not read CONTROL file "' + args.controlfile + '"'
     106            print 'Either it does not exist or its syntax is wrong.'
     107            print 'Try "' + sys.argv[0].split('/')[-1] + \
     108                  ' -h" to print usage information'
     109            sys.exit(1)
     110
     111    env_parameter = read_ecenv(c.ecmwfdatadir + 'python/ECMWF_ENV')
     112    c.assign_args_to_control(args, env_parameter)
     113    c.assign_envs_to_control(env_parameter)
     114    c.check_conditions()
     115    prepare_flexpart(args.ppid, c)
    101116
    102117    return
    103118
    104 def prepare_flexpart(args, c):
     119def prepare_flexpart(ppid, c):
    105120    '''
    106121    @Description:
     
    112127
    113128    @Input:
    114         args: instance of ArgumentParser
    115             Contains the commandline arguments from script/program call.
     129        ppid: int
     130            Contains the ppid number of the current ECMWF job. If it is called
     131            from this script, it is "None".
    116132
    117133        c: instance of class ControlFile
     
    132148    '''
    133149
    134     if not args.ppid:
     150    if not ppid:
    135151        c.ppid = str(os.getppid())
    136152    else:
    137         c.ppid = args.ppid
     153        c.ppid = ppid
    138154
    139155    c.ecapi = ecapi
     
    148164                        day=int(c.end_date[6:]))
    149165
    150     # to deaccumulate the fluxes correctly
    151     # one day ahead of the start date and
    152     # one day after the end date is needed
    153     startm1 = start - datetime.timedelta(days=1)
    154 #    endp1 = end + datetime.timedelta(days=1)
    155 
    156     # get all files with flux data to be deaccumulated
    157     inputfiles = UioFiles('*OG_acc_SL*.' + c.ppid + '.*')
    158     inputfiles.list_files(c.inputdir)
     166    # assign starting date minus 1 day
     167    # since for basetime 00 we need the 12 hours upfront
     168    # (the day before from 12 UTC to current day 00 UTC)
     169    if c.basetime == '00':
     170        start = start - datetime.timedelta(days=1)
     171
     172    print 'Prepare ' + start.strftime("%Y%m%d") + \
     173           "/to/" + end.strftime("%Y%m%d")
    159174
    160175    # create output dir if necessary
    161176    if not os.path.exists(c.outputdir):
    162177        os.makedirs(c.outputdir)
     178
     179    # get all files with flux data to be deaccumulated
     180    inputfiles = UioFiles(c.inputdir, '*OG_acc_SL*.' + c.ppid + '.*')
    163181
    164182    # deaccumulate the flux data
     
    167185    flexpart.deacc_fluxes(inputfiles, c)
    168186
    169     print 'Prepare ' + start.strftime("%Y%m%d") + \
    170           "/to/" + end.strftime("%Y%m%d")
    171 
    172187    # get a list of all files from the root inputdir
    173     inputfiles = UioFiles('????__??.*' + c.ppid + '.*')
    174     inputfiles.list_files(c.inputdir)
    175 
    176     # produce FLEXPART-ready GRIB files and
    177     # process GRIB files -
     188    inputfiles = UioFiles(c.inputdir, '????__??.*' + c.ppid + '.*')
     189
     190    # produce FLEXPART-ready GRIB files and process them -
    178191    # copy/transfer/interpolate them or make them GRIB2
    179     if c.basetime == '00':
    180         start = startm1
    181 
    182192    flexpart = EcFlexpart(c, fluxes=False)
    183193    flexpart.create(inputfiles, c)
     
    187197    # otherwise delete temporary files
    188198    if int(c.debug) != 0:
    189         print 'Temporary files left intact'
     199        print '\nTemporary files left intact'
    190200    else:
    191201        clean_up(c)
  • python/profiling.py

    rff99eae r54a8a01  
    33#************************************************************************
    44# ToDo AP
    5 # - check of license of book content
     5# - check license of book content
    66#************************************************************************
    77#*******************************************************************************
  • python/submit.py

    rff99eae r54a8a01  
    4444import subprocess
    4545import inspect
     46import collections
    4647
    4748# software specific classes and modules from flex_extract
    48 from tools import interpret_args_and_control, normal_exit
     49import _config
     50from tools import normal_exit, get_cmdline_arguments, submit_job_to_ecserver, \
     51                  read_ecenv
    4952from get_mars_data import get_mars_data
    5053from prepare_flexpart import prepare_flexpart
    51 
    52 # add path to pythonpath so that python finds its buddies
    53 LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
    54     inspect.getfile(inspect.currentframe())))
    55 if LOCAL_PYTHON_PATH not in sys.path:
    56     sys.path.append(LOCAL_PYTHON_PATH)
     54from ControlFile import ControlFile
    5755
    5856# ------------------------------------------------------------------------------
     
    7674
    7775    called_from_dir = os.getcwd()
    78     args, c = interpret_args_and_control()
     76
     77    args = get_cmdline_arguments()
     78
     79    try:
     80        c = ControlFile(args.controlfile)
     81    except IOError:
     82        try:
     83            c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile)
     84        except IOError:
     85            print 'Could not read CONTROL file "' + args.controlfile + '"'
     86            print 'Either it does not exist or its syntax is wrong.'
     87            print 'Try "' + sys.argv[0].split('/')[-1] + \
     88                  ' -h" to print usage information'
     89            sys.exit(1)
     90
     91    env_parameter = read_ecenv(c.ecmwfdatadir + 'python/ECMWF_ENV')
     92    c.assign_args_to_control(args)
     93    c.assign_envs_to_control(env_parameter)
     94    c.check_conditions()
    7995
    8096    # on local side
     97    # on ECMWF server this would be the local side
    8198    if args.queue is None:
    8299        if c.inputdir[0] != '/':
     
    84101        if c.outputdir[0] != '/':
    85102            c.outputdir = os.path.join(called_from_dir, c.outputdir)
    86         get_mars_data(args, c)
    87         prepare_flexpart(args, c)
    88         normal_exit(c)
     103        get_mars_data(c)
     104        prepare_flexpart(args.ppid, c)
     105        normal_exit(c.mailfail, 'Done!')
    89106    # on ECMWF server
    90107    else:
     
    125142    '''
    126143
    127     # read template file and split from newline signs
     144    # read template file and get index for CONTROL input
    128145    with open(jtemplate) as f:
    129146        lftext = f.read().split('\n')
    130         insert_point = lftext.index('EOF')
     147    insert_point = lftext.index('EOF')
    131148
    132     # put all parameters of ControlFile instance into a list
    133     clist = c.to_list() # ondemand
    134     colist = []  # operational
    135     mt = 0
     149    if not c.basetime:
     150    # --------- create on demand job script ------------------------------------
     151        if c.maxstep > 24:
     152            print '---- Pure forecast mode! ----'
     153        else:
     154            print '---- On-demand mode! ----'
     155        job_file = jtemplate[:-4] + 'ksh'
     156        clist = c.to_list()
    136157
    137     for elem in clist:
    138         if 'maxstep' in elem:
    139             mt = int(elem.split(' ')[1])
     158        lftextondemand = lftext[:insert_point] + clist + lftext[insert_point:]
    140159
    141     for elem in clist:
    142         if 'start_date' in elem:
    143             elem = 'start_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
    144         if 'end_date' in elem:
    145             elem = 'end_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
    146         if 'base_time' in elem:
    147             elem = 'base_time ' + '${MSJ_BASETIME}'
    148         if 'time' in elem and mt > 24:
    149             elem = 'time ' + '${MSJ_BASETIME} {MSJ_BASETIME}'
    150         colist.append(elem)
     160        with open(job_file, 'w') as f:
     161            f.write('\n'.join(lftextondemand))
    151162
    152     lftextondemand = lftext[:insert_point] + clist + lftext[insert_point + 2:]
    153     lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:]
     163        submit_job_to_ecserver('', queue, job_file)
    154164
    155     with open('job.ksh', 'w') as h:
    156         h.write('\n'.join(lftextondemand))
     165    else:
     166    # --------- create operational job script ----------------------------------
     167        print '---- Operational mode! ----'
     168        job_file = jtemplate[:-5] + 'oper.ksh'
     169        #colist = []
    157170
    158     with open('joboper.ksh', 'w') as h:
    159         h.write('\n'.join(lftextoper))
     171        if c.maxstep:
     172            mt = int(c.maxstep)
     173        else:
     174            mt = 0
    160175
    161     # submit job script to queue
    162     try:
    163         p = subprocess.check_call(['ecaccess-job-submit', '-queueName',
    164                                    queue, 'job.ksh'])
    165     except subprocess.CalledProcessError as e:
    166         print 'ecaccess-job-submit failed!'
    167         print 'Error Message: '
    168         print e.output
    169         exit(1)
     176        c.start_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
     177        c.end_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
     178        c.base_time = '${MSJ_BASETIME}'
     179        if mt > 24:
     180            c.time = '${MSJ_BASETIME} {MSJ_BASETIME}'
    170181
     182        colist = c.to_list()
     183
     184        lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:]
     185
     186        with open(job_file, 'w') as f:
     187            f.write('\n'.join(lftextoper))
     188
     189        submit_job_to_ecserver('', queue, job_file)
     190
     191    # --------------------------------------------------------------------------
    171192    print 'You should get an email with subject flex.hostname.pid'
    172193
  • python/test_suite.py

    rff99eae r54a8a01  
    11#!/usr/bin/env python
    22# -*- coding: utf-8 -*-
    3 #************************************************************************
    4 # ToDo AP
    5 # - provide more tests
    6 # - provide more documentation
    7 #************************************************************************
    8 
    93#*******************************************************************************
    104# @Author: Leopold Haimberger (University of Vienna)
     
    2519#
    2620# @Program Functionality:
    27 #    This script triggers the ECMWFDATA test suite. Call with
     21#    This script triggers the flex_extract test suite. Call with
    2822#    test_suite.py [test group]
    2923#
  • python/tools.py

    rff99eae r54a8a01  
    11#!/usr/bin/env python
    22# -*- coding: utf-8 -*-
    3 #************************************************************************
    4 # ToDo AP
    5 # - check my_error
    6 # - check normal_exit
    7 # - check get_list_as_string
    8 # - seperate args and control interpretation
    9 #************************************************************************
    103#*******************************************************************************
    114# @Author: Anne Philipp (University of Vienna)
     
    2619#        - moved all functions from file Flexparttools to this file tools
    2720#        - added function get_list_as_string
     21#        - seperated args and control interpretation
    2822#
    2923# @License:
     
    3832#
    3933# @Module Content:
    40 #    - interpret_args_and_control
     34#    - get_cmdline_arguments
    4135#    - clean_up
    4236#    - my_error
     
    4741#    - to_param_id
    4842#    - get_list_as_string
     43#    - make_dir
    4944#
    5045#*******************************************************************************
     
    5752import sys
    5853import glob
    59 import inspect
    6054import subprocess
    6155import traceback
    6256from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
    63 import numpy as np
    64 
    65 # software specific class from flex_extract
    66 from ControlFile import ControlFile
    6757
    6858# ------------------------------------------------------------------------------
     
    7060# ------------------------------------------------------------------------------
    7161
    72 def interpret_args_and_control():
    73     '''
    74     @Description:
    75         Assigns the command line arguments and reads CONTROL file
    76         content. Apply default values for non mentioned arguments.
     62def get_cmdline_arguments():
     63    '''
     64    @Description:
     65        Decomposes the command line arguments and assigns them to variables.
     66        Apply default values for non mentioned arguments.
    7767
    7868    @Input:
     
    8272        args: instance of ArgumentParser
    8373            Contains the commandline arguments from script/program call.
    84 
    85         c: instance of class ControlFile
    86             Contains all necessary information of a CONTROL file. The parameters
    87             are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
    88             NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
    89             RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
    90             SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
    91             ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, DEBUG, INPUTDIR,
    92             OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
    93             For more information about format and content of the parameter see
    94             documentation.
    95 
    96     '''
     74    '''
     75
    9776    parser = ArgumentParser(description='Retrieve FLEXPART input from \
    98                             ECMWF MARS archive',
     77                                ECMWF MARS archive',
    9978                            formatter_class=ArgumentDefaultsHelpFormatter)
    10079
    10180    # the most important arguments
    102     parser.add_argument("--start_date", dest="start_date",
     81    parser.add_argument("--start_date", dest="start_date", default=None,
    10382                        help="start date YYYYMMDD")
    104     parser.add_argument("--end_date", dest="end_date",
     83    parser.add_argument("--end_date", dest="end_date", default=None,
    10584                        help="end_date YYYYMMDD")
    10685    parser.add_argument("--date_chunk", dest="date_chunk", default=None,
     
    10887
    10988    # some arguments that override the default in the CONTROL file
    110     parser.add_argument("--basetime", dest="basetime",
     89    parser.add_argument("--basetime", dest="basetime", default=None,
    11190                        help="base such as 00/12 (for half day retrievals)")
    112     parser.add_argument("--step", dest="step",
     91    parser.add_argument("--step", dest="step", default=None,
    11392                        help="steps such as 00/to/48")
    114     parser.add_argument("--levelist", dest="levelist",
     93    parser.add_argument("--levelist", dest="levelist", default=None,
    11594                        help="Vertical levels to be retrieved, e.g. 30/to/60")
    116     parser.add_argument("--area", dest="area",
     95    parser.add_argument("--area", dest="area", default=None,
    11796                        help="area defined as north/west/south/east")
    11897
     
    123102                        help="root directory for storing output files")
    124103    parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
     104                        default=None,
    125105                        help="FLEXPART root directory (to find grib2flexpart \
    126                         and COMMAND file)\n Normally ECMWFDATA resides in \
     106                        and COMMAND file)\n Normally flex_extract resides in \
    127107                        the scripts directory of the FLEXPART distribution")
    128108
    129109    # this is only used by prepare_flexpart.py to rerun a postprocessing step
    130     parser.add_argument("--ppid", dest="ppid",
    131                         help="Specify parent process id for \
     110    parser.add_argument("--ppid", dest="ppid", default=None,
     111                        help="specify parent process id for \
    132112                        rerun of prepare_flexpart")
    133113
     
    136116                        default="job.temp",
    137117                        help="job template file for submission to ECMWF")
    138     parser.add_argument("--queue", dest="queue",
     118    parser.add_argument("--queue", dest="queue", default=None,
    139119                        help="queue for submission to ECMWF \
    140120                        (e.g. ecgate or cca )")
     
    142122                        default='CONTROL.temp',
    143123                        help="file with CONTROL parameters")
    144     parser.add_argument("--debug", dest="debug", default=0,
    145                         help="Debug mode - leave temporary files intact")
     124    parser.add_argument("--debug", dest="debug", default=None,
     125                        help="debug mode - leave temporary files intact")
    146126
    147127    args = parser.parse_args()
    148128
    149     # create instance of ControlFile for specified controlfile
    150     # and assign the parameters (and default values if necessary)
    151     try:
    152         c = ControlFile(args.controlfile)
    153     except IOError:
    154         try:
    155             LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
    156                 inspect.getfile(inspect.currentframe())))
    157             c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile)
    158         except IOError:
    159             print 'Could not read CONTROL file "' + args.controlfile + '"'
    160             print 'Either it does not exist or its syntax is wrong.'
    161             print 'Try "' + sys.argv[0].split('/')[-1] + \
    162                   ' -h" to print usage information'
    163             exit(1)
    164 
    165     # check for having at least a starting date
    166     if  args.start_date is None and getattr(c, 'start_date') is None:
    167         print 'start_date specified neither in command line nor \
    168                in CONTROL file ' + args.controlfile
    169         print 'Try "' + sys.argv[0].split('/')[-1] + \
    170               ' -h" to print usage information'
    171         exit(1)
    172 
    173     # save all existing command line parameter to the ControlFile instance
    174     # if parameter is not specified through the command line or CONTROL file
    175     # set default values
    176     if args.start_date is not None:
    177         c.start_date = args.start_date
    178     if args.end_date is not None:
    179         c.end_date = args.end_date
    180     if c.end_date is None:
    181         c.end_date = c.start_date
    182     if args.date_chunk is not None:
    183         c.date_chunk = args.date_chunk
    184 
    185     if not hasattr(c, 'debug'):
    186         c.debug = args.debug
    187 
    188     if args.inputdir is None and args.outputdir is None:
    189         c.inputdir = '../work'
    190         c.outputdir = '../work'
    191     else:
    192         if args.inputdir is not None:
    193             c.inputdir = args.inputdir
    194         if args.outputdir is None:
    195             c.outputdir = args.inputdir
    196         if args.outputdir is not None:
    197             c.outputdir = args.outputdir
    198         if args.inputdir is None:
    199             c.inputdir = args.outputdir
    200 
    201     if hasattr(c, 'outputdir') is False and args.outputdir is None:
    202         c.outputdir = c.inputdir
    203     else:
    204         if args.outputdir is not None:
    205             c.outputdir = args.outputdir
    206 
    207     if args.area is not None:
    208         afloat = '.' in args.area
    209         l = args.area.split('/')
    210         if afloat:
    211             for i, item in enumerate(l):
    212                 item = str(int(float(item) * 1000))
    213         c.upper, c.left, c.lower, c.right = l
    214 
    215     # NOTE: basetime activates the ''operational mode''
    216     if args.basetime is not None:
    217         c.basetime = args.basetime
    218 
    219     if args.step is not None:
    220         l = args.step.split('/')
    221         if 'to' in args.step.lower():
    222             if 'by' in args.step.lower():
    223                 ilist = np.arange(int(l[0]), int(l[2]) + 1, int(l[4]))
    224                 c.step = ['{:0>3}'.format(i) for i in ilist]
    225             else:
    226                 my_error(None, args.step + ':\n' +
    227                          'please use "by" as well if "to" is used')
    228         else:
    229             c.step = l
    230 
    231     if args.levelist is not None:
    232         c.levelist = args.levelist
    233         if 'to' in c.levelist:
    234             c.level = c.levelist.split('/')[2]
    235         else:
    236             c.level = c.levelist.split('/')[-1]
    237 
    238     if args.flexpart_root_scripts is not None:
    239         c.flexpart_root_scripts = args.flexpart_root_scripts
    240 
    241     return args, c
    242 
     129    return args
     130
     131def read_ecenv(filename):
     132    '''
     133    @Description:
     134        Reads the file into a dictionary where the key values are the parameter
     135        names.
     136
     137    @Input:
     138        filename: string
     139            Name of file where the ECMWV environment parameters are stored.
     140
     141    @Return:
     142        envs: dict
     143    '''
     144    envs= {}
     145    print filename
     146    with open(filename, 'r') as f:
     147        for line in f:
     148            data = line.strip().split()
     149            envs[str(data[0])] = str(data[1])
     150
     151    return envs
    243152
    244153def clean_up(c):
     
    269178
    270179    cleanlist = glob.glob(c.inputdir + "/*")
    271     for cl in cleanlist:
    272         if c.prefix not in cl:
    273             silent_remove(cl)
     180    for clist in cleanlist:
     181        if c.prefix not in clist:
     182            silent_remove(clist)
    274183        if c.ecapi is False and (c.ectrans == '1' or c.ecstorage == '1'):
    275             silent_remove(cl)
     184            silent_remove(clist)
    276185
    277186    print "Done"
     
    280189
    281190
    282 def my_error(c, message='ERROR'):
     191def my_error(users, message='ERROR'):
    283192    '''
    284193    @Description:
     
    287196
    288197    @Input:
    289         c: instance of class ControlFile
    290             Contains all the parameters of CONTROL file, which are e.g.:
    291             DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
    292             STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
    293             LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
    294             OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
    295             ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
    296             MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
    297             DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
    298 
    299             For more information about format and content of the parameter
    300             see documentation.
     198        user: list of strings
     199            Contains all email addresses which should be notified.
     200            It might also contain just the ecmwf user name which wil trigger
     201            mailing to the associated email address for this user.
    301202
    302203        message: string, optional
     
    310211
    311212    # comment if user does not want email notification directly from python
    312     try:
    313         target = []
    314         target.extend(c.mailfail)
    315     except AttributeError:
    316         target = []
    317         target.extend(os.getenv('USER'))
    318 
    319     for t in target:
    320         p = subprocess.Popen(['mail', '-s ECMWFDATA v7.0 ERROR',
    321                               os.path.expandvars(t)],
    322                              stdin=subprocess.PIPE,
    323                              stdout=subprocess.PIPE,
    324                              stderr=subprocess.PIPE,
    325                              bufsize=1)
    326         tr = '\n'.join(traceback.format_stack())
    327         pout = p.communicate(input=message + '\n\n' + tr)[0]
    328         print 'Email sent to ' + os.path.expandvars(t) + ' ' + pout.decode()
    329 
    330     exit(1)
    331 
    332     return
    333 
    334 
    335 def normal_exit(c, message='Done!'):
     213    for user in users:
     214        if '${USER}' in user:
     215            user = os.getenv('USER')
     216        try:
     217            p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ERROR',
     218                                  os.path.expandvars(user)],
     219                                 stdin=subprocess.PIPE,
     220                                 stdout=subprocess.PIPE,
     221                                 stderr=subprocess.PIPE,
     222                                 bufsize=1)
     223            trace = '\n'.join(traceback.format_stack())
     224            pout = p.communicate(input=message + '\n\n' + trace)[0]
     225        except ValueError as e:
     226            print 'ERROR: ', e
     227            sys.exit('Email could not be sent!')
     228        else:
     229            print 'Email sent to ' + os.path.expandvars(user) + ' ' + \
     230                  pout.decode()
     231
     232    sys.exit(1)
     233
     234    return
     235
     236
     237def normal_exit(users, message='Done!'):
    336238    '''
    337239    @Description:
     
    339241
    340242    @Input:
    341         c: instance of class ControlFile
    342             Contains all the parameters of CONTROL file, which are e.g.:
    343             DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
    344             STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
    345             LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
    346             OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
    347             ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
    348             MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
    349             DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
    350 
    351             For more information about format and content of the parameter
    352             see documentation.
     243        user: list of strings
     244            Contains all email addresses which should be notified.
     245            It might also contain just the ecmwf user name which wil trigger
     246            mailing to the associated email address for this user.
    353247
    354248        message: string, optional
     
    362256
    363257    # comment if user does not want notification directly from python
    364     try:
    365         target = []
    366         target.extend(c.mailops)
    367         for t in target:
    368             p = subprocess.Popen(['mail', '-s ECMWFDATA v7.0 normal exit',
    369                                   os.path.expandvars(t)],
     258    for user in users:
     259        if '${USER}' in user:
     260            user = os.getenv('USER')
     261        try:
     262            p = subprocess.Popen(['mail', '-s flex_extract_v7.1 normal exit',
     263                                  os.path.expandvars(user)],
    370264                                 stdin=subprocess.PIPE,
    371265                                 stdout=subprocess.PIPE,
     
    373267                                 bufsize=1)
    374268            pout = p.communicate(input=message+'\n\n')[0]
    375             print 'Email sent to ' + os.path.expandvars(t) + ' ' + pout.decode()
    376     finally:
    377         pass
     269        except ValueError as e:
     270            print 'ERROR: ', e
     271            print 'Email could not be sent!'
     272        else:
     273            print 'Email sent to ' + os.path.expandvars(user) + ' ' + \
     274                  pout.decode()
    378275
    379276    return
     
    433330    except OSError as e:
    434331        # this would be "except OSError, e:" before Python 2.6
    435         if e.errno is not errno.ENOENT:
     332        if e.errno != errno.ENOENT:
    436333            # errno.ENOENT  =  no such file or directory
    437334            raise  # re-raise exception if a different error occured
     
    521418
    522419    return str_of_list
     420
     421def make_dir(directory):
     422    '''
     423    @Description:
     424        Creates a directory and gives a warning if the directory
     425        already exists. The program stops only if there is another problem.
     426
     427    @Input:
     428        directory: string
     429            The directory path which should be created.
     430
     431    @Return:
     432        <nothing>
     433    '''
     434    try:
     435        os.makedirs(directory)
     436    except OSError as e:
     437        if e.errno != errno.EEXIST:
     438            # errno.EEXIST = directory already exists
     439            raise # re-raise exception if a different error occured
     440        else:
     441            print 'WARNING: Directory {0} already exists!'.format(directory)
     442
     443    return
     444
     445def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
     446    '''
     447    @Description:
     448        Uses the ecaccess command to send a file to the ECMWF servers.
     449        Catches and prints the error if it failed.
     450
     451    @Input:
     452        ecd: string
     453            The path were the file is to be stored.
     454
     455        filename: string
     456            The name of the file to send to the ECMWF server.
     457
     458        target: string
     459            The target where the file should be sent to, e.g. the queue.
     460
     461        ecuid: string
     462            The user id on ECMWF server.
     463
     464        ecgid: string
     465            The group id on ECMWF server.
     466
     467    @Return:
     468        <nothing>
     469    '''
     470
     471    try:
     472        subprocess.check_call(['ecaccess-file-put',
     473                               ecd + '../' + filename,
     474                               target + ':/home/ms/' +
     475                               ecgid + '/' + ecuid +
     476                               '/' + filename])
     477    except subprocess.CalledProcessError as e:
     478        print 'ERROR:'
     479        print e
     480        sys.exit('ecaccess-file-put failed!\n' + \
     481                 'Probably the eccert key has expired.')
     482
     483    return
     484
     485def submit_job_to_ecserver(ecd, target, jobname):
     486    '''
     487    @Description:
     488        Uses ecaccess to submit a job to the ECMWF server.
     489        Catches and prints the error if one arise.
     490
     491    @Input:
     492        ecd: string
     493            The path were the file is to be stored.
     494
     495        target: string
     496            The target where the file should be sent to, e.g. the queue.
     497
     498        jobname: string
     499            The name of the jobfile to be submitted to the ECMWF server.
     500
     501    @Return:
     502        <nothing>
     503    '''
     504
     505    try:
     506        subprocess.check_call(['ecaccess-job-submit',
     507                               '-queueName', target,
     508                               jobname])
     509    except subprocess.CalledProcessError as e:
     510        print '... ERROR CODE: ', e.returncode
     511        sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
     512
     513    return
  • pythontest/TestTools.py

    refdb01a r54a8a01  
    44import unittest
    55import sys
     6import os
    67sys.path.append('../python')
    7 from Tools import init128, toparamId
     8from tools import init128, to_param_id, my_error, read_ecenv
    89
    910
     
    2526
    2627
    27     def test_toparamId(self):
     28    def test_to_param_id(self):
    2829        '''
    2930        '''
    3031        table128 = init128('../grib_templates/ecmwf_grib1_table_128')
    31         pars = toparamId("T/SP/LSP/SSHF", table128)
     32        pars = to_param_id("T/SP/LSP/SSHF", table128)
    3233        for par in pars:
    3334            self.assertIn(par, [130, 134, 142, 146])
     35
     36    def test_error_notifcation(self):
     37        '''
     38        '''
     39        with self.assertRaises(SystemExit) as re:
     40            my_error(['${USER}', 'anne.philipp@univie.ac.at'], 'Failed!')
     41        self.assertEqual(re.exception.code, 1)
     42
     43    def test_read_ecenv(self):
     44
     45        envs_ref = {'ECUID': 'km4a',
     46                    'ECGID': 'at',
     47                    'GATEWAY': 'srvx8.img.univie.ac.at',
     48                    'DESTINATION': 'annep@genericSftp'
     49                   }
     50        envs = read_ecenv(os.getcwd() + '/TestData/ECMWF_ENV')
     51
     52        self.assertDictEqual(envs_ref, envs)
     53
     54
     55
    3456
    3557
  • pythontest/TestUIOFiles.py

    refdb01a r54a8a01  
    66import sys
    77sys.path.append('../python')
    8 import UIOFiles
     8import UioFiles
    99
    1010
    11 class TestUIOFiles(unittest.TestCase):
     11class TestUioFiles(unittest.TestCase):
    1212    '''
    1313    Test class to test the UIOFiles methods.
     
    5252
    5353        # Initialise and collect filenames
    54         files = UIOFiles.UIOFiles(['.grb'])
    55         files.listFiles(self.testpath, '*')
     54        files = UioFiles.UioFiles(['.grb'])
     55        files.list_files(self.testpath, '*')
    5656        # get the basename to just check for equality of filenames
    5757        filelist = [os.path.basename(f) for f in files.files]
Note: See TracChangeset for help on using the changeset viewer.
hosted by ZAMG