Changeset 02c8c50 in flex_extract.git for python


Ignore:
Timestamp:
Mar 18, 2018, 6:27:28 PM (6 years ago)
Author:
Anne Philipp <bscannephilipp@…>
Branches:
master, ctbto, dev
Children:
efdb01a
Parents:
64cf353
Message:

more changes in PEP8 style and slight modifications in coding style and naming. More documentation of functions.

Location:
python
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • python/FlexpartTools.py

    r64cf353 r02c8c50  
    44# TODO AP
    55#AP
    6 # - Functionality Provided is not correct for this file
     6# - Functionality Description Provided is not correct for this file
    77# - localpythonpath should not be set in module load section!
    88# - Change History ist nicht angepasst ans File! Original geben lassen
    99# - def myerror muss angepasst werden da derzeit manuelle modifikation notwendig
    10 # - def --init-- marsretrieval class, remove dataset and expect?!
    11 # - the EIFlexpart class is jsut for EI ? So we should maybe create classes
    12 #   for each possible data set so the boundarys of the attributes can be
    13 #   validated!
     10# - the gaussian keyword in mars retrieval should be removed!!! but kept in
     11#   control file for the reason of calculation of eta dot from gaussian grid
     12# - call of convert in eigene Funktion auslagern
     13
    1414#************************************************************************
    1515"""
    … …  
    9696            RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
    9797            SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
    98             ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR
     98            ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, DEBUG, INPUTDIR,
     99            OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
    99100            For more information about format and content of the parameter see
    100101            documentation.
    … …  
    105106                            formatter_class=ArgumentDefaultsHelpFormatter)
    106107
    107 # the most important arguments
     108    # the most important arguments
    108109    parser.add_argument("--start_date", dest="start_date",
    109110                        help="start date YYYYMMDD")
    … …  
    113114                        help="# of days to be retrieved at once")
    114115
    115 # some arguments that override the default in the control file
     116    # some arguments that override the default in the control file
    116117    parser.add_argument("--basetime", dest="basetime",
    117118                        help="base such as 00/12 (for half day retrievals)")
    … …  
    123124                        help="area defined as north/west/south/east")
    124125
    125 # set the working directories
     126    # set the working directories
    126127    parser.add_argument("--inputdir", dest="inputdir", default=None,
    127128                        help="root directory for storing intermediate files")
    … …  
    133134                        the scripts directory of the FLEXPART distribution")
    134135
    135 # this is only used by prepareFLEXPART.py to rerun a postprocessing step
     136    # this is only used by prepareFLEXPART.py to rerun a postprocessing step
    136137    parser.add_argument("--ppid", dest="ppid",
    137138                        help="Specify parent process id for \
    138139                        rerun of prepareFLEXPART")
    139140
    140 # arguments for job submission to ECMWF, only needed by submit.py
     141    # arguments for job submission to ECMWF, only needed by submit.py
    141142    parser.add_argument("--job_template", dest='job_template',
    142143                        default="job.temp",
    143144                        help="job template file for submission to ECMWF")
    144     #parser.add_argument("--remote", dest="remote",
    145                         #help="target for submission to ECMWF \
    146                         #(ecgate or cca etc.)")
    147145    parser.add_argument("--queue", dest="queue",
    148146                        help="queue for submission to ECMWF \
    … …  
    156154    args = parser.parse_args()
    157155
     156    # create instance of Control for specified controlfile
     157    # and assign the parameters (and default values if necessary)
    158158    try:
    159159        c = Control(args.controlfile)
    … …  
    168168            exit(1)
    169169
     170    # check for having at least a starting date
    170171    if  args.start_date is None and getattr(c, 'start_date') is None:
    171172        print('start_date specified neither in command line nor \
    … …  
    175176        exit(1)
    176177
     178    # save all existing command line parameter to the Control instance
     179    # if parameter is not specified through the command line or CONTROL file
     180    # set default values
    177181    if args.start_date is not None:
    178182        c.start_date = args.start_date
    … …  
    214218        c.upper, c.left, c.lower, c.right = l
    215219
    216 # basetime aktiviert den ''operational mode''
     220    # NOTE: basetime activates the ''operational mode''
    217221    if args.basetime is not None:
    218222        c.basetime = args.basetime
    … …  
    357361    @Input:
    358362        c: instance of class Control
    359             Contains all the parameters of control files, which are:
    360             DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
    361             NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL,
    362             LEVELIST, RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA,
    363             ETADIFF, DPDETA, SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX,
    364             ECSTORAGE, ECTRANS, ECFSDIR, MAILOPS, MAILFAIL,
    365             GRIB2FLEXPART, FLEXPARTDIR
     363            Contains all the parameters of control files, which are e.g.:
     364            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
     365            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
     366            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
     367            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
     368            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
     369            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
     370            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
     371
    366372            For more information about format and content of the parameter
    367373            see documentation.
    … …  
    388394    '''
    389395    @Description:
    390         Print error message.
     396        Prints a specified error message which can be passed to the function
     397        before exiting the program.
    391398
    392399    @Input:
    393400        c: instance of class Control
    394             Contains all the parameters of control files, which are:
    395             DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
    396             NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL,
    397             LEVELIST, RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA,
    398             ETADIFF, DPDETA, SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX,
    399             ECSTORAGE, ECTRANS, ECFSDIR, MAILOPS, MAILFAIL,
    400             GRIB2FLEXPART, FLEXPARTDIR
     401            Contains all the parameters of control files, which are e.g.:
     402            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
     403            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
     404            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
     405            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
     406            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
     407            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
     408            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
     409
    401410            For more information about format and content of the parameter
    402411            see documentation.
    403412
    404         message: string
     413        message: string, optional
    405414            Error message. Default value is "ERROR".
    406415
    … …  
    436445    '''
    437446    @Description:
    438         Print exit message.
     447        Prints a specific exit message which can be passed to the function.
    439448
    440449    @Input:
    441450        c: instance of class Control
    442             Contains all the parameters of control files, which are:
    443             DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
    444             NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL,
    445             LEVELIST, RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA,
    446             ETADIFF, DPDETA, SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX,
    447             ECSTORAGE, ECTRANS, ECFSDIR, MAILOPS, MAILFAIL,
    448             GRIB2FLEXPART, FLEXPARTDIR
     451            Contains all the parameters of control files, which are e.g.:
     452            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
     453            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
     454            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
     455            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
     456            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
     457            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
     458            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
     459
    449460            For more information about format and content of the parameter
    450461            see documentation.
    451462
    452         message: string
     463        message: string, optional
    453464            Message for exiting program. Default value is "Done!".
    454465
    … …  
    481492    '''
    482493    @Description:
     494        This method is taken from an example at the ECMWF wiki website.
     495        https://software.ecmwf.int/wiki/display/GRIB/index.py; 2018-03-16
     496
     497        This method combines the single characters of the passed arguments
     498        with each other. So that each character of each argument value
     499        will be combined with each character of the other arguments as a tuple.
     500
     501        Example:
     502        product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
     503        product(range(2), repeat = 3) --> 000 001 010 011 100 101 110 111
    483504
    484505    @Input:
     506        *args: tuple
     507            Positional arguments (arbitrary number).
     508
     509        **kwds: dictionary
     510            Contains all the keyword arguments from *args.
    485511
    486512    @Return:
    487 
    488     '''
    489     # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
    490     # product(range(2), repeat = 3) --> 000 001 010 011 100 101 110 111
     513        prod: tuple
     514            Return will be done with "yield". A tuple of combined arguments.
     515            See example in description above.
     516    '''
     517
    491518    pools = map(tuple, args) * kwds.get('repeat', 1)
    492519    result = [[]]
    … …  
    527554    '''
    528555    @Description:
    529         Opens and reads the grib table 128 file.
     556        Opens and reads the grib file with table 128 information.
    530557
    531558    @Input:
    … …  
    584611    '''
    585612    @Description:
     613        Interpolation of deaccumulated fluxes of an ECMWF model FG field
     614        using a cubic polynomial solution which conserves the integrals
     615        of the fluxes within each timespan.
     616        Disaggregation is done for a list of 4 values to generate a new
     617        interpolated value which is output at the central point of the 4
     618        accumulation timespans.
    586619
    587620    @Input:
     621        alist: list of size 4, float
     622            List of 4 flux values.
    588623
    589624    @Return:
     625        nvalue: float
     626            New value for the second position of the disaggregated
     627            fluxes field.
    590628
    591629    '''
    … …  
    594632    pyc = alist[1] - alist[0] - 7. * pya / 2. - 2. * pyb
    595633    pyd = alist[0] - pya / 4. - pyb / 3. - pyc / 2.
    596     sfeld = 8. * pya + 4. * pyb + 2. * pyc + pyd
    597 
    598     return sfeld
     634    nvalue = 8. * pya + 4. * pyb + 2. * pyc + pyd
     635
     636    return nvalue
    599637
    600638
    … …  
    602640    '''
    603641    @Description:
     642        Interpolation of deaccumulated precipitation fiels of the ECMWF fields
     643        using a modified linear solution.
    604644        Disaggregate a list of 4 precipitation values to generate a new value
    605         for the second position of the 4 value list. This is used for
    606         precipitation fields.
     645        for the second position of the 4 value list. The interpolated values
     646        are output at the central point of the 4 accumulation timespans
     647        This is used for precipitation fields.
    607648
    608649    @Input:
    … …  
    611652
    612653    @Return:
    613         sfeld: float
     654        nvalue: float
    614655            New value for the second position of the disaggregated
    615656            precipitation field.
    … …  
    630671    mask = xb + xd > 0.
    631672    xbd[mask] = xb[mask] * xc[mask] / (xb[mask] + xd[mask])
    632     sfeld = xac + xbd
    633 
    634     return sfeld
     673    nvalue = xac + xbd
     674
     675    return nvalue
    635676
    636677
    637678class Control:
    638679    '''
    639     Class containing the information of the ECMWFDATA control file
    640 
    641     Contains all the parameters of control files, which are:
    642     DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
    643     NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL,
    644     LEVELIST, RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA,
    645     ETADIFF, DPDETA, SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX,
    646     ECSTORAGE, ECTRANS, ECFSDIR, MAILOPS, MAILFAIL,
    647     GRIB2FLEXPART, FLEXPARTDIR
     680    Class containing the information of the ECMWFDATA control file.
     681
     682    Contains all the parameters of control files, which are e.g.:
     683    DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
     684    STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
     685    LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
     686    OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
     687    ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
     688    MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR,
     689    BASETIME, DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
     690
    648691    For more information about format and content of the parameter
    649692    see documentation.
     693
    650694    '''
    651695
    … …  
    654698        @Description:
    655699            Initialises the instance of Control class and defines and
    656             assign all controlfile variables.
     700            assign all controlfile variables. Set default values if
     701            parameter was not in CONTROL file.
    657702
    658703        @Input:
    … …  
    666711            <nothing>
    667712        '''
     713
     714        # read whole CONTROL file
    668715        with open(filename) as f:
    669716            fdata = f.read().split('\n')
     717
     718        # go through every line and store parameter
     719        # as class variable
    670720        for ldata in fdata:
    671721            data = ldata.split()
    … …  
    714764                pass
    715765
     766        # check a couple of necessary attributes if they contain values
     767        # otherwise set default values
    716768        if not hasattr(self, 'start_date'):
    717769            self.start_date = None
    … …  
    741793
    742794        if not hasattr(self, 'maxstep'):
     795            # find out maximum step
    743796            self.maxstep = 0
    744797            for s in self.step:
    … …  
    756809        if not hasattr(self, 'date_chunk'):
    757810            self.date_chunk = '3'
    758 
    759         self.ecmwfdatadir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))+'/../' # script directory
    760 #AP wieso nicht abfragen? Wieso immer setzen?
    761 #        if not hasattr(self,'exedir'):
    762         self.exedir = self.ecmwfdatadir + 'src/'
    763811        if not hasattr(self, 'grib2flexpart'):
    764812            self.grib2flexpart = '0'
     813
     814        # script directory
     815        self.ecmwfdatadir = os.path.dirname(os.path.abspath(
     816                                            inspect.getfile(
     817                                            inspect.currentframe()))) + '/../'
     818        # Fortran source directory
     819        self.exedir = self.ecmwfdatadir + 'src/'
     820
     821        # FLEXPART directory
    765822        if not hasattr(self, 'flexpart_root_scripts'):
    766823            self.flexpart_root_scripts = self.ecmwfdatadir
    767824
    768825        return
     826
    769827
    770828    def __str__(self):
    … …  
    772830        @Description:
    773831            Prepares a single string with all the comma seperated Control
    774             class attributes including their values. The string has the format
    775 #AP            ????????????????? anschaun
     832            class attributes including their values.
     833
     834            Example:
     835            {'kids': 0, 'name': 'Dog', 'color': 'Spotted',
     836             'age': 10, 'legs': 2, 'smell': 'Alot'}
    776837
    777838        @Input:
    … …  
    782843            string of Control class attributes with their values
    783844        '''
     845
    784846        attrs = vars(self)
    785         # {'kids': 0, 'name': 'Dog', 'color': 'Spotted', 'age': 10, 'legs': 2, 'smell': 'Alot'}
    786         # now dump this in some way or another
     847
    787848        return ', '.join("%s: %s" % item for item in attrs.items())
    788849
    … …  
    829890
    830891class MARSretrieval:
    831     'class for MARS retrievals'
     892    '''
     893    Class for submitting MARS retrievals.
     894
     895    A description of MARS keywords/arguments and examples of their
     896    values can be found here:
     897    https://software.ecmwf.int/wiki/display/UDOC/\
     898                   Identification+keywords#Identificationkeywords-class
     899
     900    '''
    832901
    833902    def __init__(self, server, marsclass = "ei", type = "", levtype = "",
    … …  
    835904                 area = "", time = "", step = "", expver = "1", number = "",
    836905                 accuracy = "", grid = "", gaussian = "", target = "",
    837                  param = "", dataset = "", expect = ""):
     906                 param = ""):
    838907        '''
    839908        @Description:
     909            Initialises the instance of the MARSretrieval class and
     910            defines and assigns a set of the necessary retrieval parameters
     911            for the FLEXPART input data.
     912            A description of MARS keywords/arguments, their dependencies
     913            on each other and examples of their values can be found here:
     914
     915            https://software.ecmwf.int/wiki/display/UDOC/MARS+keywords
    840916
    841917        @Input:
    842             self
    843             server
    844             marsclass = "ei"
    845             type = ""
    846             levtype = ""
    847             levelist = ""
    848             repres = ""
    849             date = ""
    850             resol = ""
    851             stream = ""
    852             area = ""
    853             time = ""
    854             step = ""
    855             expver = "1"
    856             number = ""
    857             accuracy = ""
    858             grid = ""
    859             gaussian = ""
    860             target = ""
    861             param = ""
    862             dataset = ""
    863             expect = ""
     918            self: instance of MARSretrieval
     919                For description see class documentation.
     920
     921            server: instance of ECMWFService (from ECMWF Web-API)
     922                This is the connection to the ECMWF data servers.
     923                It is needed for the pythonic access of ECMWF data.
     924
     925            marsclass: string, optional
     926                Characterisation of dataset. E.g. EI (ERA-Interim),
     927                E4 (ERA40), OD (Operational archive), ea (ERA5).
     928                Default is the ERA-Interim dataset "ei".
     929
     930            type: string, optional
     931                Determines the type of fields to be retrieved.
     932                Selects between observations, images or fields.
     933                Examples for fields: Analysis (an), Forecast (fc),
     934                Perturbed Forecast (pf), Control Forecast (cf) and so on.
     935                Default is an empty string.
     936
     937            levtype: string, optional
     938                Denotes type of level. Has a direct implication on valid
     939                levelist values!
     940                E.g. model level (ml), pressure level (pl), surface (sfc),
     941                potential vorticity (pv), potential temperature (pt)
     942                and depth (dp).
     943                Default is an empty string.
     944
     945            levelist: string, optional
     946                Specifies the required levels. It has to have a valid
     947                correspondence to the selected levtype.
     948                Examples: model level: 1/to/137, pressure levels: 500/to/1000
     949                Default is an empty string.
     950
     951            repres: string, optional
     952                Selects the representation of the archived data.
     953                E.g. sh - spherical harmonics, gg - Gaussian grid,
     954                ll - latitude/longitude, ...
     955                Default is an empty string.
     956
     957            date: string, optional
     958                Specifies the Analysis date, the Forecast base date or
     959                Observations date. Valid formats are:
     960                Absolute as YYYY-MM-DD or YYYYMMDD.
     961                Default is an empty string.
     962
     963            resol: string, optional
     964                Specifies the desired triangular truncation of retrieved data,
     965                before carrying out any other selected post-processing.
     966                The default is automatic truncation (auto), by which the lowest
     967                resolution compatible with the value specified in grid is
     968                automatically selected for the retrieval.
     969                Users wanting to perform post-processing from full spectral
     970                resolution should specify Archived Value (av).
     971                The following are examples of existing resolutions found in
     972                the archive: 63, 106, 159, 213, 255, 319, 399, 511, 799 or 1279.
     973                This keyword has no meaning/effect if the archived data is
     974                not in spherical harmonics representation.
     975                The best selection can be found here:
     976                https://software.ecmwf.int/wiki/display/UDOC/\
     977                      Retrieve#Retrieve-Truncationbeforeinterpolation
     978                Default is an empty string.
     979
     980            stream: string, optional
     981                Identifies the forecasting system used to generate the data.
     982                E.g. oper (Atmospheric model), enfo (Ensemble forecats), ...
     983                Default is an empty string.
     984
     985            area: string, optional
     986                Specifies the desired sub-area of data to be extracted.
     987                Areas can be defined to wrap around the globe.
     988
     989                Latitude values must be given as signed numbers, with:
     990                    north latitudes (i.e. north of the equator)
     991                        being positive (e.g: 40.5)
     992                    south latitutes (i.e. south of the equator)
     993                        being negative (e.g: -50.5)
     994                Longtitude values must be given as signed numbers, with:
     995                    east longitudes (i.e. east of the 0 degree meridian)
     996                        being positive (e.g: 35.0)
     997                    west longitudes (i.e. west of the 0 degree meridian)
     998                        being negative (e.g: -20.5)
     999
     1000                E.g.: North/West/South/East
     1001                Default is an empty string.
     1002
     1003            time: string, optional
     1004                Specifies the time of the data in hours and minutes.
     1005                Valid values depend on the type of data: Analysis time,
     1006                Forecast base time or First guess verification time
     1007                (all usually at synoptic hours: 00, 06, 12 and 18 ).
     1008                Observation time (any combination in hours and minutes is valid,
     1009                subject to data availability in the archive).
     1010                The syntax is HHMM or HH:MM. If MM is omitted it defaults to 00.
     1011                Default is an empty string.
     1012
     1013            step: string, optional
     1014                Specifies the forecast time step from forecast base time.
     1015                Valid values are hours (HH) from forecast base time. It also
     1016                specifies the length of the forecast which verifies at
     1017                First Guess time.
     1018                E.g. 1/3/6-hourly
     1019                Default is an empty string.
     1020
     1021            expver: string, optional
     1022                The version of the dataset. Each experiment is assigned a
     1023                unique code (version). Production data is assigned 1 or 2,
     1024                and experimental data in Operations 11, 12 ,...
     1025                Research or Member State's experiments have a four letter
     1026                experiment identifier.
     1027                Default is "1".
     1028
     1029            number: string, optional
     1030                Selects the member in ensemble forecast run. (Only then it
     1031                is necessary.) It has a different meaning depending on
     1032                the type of data.
     1033                E.g. Perturbed Forecasts: specifies the Ensemble forecast member
     1034                Default is an empty string.
     1035
     1036            accuracy: string, optional
     1037                Specifies the number of bits per value to be used in the
     1038                generated GRIB coded fields.
     1039                A positive integer may be given to specify the preferred number
     1040                of bits per packed value. This must not be greater than the
     1041                number of bits normally used for a Fortran integer on the
     1042                processor handling the request (typically 32 or 64 bit).
     1043                Within a compute request the accuracy of the original fields
     1044                can be passed to the result field by specifying accuracy=av.
     1045                Default is an empty string.
     1046
     1047            grid: string, optional
     1048                Specifies the output grid which can be either a Gaussian grid
     1049                or a Latitude/Longitude grid. MARS requests specifying
     1050                grid=av will return the archived model grid.
     1051
     1052                Lat/Lon grid: The grid spacing needs to be an integer
     1053                fraction of 90 degrees e.g. grid = 0.5/0.5
     1054
     1055                Gaussian grid: specified by a letter denoting the type of
     1056                Gaussian grid followed by an integer (the grid number)
     1057                representing the number of lines between the Pole and Equator,
     1058                e.g.
     1059                grid = F160 - full (or regular) Gaussian grid with
     1060                       160 latitude lines between the pole and equator
     1061                grid = N320 - ECMWF original reduced Gaussian grid with
     1062                       320 latitude lines between the pole and equator,
     1063                       see Reduced Gaussian Grids for grid numbers used at ECMWF
     1064                grid = O640 - ECMWF octahedral (reduced) Gaussian grid with
     1065                       640 latitude lines between the pole and equator
     1066                Default is an empty string.
     1067
     1068            gaussian: string, optional
     1069                This parameter is deprecated and should no longer be used.
     1070                Specifies the desired type of Gaussian grid for the output.
     1071                Valid Gaussian grids are quasi-regular (reduced) or regular.
     1072                Keyword gaussian can only be specified together with
     1073                keyword grid. Gaussian without grid has no effect.
     1074                Default is an empty string.
     1075
     1076            target: string, optional
     1077                Specifies a file into which data is to be written after
     1078                retrieval or manipulation. Path names should always be
     1079                enclosed in double quotes. The MARS client supports automatic
     1080                generation of multiple target files using MARS keywords
     1081                enclosed in square brackets [ ].  If the environment variable
     1082                MARS_MULTITARGET_STRICT_FORMAT is set to 1 before calling mars,
     1083                the keyword values will be used in the filename as shown by
     1084                the ecCodes GRIB tool grib_ls -m, e.g. with
     1085                MARS_MULTITARGET_STRICT_FORMAT set to 1 the keywords time,
     1086                expver and param will be formatted as 0600, 0001 and 129.128
     1087                rather than 600, 1 and 129.
     1088                Default is an empty string.
     1089
     1090            param: string, optional
     1091                Specifies the meteorological parameter.
     1092                The list of meteorological parameters in MARS is extensive.
     1093                Their availability is directly related to their meteorological
     1094                meaning and, therefore, the rest of directives specified
     1095                in the MARS request.
     1096                Meteorological parameters can be specified by their
     1097                GRIB code (param=130), their mnemonic (param=t) or
     1098                full name (param=temperature).
     1099                The list of parameter should be seperated by a "/"-sign.
     1100                E.g. 130/131/133
     1101                Default is an empty string.
    8641102
    8651103        @Return:
    … …  
    8671105        '''
    8681106
    869 #        self.dataset = dataset
     1107        self.server = server
    8701108        self.marsclass = marsclass
    8711109        self.type = type
    … …  
    8801118        self.step = step
    8811119        self.expver = expver
    882         self.target = target
    883         self.param = param
    8841120        self.number = number
    8851121        self.accuracy = accuracy
    8861122        self.grid = grid
    8871123        self.gaussian = gaussian
    888 #    self.expect = expect
    889         self.server = server
     1124        self.target = target
     1125        self.param = param
    8901126
    8911127        return
     1128
    8921129
    8931130    def displayInfo(self):
    8941131        '''
    8951132        @Description:
    896             .
     1133            Prints all class attributes and their values.
    8971134
    8981135        @Input:
    899             self:
     1136            self: instance of MARSretrieval
     1137                For description see class documentation.
    9001138
    9011139        @Return:
    9021140            <nothing>
    9031141        '''
     1142        # Get all class attributes and their values as a dictionary
    9041143        attrs = vars(self)
     1144
     1145        # iterate through all attributes and print them
     1146        # with their corresponding values
    9051147        for item in attrs.items():
    9061148            if item[0] in ('server'):
    … …  
    9141156        '''
    9151157        @Description:
    916             .
     1158            Submits a MARS retrieval. Depending on the existence of
     1159            ECMWF Web-API it is submitted via Python or a
     1160            subprocess in the Shell. The parameter for the mars retrieval
     1161            are taken from the defined class attributes.
    9171162
    9181163        @Input:
    919             self:
     1164            self: instance of MARSretrieval
     1165                For description see class documentation.
    9201166
    9211167        @Return:
    9221168            <nothing>
    9231169        '''
     1170        # Get all class attributes and their values as a dictionary
    9241171        attrs = vars(self)
    925     #   self.server.retrieve(dicolist)
     1172
     1173        # convert the dictionary of attributes into a comma
     1174        # seperated list of attributes with their values
     1175        # needed for the retrieval call
    9261176        s = 'ret'
    9271177        for k, v in attrs.iteritems():
    … …  
    9371187                s = s + ',' + k + '=' + str(v)
    9381188
    939         if self.server !=  False:
     1189        # MARS request via Python script
     1190        if self.server is not False:
    9401191            try:
    941                 self.server.execute(s,target)
     1192                self.server.execute(s, target)
    9421193            except:
    943                 print('MARS Request failed, have you already registered at apps.ecmwf.int?')
     1194                print('MARS Request failed, \
     1195                    have you already registered at apps.ecmwf.int?')
    9441196                raise IOError
    9451197            if os.stat(target).st_size == 0:
    9461198                print('MARS Request returned no data - please check request')
    9471199                raise IOError
     1200        # MARS request via extra process in shell
    9481201        else:
    9491202            s += ',target = "' + target + '"'
    9501203            p = subprocess.Popen(['mars'], stdin=subprocess.PIPE,
    9511204                                 stdout=subprocess.PIPE,
    952                                  stderr=subprocess.PIPE, bufsize = 1)
     1205                                 stderr=subprocess.PIPE, bufsize=1)
    9531206            pout = p.communicate(input=s)[0]
    9541207            print(pout.decode())
    … …  
    9641217
    9651218
    966 ##############################################################
    967 ##############################################################
    968 class EIFlexpart:
    969     '''
    970     Class to retrieve Era Interim data for running FLEXPART
    971     '''
    972 #AP change class name to ECFlexpart
    973     def __init__(self, c, fluxes=False):
     1219
     1220
     1221class ECFlexpart:
     1222    '''
     1223    Class to retrieve ECMWF data specific for running FLEXPART.
     1224    '''
     1225    def __init__(self, c, fluxes=False): #done/ verstehen
    9741226        '''
    9751227        @Description:
    976             Creates an object/instance of EIFlexpart with the
     1228            Creates an object/instance of ECFlexpart with the
    9771229            associated settings of its attributes for the retrieval.
    9781230
    9791231        @Input:
    980             self: instance of EIFlexpart
     1232            self: instance of ECFlexpart
    9811233                The current object of the class.
    9821234
    9831235            c: instance of class Control
    984                 Contains all the parameters of control files, which are:
    985                 DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
    986                 NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL,
    987                 LEVELIST, RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA,
    988                 ETADIFF, DPDETA, SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX,
    989                 ECSTORAGE, ECTRANS, ECFSDIR, MAILOPS, MAILFAIL,
    990                 GRIB2FLEXPART, FLEXPARTDIR
     1236                Contains all the parameters of control files, which are e.g.:
     1237                DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
     1238                STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
     1239                LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
     1240                OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
     1241                ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
     1242                MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
     1243                DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
     1244
    9911245                For more information about format and content of the parameter
    9921246                see documentation.
    … …  
    10001254            <nothing>
    10011255        '''
     1256
    10021257        # different mars types for retrieving reanalysis data for flexpart
    1003 
    10041258        self.types = dict()
    10051259        try:
    … …  
    11161370
    11171371
    1118         # Now comes the nasty part that deals with the different scenarios we have:
     1372        # Now comes the nasty part that deals with the different
     1373        # scenarios we have:
    11191374        # 1) Calculation of etadot on
    11201375        #    a) Gaussian grid
    … …  
    11741429                    if '/D' not in self.params['OG__ML'][0]:
    11751430                        self.params['OG__ML'][0] += '/D'
    1176         #            wrf_sfc = 'sp/msl/skt/2t/10u/10v/2d/z/lsm/sst/ci/sd/stl1/ /
    1177         #                       stl2/stl3/stl4/swvl1/swvl2/swvl3/swvl4'.upper()
     1431                    #wrf_sfc = 'sp/msl/skt/2t/10u/10v/2d/z/lsm/sst/ci/sd/stl1/ /
     1432                    #           stl2/stl3/stl4/swvl1/swvl2/swvl3/swvl4'.upper()
    11781433                    wrf_sfc = '134/235/167/165/166/168/129/172/34/31/141/ \
    11791434                               139/170/183/236/39/40/41/42'.upper()
    … …  
    11931448
    11941449
    1195     def create_namelist(self, c, filename):
     1450    def write_namelist(self, c, filename): #done
    11961451        '''
    11971452        @Description:
    1198             Creates a namelist file in the temporary directory.
    1199             It will contain values for maxl, maxb, mlevel,
     1453            Creates a namelist file in the temporary directory and writes
     1454            the following values to it: maxl, maxb, mlevel,
    12001455            mlevelist, mnauf, metapar, rlo0, rlo1, rla0, rla1,
    12011456            momega, momegadiff, mgauss, msmooth, meta, metadiff, mdpdeta
    12021457
    12031458        @Input:
    1204             self: instance of EIFlexpart
     1459            self: instance of ECFlexpart
    12051460                The current object of the class.
    12061461
    12071462            c: instance of class Control
    1208                 Contains all the parameters of control files, which are:
    1209                 DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
    1210                 NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL,
    1211                 LEVELIST, RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA,
    1212                 ETADIFF, DPDETA, SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX,
    1213                 ECSTORAGE, ECTRANS, ECFSDIR, MAILOPS, MAILFAIL,
    1214                 GRIB2FLEXPART, FLEXPARTDIR
     1463                Contains all the parameters of control files, which are e.g.:
     1464                DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
     1465                STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
     1466                LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
     1467                OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
     1468                ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
     1469                MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
     1470                DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
     1471
    12151472                For more information about format and content of the parameter
    12161473                see documentation.
    … …  
    12221479            <nothing>
    12231480        '''
     1481
    12241482        self.inputdir = c.inputdir
    12251483        area = asarray(self.area.split('/')).astype(float)
    … …  
    12321490        maxb = int((area[0] - area[2]) / grid[0]) + 1
    12331491
    1234         f = open(self.inputdir + '/' + filename, 'w')
    1235         f.write('&NAMGEN\n')
    1236         f.write(',\n  '.join(['maxl = ' + str(maxl), 'maxb = ' + str(maxb),
    1237                 'mlevel = ' + self.level,
    1238                 'mlevelist = ' + '"' + self.levelist + '"',
    1239                 'mnauf = ' + self.resol, 'metapar = ' + '77',
    1240                 'rlo0 = ' + str(area[1]), 'rlo1 = ' + str(area[3]),
    1241                 'rla0 = ' + str(area[2]), 'rla1 = ' + str(area[0]),
    1242                 'momega = ' + c.omega, 'momegadiff = ' + c.omegadiff,
    1243                 'mgauss = ' + c.gauss, 'msmooth = ' + c.smooth,
    1244                 'meta = ' + c.eta, 'metadiff = ' + c.etadiff,
    1245                 'mdpdeta = ' + c.dpdeta]))
    1246 
    1247         f.write('\n/\n')
    1248         f.close()
     1492        with open(self.inputdir + '/' + filename, 'w') as f:
     1493            f.write('&NAMGEN\n')
     1494            f.write(',\n  '.join(['maxl = ' + str(maxl), 'maxb = ' + str(maxb),
     1495                    'mlevel = ' + self.level,
     1496                    'mlevelist = ' + '"' + self.levelist + '"',
     1497                    'mnauf = ' + self.resol, 'metapar = ' + '77',
     1498                    'rlo0 = ' + str(area[1]), 'rlo1 = ' + str(area[3]),
     1499                    'rla0 = ' + str(area[2]), 'rla1 = ' + str(area[0]),
     1500                    'momega = ' + c.omega, 'momegadiff = ' + c.omegadiff,
     1501                    'mgauss = ' + c.gauss, 'msmooth = ' + c.smooth,
     1502                    'meta = ' + c.eta, 'metadiff = ' + c.etadiff,
     1503                    'mdpdeta = ' + c.dpdeta]))
     1504
     1505            f.write('\n/\n')
     1506
    12491507        return
    12501508
    … …  
    12551513
    12561514        @Input:
    1257             self: instance of EIFlexpart
     1515            self: instance of ECFlexpart
    12581516
    12591517            server: instance of ECMWFService
    … …  
    12631521            times:
    12641522
    1265             inputdir: string
     1523            inputdir: string, optional
    12661524                Default string is empty ('').
    12671525
    … …  
    13011559                mfdate = self.dates
    13021560                mfstream = self.stream
    1303                 mftarget = self.inputdir+"/"+ftype+pk+'.'+self.dates.split('/')[0]+'.'+str(os.getppid())+'.'+str(os.getpid())+".grb"
     1561                mftarget = self.inputdir + "/" + ftype + pk + '.' + \
     1562                           self.dates.split('/')[0] + '.' + str(os.getppid()) +\
     1563                           '.' + str(os.getpid()) + ".grb"
    13041564                if pk == 'OG__SL':
    13051565                    pass
    … …  
    13261586
    13271587                if self.basetime is None:
    1328                     MR =  MARSretrieval(self.server, dataset = self.dataset, marsclass = self.marsclass, stream = mfstream,
    1329                               type = mftype, levtype = pv[1], levelist = pv[2],resol = self.resol, gaussian = gaussian,
    1330                               accuracy = self.accuracy,grid = pv[3],target = mftarget,area = area,
    1331                               date = mfdate, time = mftime,number = self.number,step = mfstep, expver = self.expver, param = pv[0])
     1588                    MR =  MARSretrieval(self.server,
     1589                            marsclass=self.marsclass, stream=mfstream,
     1590                            type=mftype, levtype=pv[1], levelist=pv[2],
     1591                            resol=self.resol, gaussian=gaussian,
     1592                            accuracy=self.accuracy, grid=pv[3],
     1593                            target=mftarget, area=area, date=mfdate,
     1594                            time=mftime, number=self.number, step=mfstep,
     1595                            expver=self.expver, param=pv[0])
    13321596
    13331597                    MR.displayInfo()
    … …  
    13451609                    if 'by' in mftime:
    13461610                        tm1 = 2
    1347                     maxtime = datetime.datetime.strptime(mfdate.split('/')[-1]+mftime.split('/')[tm1],'%Y%m%d%H')+ \
    1348                     datetime.timedelta(hours = int(mfstep.split('/')[sm1]))
    1349 
    1350                     elimit = datetime.datetime.strptime(mfdate.split('/')[-1]+self.basetime,'%Y%m%d%H')
     1611                    maxtime = datetime.datetime.strptime(
     1612                                mfdate.split('/')[-1] + mftime.split('/')[tm1],
     1613                                '%Y%m%d%H') + datetime.timedelta(
     1614                                hours=int(mfstep.split('/')[sm1]))
     1615
     1616                    elimit = datetime.datetime.strptime(
     1617                                mfdate.split('/')[-1] +
     1618                                self.basetime, '%Y%m%d%H')
    13511619
    13521620                    if self.basetime == '12':
    … …  
    13571625                # if maxtime-elimit<12h reduce step for last time
    13581626                # A split of the MARS job into 2 is likely necessary.
    1359                             maxtime = elimit-datetime.timedelta(hours = 24)
    1360                             mfdate = '/'.join(('/'.join(mfdate.split('/')[:-1]),datetime.datetime.strftime(maxtime,'%Y%m%d')))
    1361 
    1362                             MR =  MARSretrieval(self.server, dataset = self.dataset, marsclass = self.marsclass, stream = self.stream,
    1363                                                 type = mftype, levtype = pv[1], levelist = pv[2],resol = self.resol, gaussian = gaussian,
    1364                                                 accuracy = self.accuracy,grid = pv[3],target = mftarget,area = area,
    1365                                                 date = mfdate, time = mftime,number = self.number,step = mfstep, expver = self.expver, param = pv[0])
     1627                            maxtime = elimit-datetime.timedelta(hours=24)
     1628                            mfdate = '/'.join(('/'.join(mfdate.split('/')[:-1]),
     1629                                                datetime.datetime.strftime(
     1630                                                maxtime, '%Y%m%d')))
     1631
     1632                            MR = MARSretrieval(self.server,
     1633                                            marsclass=self.marsclass,
     1634                                            stream=self.stream, type=mftype,
     1635                                            levtype=pv[1], levelist=pv[2],
     1636                                            resol=self.resol, gaussian=gaussian,
     1637                                            accuracy=self.accuracy, grid=pv[3],
     1638                                            target=mftarget, area=area,
     1639                                            date=mfdate, time=mftime,
     1640                                            number=self.number, step=mfstep,
     1641                                            expver=self.expver, param=pv[0])
    13661642
    13671643                            MR.displayInfo()
    13681644                            MR.dataRetrieve()
    13691645
    1370                             maxtime = elimit-datetime.timedelta(hours = 12)
    1371                             mfdate = datetime.datetime.strftime(maxtime,'%Y%m%d')
     1646                            maxtime = elimit - datetime.timedelta(hours=12)
     1647                            mfdate = datetime.datetime.strftime(maxtime,
     1648                                                                '%Y%m%d')
    13721649                            mftime = '00'
    1373                             mftarget = self.inputdir+"/"+ftype+pk+'.'+mfdate+'.'+str(os.getppid())+'.'+str(os.getpid())+".grb"
    1374 
    1375                             MR =  MARSretrieval(self.server, dataset = self.dataset, marsclass = self.marsclass, stream = self.stream,
    1376                                                 type = mftype, levtype = pv[1], levelist = pv[2],resol = self.resol, gaussian = gaussian,
    1377                                                 accuracy = self.accuracy,grid = pv[3],target = mftarget,area = area,
    1378                                                 date = mfdate, time = mftime,number = self.number,step = mfstep, expver = self.expver, param = pv[0])
     1650                            mftarget = self.inputdir + "/" + ftype + pk + \
     1651                                       '.' + mfdate + '.' + str(os.getppid()) +\
     1652                                       '.' + str(os.getpid()) + ".grb"
     1653
     1654                            MR = MARSretrieval(self.server,
     1655                                            marsclass=self.marsclass,
     1656                                            stream=self.stream, type=mftype,
     1657                                            levtype=pv[1], levelist=pv[2],
     1658                                            resol=self.resol, gaussian=gaussian,
     1659                                            accuracy=self.accuracy, grid=pv[3],
     1660                                            target=mftarget, area=area,
     1661                                            date=mfdate, time=mftime,
     1662                                            number=self.number, step=mfstep,
     1663                                            expver=self.expver, param=pv[0])
    13791664
    13801665                            MR.displayInfo()
    13811666                            MR.dataRetrieve()
    13821667                        else:
    1383                             MR =  MARSretrieval(self.server, dataset = self.dataset, marsclass = self.marsclass, stream = self.stream,
    1384                                         type = mftype, levtype = pv[1], levelist = pv[2],resol = self.resol, gaussian = gaussian,
    1385                                         accuracy = self.accuracy,grid = pv[3],target = mftarget,area = area,
    1386                                         date = mfdate, time = mftime,number = self.number,step = mfstep, expver = self.expver, param = pv[0])
     1668                            MR = MARSretrieval(self.server,
     1669                                            marsclass=self.marsclass,
     1670                                            stream=self.stream, type=mftype,
     1671                                            levtype=pv[1], levelist=pv[2],
     1672                                            resol=self.resol, gaussian=gaussian,
     1673                                            accuracy=self.accuracy, grid=pv[3],
     1674                                            target=mftarget, area=area,
     1675                                            date=mfdate, time=mftime,
     1676                                            number=self.number, step=mfstep,
     1677                                            expver=self.expver, param=pv[0])
    13871678
    13881679                            MR.displayInfo()
    13891680                            MR.dataRetrieve()
    13901681                    else:
    1391                         maxtime = elimit-datetime.timedelta(hours = 24)
     1682                        maxtime = elimit - datetime.timedelta(hours=24)
    13921683                        mfdate = datetime.datetime.strftime(maxtime,'%Y%m%d')
    13931684
    … …  
    13961687                        if '/' in mftime:
    13971688                            times = mftime.split('/')
    1398                             while int(times[0])+int(mfstep.split('/')[0])< = 12 and pk! = 'OG_OROLSM__SL' and 'acc' not in pk:
     1689                            while ((int(times[0]) +
     1690                                   int(mfstep.split('/')[0]) <= 12) and
     1691                                  (pk != 'OG_OROLSM__SL') and 'acc' not in pk):
    13991692                                times = times[1:]
    1400                             if len(times)>1:
     1693                            if len(times) > 1:
    14011694                                mftime = '/'.join(times)
    14021695                            else:
    14031696                                mftime = times[0]
    14041697
    1405                         MR =  MARSretrieval(self.server, dataset = self.dataset, marsclass = self.marsclass, stream = self.stream,
    1406                                   type = mftype, levtype = pv[1], levelist = pv[2],resol = self.resol, gaussian = gaussian,
    1407                                   accuracy = self.accuracy,grid = pv[3],target = mftarget,area = area,
    1408                                   date = mfdate, time = mftime,number = self.number,step = mfstep, expver = self.expver, param = pv[0])
     1698                        MR = MARSretrieval(self.server,
     1699                                        marsclass=self.marsclass,
     1700                                        stream=self.stream, type=mftype,
     1701                                        levtype=pv[1], levelist=pv[2],
     1702                                        resol=self.resol, gaussian=gaussian,
     1703                                        accuracy=self.accuracy, grid=pv[3],
     1704                                        target=mftarget, area=area,
     1705                                        date=mfdate, time=mftime,
     1706                                        number=self.number, step=mfstep,
     1707                                        expver=self.expver, param=pv[0])
    14091708
    14101709                        MR.displayInfo()
    14111710                        MR.dataRetrieve()
    14121711
    1413                         if int(mftimesave.split('/')[0]) == 0 and int(mfstep.split('/')[0]) == 0 and pk! = 'OG_OROLSM__SL':
     1712                        if (int(mftimesave.split('/')[0]) == 0 and
     1713                            int(mfstep.split('/')[0]) == 0 and
     1714                            pk != 'OG_OROLSM__SL'):
    14141715                            mfdate = datetime.datetime.strftime(elimit,'%Y%m%d')
    14151716                            mftime = '00'
    14161717                            mfstep = '000'
    1417                             mftarget = self.inputdir+"/"+ftype+pk+'.'+mfdate+'.'+str(os.getppid())+'.'+str(os.getpid())+".grb"
    1418 
    1419                             MR =  MARSretrieval(self.server, dataset = self.dataset, marsclass = self.marsclass, stream = self.stream,
    1420                                           type = mftype, levtype = pv[1], levelist = pv[2],resol = self.resol, gaussian = gaussian,
    1421                                           accuracy = self.accuracy,grid = pv[3],target = mftarget,area = area,
    1422                                           date = mfdate, time = mftime,number = self.number,step = mfstep, expver = self.expver, param = pv[0])
     1718                            mftarget = self.inputdir + "/" + ftype + pk + \
     1719                                       '.' + mfdate + '.' + str(os.getppid()) +\
     1720                                       '.' + str(os.getpid()) + ".grb"
     1721
     1722                            MR = MARSretrieval(self.server,
     1723                                        marsclass=self.marsclass,
     1724                                        stream=self.stream, type=mftype,
     1725                                        levtype=pv[1], levelist=pv[2],
     1726                                        resol=self.resol, gaussian=gaussian,
     1727                                        accuracy=self.accuracy, grid=pv[3],
     1728                                        target=mftarget, area=area,
     1729                                        date=mfdate, time=mftime,
     1730                                        number=self.number, step=mfstep,
     1731                                        expver=self.expver, param=pv[0])
    14231732
    14241733                            MR.displayInfo()
    … …  
    14301739
    14311740
    1432     def getFlexpartTime(self, type, step, time):
    1433 #AP remove this function, no longer needed
     1741    def process_output(self, c): #done
    14341742        '''
    14351743        @Description:
    1436             ????
    1437 #AP wozu? wird das überhaupt noch gebraucht? in create auskommentiert
     1744            The grib files are postprocessed depending on selection in
     1745            control file. The following modifications might be done if
     1746            properly switched in control file:
     1747            GRIB2 - Conversion to GRIB2
     1748            ECTRANS - Transfer of files to gateway server
     1749            ECSTORAGE - Storage at ECMWF server
     1750            GRIB2FLEXPART - Conversion of GRIB files to FLEXPART binary format
    14381751
    14391752        @Input:
    1440             self: instance of EIFlexpart
     1753            self: instance of ECFlexpart
    14411754                The current object of the class.
    14421755
    1443             type: string
    1444                 Type of the data field. E.g. FC - forecast or
    1445                 AN - analysis
    1446 
    1447             step: integer
    1448                 Forecast step in hours.
    1449 
    1450             time: integer
    1451                 Time in hours.
    1452 
    1453         @Return:
    1454             cflextime:
    1455 
    1456         '''
    1457         cstep = '{:0>3}'.format(step)
    1458         ctime = '{:0>2}'.format(int(time / 100))
    1459         ctype = str(type).upper()
    1460 
    1461         myinfo = [ctype, ctime, cstep]
    1462         cflextime = None
    1463         for t, marsinfo in self.mars.items():
    1464             if myinfo == marsinfo:
    1465                 cflextime = t
    1466 
    1467         return cflextime
    1468 
    1469     def process_output(self, c):
    1470         '''
    1471         @Description:
    1472 
    1473 
    1474         @Input:
    1475             self: instance of EIFlexpart
    1476                 The current object of the class.
    1477 
    14781756            c: instance of class Control
    1479                 Contains all the parameters of control files, which are:
    1480                 DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
    1481                 NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL,
    1482                 LEVELIST, RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA,
    1483                 ETADIFF, DPDETA, SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX,
    1484                 ECSTORAGE, ECTRANS, ECFSDIR, MAILOPS, MAILFAIL,
    1485                 GRIB2FLEXPART, FLEXPARTDIR
     1757                Contains all the parameters of control files, which are e.g.:
     1758                DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
     1759                STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
     1760                LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
     1761                OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
     1762                ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
     1763                MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
     1764                DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
     1765
    14861766                For more information about format and content of the parameter
    14871767                see documentation.
    … …  
    14921772        '''
    14931773
    1494         print 'Postprocessing:\n Format: {}\n'.format(c.format)
     1774        print('Postprocessing:\n Format: {}\n'.format(c.format))
     1775
    14951776        if c.ecapi is False:
    1496             print 'ecstorage: {}\n ecfsdir: {}\n'.format(c.ecstorage, c.ecfsdir)
     1777            print('ecstorage: {}\n ecfsdir: {}\n'.
     1778                  format(c.ecstorage, c.ecfsdir))
    14971779            if not hasattr(c, 'gateway'):
    14981780                c.gateway = os.getenv('GATEWAY')
    14991781            if not hasattr(c, 'destination'):
    15001782                c.destination = os.getenv('DESTINATION')
    1501             print 'ectrans: {}\n gateway: {}\n destination: {}\n '}
    1502                     .format(c.ectrans, c.gateway, c.destination)
    1503         print 'Output filelist: \n', self.outputfilelist
     1783            print('ectrans: {}\n gateway: {}\n destination: {}\n '
     1784                    .format(c.ectrans, c.gateway, c.destination))
     1785
     1786        print('Output filelist: \n', self.outputfilelist)
    15041787
    15051788        if c.format.lower() == 'grib2':
    15061789            for ofile in self.outputfilelist:
    1507                 p = subprocess.check_call(['grib_set', '-s', 'edition = 2, \
    1508                                         productDefinitionTemplateNumber = 8',
    1509                                         ofile, ofile + '_2'])
     1790                p = subprocess.check_call(['grib_set', '-s', 'edition=2, \
     1791                                            productDefinitionTemplateNumber=8',
     1792                                            ofile, ofile + '_2'])
    15101793                p = subprocess.check_call(['mv', ofile + '_2', ofile])
    15111794
    … …  
    15151798                                           c.gateway, '-remote', c.destination,
    15161799                                           '-source', ofile])
    1517                 print 'ectrans:',p
     1800                print('ectrans:', p)
     1801
    15181802        if int(c.ecstorage) == 1 and c.ecapi is False:
    15191803            for ofile in self.outputfilelist:
    … …  
    15211805                                           os.path.expandvars(c.ecfsdir)])
    15221806
    1523 #    20131107 000000      EN13110700              ON DISC
    15241807        if c.outputdir != c.inputdir:
    15251808            for ofile in self.outputfilelist:
    15261809                p = subprocess.check_call(['mv', ofile, c.outputdir])
    15271810
     1811        # prepare environment for the grib2flexpart run
     1812        # to convert grib to flexpart binary
    15281813        if c.grib2flexpart == '1':
    1529             f = open(c.outputdir + '/' + 'AVAILABLE', 'w')
     1814
     1815            # generate AVAILABLE file
     1816            # Example of AVAILABLE file data
     1817            # 20131107 000000      EN13110700              ON DISC
    15301818            clist = []
    1531             for ofile in self.outputfilelist:  # generate AVAILABLE file
     1819            for ofile in self.outputfilelist:
    15321820                fname = ofile.split('/')
    15331821                if '.' in fname[-1]:
    … …  
    15441832                             fname[-1] + ' '*14 + 'ON DISC')
    15451833            clist.sort()
    1546             f.write('\n'.join(clist) + '\n')
    1547             f.close()
    1548 
     1834            with open(c.outputdir + '/' + 'AVAILABLE', 'w') as f:
     1835                f.write('\n'.join(clist) + '\n')
     1836
     1837            # generate pathnames file
    15491838            pwd = os.path.abspath(c.outputdir)
    1550             f = open(pwd + '/pathnames','w')
    1551             f.write(pwd + '/Options/\n')
    1552             f.write(pwd + '/\n')
    1553             f.write(pwd + '/\n')
    1554             f.write(pwd + '/AVAILABLE\n')
    1555             f.write(' = == = == = == = == = == ==  = \n')
    1556             f.close()
    1557 
     1839            with open(pwd + '/pathnames','w') as f:
     1840                f.write(pwd + '/Options/\n')
     1841                f.write(pwd + '/\n')
     1842                f.write(pwd + '/\n')
     1843                f.write(pwd + '/AVAILABLE\n')
     1844                f.write(' = == = == = == = == = == ==  = \n')
     1845
     1846            # create Options dir if necessary
    15581847            if not os.path.exists(pwd + '/Options'):
    15591848                os.makedirs(pwd+'/Options')
    1560             f = open(os.path.expandvars(
     1849
     1850            # read template COMMAND file
     1851            with open(os.path.expandvars(
    15611852                     os.path.expanduser(c.flexpart_root_scripts)) +
    1562                      '/../Options/COMMAND', 'r')
    1563             lflist = f.read().split('\n')
     1853                     '/../Options/COMMAND', 'r') as f:
     1854                lflist = f.read().split('\n')
     1855
     1856            # find index of list where to put in the
     1857            # date and time information
     1858            # usually after the LDIRECT parameter
    15641859            i = 0
    15651860            for l in lflist:
    … …  
    15681863                i += 1
    15691864
    1570             clist.sort()
     1865#            clist.sort()
     1866            # insert the date and time information of run star and end
     1867            # into the list of lines of COMMAND file
    15711868            lflist = lflist[:i+1] + \
    15721869                     [clist[0][:16], clist[-1][:16]] + \
    15731870                     lflist[i+3:]
    15741871
     1872            # write the new COMMAND file
    15751873            with open(pwd + '/Options/COMMAND', 'w') as g:
    15761874                g.write('\n'.join(lflist) + '\n')
    15771875
     1876            # change to outputdir and start the
     1877            # grib2flexpart run
     1878            # afterwards switch back to the working dir
    15781879            os.chdir(c.outputdir)
    15791880            p = subprocess.check_call([os.path.expandvars(
    … …  
    15851886        return
    15861887
    1587     def create(self, inputfiles, c):
     1888    def create(self, inputfiles, c): #done
    15881889        '''
    15891890        @Description:
    1590 #AP WOZU und ist einrueckung richtig
     1891            This method is based on the ECMWF example index.py
     1892            https://software.ecmwf.int/wiki/display/GRIB/index.py
     1893
     1894            An index file will be created which depends on the combination
     1895            of "date", "time" and "stepRange" values. This is used to iterate
     1896            over all messages in the grib files passed through the parameter
     1897            "inputfiles" to seperate specific parameters into fort.* files.
     1898            Afterwards the FORTRAN program Convert2 is called to convert
     1899            the data fields all to the same grid and put them in one file
     1900            per day.
    15911901
    15921902        @Input:
    1593             self: instance of EIFlexpart
     1903            self: instance of ECFlexpart
    15941904                The current object of the class.
    15951905
    1596             inputfiles: list of strings
    1597                 A list of filenames.
     1906            inputfiles: instance of UIOFiles
     1907                Contains a list of files.
    15981908
    15991909            c: instance of class Control
    1600                 Contains all the parameters of control files, which are:
    1601                 DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
    1602                 NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL,
    1603                 LEVELIST, RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA,
    1604                 ETADIFF, DPDETA, SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX,
    1605                 ECSTORAGE, ECTRANS, ECFSDIR, MAILOPS, MAILFAIL,
    1606                 GRIB2FLEXPART, FLEXPARTDIR
     1910                Contains all the parameters of control files, which are e.g.:
     1911                DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
     1912                STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
     1913                LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
     1914                OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
     1915                ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
     1916                MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
     1917                DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
     1918
    16071919                For more information about format and content of the parameter
    16081920                see documentation.
    … …  
    16141926        table128 = init128(c.ecmwfdatadir +
    16151927                           '/grib_templates/ecmwf_grib1_table_128')
    1616 #AP wieso wrf
    1617         wrfpars = toparamId('sp/mslp/skt/2t/10u/10v/2d/z/lsm/sst/ci/sd/ \
     1928        wrfpars = toparamId('sp/mslp/skt/2t/10u/10v/2d/z/lsm/sst/ci/sd/\
    16181929                            stl1/stl2/stl3/stl4/swvl1/swvl2/swvl3/swvl4',
    16191930                            table128)
     1931
    16201932        index_keys = ["date", "time", "step"]
    16211933        indexfile = c.inputdir + "/date_time_stepRange.idx"
    16221934        silentremove(indexfile)
    16231935        grib = GribTools(inputfiles.files)
     1936        # creates new index file
    16241937        iid = grib.index(index_keys=index_keys, index_file=indexfile)
    1625         print 'index done...'
    1626 
    1627         fdict = {'10':None, '11':None, '12':None, '13':None, '16':None,
    1628                  '17':None, '19':None, '21':None, '22':None, '20':None}
    1629         for f in fdict.keys():
    1630             silentremove(c.inputdir + "/fort." + f)
    1631 
     1938
     1939        # read values of index keys
    16321940        index_vals = []
    16331941        for key in index_keys:
    1634             key_vals = grib_index_get(iid, key)
    1635             print key_vals
    1636             index_vals.append(key_vals)
    1637 
    1638         # creates index file
     1942            index_vals.append(grib_index_get(iid, key))
     1943            print(index_vals[-1])
     1944            # index_vals looks for example like:
     1945            # index_vals[0]: ('20171106', '20171107', '20171108') ; date
     1946            # index_vals[1]: ('0', '1200', '1800', '600') ; time
     1947            # index_vals[2]: ('0', '12', '3', '6', '9') ; stepRange
     1948
     1949
    16391950        for prod in product(*index_vals):
     1951            # e.g. prod = ('20170505', '0', '12')
     1952            #             (  date    ,time, step)
     1953            # per date e.g. time = 0, 600, 1200, 1800
     1954            # per time e.g. step = 0, 3, 6, 9, 12
    16401955            for i in range(len(index_keys)):
    1641                 grib_index_select(iid, index_keys[i],  prod[i])
     1956                grib_index_select(iid, index_keys[i], prod[i])
    16421957
    16431958            gid = grib_new_from_index(iid)
    … …  
    16451960            # therefore save in hid
    16461961            hid = gid
    1647             cflextime = None
    1648             for k, f in fdict.iteritems():
    1649                 fdict[k] = open(c.inputdir + '/fort.' + k, 'w')
    16501962            if gid is not None:
    16511963                cdate = str(grib_get(gid, 'date'))
    … …  
    16531965                type = grib_get(gid, 'type')
    16541966                step = grib_get(gid, 'step')
    1655         #       cflextime  =  self.getFlexpartTime(type,step, time)
     1967                # create correct timestamp from the three time informations
     1968                # date, time, step
    16561969                timestamp = datetime.datetime.strptime(
    16571970                                cdate + '{:0>2}'.format(time/100), '%Y%m%d%H')
    16581971                timestamp += datetime.timedelta(hours=int(step))
    1659         #       print gid,index_keys[i],prod[i],cdate,time,step,timestamp
    16601972
    16611973                cdateH = datetime.datetime.strftime(timestamp, '%Y%m%d%H')
    … …  
    16701982                        slimit = datetime.datetime.strptime(
    16711983                                    c.end_date + bt, '%Y%m%d%H') - \
    1672                                  datetime.timedelta(hours=12-int(c.dtime))
     1984                                    datetime.timedelta(hours=12-int(c.dtime))
    16731985                    if c.basetime == '12':
    16741986                        bt = '12'
    … …  
    16771989                                 datetime.timedelta(hours=12-int(c.dtime))
    16781990
    1679                     elimit = datetime.datetime.strptime(c.end_date + bt, '%Y%m%d%H')
     1991                    elimit = datetime.datetime.strptime(
     1992                                c.end_date + bt, '%Y%m%d%H')
    16801993
    16811994                    if timestamp < slimit or timestamp > elimit:
    … …  
    16912004                        if cdate != olddate:
    16922005                            fwrf = open(c.outputdir + '/WRF' + cdate +
    1693                                         '.{:0>2}'.format(time) + '.000.grb2', 'w')
     2006                                        '.{:0>2}'.format(time) + '.000.grb2',
     2007                                        'w')
    16942008                            olddate = cdate[:]
    16952009            except AttributeError:
    16962010                pass
    16972011
    1698 #           print 'cyear '+cyear+'/'+cmonth+'/'+'/EI'+cyear[2:4]+cmonth+cday+cflextime
     2012            # delete old fort.* files and open them newly
     2013            fdict = {'10':None, '11':None, '12':None, '13':None, '16':None,
     2014                         '17':None, '19':None, '21':None, '22':None, '20':None}
     2015            #for f in fdict.keys():
     2016            #    silentremove(c.inputdir + "/fort." + f)
     2017            for k, f in fdict.iteritems():
     2018                silentremove(c.inputdir + "/fort." + k)
     2019                fdict[k] = open(c.inputdir + '/fort.' + k, 'w')
    16992020
    17002021            savedfields = []
    … …  
    17092030                # Relative humidity (Q.grb) is used as a template only
    17102031                # so we need the first we "meet"
    1711                     fout = open(c.inputdir + '/fort.18', 'w')
    1712                     grib_write(gid, fout)
    1713                     fout.close()
     2032                    with open(c.inputdir + '/fort.18', 'w') as fout:
     2033                        grib_write(gid, fout)
    17142034                elif paramId == 131 or paramId == 132:
    17152035                    grib_write(gid, fdict['10'])
    … …  
    17242044                elif paramId in [129, 138, 155] and levtype == 'hybrid' \
    17252045                                                and c.wrf == '1':
    1726         #            print paramId,'not written'
    17272046                    pass
    17282047                elif paramId == 246 or paramId == 247:
    … …  
    17332052                        clwc += grib_get_values(gid)
    17342053                        grib_set_values(gid, clwc)
    1735             #            grib_set(gid,'shortName','qc')
    17362054                        grib_set(gid, 'paramId', 201031)
    17372055                        grib_write(gid, fdict['22'])
    … …  
    17452063                        savedfields.append(paramId)
    17462064                    else:
    1747                         print 'duplicate ' + str(paramId) + ' not written'
     2065                        print('duplicate ' + str(paramId) + ' not written')
    17482066
    17492067                try:
    17502068                    if c.wrf == '1':
     2069# die if abfrage scheint ueberfluessig da eh das gleihce ausgefuehrt wird
    17512070                        if levtype == 'hybrid':
    17522071                            if paramId in [129, 130, 131, 132, 133, 138, 155]:
    … …  
    17592078
    17602079                grib_release(gid)
    1761                 gid  = grib_new_from_index(iid)
     2080                gid = grib_new_from_index(iid)
    17622081
    17632082        for f in fdict.values():
    … …  
    17652084
    17662085        # call for CONVERT2
     2086# AUSLAGERN IN EIGENE FUNKTION
    17672087
    17682088        if hid is not None:
    … …  
    17702090            os.chdir(c.inputdir)
    17712091            if os.stat('fort.21').st_size == 0 and int(c.eta) == 1:
    1772                 print 'Parameter 77 (etadot) is missing, most likely it is \
    1773                        not available for this type or date/time\n'
    1774                 print 'Check parameters CLASS, TYPE, STREAM, START_DATE\n'
     2092                print('Parameter 77 (etadot) is missing, most likely it is \
     2093                       not available for this type or date/time\n')
     2094                print('Check parameters CLASS, TYPE, STREAM, START_DATE\n')
    17752095                myerror(c, 'fort.21 is empty while parameter eta is set \
    17762096                            to 1 in CONTROL file')
    … …  
    17902110
    17912111            fnout += suffix
    1792             print "outputfile = " + fnout
     2112            print("outputfile = " + fnout)
    17932113            self.outputfilelist.append(fnout) # needed for final processing
    17942114            fout = open(fnout, 'wb')
    … …  
    18252145
    18262146        @Input:
    1827             self: instance of EIFlexpart
     2147            self: instance of ECFlexpart
    18282148                The current object of the class.
    18292149
    1830             inputfiles: list of strings
    1831                 A list of filenames.
     2150            inputfiles: instance of UIOFiles
     2151                Contains a list of files.
    18322152
    18332153            c: instance of class Control
    1834                 Contains all the parameters of control files, which are:
    1835                 DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
    1836                 NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL,
    1837                 LEVELIST, RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA,
    1838                 ETADIFF, DPDETA, SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX,
    1839                 ECSTORAGE, ECTRANS, ECFSDIR, MAILOPS, MAILFAIL,
    1840                 GRIB2FLEXPART, FLEXPARTDIR
     2154                Contains all the parameters of control files, which are e.g.:
     2155                DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
     2156                STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
     2157                LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
     2158                OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
     2159                ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
     2160                MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
     2161                DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
     2162
    18412163                For more information about format and content of the parameter
    18422164                see documentation.
    … …  
    18492171                           '/grib_templates/ecmwf_grib1_table_128')
    18502172        pars = toparamId(self.params['OG_acc_SL'][0], table128)
     2173
    18512174        index_keys = ["date", "time", "step"]
    18522175        indexfile = c.inputdir + "/date_time_stepRange.idx"
    1853         silentremove(indexfile)  # delete, if it exists already
     2176        silentremove(indexfile)
    18542177        grib = GribTools(inputfiles.files)
     2178        # creates new index file
    18552179        iid = grib.index(index_keys=index_keys, index_file=indexfile)
    1856         print 'index done...'
    1857 
    1858         # get the date, time and step values from indexfile
     2180
     2181        # read values of index keys
    18592182        index_vals = []
    18602183        for key in index_keys:
    18612184            key_vals = grib_index_get(iid,key)
    1862             print key_vals
    1863             # have to sort the steps, therefore convert to int first
     2185            print(key_vals)
     2186            # have to sort the steps for disaggregation,
     2187            # therefore convert to int first
    18642188            if key == 'step':
    1865                 l = []
    1866                 for k in key_vals:
    1867                     l.append(int(k))
    1868                 l.sort()
    1869                 # now convert back to str and overwrite key_vals
    1870                 key_vals = []
    1871                 for k in l:
    1872                     key_vals.append(str(k))
     2189                key_vals = [int(k) for k in key_vals]
     2190                key_vals.sort()
     2191                key_vals = [str(k) for k in key_vals]
    18732192            index_vals.append(key_vals)
     2193            # index_vals looks for example like:
     2194            # index_vals[0]: ('20171106', '20171107', '20171108') ; date
     2195            # index_vals[1]: ('0', '1200', '1800', '600') ; time
     2196            # index_vals[2]: ('0', '12', '3', '6', '9') ; stepRange
    18742197
    18752198        valsdict = {}
    … …  
    18802203            svalsdict[str(p)] = []
    18812204            stepsdict[str(p)] = []
    1882 #AP muss das hier wirklich eingerückt sein?
     2205 # ab hier eien Einrückung zurück!!!!
    18832206            for prod in product(*index_vals):
     2207                # e.g. prod = ('20170505', '0', '12')
     2208                #             (  date    ,time, step)
     2209                # per date e.g. time = 0, 600, 1200, 1800
     2210                # per time e.g. step = 0, 3, 6, 9, 12
    18842211                for i in range(len(index_keys)):
    18852212                    grib_index_select(iid, index_keys[i], prod[i])
    18862213
    18872214                gid = grib_new_from_index(iid)
     2215                # do convert2 program if gid at this time is not None,
     2216                # therefore save in hid
    18882217                hid = gid
    1889                 cflextime = None
    18902218                if gid is not None:
    1891                     cdate = grib_get(gid, 'date')
    1892                     #cyear = cdate[:4]
    1893                     #cmonth = cdate[4:6]
    1894                     #cday  = cdate[6:8]
     2219                    cdate = str(grib_get(gid, 'date'))
    18952220                    time = grib_get(gid, 'time')
    18962221                    type = grib_get(gid, 'type')
    18972222                    step = grib_get(gid, 'step')
    1898                     # date+time+step-2*dtime (since interpolated value valid for step-2*dtime)
     2223                    # date+time+step-2*dtime
     2224                    #(since interpolated value valid for step-2*dtime)
    18992225                    sdate = datetime.datetime(year=cdate / 10000,
    19002226                                              month=mod(cdate, 10000) / 100,
    … …  
    19262252                g = open(gnout, 'w')
    19272253                h = open(hnout, 'w')
    1928             print "outputfile = " + fnout
     2254            print("outputfile = " + fnout)
    19292255            f = open(fnout, 'w')
    19302256
    … …  
    19522278                    if step <= int(c.dtime):
    19532279                        svdp.append(values[:] / int(c.dtime))
    1954                     else:
     2280                    else: # deaccumulate values
    19552281                        svdp.append((vdp[-1] - vdp[-2]) / int(c.dtime))
    19562282
    1957                     print cparamId, atime, step, len(values), \
    1958                           values[0], std(values)
     2283                    print(cparamId, atime, step, len(values),
     2284                          values[0], std(values))
    19592285                    # save the 1/3-hourly or specific values
    19602286                    # svdp.append(values[:])
    19612287                    sd.append(step)
     2288                    # len(svdp) correspond to the time
    19622289                    if len(svdp) >= 3:
    19632290                        if len(svdp) > 3:

  • python/GribTools.py

    r64cf353 r02c8c50  
    8686                List of keynames.
    8787
    88             wherekeynames: list of ???
     88            wherekeynames: list of ???, optional
    8989                Default value is an empty list.
    9090
    91             wherekeyvalues: list of ???
     91            wherekeyvalues: list of ???, optional
    9292                Default value is an empty list.
    9393
    … …  
    154154                List of keynames. Default is an empty list.
    155155
    156             wherekeynames: list of ???
     156            wherekeynames: list of ???, optional
    157157                Default value is an empty list.
    158158
    159             wherekeyvalues: list of ???
     159            wherekeyvalues: list of ???, optional
    160160                Default value is an empty list.
    161161
    162             strict: boolean
     162            strict: boolean, optional
    163163                Decides if everything from keynames and keyvalues
    164164                is written out the grib file (False) or only those
    165165                meeting the where statement (True). Default is False.
    166166
    167             filemode:
     167            filemode: string, optional
    168168                Sets the mode for the output file. Default is "w".
    169169
    … …  
    231231                Filename of the input file to read the grib messages from.
    232232
    233             selectWhere: boolean
     233            selectWhere: boolean, optional
    234234                Decides if to copy the keynames and values equal to (True) or
    235235                different to (False) the keynames/keyvalues list passed to the
    236236                function. Default is True.
    237237
    238             keynames: list of ???
     238            keynames: list of ???, optional
    239239                List of keynames. Default is an empty list.
    240240
    241             keyvalues: list of ???
     241            keyvalues: list of ???, optional
    242242                List of keynames. Default is an empty list.
    243243
    244             filemode:
     244            filemode: string, optional
    245245                Sets the mode for the output file. Default is "w".
    246246
    … …  
    288288        '''
    289289        @Description:
    290             Create index from a list of files if it does not exist or
     290            Create index file from a list of files if it does not exist or
    291291            read an index file.
    292292
    293293        @Input:
    294             index_keys: list of ???
     294            index_keys: list of strings, optional
     295                Contains the list of key parameter names from
     296                which the index is to be created.
    295297                Default is a list with a single entry string "mars".
    296298
    297             index_file: string
     299            index_file: string, optional
    298300                Filename where the indices are stored.
    299301                Default is "my.idx".
    … …  
    310312            print("Use existing index file: %s " % (index_file))
    311313        else:
    312 #AP does the for loop overwrite the iid all the time?
    313314            for file in self.filename:
    314315                print("Inputfile: %s " % (file))
    … …  
    317318                else:
    318319                    grib_index_add_file(self.iid, file)
    319 #AP or does the if has to be in the for loop?
    320 #AP would make more sense?
     320
    321321            if self.iid is not None:
    322322                grib_index_write(self.iid, index_file)
    323323
     324        print('... index done')
     325
    324326        return self.iid
    325327

  • python/UIOTools.py

    r64cf353 r02c8c50  
    6969
    7070            suffix: list of strings
    71                 Types of files to manipulate such as
     71                The types of files which should be manipulated such as
    7272                ['grib', 'grb', 'grib1', 'grib2', 'grb1', 'grb2']
    7373
    … …  
    7575            <nothing>
    7676        '''
     77
    7778        self.suffix = suffix
     79
    7880        return
    7981
    80     def listFiles(self, pathname, pattern):
     82    def listFiles(self, path, pattern):
    8183        '''
    8284        @Description:
    … …  
    8991                Description see class documentation.
    9092
    91             pathname: string
     93            path: string
    9294                Directory where to list the files.
    9395
    … …  
    99101            <nothing>
    100102        '''
    101 #AP pathname zu path ändern
    102 #AP is it possible for each possible file extension ? mabye regexx?
    103         # Get the absolute path of the pathname parameter
    104         pathname = os.path.abspath(pathname)
     103
     104        # Get the absolute path
     105        path = os.path.abspath(path)
    105106
    106107        # Get a list of files in pathname
    107         filesInCurDir0 = glob.glob(pathname + '/' + pattern)
     108        filesInCurDir0 = glob.glob(path + '/' + pattern)
    108109        filesInCurDir = []
    109110        for f in filesInCurDir0:
    110111            filesInCurDir.append(f.split('/')[-1])
     112
    111113        self.counter = 0
    112114        self.files = []
    113115        # Traverse through all files
    114116        for file in filesInCurDir:
    115             curFile = os.path.join(pathname, file)
     117            curFile = os.path.join(path, file)
    116118
    117119            # Check if it's a normal file or directory

  • python/getMARSdata.py

    r64cf353 r02c8c50  
    5050    sys.path.append(localpythonpath)
    5151
    52 from FlexpartTools import MARSretrieval, EIFlexpart, silentremove, \
     52from FlexpartTools import ECFlexpart, \
    5353                          Control, myerror, normalexit, \
    5454                          interpret_args_and_control
    … …  
    101101        while day<endp1:
    102102                # we need to retrieve MARS data for this period (maximum one month)
    103                 flexpart = EIFlexpart(c,fluxes=True)
     103                flexpart = ECFlexpart(c,fluxes=True)
    104104                if day+datechunk-datetime.timedelta(days=1)<endp1:
    105105                    dates= day.strftime("%Y%m%d") + "/to/" + (day+datechunk-datetime.timedelta(days=1)).strftime("%Y%m%d")
    … …  
    118118        while day<=end:
    119119                # we need to retrieve MARS data for this period (maximum one month)
    120                 flexpart = EIFlexpart(c,fluxes=True)
     120                flexpart = ECFlexpart(c,fluxes=True)
    121121                if day+datechunk-datetime.timedelta(days=1)<end:
    122122                    dates= day.strftime("%Y%m%d") + "/to/" + (day+datechunk-datetime.timedelta(days=1)).strftime("%Y%m%d")
    … …  
    138138
    139139               # we need to retrieve MARS data for this period (maximum one month)
    140             flexpart = EIFlexpart(c)
     140            flexpart = ECFlexpart(c)
    141141            if day+datechunk-datetime.timedelta(days=1)<end:
    142142                dates= day.strftime("%Y%m%d") + "/to/" + (day+datechunk-datetime.timedelta(days=1)).strftime("%Y%m%d")

  • python/install.py

    r64cf353 r02c8c50  
    6868from argparse import ArgumentParser,ArgumentDefaultsHelpFormatter
    6969from GribTools import GribTools
    70 from FlexpartTools import EIFlexpart, Control, install_args_and_control
     70from FlexpartTools import ECFlexpart, Control, install_args_and_control
    7171from getMARSdata import getMARSdata
    7272from prepareFLEXPART import prepareFLEXPART

  • python/prepareFLEXPART.py

    r64cf353 r02c8c50  
    11#!/usr/bin/env python
    2 #
    3 # This software is licensed under the terms of the Apache Licence Version 2.0
    4 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
    5 #
    6 # Functionality provided: Prepare input 3D-wind fields in hybrid coordinates + surface fields for FLEXPART runs
    7 #
    8 # Creation: October  2014 - Anne Fouilloux - University of Oslo
    9 # Extension November 2015 - Leopold Haimberger - University of Vienna for:
    10 # - using the WebAPI also for general MARS retrievals
    11 # - job submission on ecgate and cca
    12 # - job templates suitable for twice daily operational dissemination
    13 # - dividing retrievals of longer periods into digestable chunks
    14 # - retrieve also longer term forecasts, not only analyses and short term forecast data
    15 # - conversion into GRIB2
    16 # - conversion into .fp format for faster execution of FLEXPART
    17 #
    18 # Requirements:
    19 # in addition to a standard python 2.6 or 2.7 installation the following packages need to be installed
    20 # ECMWF WebMARS, gribAPI with python enabled, emoslib, ecaccess web toolkit,
    21 # all available from https://software.ecmwf.int/
    22 # dateutils
    23 # matplotlib (optional, for debugging)
    24 #
     2# -*- coding: utf-8 -*-
     3#************************************************************************
     4# TODO AP
     5#AP
     6# - Change History ist nicht angepasst ans File! Original geben lassen
     7# - wieso cleanup in main wenn es in prepareflexpart bereits abgefragt wurde?
     8#   doppelt gemoppelt?
     9# - wieso start=startm1 wenn basetime = 0 ?  wenn die fluxes nicht mehr
     10#   relevant sind? verstehe ich nicht
     11#************************************************************************
     12"""
     13@Author: Anne Fouilloux (University of Oslo)
     14
     15@Date: October 2014
     16
     17@ChangeHistory:
     18    November 2015 - Leopold Haimberger (University of Vienna):
     19        - using the WebAPI also for general MARS retrievals
     20        - job submission on ecgate and cca
     21        - job templates suitable for twice daily operational dissemination
     22        - dividing retrievals of longer periods into digestable chunks
     23        - retrieve also longer term forecasts, not only analyses and
     24          short term forecast data
     25        - conversion into GRIB2
     26        - conversion into .fp format for faster execution of FLEXPART
     27
     28    February 2018 - Anne Philipp (University of Vienna):
     29        - applied PEP8 style guide
     30        - added documentation
     31        - minor changes in programming style for consistence
     32
     33@License:
     34    (C) Copyright 2014.
     35
     36    This software is licensed under the terms of the Apache Licence Version 2.0
     37    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
     38
     39@Requirements:
     40    - A standard python 2.6 or 2.7 installation
     41    - dateutils
     42    - matplotlib (optional, for debugging)
     43    - ECMWF specific packages, all available from https://software.ecmwf.int/
     44        ECMWF WebMARS, gribAPI with python enabled, emoslib and
     45        ecaccess web toolkit
     46
     47@Description:
     48    Further documentation may be obtained from www.flexpart.eu.
     49
     50    Functionality provided:
     51        Prepare input 3D-wind fields in hybrid coordinates +
     52        surface fields for FLEXPART runs
     53"""
     54
    2555import calendar
    2656import shutil
    … …  
    2959import os,inspect,sys
    3060import socket
    31 from argparse import ArgumentParser,ArgumentDefaultsHelpFormatter
     61from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
    3262# add path to submit.py to pythonpath so that python finds its buddies
    3363localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
    … …  
    3767#from string import strip
    3868from GribTools import GribTools
    39 from FlexpartTools import EIFlexpart, Control, interpret_args_and_control, cleanup
     69from FlexpartTools import ECFlexpart, Control, interpret_args_and_control, cleanup
    4070
    4171hostname=socket.gethostname()
    … …  
    4979
    5080
    51 def prepareFLEXPART(args,c):
     81def prepareFLEXPART(args, c):
     82    '''
     83    @Description:
    5284
    5385
     86    @Input:
     87        args: instance of ArgumentParser
     88            Contains the commandline arguments from script/program call.
    5489
    55     namelist='fort.4'
     90        c: instance of class Control
     91            Contains all the parameters of control files, which are e.g.:
     92            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
     93            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
     94            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
     95            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
     96            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
     97            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
     98            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
     99
     100            For more information about format and content of the parameter
     101            see documentation.
     102
     103    @Return:
     104        <nothing>
     105    '''
    56106
    57107    if not args.ppid:
    58         c.ppid=str(os.getppid())
     108        c.ppid = str(os.getppid())
    59109    else:
    60         c.ppid=args.ppid
     110        c.ppid = args.ppid
    61111
    62     c.ecapi=ecapi
     112    c.ecapi = ecapi
    63113
    64     syear=int(c.start_date[:4])
    65     smonth=int(c.start_date[4:6])
    66     sday=int(c.start_date[6:])
    67     start = datetime.date( year = syear, month = smonth, day = sday )
    68     eyear=int(c.end_date[:4])
    69     emonth=int(c.end_date[4:6])
    70     eday=int(c.end_date[6:])
     114    # create the start and end date
     115    start = datetime.date(year=int(c.start_date[:4]),
     116                          month=int(c.start_date[4:6]),
     117                          day=int(c.start_date[6:]))
    71118
    72     end = datetime.date( year = eyear, month = emonth, day = eday )
     119    end = datetime.date(year=int(c.end_date[:4]),
     120                        month=int(c.end_date[4:6]),
     121                        day=int(c.end_date[6:]))
    73122
    74     inputfiles=UIOFiles(['.grib', '.grb', '.grib1', '.grib2', '.grb1','.grb2'])
     123    # to deaccumulated the fluxes correctly
     124    # one day ahead of the start date and
     125    # one day after the end date is needed
     126    startm1 = start - datetime.timedelta(days=1)
     127    endp1 = end + datetime.timedelta(days=1)
    75128
    76     startm1=start- datetime.timedelta(days=1)
    77     endp1=end + datetime.timedelta(days=1)
     129    # get all files with flux data to be deaccumulated
     130    inputfiles = UIOFiles(['.grib', '.grb', '.grib1',
     131                           '.grib2', '.grb1', '.grb2'])
    78132
    79     inputfiles.listFiles(c.inputdir, '*OG_acc_SL*.'+c.ppid+'.*')
     133    inputfiles.listFiles(c.inputdir, '*OG_acc_SL*.' + c.ppid + '.*')
     134
     135    # create output dir if necessary
    80136    if not os.path.exists(c.outputdir):
    81     os.makedirs(c.outputdir)
     137        os.makedirs(c.outputdir)
    82138
    83     flexpart = EIFlexpart(c,fluxes=True)
    84     flexpart.create_namelist(c,'fort.4')
     139    # deaccumulate the flux data
     140    flexpart = ECFlexpart(c, fluxes=True)
     141    flexpart.write_namelist(c, 'fort.4')
    85142    flexpart.deacc_fluxes(inputfiles, c)
    86143
    87     inputfiles=UIOFiles(['.grib', '.grb', '.grib1', '.grib2', '.grb1','.grb2'])
     144    print('Prepare ' + start.strftime("%Y%m%d") +
     145          "/to/" + end.strftime("%Y%m%d"))
    88146
    89     print 'Prepare '+start.strftime("%Y%m%d") + "/to/" + end.strftime("%Y%m%d")
    90     # we will make the list of files from the root inputdir
    91     inputfiles.listFiles(c.inputdir, '????__??.*'+c.ppid+'.*')
     147    # get a list of all files from the root inputdir
     148    inputfiles = UIOFiles(['.grib', '.grb', '.grib1',
     149                           '.grib2', '.grb1', '.grb2'])
    92150
    93     if c.basetime=='00':
    94         start=startm1
    95     flexpart = EIFlexpart(c)
    96     flexpart.create(inputfiles, c) # produce FLEXPART-ready GRIB files
    97     flexpart.process_output(c) # process GRIB files - copy/transfer/interpolate them or make them GRIB2
     151    inputfiles.listFiles(c.inputdir, '????__??.*' + c.ppid + '.*')
    98152
    99     if int(c.debug)!=0:
    100     print('Temporary files left intact')
     153    # produce FLEXPART-ready GRIB files and
     154    # process GRIB files -
     155    # copy/transfer/interpolate them or make them GRIB2
     156    if c.basetime == '00':
     157        start = startm1
     158
     159    flexpart = ECFlexpart(c, fluxes=False)
     160    flexpart.create(inputfiles, c)
     161    flexpart.process_output(c)
     162
     163    # check if in debugging mode, then store all files
     164    if int(c.debug) != 0:
     165        print('Temporary files left intact')
    101166    else:
    102     cleanup(c)
     167        cleanup(c)
     168
     169    return
    103170
    104171if __name__ == "__main__":

  • python/submit.py

    r64cf353 r02c8c50  
    6767from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
    6868from GribTools import GribTools
    69 from FlexpartTools import EIFlexpart, Control, interpret_args_and_control, normalexit, myerror
     69from FlexpartTools import ECFlexpart, Control, interpret_args_and_control, normalexit, myerror
    7070from getMARSdata import getMARSdata
    7171from prepareFLEXPART import prepareFLEXPART
    … …  
    115115
    116116        c: instance of class Control
    117             Contains all necessary information of a control file. The parameters
    118             are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
    119             NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
    120             RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
    121             SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
    122             MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR
    123             For more information about format and content of the parameter see
    124             documentation.
     117            Contains all the parameters of control files, which are e.g.:
     118            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
     119            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
     120            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
     121            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
     122            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
     123            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
     124            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
     125
     126            For more information about format and content of the parameter
     127            see documentation.
    125128
    126129        queue: string
Note: See TracChangeset for help on using the changeset viewer.
hosted by ZAMG