Changeset 54a8a01 in flex_extract.git for python/get_mars_data.py


Ignore:
Timestamp:
Aug 31, 2018, 7:50:37 AM (18 months ago)
Author:
Anne Philipp <anne.philipp@…>
Branches:
dev
Children:
597d4d1
Parents:
e1228f3
Message:

restructuring, documentations and bug fixes

File:
1 edited

Legend:

Unmodified
Added
Removed
  • python/get_mars_data.py

    rff99eae r54a8a01  
    2222#          online documentation)
    2323#        - use of UIFiles class for file selection and deletion
    24 
     24#
    2525#
    2626# @License:
     
    3939#    - main
    4040#    - get_mars_data
     41#    - do_retrievement
    4142#
    4243#*******************************************************************************
     
    5657
    5758# software specific classes and modules from flex_extract
    58 from tools import my_error, normal_exit, interpret_args_and_control
     59from tools import my_error, normal_exit, get_cmdline_arguments, read_ecenv
    5960from EcFlexpart import EcFlexpart
    6061from UioFiles import UioFiles
     
    8283        <nothing>
    8384    '''
    84     args, c = interpret_args_and_control()
     85
     86    args = get_cmdline_arguments()
     87
     88    try:
     89        c = ControlFile(args.controlfile)
     90    except IOError:
     91        try:
     92            c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile)
     93        except IOError:
     94            print 'Could not read CONTROL file "' + args.controlfile + '"'
     95            print 'Either it does not exist or its syntax is wrong.'
     96            print 'Try "' + sys.argv[0].split('/')[-1] + \
     97                  ' -h" to print usage information'
     98            sys.exit(1)
     99
     100    env_parameter = read_ecenv(c.ecmwfdatadir + 'python/ECMWF_ENV')
     101    c.assign_args_to_control(args, env_parameter)
     102    c.assign_envs_to_control(env_parameter)
     103    c.check_conditions()
     104
    85105    get_mars_data(c)
    86     normal_exit(c)
     106    normal_exit(c.mailfail, 'Done!')
    87107
    88108    return
     
    129149    print 'ecapi: ', c.ecapi
    130150
     151    # basetime geht rückwärts
     152
     153    # if basetime 00
     154    # dann wird von 12 am vortag bis 00 am start tag geholt
     155    # aber ohne 12 selbst sondern 12 + step
     156
     157    # if basetime 12
     158    # dann wird von 00 + step bis 12 am start tag geholt
     159
     160    # purer forecast wird vorwärts bestimmt.
     161    # purer forecast mode ist dann wenn  größer 24 stunden
     162    # wie kann das noch festgestellt werden ????
     163    # nur FC und steps mehr als 24 ?
     164    # die einzige problematik beim reinen forecast ist die benennung der files!
     165    # also sobald es Tagesüberschneidungen gibt
     166    # allerdings ist das relevant und ersichtlich an den NICHT FLUSS DATEN
     167
     168
    131169    # set start date of retrieval period
    132170    start = datetime.date(year=int(c.start_date[:4]),
     
    134172                          day=int(c.start_date[6:]))
    135173    startm1 = start - datetime.timedelta(days=1)
    136     if c.basetime == '00':
    137         start = startm1
    138174
    139175    # set end date of retrieval period
     
    141177                        month=int(c.end_date[4:6]),
    142178                        day=int(c.end_date[6:]))
     179
     180    # set time period for one single retrieval
     181    datechunk = datetime.timedelta(days=int(c.date_chunk))
     182
     183    if c.basetime == '00':
     184        start = startm1
     185
    143186    if c.basetime == '00' or c.basetime == '12':
     187        # endp1 = end + datetime.timedelta(days=1)
     188        endp1 = end
     189    else:
     190        # endp1 = end + datetime.timedelta(days=2)
    144191        endp1 = end + datetime.timedelta(days=1)
    145     else:
    146         endp1 = end + datetime.timedelta(days=2)
    147 
    148     # set time period of one single retrieval
    149     datechunk = datetime.timedelta(days=int(c.date_chunk))
    150192
    151193    # --------------  flux data ------------------------------------------------
    152194    print 'removing old flux content of ' + c.inputdir
    153     tobecleaned = UioFiles('*_acc_*.' + str(os.getppid()) + '.*.grb')
    154     tobecleaned.list_files(c.inputdir)
     195    tobecleaned = UioFiles(c.inputdir,
     196                           '*_acc_*.' + str(os.getppid()) + '.*.grb')
    155197    tobecleaned.delete_files()
    156198
    157     # if forecast for maximum one day (upto 23h) are to be retrieved,
     199    # if forecast for maximum one day (upto 24h) are to be retrieved,
    158200    # collect accumulation data (flux data)
    159201    # with additional days in the beginning and at the end
    160202    # (used for complete disaggregation of original period)
    161     if c.maxstep < 24:
    162         day = startm1
    163         while day < endp1:
    164             # retrieve MARS data for the whole period
    165             flexpart = EcFlexpart(c, fluxes=True)
    166             tmpday = day + datechunk - datetime.timedelta(days=1)
    167             if tmpday < endp1:
    168                 dates = day.strftime("%Y%m%d") + "/to/" + \
    169                         tmpday.strftime("%Y%m%d")
    170             else:
    171                 dates = day.strftime("%Y%m%d") + "/to/" + \
    172                         end.strftime("%Y%m%d")
    173 
    174             print "retrieve " + dates + " in dir " + c.inputdir
    175 
    176             try:
    177                 flexpart.retrieve(server, dates, c.inputdir)
    178             except IOError:
    179                 my_error(c, 'MARS request failed')
    180 
    181             day += datechunk
     203    if c.maxstep <= 24:
     204        do_retrievement(c, server, startm1, endp1, datechunk, fluxes=True)
    182205
    183206    # if forecast data longer than 24h are to be retrieved,
     
    187210    # exact time period with boundary conditions)
    188211    else:
    189         day = start
    190         while day <= end:
    191             # retrieve MARS data for the whole period
    192             flexpart = EcFlexpart(c, fluxes=True)
    193             tmpday = day + datechunk - datetime.timedelta(days=1)
    194             if tmpday < end:
    195                 dates = day.strftime("%Y%m%d") + "/to/" + \
    196                         tmpday.trftime("%Y%m%d")
    197             else:
    198                 dates = day.strftime("%Y%m%d") + "/to/" + \
    199                         end.strftime("%Y%m%d")
    200 
    201             print "retrieve " + dates + " in dir " + c.inputdir
    202 
    203             try:
    204                 flexpart.retrieve(server, dates, c.inputdir)
    205             except IOError:
    206                 my_error(c, 'MARS request failed')
    207 
    208             day += datechunk
     212        do_retrievement(c, server, start, end, datechunk, fluxes=True)
    209213
    210214    # --------------  non flux data --------------------------------------------
    211215    print 'removing old non flux content of ' + c.inputdir
    212     tobecleaned = UioFiles('*__*.' + str(os.getppid()) + '.*.grb')
    213     tobecleaned.list_files(c.inputdir)
     216    tobecleaned = UioFiles(c.inputdir,
     217                           '*__*.' + str(os.getppid()) + '.*.grb')
    214218    tobecleaned.delete_files()
     219
     220    do_retrievement(c, server, start, end, datechunk, fluxes=False)
     221
     222    return
     223
     224def do_retrievement(c, server, start, end, delta_t, fluxes=False):
     225    '''
     226    @Description:
     227        Divides the complete retrieval period in smaller chunks and
     228        retrieves the data from MARS.
     229
     230    @Input:
     231        c: instance of ControlFile
     232            Contains all the parameters of CONTROL file, which are e.g.:
     233            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
     234            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
     235            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
     236            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
     237            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
     238            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
     239            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
     240
     241            For more information about format and content of the parameter
     242            see documentation.
     243
     244        server: instance of ECMWFService
     245            The server connection to ECMWF
     246
     247        start: instance of datetime
     248            The start date of the retrieval.
     249
     250        end: instance of datetime
     251            The end date of the retrieval.
     252
     253        delta_t: instance of datetime
     254            Delta_t +1 is the maximal time period of a single
     255            retrieval.
     256
     257        fluxes: boolean, optional
     258            Decides if the flux parameters are to be retrieved or
     259            the rest of the parameter list.
     260            Default value is False.
     261
     262    @Return:
     263        <nothing>
     264    '''
     265
     266    # since actual day also counts as one day,
     267    # we only need to add datechunk - 1 days to retrieval
     268    # for a period
     269    delta_t_m1 = delta_t - datetime.timedelta(days=1)
    215270
    216271    day = start
    217272    while day <= end:
    218         # retrieve all non flux MARS data for the whole period
    219         flexpart = EcFlexpart(c, fluxes=False)
    220         tmpday = day + datechunk - datetime.timedelta(days=1)
     273        flexpart = EcFlexpart(c, fluxes)
     274        tmpday = day + delta_t_m1
    221275        if tmpday < end:
    222276            dates = day.strftime("%Y%m%d") + "/to/" + \
     
    231285            flexpart.retrieve(server, dates, c.inputdir)
    232286        except IOError:
    233             my_error(c, 'MARS request failed')
    234 
    235         day += datechunk
     287            my_error(c.mailfail, 'MARS request failed')
     288
     289        day += delta_t
    236290
    237291    return
Note: See TracChangeset for help on using the changeset viewer.
hosted by ZAMG