source: flex_extract.git/Source/Python/Mods/get_mars_data.py @ 44174de

ctbtodev
Last change on this file since 44174de was 44174de, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

commented out the WRF parts since they are still under construction and added License SPDX tags

  • Property mode set to 100755
File size: 11.0 KB
RevLine 
[8463d78]1#!/usr/bin/env python3
[efdb01a]2# -*- coding: utf-8 -*-
[991df6a]3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
[ff99eae]11#        - moved the getEIdata program into a function "get_mars_data"
[991df6a]12#        - moved the AgurmentParser into a seperate function
13#        - adatpted the function for the use in flex_extract
[ff99eae]14#        - renamed file to get_mars_data
[991df6a]15#
16#    February 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added structured documentation
19#        - minor changes in programming style for consistence
20#        - added function main and moved function calls vom __main__ there
21#          (necessary for better documentation with docstrings for later
22#          online documentation)
23#        - use of UIFiles class for file selection and deletion
[6f951ca]24#        - seperated get_mars_data function into several smaller pieces:
25#          write_reqheader, mk_server, mk_dates, remove_old, do_retrievment
[991df6a]26#
27# @License:
[6f951ca]28#    (C) Copyright 2014-2019.
29#    Anne Philipp, Leopold Haimberger
[991df6a]30#
[44174de]31#    SPDX-License-Identifier: CC-BY-4.0
32#
[6f951ca]33#    This work is licensed under the Creative Commons Attribution 4.0
34#    International License. To view a copy of this license, visit
35#    http://creativecommons.org/licenses/by/4.0/ or send a letter to
36#    Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
[991df6a]37#*******************************************************************************
[6f951ca]38'''This script extracts MARS data from ECMWF servers.
[96e1533]39
[6f951ca]40At first, the necessary parameters from command line and CONTROL files are
41extracted. They define the data set to be extracted from MARS.
[96e1533]42
43This file can also be imported as a module and contains the following
44functions:
45
46    * main - the main function of the script
[6f951ca]47    * get_mars_data - overall control of ECMWF data retrievment
48    * write_reqheader - writes the header into the mars_request file
49    * mk_server - creates the server connection to ECMWF servers
50    * mk_dates - defines the start and end date
51    * remove_old - deletes old retrieved grib files
52    * do_retrievement - creates individual retrievals
53
54Type: get_mars_data.py --help
55to get information about command line parameters.
56Read the documentation for usage instructions.
57'''
[efdb01a]58# ------------------------------------------------------------------------------
59# MODULES
60# ------------------------------------------------------------------------------
[7e25255]61from __future__ import print_function
62
[991df6a]63import os
64import sys
65import inspect
[ca867de]66from datetime import datetime, timedelta
[ff99eae]67
68# software specific classes and modules from flex_extract
[6f951ca]69# add path to local main python path for flex_extract to get full access
[70fee58]70sys.path.append(os.path.dirname(os.path.abspath(
71    inspect.getfile(inspect.currentframe()))) + '/../')
[2fb99de]72import _config
[ba99230]73from Mods.tools import (setup_controldata, my_error, normal_exit, get_cmdline_args,
[5bad6ec]74                   read_ecenv, make_dir)
[ba99230]75from Classes.EcFlexpart import EcFlexpart
76from Classes.UioFiles import UioFiles
77from Classes.MarsRetrieval import MarsRetrieval
[ca867de]78
79try:
[f20af73]80    ec_api = True
[ca867de]81    import ecmwfapi
82except ImportError:
[f20af73]83    ec_api = False
84
85try:
86    cds_api = True
87    import cdsapi
88except ImportError:
89    cds_api = False
[efdb01a]90# ------------------------------------------------------------------------------
91# FUNCTION
92# ------------------------------------------------------------------------------
[991df6a]93def main():
[274f9ef]94    '''Controls the program to get data out of mars.
95
96    This is done if it is called directly from command line.
97    Then it also takes program call arguments and control file input.
[295ff45]98
[274f9ef]99    Parameters
100    ----------
[991df6a]101
[274f9ef]102    Return
103    ------
[991df6a]104
105    '''
[54a8a01]106
[f20af73]107    c, _, _, _ = setup_controldata()
[ff99eae]108    get_mars_data(c)
[f20af73]109    normal_exit('Retrieving MARS data: Done!')
[d69b677]110
[991df6a]111    return
112
[ff99eae]113def get_mars_data(c):
[274f9ef]114    '''Retrieves the EC data needed for a FLEXPART simulation.
115
116    Start and end dates for retrieval period is set. Retrievals
117    are divided into smaller periods if necessary and datechunk parameter
118    is set.
119
120    Parameters
121    ----------
[6f951ca]122    c : ControlFile
[274f9ef]123        Contains all the parameters of CONTROL file and
124        command line.
125
126    Return
127    ------
128
[991df6a]129    '''
[f20af73]130    c.ec_api = ec_api
131    c.cds_api = cds_api
[d69b677]132
133    if not os.path.exists(c.inputdir):
[5bad6ec]134        make_dir(c.inputdir)
[991df6a]135
[403cbf1]136    if c.request == 0:
[2fb99de]137        print("Retrieving EC data!")
[403cbf1]138    else:
139        if c.request == 1:
140            print("Printing mars requests!")
141        elif c.request == 2:
142            print("Retrieving EC data and printing mars request!")
[631ba13]143        write_reqheader(os.path.join(c.inputdir, _config.FILE_MARS_REQUESTS))
[2fb99de]144
145    print("start date %s " % (c.start_date))
146    print("end date %s " % (c.end_date))
[d69b677]147
[631ba13]148    server = mk_server(c)
149
150    # if data are to be retrieved, clean up any old grib files
151    if c.request == 0 or c.request == 2:
[1d15e27]152        remove_old('*grb', c.inputdir)
[631ba13]153
154    # --------------  flux data ------------------------------------------------
[45c6337]155    start, end, datechunk = mk_dates(c, fluxes=True)
[631ba13]156    do_retrievement(c, server, start, end, datechunk, fluxes=True)
157
158    # --------------  non flux data --------------------------------------------
[45c6337]159    start, end, datechunk = mk_dates(c, fluxes=False)
[631ba13]160    do_retrievement(c, server, start, end, datechunk, fluxes=False)
161
162    return
163
164def write_reqheader(marsfile):
165    '''Writes header with column names into mars request file.
166
167    Parameters
168    ----------
[6f951ca]169    marsfile : str
[631ba13]170        Path to the mars request file.
171
172    Return
173    ------
174
175    '''
176    MR = MarsRetrieval(None, None)
177    attrs = vars(MR).copy()
178    del attrs['server']
179    del attrs['public']
180    with open(marsfile, 'w') as f:
181        f.write('request_number' + ', ')
[79729d5]182        f.write(', '.join(str(key) for key in sorted(attrs.keys())))
[631ba13]183        f.write('\n')
184
185    return
186
187def mk_server(c):
[f20af73]188    '''Creates a server connection with available python API.
189
190    Which API is used depends on availability and the dataset to be retrieved.
191    The CDS API is used for ERA5 dataset no matter if the user is a member or
192    a public user. ECMWF WebAPI is used for all other available datasets.
[631ba13]193
194    Parameters
195    ----------
[6f951ca]196    c : ControlFile
[631ba13]197        Contains all the parameters of CONTROL file and
198        command line.
199
200    Return
201    ------
[f20af73]202    server : ECMWFDataServer, ECMWFService or Client
203        Connection to ECMWF server via python interface ECMWF WebAPI or CDS API.
[631ba13]204
205    '''
[f20af73]206    if cds_api and (c.marsclass.upper() == 'EA'):
207        server = cdsapi.Client()
208        c.ec_api = False
209    elif c.ec_api:
[5bad6ec]210        if c.public:
211            server = ecmwfapi.ECMWFDataServer()
212        else:
213            server = ecmwfapi.ECMWFService("mars")
[f20af73]214        c.cds_api = False
[d69b677]215    else:
216        server = False
217
[f20af73]218    print('Using ECMWF WebAPI: ' + str(c.ec_api))
219    print('Using CDS API: ' + str(c.cds_api))
[991df6a]220
[631ba13]221    return server
[54a8a01]222
223
[45b99e6]224def check_dates_for_nonflux_fc_times(types, times):
225    '''
226    '''
227    for ty, ti in zip(types,times):
228        if ty.upper() == 'FC' and int(ti) == 18:
229            return True
230    return False
231
232
[631ba13]233def mk_dates(c, fluxes):
234    '''Prepares start and end date depending on flux or non flux data.
[54a8a01]235
[631ba13]236    If forecast for maximum one day (upto 24h) are to be retrieved, then
237    collect accumulation data (flux data) with additional days in the
238    beginning and at the end (used for complete disaggregation of
239    original period)
[54a8a01]240
[631ba13]241    If forecast data longer than 24h are to be retrieved, then
242    collect accumulation data (flux data) with the exact start and end date
243    (disaggregation will be done for the exact time period with
244    boundary conditions)
245
246    Since for basetime the extraction contains the 12 hours upfront,
247    if basetime is 0, the starting date has to be the day before and
248
249    Parameters
250    ----------
[6f951ca]251    c : ControlFile
[631ba13]252        Contains all the parameters of CONTROL file and
253        command line.
254
[6f951ca]255    fluxes : boolean, optional
[631ba13]256        Decides if the flux parameter settings are stored or
257        the rest of the parameter list.
258        Default value is False.
259
260    Return
261    ------
[6f951ca]262    start : datetime
[631ba13]263        The start date of the retrieving data set.
264
[6f951ca]265    end : datetime
[631ba13]266        The end date of the retrieving data set.
267
[6f951ca]268    chunk : datetime
[631ba13]269        Time period in days for one single mars retrieval.
270
271    '''
[ca867de]272    start = datetime.strptime(c.start_date, '%Y%m%d')
273    end = datetime.strptime(c.end_date, '%Y%m%d')
[631ba13]274    chunk = timedelta(days=int(c.date_chunk))
275
[d4696e0]276    if c.basetime == 0:
277        start = start - timedelta(days=1)
[efdb01a]278
[7e25255]279    if c.purefc and fluxes and c.maxstep < 24:
280        start = start - timedelta(days=1)
281        end = end + timedelta(days=1)
282
[d4696e0]283    if not c.purefc and fluxes and not c.basetime == 0:
[ca867de]284        start = start - timedelta(days=1)
[631ba13]285        end = end + timedelta(days=1)
[ca867de]286
[45b99e6]287    # if we have non-flux forecast data starting at 18 UTC
288    # we need to start retrieving data one day in advance
289    if not fluxes and check_dates_for_nonflux_fc_times(c.type, c.time):
290        start = start - timedelta(days=1)
291
[631ba13]292    return start, end, chunk
[54a8a01]293
[1d15e27]294def remove_old(pattern, inputdir):
[3f36e42]295    '''Deletes old retrieval files from current input directory
296    matching the pattern.
[54a8a01]297
[631ba13]298    Parameters
299    ----------
[6f951ca]300    pattern : str
[631ba13]301        The sub string pattern which identifies the files to be deleted.
[991df6a]302
[6f951ca]303    inputdir : str, optional
[1d15e27]304        Path to the directory where the retrieved data is stored.
305
[631ba13]306    Return
307    ------
[991df6a]308
[631ba13]309    '''
[1d15e27]310    print('... removing old content of ' + inputdir)
[631ba13]311
[1d15e27]312    tobecleaned = UioFiles(inputdir, pattern)
[631ba13]313    tobecleaned.delete_files()
[54a8a01]314
315    return
316
[631ba13]317
[54a8a01]318def do_retrievement(c, server, start, end, delta_t, fluxes=False):
[274f9ef]319    '''Divides the complete retrieval period in smaller chunks and
320    retrieves the data from MARS.
321
322    Parameters
323    ----------
[6f951ca]324    c : ControlFile
[274f9ef]325        Contains all the parameters of CONTROL file and
326        command line.
[54a8a01]327
[6f951ca]328    server : ECMWFService or ECMWFDataServer
[274f9ef]329            The server connection to ECMWF.
[54a8a01]330
[6f951ca]331    start : datetime
[274f9ef]332        The start date of the retrieval.
[54a8a01]333
[6f951ca]334    end : datetime
[274f9ef]335        The end date of the retrieval.
[54a8a01]336
[6f951ca]337    delta_t : datetime
[274f9ef]338        Delta_t + 1 is the maximal time period of a single
339        retrieval.
[54a8a01]340
[6f951ca]341    fluxes : boolean, optional
[274f9ef]342        Decides if the flux parameters are to be retrieved or
343        the rest of the parameter list.
344        Default value is False.
[54a8a01]345
[274f9ef]346    Return
347    ------
[54a8a01]348
349    '''
350
351    # since actual day also counts as one day,
352    # we only need to add datechunk - 1 days to retrieval
353    # for a period
[ca867de]354    delta_t_m1 = delta_t - timedelta(days=1)
[54a8a01]355
[efdb01a]356    day = start
357    while day <= end:
[54a8a01]358        flexpart = EcFlexpart(c, fluxes)
359        tmpday = day + delta_t_m1
[ff99eae]360        if tmpday < end:
361            dates = day.strftime("%Y%m%d") + "/to/" + \
362                    tmpday.strftime("%Y%m%d")
363        else:
364            dates = day.strftime("%Y%m%d") + "/to/" + \
365                    end.strftime("%Y%m%d")
[efdb01a]366
[2fb99de]367        print("... retrieve " + dates + " in dir " + c.inputdir)
[64cf353]368
[ff99eae]369        try:
[5bad6ec]370            flexpart.retrieve(server, dates, c.public, c.request, c.inputdir)
[ff99eae]371        except IOError:
[f20af73]372            my_error('MARS request failed')
[efdb01a]373
[54a8a01]374        day += delta_t
[64cf353]375
[efdb01a]376    return
[d69b677]377
378if __name__ == "__main__":
[991df6a]379    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG