source: flex_extract.git/source/python/mods/get_mars_data.py @ dbf9d89

ctbtodev
Last change on this file since dbf9d89 was dbf9d89, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

fixed bug to address purefc variable of c instance

  • Property mode set to 100755
File size: 9.8 KB
RevLine 
[d69b677]1#!/usr/bin/env python
[efdb01a]2# -*- coding: utf-8 -*-
[991df6a]3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
[ff99eae]11#        - moved the getEIdata program into a function "get_mars_data"
[991df6a]12#        - moved the AgurmentParser into a seperate function
13#        - adatpted the function for the use in flex_extract
[ff99eae]14#        - renamed file to get_mars_data
[991df6a]15#
16#    February 2018 - Anne Philipp (University of Vienna):
17#        - applied PEP8 style guide
18#        - added structured documentation
19#        - minor changes in programming style for consistence
20#        - added function main and moved function calls vom __main__ there
21#          (necessary for better documentation with docstrings for later
22#          online documentation)
23#        - use of UIFiles class for file selection and deletion
[54a8a01]24#
[991df6a]25#
26# @License:
27#    (C) Copyright 2014-2018.
28#
29#    This software is licensed under the terms of the Apache Licence Version 2.0
30#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
31#
32# @Program Functionality:
33#    This program can be used as a module in the whole flex_extract process
34#    or can be run by itself to just extract MARS data from ECMWF. To do so,
35#    a couple of necessary parameters has to be passed with the program call.
36#    See documentation for more details.
37#
38# @Program Content:
39#    - main
[ff99eae]40#    - get_mars_data
[54a8a01]41#    - do_retrievement
[991df6a]42#
43#*******************************************************************************
[96e1533]44"""ToDo: Name of litte program
[991df6a]45
[96e1533]46ToDo: Add desccription
47
48ToDo: Add Conditions
49
50This script requires that `...` be installed within the Python
51environment you are running this script in.
52
53This file can also be imported as a module and contains the following
54functions:
55
56    * get_mars_data -
57    * do_retrievement -
58    * main - the main function of the script
59"""
[efdb01a]60# ------------------------------------------------------------------------------
61# MODULES
62# ------------------------------------------------------------------------------
[991df6a]63import os
64import sys
65import inspect
[ca867de]66from datetime import datetime, timedelta
[ff99eae]67
68# software specific classes and modules from flex_extract
[70fee58]69sys.path.append(os.path.dirname(os.path.abspath(
70    inspect.getfile(inspect.currentframe()))) + '/../')
[2fb99de]71import _config
[5bad6ec]72from tools import (my_error, normal_exit, get_cmdline_arguments,
73                   read_ecenv, make_dir)
[25b14be]74from classes.EcFlexpart import EcFlexpart
75from classes.UioFiles import UioFiles
[403cbf1]76from classes.MarsRetrieval import MarsRetrieval
[ca867de]77
78try:
79    ecapi = True
80    import ecmwfapi
81except ImportError:
82    ecapi = False
[efdb01a]83# ------------------------------------------------------------------------------
84# FUNCTION
85# ------------------------------------------------------------------------------
[991df6a]86def main():
[274f9ef]87    '''Controls the program to get data out of mars.
88
89    This is done if it is called directly from command line.
90    Then it also takes program call arguments and control file input.
[295ff45]91
[274f9ef]92    Parameters
93    ----------
[991df6a]94
[274f9ef]95    Return
96    ------
[991df6a]97
98    '''
[54a8a01]99
100    args = get_cmdline_arguments()
[4971f63]101    c = ControlFile(args.controlfile)
[54a8a01]102
[2fb99de]103    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
[295ff45]104    c.assign_args_to_control(args)
[54a8a01]105    c.assign_envs_to_control(env_parameter)
[2fb99de]106    c.check_conditions(args.queue)
[54a8a01]107
[ff99eae]108    get_mars_data(c)
[96e1533]109    normal_exit(c.mailops, c.queue, 'Done!')
[d69b677]110
[991df6a]111    return
112
[ff99eae]113def get_mars_data(c):
[274f9ef]114    '''Retrieves the EC data needed for a FLEXPART simulation.
115
116    Start and end dates for retrieval period is set. Retrievals
117    are divided into smaller periods if necessary and datechunk parameter
118    is set.
119
120    Parameters
121    ----------
122    c : :obj:`ControlFile`
123        Contains all the parameters of CONTROL file and
124        command line.
125
126    Return
127    ------
128
[991df6a]129    '''
[631ba13]130    c.ecapi = ecapi
[d69b677]131
132    if not os.path.exists(c.inputdir):
[5bad6ec]133        make_dir(c.inputdir)
[991df6a]134
[403cbf1]135    if c.request == 0:
[2fb99de]136        print("Retrieving EC data!")
[403cbf1]137    else:
138        if c.request == 1:
139            print("Printing mars requests!")
140        elif c.request == 2:
141            print("Retrieving EC data and printing mars request!")
[631ba13]142        write_reqheader(os.path.join(c.inputdir, _config.FILE_MARS_REQUESTS))
[2fb99de]143
144    print("start date %s " % (c.start_date))
145    print("end date %s " % (c.end_date))
[d69b677]146
[631ba13]147    server = mk_server(c)
148
149    # if data are to be retrieved, clean up any old grib files
150    if c.request == 0 or c.request == 2:
[1d15e27]151        remove_old('*grb', c.inputdir)
[631ba13]152
153    # --------------  flux data ------------------------------------------------
[45c6337]154    start, end, datechunk = mk_dates(c, fluxes=True)
[631ba13]155    do_retrievement(c, server, start, end, datechunk, fluxes=True)
156
157    # --------------  non flux data --------------------------------------------
[45c6337]158    start, end, datechunk = mk_dates(c, fluxes=False)
[631ba13]159    do_retrievement(c, server, start, end, datechunk, fluxes=False)
160
161    return
162
163def write_reqheader(marsfile):
164    '''Writes header with column names into mars request file.
165
166    Parameters
167    ----------
168    marsfile : :obj:`string`
169        Path to the mars request file.
170
171    Return
172    ------
173
174    '''
175    MR = MarsRetrieval(None, None)
176    attrs = vars(MR).copy()
177    del attrs['server']
178    del attrs['public']
179    with open(marsfile, 'w') as f:
180        f.write('request_number' + ', ')
181        f.write(', '.join(str(key) for key in sorted(attrs.iterkeys())))
182        f.write('\n')
183
184    return
185
186def mk_server(c):
187    '''Creates server connection if ECMWF WebAPI is available.
188
189    Parameters
190    ----------
191    c : :obj:`ControlFile`
192        Contains all the parameters of CONTROL file and
193        command line.
194
195    Return
196    ------
197    server : :obj:`ECMWFDataServer` or :obj:`ECMWFService`
198        Connection to ECMWF server via python interface ECMWF WebAPI.
199
200    '''
201    if c.ecapi:
[5bad6ec]202        if c.public:
203            server = ecmwfapi.ECMWFDataServer()
204        else:
205            server = ecmwfapi.ECMWFService("mars")
[d69b677]206    else:
207        server = False
208
[2fb99de]209    print('Using ECMWF WebAPI: ' + str(c.ecapi))
[991df6a]210
[631ba13]211    return server
[54a8a01]212
213
[631ba13]214def mk_dates(c, fluxes):
215    '''Prepares start and end date depending on flux or non flux data.
[54a8a01]216
[631ba13]217    If forecast for maximum one day (upto 24h) are to be retrieved, then
218    collect accumulation data (flux data) with additional days in the
219    beginning and at the end (used for complete disaggregation of
220    original period)
[54a8a01]221
[631ba13]222    If forecast data longer than 24h are to be retrieved, then
223    collect accumulation data (flux data) with the exact start and end date
224    (disaggregation will be done for the exact time period with
225    boundary conditions)
226
227    Since for basetime the extraction contains the 12 hours upfront,
228    if basetime is 0, the starting date has to be the day before and
229
230    Parameters
231    ----------
232    c : :obj:`ControlFile`
233        Contains all the parameters of CONTROL file and
234        command line.
235
236    fluxes : :obj:`boolean`, optional
237        Decides if the flux parameter settings are stored or
238        the rest of the parameter list.
239        Default value is False.
240
241    Return
242    ------
243    start : :obj:`datetime`
244        The start date of the retrieving data set.
245
246    end : :obj:`datetime`
247        The end date of the retrieving data set.
248
249    chunk : :obj:`datetime`
250        Time period in days for one single mars retrieval.
251
252    '''
[ca867de]253    start = datetime.strptime(c.start_date, '%Y%m%d')
254    end = datetime.strptime(c.end_date, '%Y%m%d')
[631ba13]255    chunk = timedelta(days=int(c.date_chunk))
256
257    if c.basetime:
258        if c.basetime == '00':
259            start = start - timedelta(days=1)
[efdb01a]260
[dbf9d89]261    if not c.purefc and fluxes:
[ca867de]262        start = start - timedelta(days=1)
[631ba13]263        end = end + timedelta(days=1)
[ca867de]264
[631ba13]265    return start, end, chunk
[54a8a01]266
[1d15e27]267def remove_old(pattern, inputdir):
[3f36e42]268    '''Deletes old retrieval files from current input directory
269    matching the pattern.
[54a8a01]270
[631ba13]271    Parameters
272    ----------
273    pattern : :obj:`string`
274        The sub string pattern which identifies the files to be deleted.
[991df6a]275
[1d15e27]276    inputdir : :obj:`string`, optional
277        Path to the directory where the retrieved data is stored.
278
[631ba13]279    Return
280    ------
[991df6a]281
[631ba13]282    '''
[1d15e27]283    print('... removing old content of ' + inputdir)
[631ba13]284
[1d15e27]285    tobecleaned = UioFiles(inputdir, pattern)
[631ba13]286    tobecleaned.delete_files()
[54a8a01]287
288    return
289
[631ba13]290
[54a8a01]291def do_retrievement(c, server, start, end, delta_t, fluxes=False):
[274f9ef]292    '''Divides the complete retrieval period in smaller chunks and
293    retrieves the data from MARS.
294
295    Parameters
296    ----------
297    c : :obj:`ControlFile`
298        Contains all the parameters of CONTROL file and
299        command line.
[54a8a01]300
[274f9ef]301    server : :obj:`ECMWFService`
302            The server connection to ECMWF.
[54a8a01]303
[274f9ef]304    start : :obj:`datetime`
305        The start date of the retrieval.
[54a8a01]306
[274f9ef]307    end : :obj:`datetime`
308        The end date of the retrieval.
[54a8a01]309
[274f9ef]310    delta_t : :obj:`datetime`
311        Delta_t + 1 is the maximal time period of a single
312        retrieval.
[54a8a01]313
[274f9ef]314    fluxes : :obj:`boolean`, optional
315        Decides if the flux parameters are to be retrieved or
316        the rest of the parameter list.
317        Default value is False.
[54a8a01]318
[274f9ef]319    Return
320    ------
[54a8a01]321
322    '''
323
324    # since actual day also counts as one day,
325    # we only need to add datechunk - 1 days to retrieval
326    # for a period
[ca867de]327    delta_t_m1 = delta_t - timedelta(days=1)
[54a8a01]328
[efdb01a]329    day = start
330    while day <= end:
[54a8a01]331        flexpart = EcFlexpart(c, fluxes)
332        tmpday = day + delta_t_m1
[ff99eae]333        if tmpday < end:
334            dates = day.strftime("%Y%m%d") + "/to/" + \
335                    tmpday.strftime("%Y%m%d")
336        else:
337            dates = day.strftime("%Y%m%d") + "/to/" + \
338                    end.strftime("%Y%m%d")
[efdb01a]339
[2fb99de]340        print("... retrieve " + dates + " in dir " + c.inputdir)
[64cf353]341
[ff99eae]342        try:
[5bad6ec]343            flexpart.retrieve(server, dates, c.public, c.request, c.inputdir)
[ff99eae]344        except IOError:
[54a8a01]345            my_error(c.mailfail, 'MARS request failed')
[efdb01a]346
[54a8a01]347        day += delta_t
[64cf353]348
[efdb01a]349    return
[d69b677]350
351if __name__ == "__main__":
[991df6a]352    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG