source: flex_extract.git/python/prepare_flexpart.py @ 2fb99de

devfeature/makefilesorigin/task/language-editingtask/language-editing
Last change on this file since 2fb99de was 2fb99de, checked in by Anne Philipp <anne.philipp@…>, 22 months ago

introduced config with path definitions and changed py files accordingly; Installation works; some tests were added for tarball making; Problems in submission to ecgate

  • Property mode set to 100755
File size: 6.7 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - using the WebAPI also for general MARS retrievals
12#        - job submission on ecgate and cca
13#        - job templates suitable for twice daily operational dissemination
14#        - dividing retrievals of longer periods into digestable chunks
15#        - retrieve also longer term forecasts, not only analyses and
16#          short term forecast data
17#        - conversion into GRIB2
18#        - conversion into .fp format for faster execution of FLEXPART
19#
20#    February 2018 - Anne Philipp (University of Vienna):
21#        - applied PEP8 style guide
22#        - added documentation
23#        - minor changes in programming style for consistence
24#        - BUG: removed call of clean_up-Function after call of
25#               prepareFlexpart in main since it is already called in
26#               prepareFlexpart at the end!
27#        - created function main and moved the two function calls for
28#          arguments and prepare_flexpart into it
29#
30# @License:
31#    (C) Copyright 2014-2018.
32#
33#    This software is licensed under the terms of the Apache Licence Version 2.0
34#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
35#
36# @Program Functionality:
37#    This program prepares the final version of the grib files which are
38#    then used by FLEXPART. It converts the bunch of grib files extracted
39#    via get_mars_data by doing for example the necessary conversion to get
40#    consistent grids or the disaggregation of flux data. Finally, the
41#    program combines the data fields in files per available hour with the
42#    naming convention xxYYMMDDHH, where xx should be 2 arbitrary letters
43#    (mostly xx is chosen to be "EN").
44#
45# @Program Content:
46#    - main
47#    - prepare_flexpart
48#
49#*******************************************************************************
50
51# ------------------------------------------------------------------------------
52# MODULES
53# ------------------------------------------------------------------------------
54import datetime
55import os
56import inspect
57import sys
58import socket
59
60# software specific classes and modules from flex_extract
61import _config
62from UioFiles import UioFiles
63from tools import clean_up, get_cmdline_arguments, read_ecenv
64from EcFlexpart import EcFlexpart
65
66ecapi = 'ecmwf' not in socket.gethostname()
67try:
68    if ecapi:
69        import ecmwfapi
70except ImportError:
71    ecapi = False
72
73# ------------------------------------------------------------------------------
74# FUNCTION
75# ------------------------------------------------------------------------------
76def main():
77    '''
78    @Description:
79        If prepare_flexpart is called from command line, this function controls
80        the program flow and calls the argumentparser function and
81        the prepare_flexpart function for preparation of GRIB data for FLEXPART.
82
83    @Input:
84        <nothing>
85
86    @Return:
87        <nothing>
88    '''
89
90    args = get_cmdline_arguments()
91
92    try:
93        c = ControlFile(args.controlfile)
94    except IOError:
95        print('Could not read CONTROL file "' + args.controlfile + '"')
96        print('Either it does not exist or its syntax is wrong.')
97        print('Try "' + sys.argv[0].split('/')[-1] + \
98              ' -h" to print usage information')
99        sys.exit(1)
100
101    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
102    c.assign_args_to_control(args, env_parameter)
103    c.assign_envs_to_control(env_parameter)
104    c.check_conditions(args.queue)
105    prepare_flexpart(args.ppid, c)
106
107    return
108
109def prepare_flexpart(ppid, c):
110    '''
111    @Description:
112        Lists all grib files retrieved from MARS with get_mars_data and
113        uses prepares data for the use in FLEXPART. Specific data fields
114        are converted to a different grid and the flux data are going to be
115        disaggregated. The data fields are collected by hour and stored in
116        a file with a specific FLEXPART relevant naming convention.
117
118    @Input:
119        ppid: int
120            Contains the ppid number of the current ECMWF job. If it is called
121            from this script, it is "None".
122
123        c: instance of class ControlFile
124            Contains all the parameters of CONTROL file, which are e.g.:
125            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
126            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
127            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
128            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
129            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
130            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
131            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
132
133            For more information about format and content of the parameter
134            see documentation.
135
136    @Return:
137        <nothing>
138    '''
139
140    if not ppid:
141        c.ppid = str(os.getppid())
142    else:
143        c.ppid = ppid
144
145    c.ecapi = ecapi
146
147    # create the start and end date
148    start = datetime.date(year=int(c.start_date[:4]),
149                          month=int(c.start_date[4:6]),
150                          day=int(c.start_date[6:]))
151
152    end = datetime.date(year=int(c.end_date[:4]),
153                        month=int(c.end_date[4:6]),
154                        day=int(c.end_date[6:]))
155
156    # assign starting date minus 1 day
157    # since for basetime 00 we need the 12 hours upfront
158    # (the day before from 12 UTC to current day 00 UTC)
159    if c.basetime == '00':
160        start = start - datetime.timedelta(days=1)
161
162    print('Prepare ' + start.strftime("%Y%m%d") +
163           "/to/" + end.strftime("%Y%m%d"))
164
165    # create output dir if necessary
166    if not os.path.exists(c.outputdir):
167        os.makedirs(c.outputdir)
168
169    # get all files with flux data to be deaccumulated
170    inputfiles = UioFiles(c.inputdir, '*OG_acc_SL*.' + c.ppid + '.*')
171
172    # deaccumulate the flux data
173    flexpart = EcFlexpart(c, fluxes=True)
174    flexpart.write_namelist(c, _config.FILE_NAMELIST)
175    flexpart.deacc_fluxes(inputfiles, c)
176
177    # get a list of all files from the root inputdir
178    inputfiles = UioFiles(c.inputdir, '????__??.*' + c.ppid + '.*')
179
180    # produce FLEXPART-ready GRIB files and process them -
181    # copy/transfer/interpolate them or make them GRIB2
182    flexpart = EcFlexpart(c, fluxes=False)
183    flexpart.create(inputfiles, c)
184    flexpart.process_output(c)
185
186    # check if in debugging mode, then store all files
187    # otherwise delete temporary files
188    if int(c.debug) != 0:
189        print('\nTemporary files left intact')
190    else:
191        clean_up(c)
192
193    return
194
195if __name__ == "__main__":
196    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG