source: flex_extract.git/source/python/submit.py @ f20af73

dev
Last change on this file since f20af73 was f20af73, checked in by Anne Philipp <anne.philipp@…>, 5 months ago

added CDS API support for ERA5 instead of ECMWFAPI / refactored setup of CONTROL parameter because for local version it wanted to use not installed ECMWF_ENV file.

  • Property mode set to 100755
File size: 8.7 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - job submission on ecgate and cca
12#        - job templates suitable for twice daily operational dissemination
13#
14#    February 2018 - Anne Philipp (University of Vienna):
15#        - applied PEP8 style guide
16#        - added documentation
17#        - minor changes in programming style (for consistence)
18#        - changed path names to variables from config file
19#        - added option for writing mars requests to extra file
20#          additionally, as option without submitting the mars jobs
21#        - splitted submit function to use genshi templates for the
22#          job script and avoid code duplication
23#
24# @License:
25#    (C) Copyright 2014-2019.
26#    Anne Philipp, Leopold Haimberger
27#
28#    This work is licensed under the Creative Commons Attribution 4.0
29#    International License. To view a copy of this license, visit
30#    http://creativecommons.org/licenses/by/4.0/ or send a letter to
31#    Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
32#*******************************************************************************
33'''This script allows the user to extract meteorological fields from the ECMWF.
34
35It prepares the settings for retrieving the data from ECMWF servers and
36process the resulting files to prepare them for the use with FLEXPART or
37FLEXTRA.
38
39If it is supposed to work locally then it works through the necessary
40functions get_mars_data and prepare_flexpart. Otherwise it prepares
41a job script (korn shell) which will do the necessary work on the
42ECMWF server. This script will de submitted via the ecaccess command
43ecaccess-job-submit.
44
45This file can also be imported as a module which then contains the following
46functions:
47
48    * main - the main function of the script
49    * submit - calls mk_jobscript depending on operation mode and submits its
50    * mk_jobscript - creates the job script from a template
51
52Type: submit.py --help
53to get information about command line parameters.
54Read the documentation for usage instructions.
55'''
56
57# ------------------------------------------------------------------------------
58# MODULES
59# ------------------------------------------------------------------------------
60import os
61import sys
62import subprocess
63import inspect
64import collections
65from datetime import datetime, timedelta
66
67# software specific classes and modules from flex_extract
68import _config
69from mods.tools import (setup_controldata, normal_exit, get_cmdline_args,
70                        submit_job_to_ecserver, read_ecenv)
71from mods.get_mars_data import get_mars_data
72from mods.prepare_flexpart import prepare_flexpart
73from classes.ControlFile import ControlFile
74
75# ------------------------------------------------------------------------------
76# METHODS
77# ------------------------------------------------------------------------------
78
79def main():
80    '''Get the arguments from script call and from CONTROL file.
81    Decides from the argument "queue" if the local version
82    is done "queue=None" or the gateway version with "queue=ecgate"
83    or "queue=cca".
84
85    Parameters
86    ----------
87
88    Return
89    ------
90
91    '''
92
93    c, ppid, queue, job_template = setup_controldata()
94
95    # on local side
96    # starting from an ECMWF server this would also be the local side
97    called_from_dir = os.getcwd()
98    if queue is None:
99        if c.inputdir[0] != '/':
100            c.inputdir = os.path.join(called_from_dir, c.inputdir)
101        if c.outputdir[0] != '/':
102            c.outputdir = os.path.join(called_from_dir, c.outputdir)
103        get_mars_data(c)
104        if c.request == 0 or c.request == 2:
105            prepare_flexpart(ppid, c)
106            exit_message = 'FLEX_EXTRACT IS DONE!'
107        else:
108            exit_message = 'PRINTING MARS_REQUESTS DONE!'
109    # send files to ECMWF server
110    else:
111        submit(job_template, c, queue)
112        exit_message = 'FLEX_EXTRACT JOB SCRIPT IS SUBMITED!'
113
114    normal_exit(exit_message)
115
116    return
117
118def submit(jtemplate, c, queue):
119    '''Prepares the job script and submits it to the specified queue.
120
121    Parameters
122    ----------
123    jtemplate : str
124        Job template file from sub-directory "_templates" for
125        submission to ECMWF. It contains all necessary
126        module and variable settings for the ECMWF environment as well as
127        the job call and mail report instructions.
128        Default is "job.temp".
129
130    c : ControlFile
131        Contains all the parameters of CONTROL file and
132        command line.
133
134    queue : str
135        Name of queue for submission to ECMWF (e.g. ecgate or cca )
136
137    Return
138    ------
139
140    '''
141
142    if not c.oper:
143    # --------- create on demand job script ------------------------------------
144        if c.purefc:
145            print('---- Pure forecast mode! ----')
146        else:
147            print('---- On-demand mode! ----')
148
149        job_file = os.path.join(_config.PATH_JOBSCRIPTS,
150                                jtemplate[:-5] + '.ksh')
151
152        # divide time periode into specified number of job chunks
153        # to have multiple job scripts
154        if c.job_chunk:
155            start = datetime.strptime(c.start_date, '%Y%m%d')
156            end = datetime.strptime(c.end_date, '%Y%m%d')
157            chunk = timedelta(days=c.job_chunk)
158            oneday = timedelta(days=1)
159
160            while start <= end:
161                if (start + chunk) <= end:
162                    c.end_date = (start + chunk - oneday).strftime("%Y%m%d")
163                else:
164                    c.end_date = end.strftime("%Y%m%d")
165
166                clist = c.to_list()
167                mk_jobscript(jtemplate, job_file, clist)
168
169                job_id = submit_job_to_ecserver(queue, job_file)
170                print('The job id is: ' + str(job_id.strip()))
171
172                start = start + chunk
173                c.start_date = start.strftime("%Y%m%d")
174        # submit a single job script
175        else:
176            clist = c.to_list()
177
178            mk_jobscript(jtemplate, job_file, clist)
179
180            job_id = submit_job_to_ecserver(queue, job_file)
181            print('The job id is: ' + str(job_id.strip()))
182
183    else:
184    # --------- create operational job script ----------------------------------
185        print('---- Operational mode! ----')
186
187        job_file = os.path.join(_config.PATH_JOBSCRIPTS,
188                                jtemplate[:-5] + 'oper.ksh')
189
190        c.start_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
191        c.end_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
192        c.basetime = '${MSJ_BASETIME}'
193        if c.maxstep > 24:
194            c.time = '${MSJ_BASETIME} {MSJ_BASETIME}'
195
196        clist = c.to_list()
197
198        mk_jobscript(jtemplate, job_file, clist)
199
200        job_id = submit_job_to_ecserver(queue, job_file)
201        print('The job id is: ' + str(job_id.strip()))
202
203    print('You should get an email per job with subject flex.hostname.pid')
204
205    return
206
207def mk_jobscript(jtemplate, job_file, clist):
208    '''Creates the job script from template.
209
210    Parameters
211    ----------
212    jtemplate : str
213        Job template file from sub-directory "_templates" for
214        submission to ECMWF. It contains all necessary
215        module and variable settings for the ECMWF environment as well as
216        the job call and mail report instructions.
217        Default is "job.temp".
218
219    job_file : str
220        Path to the job script file.
221
222    clist : list of str
223        Contains all necessary parameters for ECMWF CONTROL file.
224
225    Return
226    ------
227
228    '''
229    from genshi.template.text import NewTextTemplate
230    from genshi.template import  TemplateLoader
231    from genshi.template.eval import UndefinedError
232
233    # load template and insert control content as list
234    try:
235        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
236        control_template = loader.load(jtemplate,
237                                       cls=NewTextTemplate)
238
239        stream = control_template.generate(control_content=clist)
240    except UndefinedError as e:
241        print('... ERROR ' + str(e))
242
243        sys.exit('\n... error occured while trying to generate jobscript')
244    except OSError as e:
245        print('... ERROR CODE: ' + str(e.errno))
246        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
247
248        sys.exit('\n... error occured while trying to generate jobscript')
249
250    # create jobscript file
251    try:
252        with open(job_file, 'w') as f:
253            f.write(stream.render('text'))
254    except OSError as e:
255        print('... ERROR CODE: ' + str(e.errno))
256        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
257
258        sys.exit('\n... error occured while trying to write ' + job_file)
259
260    return
261
262
263if __name__ == "__main__":
264    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG