source: flex_extract.git/source/python/submit.py @ e811e1a

ctbtodev
Last change on this file since e811e1a was fc05fbd, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

BugFix? for job_chunk param. start_date counts as one full day and therefore we have to subtract one day from the 'inbetween' end_date to have to correct number of days per job.

  • Property mode set to 100755
File size: 8.9 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - job submission on ecgate and cca
12#        - job templates suitable for twice daily operational dissemination
13#
14#    February 2018 - Anne Philipp (University of Vienna):
15#        - applied PEP8 style guide
16#        - added documentation
17#        - minor changes in programming style (for consistence)
18#        - changed path names to variables from config file
19#        - added option for writing mars requests to extra file
20#          additionally, as option without submitting the mars jobs
21#        - splitted submit function to use genshi templates for the
22#          job script and avoid code duplication
23#
24# @License:
25#    (C) Copyright 2014-2019.
26#    Anne Philipp, Leopold Haimberger
27#
28#    This work is licensed under the Creative Commons Attribution 4.0
29#    International License. To view a copy of this license, visit
30#    http://creativecommons.org/licenses/by/4.0/ or send a letter to
31#    Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
32#*******************************************************************************
33'''This script allows the user to extract meteorological fields from the ECMWF.
34
35It prepares the settings for retrieving the data from ECMWF servers and
36process the resulting files to prepare them for the use with FLEXPART or
37FLEXTRA.
38
39If it is supposed to work locally then it works through the necessary
40functions get_mars_data and prepare_flexpart. Otherwise it prepares
41a job script (korn shell) which will do the necessary work on the
42ECMWF server. This script will de submitted via the ecaccess command
43ecaccess-job-submit.
44
45This file can also be imported as a module which then contains the following
46functions:
47
48    * main - the main function of the script
49    * submit - calls mk_jobscript depending on operation mode and submits its
50    * mk_jobscript - creates the job script from a template
51
52Type: submit.py --help
53to get information about command line parameters.
54Read the documentation for usage instructions.
55'''
56
57# ------------------------------------------------------------------------------
58# MODULES
59# ------------------------------------------------------------------------------
60import os
61import sys
62import subprocess
63import inspect
64import collections
65from datetime import datetime, timedelta
66
67# software specific classes and modules from flex_extract
68import _config
69from mods.tools import (normal_exit, get_cmdline_args,
70                        submit_job_to_ecserver, read_ecenv)
71from mods.get_mars_data import get_mars_data
72from mods.prepare_flexpart import prepare_flexpart
73from classes.ControlFile import ControlFile
74
75# ------------------------------------------------------------------------------
76# METHODS
77# ------------------------------------------------------------------------------
78
79def main():
80    '''Get the arguments from script call and from CONTROL file.
81    Decides from the argument "queue" if the local version
82    is done "queue=None" or the gateway version with "queue=ecgate"
83    or "queue=cca".
84
85    Parameters
86    ----------
87
88    Return
89    ------
90
91    '''
92
93    args = get_cmdline_args()
94    c = ControlFile(args.controlfile)
95
96    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
97    c.assign_args_to_control(args)
98    c.assign_envs_to_control(env_parameter)
99    c.check_conditions(args.queue)
100
101    # on local side
102    # on ECMWF server this would also be the local side
103    called_from_dir = os.getcwd()
104    if args.queue is None:
105        if c.inputdir[0] != '/':
106            c.inputdir = os.path.join(called_from_dir, c.inputdir)
107        if c.outputdir[0] != '/':
108            c.outputdir = os.path.join(called_from_dir, c.outputdir)
109        get_mars_data(c)
110        if c.request == 0 or c.request == 2:
111            prepare_flexpart(args.ppid, c)
112            exit_message = 'FLEX_EXTRACT IS DONE!'
113        else:
114            exit_message = 'PRINTING MARS_REQUESTS DONE!'
115    # send files to ECMWF server
116    else:
117        submit(args.job_template, c, args.queue)
118        exit_message = 'FLEX_EXTRACT JOB SCRIPT IS SUBMITED!'
119
120    normal_exit(exit_message)
121
122    return
123
124def submit(jtemplate, c, queue):
125    '''Prepares the job script and submits it to the specified queue.
126
127    Parameters
128    ----------
129    jtemplate : str
130        Job template file from sub-directory "_templates" for
131        submission to ECMWF. It contains all necessary
132        module and variable settings for the ECMWF environment as well as
133        the job call and mail report instructions.
134        Default is "job.temp".
135
136    c : ControlFile
137        Contains all the parameters of CONTROL file and
138        command line.
139
140    queue : str
141        Name of queue for submission to ECMWF (e.g. ecgate or cca )
142
143    Return
144    ------
145
146    '''
147
148    if not c.oper:
149    # --------- create on demand job script ------------------------------------
150        if c.purefc:
151            print('---- Pure forecast mode! ----')
152        else:
153            print('---- On-demand mode! ----')
154
155        job_file = os.path.join(_config.PATH_JOBSCRIPTS,
156                                jtemplate[:-5] + '.ksh')
157
158        # divide time periode into specified number of job chunks
159        # to have multiple job scripts
160        if c.job_chunk:
161            start = datetime.strptime(c.start_date, '%Y%m%d')
162            end = datetime.strptime(c.end_date, '%Y%m%d')
163            chunk = timedelta(days=c.job_chunk)
164            oneday = timedelta(days=1)
165
166            while start <= end:
167                if (start + chunk) <= end:
168                    c.end_date = (start + chunk - oneday).strftime("%Y%m%d")
169                else:
170                    c.end_date = end.strftime("%Y%m%d")
171
172                clist = c.to_list()
173                mk_jobscript(jtemplate, job_file, clist)
174
175                job_id = submit_job_to_ecserver(queue, job_file)
176                print('The job id is: ' + str(job_id.strip()))
177
178                start = start + chunk
179                c.start_date = start.strftime("%Y%m%d")
180        # submit a single job script
181        else:
182            clist = c.to_list()
183
184            mk_jobscript(jtemplate, job_file, clist)
185
186            job_id = submit_job_to_ecserver(queue, job_file)
187            print('The job id is: ' + str(job_id.strip()))
188
189    else:
190    # --------- create operational job script ----------------------------------
191        print('---- Operational mode! ----')
192
193        job_file = os.path.join(_config.PATH_JOBSCRIPTS,
194                                jtemplate[:-5] + 'oper.ksh')
195
196        c.start_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
197        c.end_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
198        c.basetime = '${MSJ_BASETIME}'
199        if c.maxstep > 24:
200            c.time = '${MSJ_BASETIME} {MSJ_BASETIME}'
201
202        clist = c.to_list()
203
204        mk_jobscript(jtemplate, job_file, clist)
205
206        job_id = submit_job_to_ecserver(queue, job_file)
207        print('The job id is: ' + str(job_id.strip()))
208
209    print('You should get an email per job with subject flex.hostname.pid')
210
211    return
212
213def mk_jobscript(jtemplate, job_file, clist):
214    '''Creates the job script from template.
215
216    Parameters
217    ----------
218    jtemplate : str
219        Job template file from sub-directory "_templates" for
220        submission to ECMWF. It contains all necessary
221        module and variable settings for the ECMWF environment as well as
222        the job call and mail report instructions.
223        Default is "job.temp".
224
225    job_file : str
226        Path to the job script file.
227
228    clist : list of str
229        Contains all necessary parameters for ECMWF CONTROL file.
230
231    Return
232    ------
233
234    '''
235    from genshi.template.text import NewTextTemplate
236    from genshi.template import  TemplateLoader
237    from genshi.template.eval import UndefinedError
238
239    # load template and insert control content as list
240    try:
241        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
242        control_template = loader.load(jtemplate,
243                                       cls=NewTextTemplate)
244
245        stream = control_template.generate(control_content=clist)
246    except UndefinedError as e:
247        print('... ERROR ' + str(e))
248
249        sys.exit('\n... error occured while trying to generate jobscript')
250    except OSError as e:
251        print('... ERROR CODE: ' + str(e.errno))
252        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
253
254        sys.exit('\n... error occured while trying to generate jobscript')
255
256    # create jobscript file
257    try:
258        with open(job_file, 'w') as f:
259            f.write(stream.render('text'))
260    except OSError as e:
261        print('... ERROR CODE: ' + str(e.errno))
262        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
263
264        sys.exit('\n... error occured while trying to write ' + job_file)
265
266    return
267
268
269if __name__ == "__main__":
270    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG