source: flex_extract.git/source/python/submit.py @ a4531f1

ctbtodev
Last change on this file since a4531f1 was a4531f1, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

exchanged all if conditions for pure fc from 'maxstep > < 12/24' into purefc(==True)

  • Property mode set to 100755
File size: 7.2 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - job submission on ecgate and cca
12#        - job templates suitable for twice daily operational dissemination
13#
14#    February 2018 - Anne Philipp (University of Vienna):
15#        - applied PEP8 style guide
16#        - added documentation
17#        - minor changes in programming style (for consistence)
18#        - changed path names to variables from config file
19#        - added option for writing mars requests to extra file
20#          additionally,as option without submitting the mars jobs
21#
22# @License:
23#    (C) Copyright 2014-2018.
24#
25#    This software is licensed under the terms of the Apache Licence Version 2.0
26#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
27#
28# @Program Functionality:
29#    This program is the main program of flex_extract and controls the
30#    program flow.
31#    If it is supposed to work locally then it works through the necessary
32#    functions get_mars_data and prepareFlexpart. Otherwise it prepares
33#    a shell job script which will do the necessary work on the
34#    ECMWF server and is submitted via ecaccess-job-submit.
35#
36# @Program Content:
37#    - main
38#    - submit
39#
40#*******************************************************************************
41
42# ------------------------------------------------------------------------------
43# MODULES
44# ------------------------------------------------------------------------------
45import os
46import sys
47import subprocess
48import inspect
49import collections
50
51# software specific classes and modules from flex_extract
52import _config
53from mods.tools import (normal_exit, get_cmdline_arguments,
54                        submit_job_to_ecserver, read_ecenv)
55from mods.get_mars_data import get_mars_data
56from mods.prepare_flexpart import prepare_flexpart
57from classes.ControlFile import ControlFile
58
59# ------------------------------------------------------------------------------
60# FUNCTIONS
61# ------------------------------------------------------------------------------
62
63def main():
64    '''Get the arguments from script call and from CONTROL file.
65    Decides from the argument "queue" if the local version
66    is done "queue=None" or the gateway version with "queue=ecgate"
67    or "queue=cca".
68
69    Parameters
70    ----------
71
72    Return
73    ------
74
75    '''
76
77    args = get_cmdline_arguments()
78    c = ControlFile(args.controlfile)
79
80    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
81    c.assign_args_to_control(args)
82    c.assign_envs_to_control(env_parameter)
83    c.check_conditions(args.queue)
84
85    # on local side
86    # on ECMWF server this would also be the local side
87    called_from_dir = os.getcwd()
88    if args.queue is None:
89        if c.inputdir[0] != '/':
90            c.inputdir = os.path.join(called_from_dir, c.inputdir)
91        if c.outputdir[0] != '/':
92            c.outputdir = os.path.join(called_from_dir, c.outputdir)
93        get_mars_data(c)
94        if c.request == 0 or c.request == 2:
95            prepare_flexpart(args.ppid, c)
96            exit_message = 'FLEX_EXTRACT IS DONE!'
97        else:
98            exit_message = 'PRINTING MARS_REQUESTS DONE!'
99    # send files to ECMWF server
100    else:
101        submit(args.job_template, c, args.queue)
102        exit_message = 'FLEX_EXTRACT JOB SCRIPT IS SUBMITED!'
103
104    normal_exit(exit_message)
105
106    return
107
108def submit(jtemplate, c, queue):
109    '''Prepares the job script and submits it to the specified queue.
110
111    Parameters
112    ----------
113    jtemplate : :obj:`string`
114        Job template file from sub-directory "_templates" for
115        submission to ECMWF. It contains all necessary
116        module and variable settings for the ECMWF environment as well as
117        the job call and mail report instructions.
118        Default is "job.temp".
119
120    c : :obj:`ControlFile`
121        Contains all the parameters of CONTROL file and
122        command line.
123
124    queue : :obj:`string`
125        Name of queue for submission to ECMWF (e.g. ecgate or cca )
126
127    Return
128    ------
129
130    '''
131
132    if not c.basetime:
133    # --------- create on demand job script ------------------------------------
134        if c.purefc:
135            print('---- Pure forecast mode! ----')
136        else:
137            print('---- On-demand mode! ----')
138
139        job_file = os.path.join(_config.PATH_JOBSCRIPTS,
140                                jtemplate[:-5] + '.ksh')
141
142        clist = c.to_list()
143
144        mk_jobscript(jtemplate, job_file, clist)
145
146    else:
147    # --------- create operational job script ----------------------------------
148        print('---- Operational mode! ----')
149
150        job_file = os.path.join(_config.PATH_JOBSCRIPTS,
151                                jtemplate[:-5] + 'oper.ksh')
152
153        c.start_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
154        c.end_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
155        c.base_time = '${MSJ_BASETIME}'
156        if c.maxstep > 24:
157            c.time = '${MSJ_BASETIME} {MSJ_BASETIME}'
158
159        clist = c.to_list()
160
161        mk_jobscript(jtemplate, job_file, clist)
162
163    # --------- submit the job_script to the ECMWF server
164    job_id = submit_job_to_ecserver(queue, job_file)
165    print('The job id is: ' + str(job_id.strip()))
166    print('You should get an email with subject flex.hostname.pid')
167
168    return
169
170def mk_jobscript(jtemplate, job_file, clist):
171    '''Creates the job script from template.
172
173    Parameters
174    ----------
175    jtemplate : :obj:`string`
176        Job template file from sub-directory "_templates" for
177        submission to ECMWF. It contains all necessary
178        module and variable settings for the ECMWF environment as well as
179        the job call and mail report instructions.
180        Default is "job.temp".
181
182    job_file : :obj:`string`
183        Path to the job script file.
184
185    clist : :obj:`list` of :obj:`string`
186        Contains all necessary parameters for ECMWF CONTROL file.
187
188    Return
189    ------
190
191    '''
192    from genshi.template.text import NewTextTemplate
193    from genshi.template import  TemplateLoader
194    from genshi.template.eval import UndefinedError
195
196    # load template and insert control content as list
197    try:
198        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
199        control_template = loader.load(jtemplate,
200                                       cls=NewTextTemplate)
201
202        stream = control_template.generate(control_content=clist)
203    except UndefinedError as e:
204        print('... ERROR ' + str(e))
205
206        sys.exit('\n... error occured while trying to generate jobscript')
207    except OSError as e:
208        print('... ERROR CODE: ' + str(e.errno))
209        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
210
211        sys.exit('\n... error occured while trying to generate jobscript')
212
213    # create jobscript file
214    try:
215        with open(job_file, 'w') as f:
216            f.write(stream.render('text'))
217    except OSError as e:
218        print('... ERROR CODE: ' + str(e.errno))
219        print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
220
221        sys.exit('\n... error occured while trying to write ' + job_file)
222
223    return
224
225if __name__ == "__main__":
226    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG