source: flex_extract.git/source/python/submit.py @ 7b4e39e

ctbtodev
Last change on this file since 7b4e39e was 4971f63, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

eliminated some redundancy and exchanged CONTROL2 with the config-filename

  • Property mode set to 100755
File size: 6.5 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - job submission on ecgate and cca
12#        - job templates suitable for twice daily operational dissemination
13#
14#    February 2018 - Anne Philipp (University of Vienna):
15#        - applied PEP8 style guide
16#        - added documentation
17#        - minor changes in programming style (for consistence)
18#        - changed path names to variables from config file
19#        - added option for writing mars requests to extra file
20#          additionally,as option without submitting the mars jobs
21#
22# @License:
23#    (C) Copyright 2014-2018.
24#
25#    This software is licensed under the terms of the Apache Licence Version 2.0
26#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
27#
28# @Program Functionality:
29#    This program is the main program of flex_extract and controls the
30#    program flow.
31#    If it is supposed to work locally then it works through the necessary
32#    functions get_mars_data and prepareFlexpart. Otherwise it prepares
33#    a shell job script which will do the necessary work on the
34#    ECMWF server and is submitted via ecaccess-job-submit.
35#
36# @Program Content:
37#    - main
38#    - submit
39#
40#*******************************************************************************
41
42# ------------------------------------------------------------------------------
43# MODULES
44# ------------------------------------------------------------------------------
45import os
46import sys
47import subprocess
48import inspect
49import collections
50
51# software specific classes and modules from flex_extract
52import _config
53from mods.tools import (normal_exit, get_cmdline_arguments,
54                        submit_job_to_ecserver, read_ecenv)
55from mods.get_mars_data import get_mars_data
56from mods.prepare_flexpart import prepare_flexpart
57from classes.ControlFile import ControlFile
58
59# ------------------------------------------------------------------------------
60# FUNCTIONS
61# ------------------------------------------------------------------------------
62
63def main():
64    '''
65    @Description:
66        Get the arguments from script call and from CONTROL file.
67        Decides from the argument "queue" if the local version
68        is done "queue=None" or the gateway version with "queue=ecgate"
69        or "queue=cca".
70
71    @Input:
72        <nothing>
73
74    @Return:
75        <nothing>
76    '''
77
78    args = get_cmdline_arguments()
79    c = ControlFile(args.controlfile)
80
81    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
82    c.assign_args_to_control(args)
83    c.assign_envs_to_control(env_parameter)
84    c.check_conditions(args.queue)
85
86    # on local side
87    # on ECMWF server this would also be the local side
88    called_from_dir = os.getcwd()
89    if not args.queue:
90        if c.inputdir[0] != '/':
91            c.inputdir = os.path.join(called_from_dir, c.inputdir)
92        if c.outputdir[0] != '/':
93            c.outputdir = os.path.join(called_from_dir, c.outputdir)
94        get_mars_data(c)
95        if c.request == 0 or c.request == 2:
96            prepare_flexpart(args.ppid, c)
97            normal_exit(c.mailfail, 'FLEX_EXTRACT IS DONE!')
98        else:
99            normal_exit(c.mailfail, 'PRINTING MARS_REQUESTS DONE!')
100    # send files to ECMWF server and install there
101    else:
102        submit(args.job_template, c, args.queue)
103
104    return
105
106def submit(jtemplate, c, queue):
107    '''
108    @Description:
109        Prepares the job script and submit it to the specified queue.
110
111    @Input:
112        jtemplate: string
113            Job template file from sub-directory "_templates" for
114            submission to ECMWF. It contains all necessary
115            module and variable settings for the ECMWF environment as well as
116            the job call and mail report instructions.
117            Default is "job.temp".
118
119        c: instance of class ControlFile
120            Contains all the parameters of CONTROL file, which are e.g.:
121            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
122            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
123            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
124            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
125            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
126            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
127            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
128
129            For more information about format and content of the parameter
130            see documentation.
131
132        queue: string
133            Name of queue for submission to ECMWF (e.g. ecgate or cca )
134
135    @Return:
136        <nothing>
137    '''
138
139    # read template file and get index for CONTROL input
140    with open(os.path.join(_config.PATH_TEMPLATES, jtemplate)) as f:
141        lftext = f.read().split('\n')
142    insert_point = lftext.index('EOF')
143
144    if not c.basetime:
145    # --------- create on demand job script ------------------------------------
146        if c.maxstep > 24:
147            print('---- Pure forecast mode! ----')
148        else:
149            print('---- On-demand mode! ----')
150        job_file = os.path.join(_config.PATH_JOBSCRIPTS,
151                                jtemplate[:-4] + 'ksh')
152        clist = c.to_list()
153
154        lftextondemand = lftext[:insert_point] + clist + lftext[insert_point:]
155
156        with open(job_file, 'w') as f:
157            f.write('\n'.join(lftextondemand))
158
159        submit_job_to_ecserver(queue, job_file)
160
161    else:
162    # --------- create operational job script ----------------------------------
163        print('---- Operational mode! ----')
164        job_file = os.path.join(_config.PATH_JOBSCRIPTS,
165                                jtemplate[:-5] + 'oper.ksh')
166
167        if c.maxstep:
168            mt = int(c.maxstep)
169        else:
170            mt = 0
171
172        c.start_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
173        c.end_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
174        c.base_time = '${MSJ_BASETIME}'
175        if mt > 24:
176            c.time = '${MSJ_BASETIME} {MSJ_BASETIME}'
177
178        colist = c.to_list()
179
180        lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:]
181
182        with open(job_file, 'w') as f:
183            f.write('\n'.join(lftextoper))
184
185        submit_job_to_ecserver(queue, job_file)
186
187    # --------------------------------------------------------------------------
188    print('You should get an email with subject flex.hostname.pid')
189
190    return
191
192if __name__ == "__main__":
193    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG