source: flex_extract.git/python/submit.py @ 97e09f4

ctbtodev
Last change on this file since 97e09f4 was 763fcf6, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

corrected function argument list of submit_job_to_ecgate

  • Property mode set to 100755
File size: 6.4 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - job submission on ecgate and cca
12#        - job templates suitable for twice daily operational dissemination
13#
14#    February 2018 - Anne Philipp (University of Vienna):
15#        - applied PEP8 style guide
16#        - added documentation
17#        - minor changes in programming style (for consistence)
18#
19# @License:
20#    (C) Copyright 2014-2018.
21#
22#    This software is licensed under the terms of the Apache Licence Version 2.0
23#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
24#
25# @Program Functionality:
26#    This program is the main program of flex_extract and controls the
27#    program flow.
28#    If it is supposed to work locally then it works through the necessary
29#    functions get_mars_data and prepareFlexpart. Otherwise it prepares
30#    a shell job script which will do the necessary work on the
31#    ECMWF server and is submitted via ecaccess-job-submit.
32#
33# @Program Content:
34#    - main
35#    - submit
36#
37#*******************************************************************************
38
39# ------------------------------------------------------------------------------
40# MODULES
41# ------------------------------------------------------------------------------
42import os
43import sys
44import subprocess
45import inspect
46import collections
47
48# software specific classes and modules from flex_extract
49import _config
50from tools import normal_exit, get_cmdline_arguments, submit_job_to_ecserver, \
51                  read_ecenv
52from get_mars_data import get_mars_data
53from prepare_flexpart import prepare_flexpart
54from ControlFile import ControlFile
55
56# ------------------------------------------------------------------------------
57# FUNCTIONS
58# ------------------------------------------------------------------------------
59
60def main():
61    '''
62    @Description:
63        Get the arguments from script call and from CONTROL file.
64        Decides from the argument "queue" if the local version
65        is done "queue=None" or the gateway version with "queue=ecgate"
66        or "queue=cca".
67
68    @Input:
69        <nothing>
70
71    @Return:
72        <nothing>
73    '''
74
75    called_from_dir = os.getcwd()
76
77    args = get_cmdline_arguments()
78
79    try:
80        c = ControlFile(args.controlfile)
81    except IOError:
82        try:
83            c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile)
84        except IOError:
85            print 'Could not read CONTROL file "' + args.controlfile + '"'
86            print 'Either it does not exist or its syntax is wrong.'
87            print 'Try "' + sys.argv[0].split('/')[-1] + \
88                  ' -h" to print usage information'
89            sys.exit(1)
90
91    env_parameter = read_ecenv(c.ecmwfdatadir + 'python/ECMWF_ENV')
92    c.assign_args_to_control(args)
93    c.assign_envs_to_control(env_parameter)
94    c.check_conditions()
95
96    # on local side
97    # on ECMWF server this would be the local side
98    if args.queue is None:
99        if c.inputdir[0] != '/':
100            c.inputdir = os.path.join(called_from_dir, c.inputdir)
101        if c.outputdir[0] != '/':
102            c.outputdir = os.path.join(called_from_dir, c.outputdir)
103        get_mars_data(c)
104        prepare_flexpart(args.ppid, c)
105        normal_exit(c.mailfail, 'Done!')
106    # on ECMWF server
107    else:
108        submit(args.job_template, c, args.queue)
109
110    return
111
112def submit(jtemplate, c, queue):
113    '''
114    @Description:
115        Prepares the job script and submit it to the specified queue.
116
117    @Input:
118        jtemplate: string
119            Job template file for submission to ECMWF. It contains all necessary
120            module and variable settings for the ECMWF environment as well as
121            the job call and mail report instructions.
122            Default is "job.temp".
123
124        c: instance of class ControlFile
125            Contains all the parameters of CONTROL file, which are e.g.:
126            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
127            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
128            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
129            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
130            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
131            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
132            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
133
134            For more information about format and content of the parameter
135            see documentation.
136
137        queue: string
138            Name of queue for submission to ECMWF (e.g. ecgate or cca )
139
140    @Return:
141        <nothing>
142    '''
143
144    # read template file and get index for CONTROL input
145    with open(jtemplate) as f:
146        lftext = f.read().split('\n')
147    insert_point = lftext.index('EOF')
148
149    if not c.basetime:
150    # --------- create on demand job script ------------------------------------
151        if c.maxstep > 24:
152            print '---- Pure forecast mode! ----'
153        else:
154            print '---- On-demand mode! ----'
155        job_file = jtemplate[:-4] + 'ksh'
156        clist = c.to_list()
157
158        lftextondemand = lftext[:insert_point] + clist + lftext[insert_point:]
159
160        with open(job_file, 'w') as f:
161            f.write('\n'.join(lftextondemand))
162
163        result_code = submit_job_to_ecserver(queue, job_file)
164
165    else:
166    # --------- create operational job script ----------------------------------
167        print '---- Operational mode! ----'
168        job_file = jtemplate[:-5] + 'oper.ksh'
169        #colist = []
170
171        if c.maxstep:
172            mt = int(c.maxstep)
173        else:
174            mt = 0
175
176        c.start_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
177        c.end_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
178        c.base_time = '${MSJ_BASETIME}'
179        if mt > 24:
180            c.time = '${MSJ_BASETIME} {MSJ_BASETIME}'
181
182        colist = c.to_list()
183
184        lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:]
185
186        with open(job_file, 'w') as f:
187            f.write('\n'.join(lftextoper))
188
189        result_code = submit_job_to_ecserver(queue, job_file)
190
191    # --------------------------------------------------------------------------
192    print 'You should get an email with subject flex.hostname.pid'
193
194    return
195
196if __name__ == "__main__":
197    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG