source: flex_extract.git/python/submit.py @ e1228f3

ctbtodev
Last change on this file since e1228f3 was ff99eae, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

completed application of pep8 style guide and pylint investigations. added documentation almost everywhere

  • Property mode set to 100755
File size: 5.7 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - job submission on ecgate and cca
12#        - job templates suitable for twice daily operational dissemination
13#
14#    February 2018 - Anne Philipp (University of Vienna):
15#        - applied PEP8 style guide
16#        - added documentation
17#        - minor changes in programming style (for consistence)
18#
19# @License:
20#    (C) Copyright 2014-2018.
21#
22#    This software is licensed under the terms of the Apache Licence Version 2.0
23#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
24#
25# @Program Functionality:
26#    This program is the main program of flex_extract and controls the
27#    program flow.
28#    If it is supposed to work locally then it works through the necessary
29#    functions get_mars_data and prepareFlexpart. Otherwise it prepares
30#    a shell job script which will do the necessary work on the
31#    ECMWF server and is submitted via ecaccess-job-submit.
32#
33# @Program Content:
34#    - main
35#    - submit
36#
37#*******************************************************************************
38
39# ------------------------------------------------------------------------------
40# MODULES
41# ------------------------------------------------------------------------------
42import os
43import sys
44import subprocess
45import inspect
46
47# software specific classes and modules from flex_extract
48from tools import interpret_args_and_control, normal_exit
49from get_mars_data import get_mars_data
50from prepare_flexpart import prepare_flexpart
51
52# add path to pythonpath so that python finds its buddies
53LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
54    inspect.getfile(inspect.currentframe())))
55if LOCAL_PYTHON_PATH not in sys.path:
56    sys.path.append(LOCAL_PYTHON_PATH)
57
58# ------------------------------------------------------------------------------
59# FUNCTIONS
60# ------------------------------------------------------------------------------
61
62def main():
63    '''
64    @Description:
65        Get the arguments from script call and from CONTROL file.
66        Decides from the argument "queue" if the local version
67        is done "queue=None" or the gateway version with "queue=ecgate"
68        or "queue=cca".
69
70    @Input:
71        <nothing>
72
73    @Return:
74        <nothing>
75    '''
76
77    called_from_dir = os.getcwd()
78    args, c = interpret_args_and_control()
79
80    # on local side
81    if args.queue is None:
82        if c.inputdir[0] != '/':
83            c.inputdir = os.path.join(called_from_dir, c.inputdir)
84        if c.outputdir[0] != '/':
85            c.outputdir = os.path.join(called_from_dir, c.outputdir)
86        get_mars_data(args, c)
87        prepare_flexpart(args, c)
88        normal_exit(c)
89    # on ECMWF server
90    else:
91        submit(args.job_template, c, args.queue)
92
93    return
94
95def submit(jtemplate, c, queue):
96    '''
97    @Description:
98        Prepares the job script and submit it to the specified queue.
99
100    @Input:
101        jtemplate: string
102            Job template file for submission to ECMWF. It contains all necessary
103            module and variable settings for the ECMWF environment as well as
104            the job call and mail report instructions.
105            Default is "job.temp".
106
107        c: instance of class ControlFile
108            Contains all the parameters of CONTROL file, which are e.g.:
109            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
110            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
111            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
112            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
113            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
114            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
115            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
116
117            For more information about format and content of the parameter
118            see documentation.
119
120        queue: string
121            Name of queue for submission to ECMWF (e.g. ecgate or cca )
122
123    @Return:
124        <nothing>
125    '''
126
127    # read template file and split from newline signs
128    with open(jtemplate) as f:
129        lftext = f.read().split('\n')
130        insert_point = lftext.index('EOF')
131
132    # put all parameters of ControlFile instance into a list
133    clist = c.to_list() # ondemand
134    colist = []  # operational
135    mt = 0
136
137    for elem in clist:
138        if 'maxstep' in elem:
139            mt = int(elem.split(' ')[1])
140
141    for elem in clist:
142        if 'start_date' in elem:
143            elem = 'start_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
144        if 'end_date' in elem:
145            elem = 'end_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
146        if 'base_time' in elem:
147            elem = 'base_time ' + '${MSJ_BASETIME}'
148        if 'time' in elem and mt > 24:
149            elem = 'time ' + '${MSJ_BASETIME} {MSJ_BASETIME}'
150        colist.append(elem)
151
152    lftextondemand = lftext[:insert_point] + clist + lftext[insert_point + 2:]
153    lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:]
154
155    with open('job.ksh', 'w') as h:
156        h.write('\n'.join(lftextondemand))
157
158    with open('joboper.ksh', 'w') as h:
159        h.write('\n'.join(lftextoper))
160
161    # submit job script to queue
162    try:
163        p = subprocess.check_call(['ecaccess-job-submit', '-queueName',
164                                   queue, 'job.ksh'])
165    except subprocess.CalledProcessError as e:
166        print 'ecaccess-job-submit failed!'
167        print 'Error Message: '
168        print e.output
169        exit(1)
170
171    print 'You should get an email with subject flex.hostname.pid'
172
173    return
174
175if __name__ == "__main__":
176    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG