source: flex_extract.git/python/submit.py @ efdb01a

ctbtodev
Last change on this file since efdb01a was efdb01a, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

whole bunch of modifications due to new structure of ECMWFDATA, added basics of documentation, minor programming corrections

  • Property mode set to 100755
File size: 5.6 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#************************************************************************
4# TODO AP
5#
6# - Change History ist nicht angepasst ans File! Original geben lassen
7# - dead code ? what to do?
8# - seperate operational and reanlysis for clarification
9# - add correct file description
10# - divide in two submits , ondemand und operational
11# -
12#************************************************************************
13"""
14@Author: Anne Fouilloux (University of Oslo)
15
16@Date: October 2014
17
18@ChangeHistory:
19    November 2015 - Leopold Haimberger (University of Vienna):
20        - job submission on ecgate and cca
21        - job templates suitable for twice daily operational dissemination
22
23    February 2018 - Anne Philipp (University of Vienna):
24        - applied PEP8 style guide
25        - added documentation
26        - minor changes in programming style (for consistence)
27
28@License:
29    (C) Copyright 2014-2018.
30
31    This software is licensed under the terms of the Apache Licence Version 2.0
32    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
33
34@Requirements:
35    - A standard python 2.6 or 2.7 installation
36
37@Description:
38    Further documentation may be obtained from www.flexpart.eu.
39
40
41"""
42# ------------------------------------------------------------------------------
43# MODULES
44# ------------------------------------------------------------------------------
45import os
46import sys
47import glob
48import subprocess
49import inspect
50# add the pythondir path so that python finds its buddies (from flex_extract)
51localpythonpath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
52sys.path.append(localpythonpath)
53
54# software specific classes and modules from flex_extract
55from Tools import interpret_args_and_control, normalexit
56from getMARSdata import getMARSdata
57from prepareFLEXPART import prepareFLEXPART
58# ------------------------------------------------------------------------------
59# FUNCTIONS
60# ------------------------------------------------------------------------------
61def main():
62    '''
63    @Description:
64        Get the arguments from script call and from Control file.
65        Decides from the argument "queue" if the local version
66        is done "queue=None" or the gateway version with "queue=ecgate"
67        or "queue=cca".
68
69    @Input:
70        <nothing>
71
72    @Return:
73        <nothing>
74    '''
75    calledfromdir = os.getcwd()
76    args, c = interpret_args_and_control()
77    # on local side
78    if args.queue is None:
79        if c.inputdir[0] != '/':
80            c.inputdir = os.path.join(calledfromdir, c.inputdir)
81        if c.outputdir[0] != '/':
82            c.outputdir = os.path.join(calledfromdir, c.outputdir)
83        getMARSdata(args, c)
84        prepareFLEXPART(args, c)
85        normalexit(c)
86    # on ECMWF server
87    else:
88        submit(args.job_template, c, args.queue)
89
90    return
91
92def submit(jtemplate, c, queue):
93    '''
94    @Description:
95        Prepares the job script and submit it to the specified queue.
96
97    @Input:
98        jtemplate: string
99            Job template file for submission to ECMWF. It contains all necessary
100            module and variable settings for the ECMWF environment as well as
101            the job call and mail report instructions.
102            Default is "job.temp".
103
104        c: instance of class Control
105            Contains all the parameters of control files, which are e.g.:
106            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
107            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
108            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
109            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
110            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
111            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
112            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
113
114            For more information about format and content of the parameter
115            see documentation.
116
117        queue: string
118            Name of queue for submission to ECMWF (e.g. ecgate or cca )
119
120    @Return:
121        <nothing>
122    '''
123
124    # read template file and split from newline signs
125    with open(jtemplate) as f:
126        lftext = f.read().split('\n')
127        insert_point = lftext.index('EOF')
128
129    # put all parameters of control instance into a list
130    clist = c.tolist()
131    colist = []  # operational
132    mt = 0
133
134#AP wieso 2 for loops?
135    for elem in clist:
136        if 'maxstep' in elem:
137            mt = int(elem.split(' ')[1])
138
139    for elem in clist:
140        if 'start_date' in elem:
141            elem = 'start_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
142        if 'end_date' in elem:
143#AP Fehler?! Muss end_date heissen
144            elem = 'start_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
145        if 'base_time' in elem:
146            elem = 'base_time ' + '${MSJ_BASETIME}'
147        if 'time' in elem and mt > 24:
148            elem = 'time ' + '${MSJ_BASETIME} {MSJ_BASETIME}'
149        colist.append(elem)
150
151    lftextondemand = lftext[:insert_point] + clist + lftext[insert_point + 2:]
152    lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:]
153
154    with open('job.ksh', 'w') as h:
155        h.write('\n'.join(lftextondemand))
156
157    with open('joboper.ksh', 'w') as h:
158        h.write('\n'.join(lftextoper))
159
160    # submit job script to queue
161    try:
162        p = subprocess.check_call(['ecaccess-job-submit', '-queueName',
163                                   queue, 'job.ksh'])
164    except:
165        print('ecaccess-job-submit failed, probably eccert has expired')
166        exit(1)
167
168    print('You should get an email with subject flex.hostname.pid')
169
170    return
171
172if __name__ == "__main__":
173    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG