source: flex_extract.git/source/python/submit.py @ 25b14be

devfeature/makefilesorigin/task/language-editingsysinstalltask/language-editing
Last change on this file since 25b14be was 25b14be, checked in by Anne Philipp <anne.philipp@…>, 2 years ago

changed whole tree structure of flex_extract to have better overview

  • Property mode set to 100755
File size: 6.8 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - job submission on ecgate and cca
12#        - job templates suitable for twice daily operational dissemination
13#
14#    February 2018 - Anne Philipp (University of Vienna):
15#        - applied PEP8 style guide
16#        - added documentation
17#        - minor changes in programming style (for consistence)
18#        - changed path names to variables from config file
19#        - added option for writing mars requests to extra file
20#          additionally,as option without submitting the mars jobs
21#
22# @License:
23#    (C) Copyright 2014-2018.
24#
25#    This software is licensed under the terms of the Apache Licence Version 2.0
26#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
27#
28# @Program Functionality:
29#    This program is the main program of flex_extract and controls the
30#    program flow.
31#    If it is supposed to work locally then it works through the necessary
32#    functions get_mars_data and prepareFlexpart. Otherwise it prepares
33#    a shell job script which will do the necessary work on the
34#    ECMWF server and is submitted via ecaccess-job-submit.
35#
36# @Program Content:
37#    - main
38#    - submit
39#
40#*******************************************************************************
41
42# ------------------------------------------------------------------------------
43# MODULES
44# ------------------------------------------------------------------------------
45import os
46import sys
47import subprocess
48import inspect
49import collections
50
51# software specific classes and modules from flex_extract
52import _config
53from mods.tools import (normal_exit, get_cmdline_arguments,
54                        submit_job_to_ecserver, read_ecenv)
55from mods.get_mars_data import get_mars_data
56from mods.prepare_flexpart import prepare_flexpart
57from classes.ControlFile import ControlFile
58
59# ------------------------------------------------------------------------------
60# FUNCTIONS
61# ------------------------------------------------------------------------------
62
63def main():
64    '''
65    @Description:
66        Get the arguments from script call and from CONTROL file.
67        Decides from the argument "queue" if the local version
68        is done "queue=None" or the gateway version with "queue=ecgate"
69        or "queue=cca".
70
71    @Input:
72        <nothing>
73
74    @Return:
75        <nothing>
76    '''
77
78    args = get_cmdline_arguments()
79
80    try:
81        c = ControlFile(args.controlfile)
82    except IOError:
83        print('Could not read CONTROL file "' + args.controlfile + '"')
84        print('Either it does not exist or its syntax is wrong.')
85        print('Try "' + sys.argv[0].split('/')[-1] + \
86              ' -h" to print usage information')
87        sys.exit(1)
88
89    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
90    c.assign_args_to_control(args)
91    c.assign_envs_to_control(env_parameter)
92    c.check_conditions(args.queue)
93
94    # on local side
95    # on ECMWF server this would also be the local side
96    called_from_dir = os.getcwd()
97    if args.queue is None:
98        if c.inputdir[0] != '/':
99            c.inputdir = os.path.join(called_from_dir, c.inputdir)
100        if c.outputdir[0] != '/':
101            c.outputdir = os.path.join(called_from_dir, c.outputdir)
102        get_mars_data(c)
103        if c.request == 0 or c.request == 2:
104            prepare_flexpart(args.ppid, c)
105            normal_exit(c.mailfail, 'FLEX_EXTRACT IS DONE!')
106        else:
107            normal_exit(c.mailfail, 'PRINTING MARS_REQUESTS DONE!')
108    # send files to ECMWF server and install there
109    else:
110        submit(args.job_template, c, args.queue)
111
112    return
113
114def submit(jtemplate, c, queue):
115    '''
116    @Description:
117        Prepares the job script and submit it to the specified queue.
118
119    @Input:
120        jtemplate: string
121            Job template file from sub-directory "_templates" for
122            submission to ECMWF. It contains all necessary
123            module and variable settings for the ECMWF environment as well as
124            the job call and mail report instructions.
125            Default is "job.temp".
126
127        c: instance of class ControlFile
128            Contains all the parameters of CONTROL file, which are e.g.:
129            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
130            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
131            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
132            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
133            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
134            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
135            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
136
137            For more information about format and content of the parameter
138            see documentation.
139
140        queue: string
141            Name of queue for submission to ECMWF (e.g. ecgate or cca )
142
143    @Return:
144        <nothing>
145    '''
146
147    # read template file and get index for CONTROL input
148    with open(os.path.join(_config.PATH_TEMPLATES, jtemplate)) as f:
149        lftext = f.read().split('\n')
150    insert_point = lftext.index('EOF')
151
152    if not c.basetime:
153    # --------- create on demand job script ------------------------------------
154        if c.maxstep > 24:
155            print('---- Pure forecast mode! ----')
156        else:
157            print('---- On-demand mode! ----')
158        job_file = os.path.join(_config.PATH_JOBSCRIPTS,
159                                jtemplate[:-4] + 'ksh')
160        clist = c.to_list()
161
162        lftextondemand = lftext[:insert_point] + clist + lftext[insert_point:]
163
164        with open(job_file, 'w') as f:
165            f.write('\n'.join(lftextondemand))
166
167        submit_job_to_ecserver(queue, job_file)
168
169    else:
170    # --------- create operational job script ----------------------------------
171        print('---- Operational mode! ----')
172        job_file = os.path.join(_config.PATH_JOBSCRIPTS,
173                                jtemplate[:-5] + 'oper.ksh')
174
175        if c.maxstep:
176            mt = int(c.maxstep)
177        else:
178            mt = 0
179
180        c.start_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
181        c.end_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
182        c.base_time = '${MSJ_BASETIME}'
183        if mt > 24:
184            c.time = '${MSJ_BASETIME} {MSJ_BASETIME}'
185
186        colist = c.to_list()
187
188        lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:]
189
190        with open(job_file, 'w') as f:
191            f.write('\n'.join(lftextoper))
192
193        submit_job_to_ecserver(queue, job_file)
194
195    # --------------------------------------------------------------------------
196    print('You should get an email with subject flex.hostname.pid')
197
198    return
199
200if __name__ == "__main__":
201    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG