source: flex_extract.git/python/pythontest/TestInstallTar/test_untar/python/submit.py @ 2fb99de

ctbtodev
Last change on this file since 2fb99de was 2fb99de, checked in by Anne Philipp <anne.philipp@…>, 6 years ago

introduced config with path definitions and changed py files accordingly; Installation works; some tests were added for tarball making; Problems in submission to ecgate

  • Property mode set to 100755
File size: 6.5 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - job submission on ecgate and cca
12#        - job templates suitable for twice daily operational dissemination
13#
14#    February 2018 - Anne Philipp (University of Vienna):
15#        - applied PEP8 style guide
16#        - added documentation
17#        - minor changes in programming style (for consistence)
18#
19# @License:
20#    (C) Copyright 2014-2018.
21#
22#    This software is licensed under the terms of the Apache Licence Version 2.0
23#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
24#
25# @Program Functionality:
26#    This program is the main program of flex_extract and controls the
27#    program flow.
28#    If it is supposed to work locally then it works through the necessary
29#    functions get_mars_data and prepareFlexpart. Otherwise it prepares
30#    a shell job script which will do the necessary work on the
31#    ECMWF server and is submitted via ecaccess-job-submit.
32#
33# @Program Content:
34#    - main
35#    - submit
36#
37#*******************************************************************************
38
39# ------------------------------------------------------------------------------
40# MODULES
41# ------------------------------------------------------------------------------
42import os
43import sys
44import subprocess
45import inspect
46import collections
47
48# software specific classes and modules from flex_extract
49import _config
50from tools import normal_exit, get_cmdline_arguments, submit_job_to_ecserver, \
51                  read_ecenv
52from get_mars_data import get_mars_data
53from prepare_flexpart import prepare_flexpart
54from ControlFile import ControlFile
55
56# ------------------------------------------------------------------------------
57# FUNCTIONS
58# ------------------------------------------------------------------------------
59
60def main():
61    '''
62    @Description:
63        Get the arguments from script call and from CONTROL file.
64        Decides from the argument "queue" if the local version
65        is done "queue=None" or the gateway version with "queue=ecgate"
66        or "queue=cca".
67
68    @Input:
69        <nothing>
70
71    @Return:
72        <nothing>
73    '''
74
75    called_from_dir = os.getcwd()
76
77    args = get_cmdline_arguments()
78
79    try:
80        c = ControlFile(args.controlfile)
81    except IOError:
82        try:
83            c = ControlFile(_config.PATH_LOCAL_PYTHON + args.controlfile)
84        except IOError:
85            print 'Could not read CONTROL file "' + args.controlfile + '"'
86            print 'Either it does not exist or its syntax is wrong.'
87            print 'Try "' + sys.argv[0].split('/')[-1] + \
88                  ' -h" to print usage information'
89            sys.exit(1)
90
91    env_parameter = read_ecenv(c.ecmwfdatadir + 'python/ECMWF_ENV')
92    c.assign_args_to_control(args)
93    c.assign_envs_to_control(env_parameter)
94    c.check_conditions()
95
96    # on local side
97    # on ECMWF server this would also be the local side
98    if args.queue is None:
99        if c.inputdir[0] != '/':
100            c.inputdir = os.path.join(called_from_dir, c.inputdir)
101        if c.outputdir[0] != '/':
102            c.outputdir = os.path.join(called_from_dir, c.outputdir)
103        get_mars_data(c)
104        if c.request == 0 or c.request == 2:
105            prepare_flexpart(args.ppid, c)
106            normal_exit(c.mailfail, 'FLEX_EXTRACT IS DONE!')
107        else:
108            normal_exit(c.mailfail, 'PRINTING MARS_REQUESTS DONE!')
109    # on ECMWF server
110    else:
111        submit(args.job_template, c, args.queue)
112
113    return
114
115def submit(jtemplate, c, queue):
116    '''
117    @Description:
118        Prepares the job script and submit it to the specified queue.
119
120    @Input:
121        jtemplate: string
122            Job template file for submission to ECMWF. It contains all necessary
123            module and variable settings for the ECMWF environment as well as
124            the job call and mail report instructions.
125            Default is "job.temp".
126
127        c: instance of class ControlFile
128            Contains all the parameters of CONTROL file, which are e.g.:
129            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
130            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
131            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
132            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
133            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
134            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
135            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
136
137            For more information about format and content of the parameter
138            see documentation.
139
140        queue: string
141            Name of queue for submission to ECMWF (e.g. ecgate or cca )
142
143    @Return:
144        <nothing>
145    '''
146
147    # read template file and get index for CONTROL input
148    with open(jtemplate) as f:
149        lftext = f.read().split('\n')
150    insert_point = lftext.index('EOF')
151
152    if not c.basetime:
153    # --------- create on demand job script ------------------------------------
154        if c.maxstep > 24:
155            print '---- Pure forecast mode! ----'
156        else:
157            print '---- On-demand mode! ----'
158        job_file = jtemplate[:-4] + 'ksh'
159        clist = c.to_list()
160
161        lftextondemand = lftext[:insert_point] + clist + lftext[insert_point:]
162
163        with open(job_file, 'w') as f:
164            f.write('\n'.join(lftextondemand))
165
166        result_code = submit_job_to_ecserver(queue, job_file)
167
168    else:
169    # --------- create operational job script ----------------------------------
170        print '---- Operational mode! ----'
171        job_file = jtemplate[:-5] + 'oper.ksh'
172        #colist = []
173
174        if c.maxstep:
175            mt = int(c.maxstep)
176        else:
177            mt = 0
178
179        c.start_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
180        c.end_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
181        c.base_time = '${MSJ_BASETIME}'
182        if mt > 24:
183            c.time = '${MSJ_BASETIME} {MSJ_BASETIME}'
184
185        colist = c.to_list()
186
187        lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:]
188
189        with open(job_file, 'w') as f:
190            f.write('\n'.join(lftextoper))
191
192        result_code = submit_job_to_ecserver(queue, job_file)
193
194    # --------------------------------------------------------------------------
195    print 'You should get an email with subject flex.hostname.pid'
196
197    return
198
199if __name__ == "__main__":
200    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG