source: flex_extract.git/source/python/submit.py @ 403cbf1

ctbtodev
Last change on this file since 403cbf1 was 403cbf1, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

added header writing to marsrequest file and corrected bug in normalexit call in submit

  • Property mode set to 100755
File size: 5.9 KB
RevLine 
[d69b677]1#!/usr/bin/env python
[64cf353]2# -*- coding: utf-8 -*-
[991df6a]3#*******************************************************************************
4# @Author: Anne Fouilloux (University of Oslo)
5#
6# @Date: October 2014
7#
8# @Change History:
9#
10#    November 2015 - Leopold Haimberger (University of Vienna):
11#        - job submission on ecgate and cca
12#        - job templates suitable for twice daily operational dissemination
13#
14#    February 2018 - Anne Philipp (University of Vienna):
15#        - applied PEP8 style guide
16#        - added documentation
17#        - minor changes in programming style (for consistence)
[2fb99de]18#        - changed path names to variables from config file
19#        - added option for writing mars requests to extra file
20#          additionally,as option without submitting the mars jobs
[991df6a]21#
22# @License:
23#    (C) Copyright 2014-2018.
24#
25#    This software is licensed under the terms of the Apache Licence Version 2.0
26#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
27#
28# @Program Functionality:
29#    This program is the main program of flex_extract and controls the
30#    program flow.
31#    If it is supposed to work locally then it works through the necessary
[ff99eae]32#    functions get_mars_data and prepareFlexpart. Otherwise it prepares
[991df6a]33#    a shell job script which will do the necessary work on the
34#    ECMWF server and is submitted via ecaccess-job-submit.
35#
36# @Program Content:
37#    - main
38#    - submit
39#
40#*******************************************************************************
[efdb01a]41
[64cf353]42# ------------------------------------------------------------------------------
43# MODULES
44# ------------------------------------------------------------------------------
[efdb01a]45import os
46import sys
[d69b677]47import subprocess
48import inspect
[54a8a01]49import collections
[efdb01a]50
51# software specific classes and modules from flex_extract
[54a8a01]52import _config
[25b14be]53from mods.tools import (normal_exit, get_cmdline_arguments,
54                        submit_job_to_ecserver, read_ecenv)
55from mods.get_mars_data import get_mars_data
56from mods.prepare_flexpart import prepare_flexpart
57from classes.ControlFile import ControlFile
[991df6a]58
[64cf353]59# ------------------------------------------------------------------------------
60# FUNCTIONS
61# ------------------------------------------------------------------------------
[991df6a]62
[d69b677]63def main():
[274f9ef]64    '''Get the arguments from script call and from CONTROL file.
65    Decides from the argument "queue" if the local version
66    is done "queue=None" or the gateway version with "queue=ecgate"
67    or "queue=cca".
68
69    Parameters
70    ----------
[d69b677]71
[274f9ef]72    Return
73    ------
[64cf353]74
75    '''
[ff99eae]76
[54a8a01]77    args = get_cmdline_arguments()
[4971f63]78    c = ControlFile(args.controlfile)
[2fb99de]79
80    env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
[54a8a01]81    c.assign_args_to_control(args)
82    c.assign_envs_to_control(env_parameter)
[2fb99de]83    c.check_conditions(args.queue)
[ff99eae]84
[efdb01a]85    # on local side
[2fb99de]86    # on ECMWF server this would also be the local side
87    called_from_dir = os.getcwd()
[295ff45]88    if args.queue is None:
[64cf353]89        if c.inputdir[0] != '/':
[ff99eae]90            c.inputdir = os.path.join(called_from_dir, c.inputdir)
[64cf353]91        if c.outputdir[0] != '/':
[ff99eae]92            c.outputdir = os.path.join(called_from_dir, c.outputdir)
[54a8a01]93        get_mars_data(c)
[2fb99de]94        if c.request == 0 or c.request == 2:
95            prepare_flexpart(args.ppid, c)
[96e1533]96            exit_message = 'FLEX_EXTRACT IS DONE!'
[2fb99de]97        else:
[96e1533]98            exit_message = 'PRINTING MARS_REQUESTS DONE!'
99    # send files to ECMWF server
[d69b677]100    else:
[64cf353]101        submit(args.job_template, c, args.queue)
102
[403cbf1]103    normal_exit(exit_message)
[96e1533]104
[64cf353]105    return
106
107def submit(jtemplate, c, queue):
[274f9ef]108    '''Prepares the job script and submit it to the specified queue.
109
110    Parameters
111    ----------
112    jtemplate : :obj:`string`
113        Job template file from sub-directory "_templates" for
114        submission to ECMWF. It contains all necessary
115        module and variable settings for the ECMWF environment as well as
116        the job call and mail report instructions.
117        Default is "job.temp".
118
119    c : :obj:`ControlFile`
120        Contains all the parameters of CONTROL file and
121        command line.
122
123    queue : :obj:`string`
124        Name of queue for submission to ECMWF (e.g. ecgate or cca )
125
126    Return
127    ------
128
[64cf353]129    '''
130
[54a8a01]131    # read template file and get index for CONTROL input
[2fb99de]132    with open(os.path.join(_config.PATH_TEMPLATES, jtemplate)) as f:
[64cf353]133        lftext = f.read().split('\n')
[54a8a01]134    insert_point = lftext.index('EOF')
135
136    if not c.basetime:
137    # --------- create on demand job script ------------------------------------
138        if c.maxstep > 24:
[2fb99de]139            print('---- Pure forecast mode! ----')
[54a8a01]140        else:
[2fb99de]141            print('---- On-demand mode! ----')
142        job_file = os.path.join(_config.PATH_JOBSCRIPTS,
143                                jtemplate[:-4] + 'ksh')
[54a8a01]144        clist = c.to_list()
145
146        lftextondemand = lftext[:insert_point] + clist + lftext[insert_point:]
147
148        with open(job_file, 'w') as f:
149            f.write('\n'.join(lftextondemand))
150
[96e1533]151        job_id = submit_job_to_ecserver(queue, job_file)
[54a8a01]152
153    else:
154    # --------- create operational job script ----------------------------------
[2fb99de]155        print('---- Operational mode! ----')
156        job_file = os.path.join(_config.PATH_JOBSCRIPTS,
157                                jtemplate[:-5] + 'oper.ksh')
[54a8a01]158
159        if c.maxstep:
160            mt = int(c.maxstep)
161        else:
162            mt = 0
163
164        c.start_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
165        c.end_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
166        c.base_time = '${MSJ_BASETIME}'
167        if mt > 24:
168            c.time = '${MSJ_BASETIME} {MSJ_BASETIME}'
169
170        colist = c.to_list()
171
172        lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:]
173
174        with open(job_file, 'w') as f:
175            f.write('\n'.join(lftextoper))
176
[96e1533]177        job_id = submit_job_to_ecserver(queue, job_file)
[d69b677]178
[54a8a01]179    # --------------------------------------------------------------------------
[96e1533]180    print('The job id is: ' + str(job_id.strip()))
[2fb99de]181    print('You should get an email with subject flex.hostname.pid')
[d69b677]182
[64cf353]183    return
[d69b677]184
185if __name__ == "__main__":
186    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG