source: flex_extract.git/python/submit.py @ 02c8c50

devfeature/makefilesorigin/task/language-editingtask/language-editing
Last change on this file since 02c8c50 was 02c8c50, checked in by Anne Philipp <bscannephilipp@…>, 2 years ago

more changes in PEP8 style and slight modifications in coding style and naming. More documentation of functions.

  • Property mode set to 100644
File size: 6.7 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#************************************************************************
4# TODO AP
5#AP
6# - Change History ist nicht angepasst ans File! Original geben lassen
7# - dead code ? what to do?
8# - seperate operational and reanlysis for clarification
9#************************************************************************
10"""
11@Author: Anne Fouilloux (University of Oslo)
12
13@Date: October 2014
14
15@ChangeHistory:
16    November 2015 - Leopold Haimberger (University of Vienna):
17        - using the WebAPI also for general MARS retrievals
18        - job submission on ecgate and cca
19        - job templates suitable for twice daily operational dissemination
20        - dividing retrievals of longer periods into digestable chunks
21        - retrieve also longer term forecasts, not only analyses and
22          short term forecast data
23        - conversion into GRIB2
24        - conversion into .fp format for faster execution of FLEXPART
25
26    February 2018 - Anne Philipp (University of Vienna):
27        - applied PEP8 style guide
28        - added documentation
29        - minor changes in programming style for consistence
30
31@License:
32    (C) Copyright 2014 UIO.
33
34    This software is licensed under the terms of the Apache Licence Version 2.0
35    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
36
37@Requirements:
38    - A standard python 2.6 or 2.7 installation
39    - dateutils
40    - matplotlib (optional, for debugging)
41    - ECMWF specific packages, all available from https://software.ecmwf.int/
42        ECMWF WebMARS, gribAPI with python enabled, emoslib and
43        ecaccess web toolkit
44
45@Description:
46    Further documentation may be obtained from www.flexpart.eu.
47
48    Functionality provided:
49        Prepare input 3D-wind fields in hybrid coordinates +
50        surface fields for FLEXPART runs
51"""
52# ------------------------------------------------------------------------------
53# MODULES
54# ------------------------------------------------------------------------------
55import calendar
56import shutil
57import datetime
58import time
59import os, sys, glob
60import subprocess
61import inspect
62# add path to submit.py to pythonpath so that python finds its buddies
63localpythonpath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
64sys.path.append(localpythonpath)
65from UIOTools import UIOFiles
66from string import strip
67from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
68from GribTools import GribTools
69from FlexpartTools import ECFlexpart, Control, interpret_args_and_control, normalexit, myerror
70from getMARSdata import getMARSdata
71from prepareFLEXPART import prepareFLEXPART
72# ------------------------------------------------------------------------------
73# FUNCTIONS
74# ------------------------------------------------------------------------------
75def main():
76    '''
77    @Description:
78        Get the arguments from script call and initialize an object from
79        Control class. Decides from the argument "queue" if the local version
80        is done "queue=None" or the gateway version "queue=ecgate".
81
82    @Input:
83        <nothing>
84
85    @Return:
86        <nothing>
87    '''
88    calledfromdir = os.getcwd()
89    args, c = interpret_args_and_control()
90    if args.queue is None:
91        if c.inputdir[0] != '/':
92            c.inputdir = os.path.join(calledfromdir, c.inputdir)
93        if c.outputdir[0] != '/':
94            c.outputdir = os.path.join(calledfromdir, c.outputdir)
95        getMARSdata(args, c)
96        prepareFLEXPART(args, c)
97        normalexit(c)
98    else:
99        submit(args.job_template, c, args.queue)
100
101    return
102
103def submit(jtemplate, c, queue):
104    #AP divide in two submits , ondemand und operational
105    '''
106    @Description:
107        Prepares the job script and submit it to the specified queue.
108
109    @Input:
110        jtemplate: string
111            Job template file for submission to ECMWF. It contains all necessary
112            module and variable settings for the ECMWF environment as well as
113            the job call and mail report instructions.
114            Default is "job.temp".
115
116        c: instance of class Control
117            Contains all the parameters of control files, which are e.g.:
118            DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
119            STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
120            LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
121            OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
122            ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
123            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
124            DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
125
126            For more information about format and content of the parameter
127            see documentation.
128
129        queue: string
130            Name of queue for submission to ECMWF (e.g. ecgate or cca )
131
132    @Return:
133        <nothing>
134    '''
135
136    # read template file and split from newline signs
137    with open(jtemplate) as f:
138        lftext = f.read().split('\n')
139        insert_point = lftext.index('EOF')
140#AP es gibt mehrere EOFs überprüfen!
141
142    # put all parameters of control instance into a list
143    clist = c.tolist()  # reanalysis (EI)
144    colist = []  # operational
145    mt = 0
146
147#AP wieso 2 for loops?
148#AP dieser part ist für die CTBTO Operational retrieves bis zum aktuellen Tag.
149    for elem in clist:
150        if 'maxstep' in elem:
151            mt = int(elem.split(' ')[1])
152
153    for elem in clist:
154        if 'start_date' in elem:
155            elem = 'start_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
156        if 'end_date' in elem:
157#AP Fehler?! Muss end_date heissen
158            elem = 'start_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
159        if 'base_time' in elem:
160            elem = 'base_time ' + '${MSJ_BASETIME}'
161        if 'time' in elem and mt > 24:
162            elem = 'time ' + '${MSJ_BASETIME} {MSJ_BASETIME}'
163        colist.append(elem)
164#AP end
165
166#AP whats the difference between clist and colist ?! What is MSJ?
167
168    lftextondemand = lftext[:insert_point] + clist + lftext[insert_point + 2:]
169    lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:]
170
171    with open('job.ksh', 'w') as h:
172        h.write('\n'.join(lftextondemand))
173
174#AP this is not used ?! what is it for?
175#maybe a differentiation is needed
176    h = open('joboper.ksh', 'w')
177    h.write('\n'.join(lftextoper))
178    h.close()
179#AP end
180
181    # submit job script to queue
182    try:
183        p = subprocess.check_call(['ecaccess-job-submit', '-queueName',
184                                   queue,' job.ksh'])
185    except:
186        print('ecaccess-job-submit failed, probably eccert has expired')
187        exit(1)
188
189    print('You should get an email with subject flex.hostname.pid')
190
191    return
192
193if __name__ == "__main__":
194    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG