source: flex_extract.git/python/submit.py @ 64cf353

ctbtodev
Last change on this file since 64cf353 was 64cf353, checked in by Anne Philipp <bscannephilipp@…>, 6 years ago

pep8 changes + documentation added + minor code style changes

  • Property mode set to 100644
File size: 6.6 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#************************************************************************
4# TODO AP
5#AP
6# - Change History ist nicht angepasst ans File! Original geben lassen
7# - dead code ? what to do?
8# - seperate operational and reanlysis for clarification
9#************************************************************************
10"""
11@Author: Anne Fouilloux (University of Oslo)
12
13@Date: October 2014
14
15@ChangeHistory:
16    November 2015 - Leopold Haimberger (University of Vienna):
17        - using the WebAPI also for general MARS retrievals
18        - job submission on ecgate and cca
19        - job templates suitable for twice daily operational dissemination
20        - dividing retrievals of longer periods into digestable chunks
21        - retrieve also longer term forecasts, not only analyses and
22          short term forecast data
23        - conversion into GRIB2
24        - conversion into .fp format for faster execution of FLEXPART
25
26    February 2018 - Anne Philipp (University of Vienna):
27        - applied PEP8 style guide
28        - added documentation
29        - minor changes in programming style for consistence
30
31@License:
32    (C) Copyright 2014 UIO.
33
34    This software is licensed under the terms of the Apache Licence Version 2.0
35    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
36
37@Requirements:
38    - A standard python 2.6 or 2.7 installation
39    - dateutils
40    - matplotlib (optional, for debugging)
41    - ECMWF specific packages, all available from https://software.ecmwf.int/
42        ECMWF WebMARS, gribAPI with python enabled, emoslib and
43        ecaccess web toolkit
44
45@Description:
46    Further documentation may be obtained from www.flexpart.eu.
47
48    Functionality provided:
49        Prepare input 3D-wind fields in hybrid coordinates +
50        surface fields for FLEXPART runs
51"""
52# ------------------------------------------------------------------------------
53# MODULES
54# ------------------------------------------------------------------------------
55import calendar
56import shutil
57import datetime
58import time
59import os, sys, glob
60import subprocess
61import inspect
62# add path to submit.py to pythonpath so that python finds its buddies
63localpythonpath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
64sys.path.append(localpythonpath)
65from UIOTools import UIOFiles
66from string import strip
67from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
68from GribTools import GribTools
69from FlexpartTools import EIFlexpart, Control, interpret_args_and_control, normalexit, myerror
70from getMARSdata import getMARSdata
71from prepareFLEXPART import prepareFLEXPART
72# ------------------------------------------------------------------------------
73# FUNCTIONS
74# ------------------------------------------------------------------------------
75def main():
76    '''
77    @Description:
78        Get the arguments from script call and initialize an object from
79        Control class. Decides from the argument "queue" if the local version
80        is done "queue=None" or the gateway version "queue=ecgate".
81
82    @Input:
83        <nothing>
84
85    @Return:
86        <nothing>
87    '''
88    calledfromdir = os.getcwd()
89    args, c = interpret_args_and_control()
90    if args.queue is None:
91        if c.inputdir[0] != '/':
92            c.inputdir = os.path.join(calledfromdir, c.inputdir)
93        if c.outputdir[0] != '/':
94            c.outputdir = os.path.join(calledfromdir, c.outputdir)
95        getMARSdata(args, c)
96        prepareFLEXPART(args, c)
97        normalexit(c)
98    else:
99        submit(args.job_template, c, args.queue)
100
101    return
102
103def submit(jtemplate, c, queue):
104    #AP divide in two submits , ondemand und operational
105    '''
106    @Description:
107        Prepares the job script and submit it to the specified queue.
108
109    @Input:
110        jtemplate: string
111            Job template file for submission to ECMWF. It contains all necessary
112            module and variable settings for the ECMWF environment as well as
113            the job call and mail report instructions.
114            Default is "job.temp".
115
116        c: instance of class Control
117            Contains all necessary information of a control file. The parameters
118            are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
119            NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
120            RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
121            SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
122            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR
123            For more information about format and content of the parameter see
124            documentation.
125
126        queue: string
127            Name of queue for submission to ECMWF (e.g. ecgate or cca )
128
129    @Return:
130        <nothing>
131    '''
132
133    # read template file and split from newline signs
134    with open(jtemplate) as f:
135        lftext = f.read().split('\n')
136        insert_point = lftext.index('EOF')
137#AP es gibt mehrere EOFs überprüfen!
138
139    # put all parameters of control instance into a list
140    clist = c.tolist()  # reanalysis (EI)
141    colist = []  # operational
142    mt = 0
143
144#AP wieso 2 for loops?
145#AP dieser part ist für die CTBTO Operational retrieves bis zum aktuellen Tag.
146    for elem in clist:
147        if 'maxstep' in elem:
148            mt = int(elem.split(' ')[1])
149
150    for elem in clist:
151        if 'start_date' in elem:
152            elem = 'start_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
153        if 'end_date' in elem:
154#AP Fehler?! Muss end_date heissen
155            elem = 'start_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
156        if 'base_time' in elem:
157            elem = 'base_time ' + '${MSJ_BASETIME}'
158        if 'time' in elem and mt > 24:
159            elem = 'time ' + '${MSJ_BASETIME} {MSJ_BASETIME}'
160        colist.append(elem)
161#AP end
162
163#AP whats the difference between clist and colist ?! What is MSJ?
164
165    lftextondemand = lftext[:insert_point] + clist + lftext[insert_point + 2:]
166    lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:]
167
168    with open('job.ksh', 'w') as h:
169        h.write('\n'.join(lftextondemand))
170
171#AP this is not used ?! what is it for?
172#maybe a differentiation is needed
173    h = open('joboper.ksh', 'w')
174    h.write('\n'.join(lftextoper))
175    h.close()
176#AP end
177
178    # submit job script to queue
179    try:
180        p = subprocess.check_call(['ecaccess-job-submit', '-queueName',
181                                   queue,' job.ksh'])
182    except:
183        print('ecaccess-job-submit failed, probably eccert has expired')
184        exit(1)
185
186    print('You should get an email with subject flex.hostname.pid')
187
188    return
189
190if __name__ == "__main__":
191    main()
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG