Changeset 64cf353 in flex_extract.git for python/submit.py


Ignore:
Timestamp:
Feb 8, 2018, 9:54:05 PM (6 years ago)
Author:
Anne Philipp <bscannephilipp@…>
Branches:
master, ctbto, dev
Children:
02c8c50
Parents:
6180177
Message:

pep8 changes + documentation added + minor code style changes

File:
1 edited

Legend:

Unmodified
Added
Removed

  • python/submit.py

    rd69b677 r64cf353  
    11#!/usr/bin/env python
    2 #             
    3 # This software is licensed under the terms of the Apache Licence Version 2.0
    4 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
    5 #
    6 # Functionality provided: Prepare input 3D-wind fields in hybrid coordinates + surface fields for FLEXPART runs
    7 #
    8 # Creation: October  2014 - Anne Fouilloux - University of Oslo
    9 # Extension November 2015 - Leopold Haimberger - University of Vienna for:
    10 # - using the WebAPI also for general MARS retrievals
    11 # - job submission on ecgate and cca
    12 # - job templates suitable for twice daily operational dissemination
    13 # - dividing retrievals of longer periods into digestable chunks
    14 # - retrieve also longer term forecasts, not only analyses and short term forecast data
    15 # - conversion into GRIB2
    16 # - conversion into .fp format for faster execution of FLEXPART
    17 #
    18 # Further documentation may be obtained from www.flexpart.eu
    19 #
    20 # Requirements:
    21 # in addition to a standard python 2.6 or 2.7 installation the following packages need to be installed
    22 # ECMWF WebMARS, gribAPI with python enabled, emoslib, ecaccess web toolkit, all available from https://software.ecmwf.int/
    23 # dateutils
    24 # matplotlib (optional, for debugging)
    25 #
    26 #
     2# -*- coding: utf-8 -*-
     3#************************************************************************
     4# TODO AP
     5#AP
     6# - Change History ist nicht angepasst ans File! Original geben lassen
     7# - dead code ? what to do?
     8# - seperate operational and reanlysis for clarification
     9#************************************************************************
     10"""
     11@Author: Anne Fouilloux (University of Oslo)
     12
     13@Date: October 2014
     14
     15@ChangeHistory:
     16    November 2015 - Leopold Haimberger (University of Vienna):
     17        - using the WebAPI also for general MARS retrievals
     18        - job submission on ecgate and cca
     19        - job templates suitable for twice daily operational dissemination
     20        - dividing retrievals of longer periods into digestable chunks
     21        - retrieve also longer term forecasts, not only analyses and
     22          short term forecast data
     23        - conversion into GRIB2
     24        - conversion into .fp format for faster execution of FLEXPART
     25
     26    February 2018 - Anne Philipp (University of Vienna):
     27        - applied PEP8 style guide
     28        - added documentation
     29        - minor changes in programming style for consistence
     30
     31@License:
     32    (C) Copyright 2014 UIO.
     33
     34    This software is licensed under the terms of the Apache Licence Version 2.0
     35    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
     36
     37@Requirements:
     38    - A standard python 2.6 or 2.7 installation
     39    - dateutils
     40    - matplotlib (optional, for debugging)
     41    - ECMWF specific packages, all available from https://software.ecmwf.int/
     42        ECMWF WebMARS, gribAPI with python enabled, emoslib and
     43        ecaccess web toolkit
     44
     45@Description:
     46    Further documentation may be obtained from www.flexpart.eu.
     47
     48    Functionality provided:
     49        Prepare input 3D-wind fields in hybrid coordinates +
     50        surface fields for FLEXPART runs
     51"""
     52# ------------------------------------------------------------------------------
     53# MODULES
     54# ------------------------------------------------------------------------------
    2755import calendar
    2856import shutil
    2957import datetime
    3058import time
    31 import os,sys,glob
     59import os, sys, glob
    3260import subprocess
    3361import inspect
    3462# add path to submit.py to pythonpath so that python finds its buddies
    35 localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
     63localpythonpath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
    3664sys.path.append(localpythonpath)
    3765from UIOTools import UIOFiles
    3866from string import strip
    39 from argparse import ArgumentParser,ArgumentDefaultsHelpFormatter
     67from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
    4068from GribTools import GribTools
    41 from FlexpartTools import EIFlexpart, Control, interpret_args_and_control, normalexit,myerror
     69from FlexpartTools import EIFlexpart, Control, interpret_args_and_control, normalexit, myerror
    4270from getMARSdata import getMARSdata
    4371from prepareFLEXPART import prepareFLEXPART
     72# ------------------------------------------------------------------------------
     73# FUNCTIONS
     74# ------------------------------------------------------------------------------
     75def main():
     76    '''
     77    @Description:
     78        Get the arguments from script call and initialize an object from
     79        Control class. Decides from the argument "queue" if the local version
     80        is done "queue=None" or the gateway version "queue=ecgate".
    4481
     82    @Input:
     83        <nothing>
    4584
    46 
    47 def main():
    48 
    49     calledfromdir=os.getcwd()
    50 #    os.chdir(localpythonpath)
    51     args,c=interpret_args_and_control()
    52     if args.queue==None:
    53         if c.inputdir[0]!='/':
    54             c.inputdir=os.path.join(calledfromdir,c.inputdir)
    55         if c.outputdir[0]!='/':
    56             c.outputdir=os.path.join(calledfromdir,c.outputdir)
    57         getMARSdata(args,c)
    58         prepareFLEXPART(args,c)
     85    @Return:
     86        <nothing>
     87    '''
     88    calledfromdir = os.getcwd()
     89    args, c = interpret_args_and_control()
     90    if args.queue is None:
     91        if c.inputdir[0] != '/':
     92            c.inputdir = os.path.join(calledfromdir, c.inputdir)
     93        if c.outputdir[0] != '/':
     94            c.outputdir = os.path.join(calledfromdir, c.outputdir)
     95        getMARSdata(args, c)
     96        prepareFLEXPART(args, c)
    5997        normalexit(c)
    6098    else:
    61         submit(args.job_template,c,args.queue)
    62        
    63        
    64 def submit(jtemplate,c,queue):
    65    
    66     f=open(jtemplate)
    67     lftext=f.read().split('\n')
    68     insert_point=lftext.index('EOF')
    69     f.close()
    70    
    71     clist=c.tolist()
    72     colist=[]
    73     mt=0
     99        submit(args.job_template, c, args.queue)
     100
     101    return
     102
     103def submit(jtemplate, c, queue):
     104    #AP divide in two submits , ondemand und operational
     105    '''
     106    @Description:
     107        Prepares the job script and submit it to the specified queue.
     108
     109    @Input:
     110        jtemplate: string
     111            Job template file for submission to ECMWF. It contains all necessary
     112            module and variable settings for the ECMWF environment as well as
     113            the job call and mail report instructions.
     114            Default is "job.temp".
     115
     116        c: instance of class Control
     117            Contains all necessary information of a control file. The parameters
     118            are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
     119            NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
     120            RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
     121            SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
     122            MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR
     123            For more information about format and content of the parameter see
     124            documentation.
     125
     126        queue: string
     127            Name of queue for submission to ECMWF (e.g. ecgate or cca )
     128
     129    @Return:
     130        <nothing>
     131    '''
     132
     133    # read template file and split from newline signs
     134    with open(jtemplate) as f:
     135        lftext = f.read().split('\n')
     136        insert_point = lftext.index('EOF')
     137#AP es gibt mehrere EOFs überprüfen!
     138
     139    # put all parameters of control instance into a list
     140    clist = c.tolist()  # reanalysis (EI)
     141    colist = []  # operational
     142    mt = 0
     143
     144#AP wieso 2 for loops?
     145#AP dieser part ist für die CTBTO Operational retrieves bis zum aktuellen Tag.
    74146    for elem in clist:
    75147        if 'maxstep' in elem:
    76             mt=int(elem.split(' ')[1])
    77            
     148            mt = int(elem.split(' ')[1])
     149
    78150    for elem in clist:
    79151        if 'start_date' in elem:
    80             elem='start_date '+'${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
     152            elem = 'start_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
    81153        if 'end_date' in elem:
    82             elem='start_date '+'${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
     154#AP Fehler?! Muss end_date heissen
     155            elem = 'start_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
    83156        if 'base_time' in elem:
    84             elem='base_time '+'${MSJ_BASETIME}'
    85         if 'time' in elem and mt>24:
    86             elem='time '+'${MSJ_BASETIME} {MSJ_BASETIME}'
     157            elem = 'base_time ' + '${MSJ_BASETIME}'
     158        if 'time' in elem and mt > 24:
     159            elem = 'time ' + '${MSJ_BASETIME} {MSJ_BASETIME}'
    87160        colist.append(elem)
    88        
    89     lftextondemand=lftext[:insert_point]+clist+lftext[insert_point+2:]
    90     lftextoper=lftext[:insert_point]+colist+lftext[insert_point+2:]
    91    
    92     h=open('job.ksh','w')
    93     h.write('\n'.join(lftextondemand)) 
     161#AP end
     162
     163#AP whats the difference between clist and colist ?! What is MSJ?
     164
     165    lftextondemand = lftext[:insert_point] + clist + lftext[insert_point + 2:]
     166    lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:]
     167
     168    with open('job.ksh', 'w') as h:
     169        h.write('\n'.join(lftextondemand))
     170
     171#AP this is not used ?! what is it for?
     172#maybe a differentiation is needed
     173    h = open('joboper.ksh', 'w')
     174    h.write('\n'.join(lftextoper))
    94175    h.close()
    95    
    96     h=open('joboper.ksh','w')
    97     h.write('\n'.join(lftextoper)) 
    98     h.close()
    99    
    100     try:   
    101         p=subprocess.check_call(['ecaccess-job-submit','-queueName',queue,'job.ksh'])
     176#AP end
     177
     178    # submit job script to queue
     179    try:
     180        p = subprocess.check_call(['ecaccess-job-submit', '-queueName',
     181                                   queue,' job.ksh'])
    102182    except:
    103         print 'ecaccess-job-submit failed, probably eccert has expired'
     183        print('ecaccess-job-submit failed, probably eccert has expired')
    104184        exit(1)
    105     #pout=p.communicate(input=s)[0]
    106     print 'You should get an email with subject flex.hostname.pid'
    107185
     186    print('You should get an email with subject flex.hostname.pid')
    108187
     188    return
    109189
    110            
    111190if __name__ == "__main__":
    112191    main()
Note: See TracChangeset for help on using the changeset viewer.
hosted by ZAMG