Index: python/install.py
===================================================================
--- python/install.py	(revision e18f4b5f53f063ab746e1307bdd3162d31cb3e8a)
+++ python/install.py	(revision e18f4b5f53f063ab746e1307bdd3162d31cb3e8a)
@@ -0,0 +1,197 @@
+#!/usr/bin/env python
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Functionality provided: Prepare input 3D-wind fields in hybrid coordinates + surface fields for FLEXPART runs
+#
+# Creation: October  2014 - Anne Fouilloux - University of Oslo
+# Extension November 2015 - Leopold Haimberger - University of Vienna for:
+# - using the WebAPI also for general MARS retrievals
+# - job submission on ecgate and cca
+# - job templates suitable for twice daily operational dissemination
+# - dividing retrievals of longer periods into digestable chunks
+# - retrieve also longer term forecasts, not only analyses and short term forecast data
+# - conversion into GRIB2
+# - conversion into .fp format for faster execution of FLEXPART
+#
+#
+# Further documentation may be obtained from www.flexpart.eu
+#
+# Requirements:
+# in addition to a standard python 2.6 or 2.7 installation the following packages need to be installed
+# ECMWF WebMARS, gribAPI with python enabled, emoslib, ecaccess web toolkit, all available from https://software.ecmwf.int/
+# dateutils
+# matplotlib (optional, for debugging)
+#
+#
+import calendar
+import shutil
+import datetime
+import time
+import os,sys,glob
+import subprocess
+import inspect
+# add path to submit.py to pythonpath so that python finds its buddies
+localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+sys.path.append(localpythonpath)
+from UIOTools import UIOFiles
+from string import strip
+from argparse import ArgumentParser,ArgumentDefaultsHelpFormatter
+from GribTools import GribTools
+from FlexpartTools import EIFlexpart, Control, install_args_and_control
+from getMARSdata import getMARSdata
+from prepareFLEXPART import prepareFLEXPART
+
+
+
+def main():
+
+    calledfromdir=os.getcwd()
+    os.chdir(localpythonpath)
+    args,c=install_args_and_control()
+#    if c.outputdir[0]!='/':
+#	c.outputdir=os.path.join(calledfromdir,c.outputdir)
+#	c.inputdir=c.outputdir
+    if args.install_target!=None:
+        install_via_gateway(c,args.install_target)
+
+    else:
+        print 'Please specify installation target (local|ecgate|cca)'
+        print 'use -h or --help for help'
+    sys.exit()
+
+def install_via_gateway(c,target):
+
+    ecd=c.flexextractdir
+    template=ecd+'python/compilejob.temp'
+    job=ecd+'python/compilejob.ksh'
+    fo=open(job,'w')
+    with open(template) as f:
+        fdata = f.read().split('\n')
+        for data in fdata:
+            if 'MAKEFILE=' in data:
+                if c.makefile is not None:
+                    data='export MAKEFILE='+c.makefile
+            if 'FLEXPART_ROOT_SCRIPTS=' in data:
+                if c.flexpart_root_scripts!='../':
+                    data='export FLEXPART_ROOT_SCRIPTS='+c.flexpart_root_scripts
+                else:
+                    data='export FLEXPART_ROOT_SCRIPTS=$HOME'
+            if target.lower()!='local':
+                if '--workdir' in data:
+                    data='#SBATCH --workdir=/scratch/ms/'+c.ecgid+'/'+c.ecuid
+                if '##PBS -o' in data:
+                    data='##PBS -o /scratch/ms/'+c.ecgid+'/'+c.ecuid+'flex_ecmwf.$Jobname.$Job_ID.out'
+                if 'FLEXPART_ROOT_SCRIPTS=' in data:
+                    if c.ec_flexpart_root_scripts!='../':
+                        data='export FLEXPART_ROOT_SCRIPTS='+c.ec_flexpart_root_scripts
+                    else:
+                        data='export FLEXPART_ROOT_SCRIPTS=$HOME'
+
+            fo.write(data+'\n')
+    f.close()
+    fo.close()
+
+    if target.lower()!='local':
+        template=ecd+'python/job.temp.o'
+        with open(template) as f:
+            fdata = f.read().split('\n')
+        f.close()
+        fo=open(template[:-2],'w')
+        for data in fdata:
+            if '--workdir' in data:
+                data='#SBATCH --workdir=/scratch/ms/'+c.ecgid+'/'+c.ecuid
+            if '##PBS -o' in data:
+                data='##PBS -o /scratch/ms/'+c.ecgid+'/'+c.ecuid+'flex_ecmwf.$Jobname.$Job_ID.out'
+            if  'export PATH=${PATH}:' in data:
+                data+=c.ec_flexpart_root_scripts+'/flex_extract_v7.0.3/python'
+
+            if 'cat>>' in data or 'cat >>' in data:
+                i=data.index('>')
+                fo.write(data[:i]+data[i+1:]+'\n')
+                fo.write('GATEWAY '+c.gateway+'\n')
+                fo.write('DESTINATION '+c.destination+'\n')
+                fo.write('EOF\n')
+
+            fo.write(data+'\n')
+        fo.close()
+
+        job=ecd+'python/ECMWF_ENV'
+        fo=open(job,'w')
+        fo.write('ECUID '+c.ecuid+'\n')
+        fo.write('ECGID '+c.ecgid+'\n')
+        fo.write('GATEWAY '+c.gateway+'\n')
+        fo.write('DESTINATION '+c.destination+'\n')
+        fo.close()
+
+
+
+    if target.lower()=='local':
+        # compile CONVERT2
+        if c.flexpart_root_scripts==None or c.flexpart_root_scripts=='../':
+            print 'Warning: FLEXPART_ROOT_SCRIPTS has not been specified'
+            print 'Only CONVERT2 will be compiled in '+ecd+'/../src'
+        else:
+            c.flexpart_root_scripts=os.path.expandvars(os.path.expanduser(c.flexpart_root_scripts))
+            if os.path.abspath(ecd)!=os.path.abspath(c.flexpart_root_scripts):
+                os.chdir('/')
+                p=subprocess.check_call(['tar','-cvf',ecd+'../flex_extract_v7.0.3.tar',ecd+'python',ecd+'grib_templates',ecd+'src'])
+                try:
+                    os.makedirs(c.flexpart_root_scripts+'/flex_extract_v7.0.3')
+                except:
+                    pass
+                os.chdir(c.flexpart_root_scripts+'/flex_extract_v7.0.3')
+                p=subprocess.check_call(['tar','-xvf',ecd+'../flex_extract_v7.0.3.tar'])
+                os.chdir(c.flexpart_root_scripts+'/flex_extract_v7.0.3/src')
+
+        os.chdir('../src')
+        print 'install flex_extract_v7.0.3 software on '+target+' in directory '+os.getcwd()
+        if c.makefile==None:
+            makefile='Makefile.local.ifort'
+        else:
+            makefile=c.makefile
+        flist=glob.glob('*.mod')+glob.glob('*.o')
+        if flist:
+            p=subprocess.check_call(['rm']+flist)
+        try:
+            print 'Using makefile: '+makefile
+            p=subprocess.check_call(['make','-f',makefile])
+            p=subprocess.check_call(['ls','-l','CONVERT2'])
+        except:
+            print 'compile failed - please edit '+makefile+' or try another Makefile in the src directory.'
+            print 'most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB and EMOSLIB must be adapted.'
+            print 'Available Makefiles:'
+            print glob.glob('Makefile*')
+
+    elif target.lower()=='ecgate':
+        os.chdir('/')
+        p=subprocess.check_call(['tar','-cvf',ecd+'../flex_extract_v7.0.3.tar',ecd+'python',ecd+'grib_templates',ecd+'src'])
+        try:
+            p=subprocess.check_call(['ecaccess-file-put',ecd+'../flex_extract_v7.0.3.tar','ecgate:/home/ms/'+c.ecgid+'/'+c.ecuid+'/flex_extract_v7.0.3.tar'])
+        except:
+            print 'ecaccess-file-put failed! Probably the eccert key has expired.'
+            exit(1)
+        p=subprocess.check_call(['ecaccess-job-submit','-queueName',target,ecd+'python/compilejob.ksh'])
+        print 'compilejob.ksh has been submitted to ecgate for installation in '+c.ec_flexpart_root_scripts+'/flex_extract_v7.0.3'
+        print 'You should get an email with subject flexcompile within the next few minutes'
+    elif target.lower()=='cca':
+        os.chdir('/')
+        p=subprocess.check_call(['tar','-cvf',ecd+'../flex_extract_v7.0.3.tar',ecd+'python',ecd+'grib_templates',ecd+'src'])
+        try:
+            p=subprocess.check_call(['ecaccess-file-put',ecd+'../flex_extract_v7.0.3.tar','cca:/home/ms/'+c.ecgid+'/'+c.ecuid+'/flex_extract_v7.0.3.tar'])
+        except:
+            print 'ecaccess-file-put failed! Probably the eccert key has expired.'
+            exit(1)
+
+        p=subprocess.check_call(['ecaccess-job-submit','-queueName',target,ecd+'python/compilejob.ksh'])
+        print 'compilejob.ksh has been submitted to cca for installation in '+c.ec_flexpart_root_scripts+'/flex_extract_v7.0.3'
+        print 'You should get an email with subject flexcompile within the next few minutes'
+    else:
+        print 'ERROR: unknown installation target ',target
+        print 'Valid targets: ecgate, cca, local'
+    return
+
+
+if __name__ == "__main__":
+    main()
