Changeset a4b6cef in flex_extract.git
- Timestamp:
- Feb 5, 2018, 2:16:58 PM (6 years ago)
- Branches:
- master, ctbto, dev
- Children:
- 507d47f
- Parents:
- b780393
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
python/install.py
rd69b677 ra4b6cef 1 1 #!/usr/bin/env python 2 # 2 # 3 3 # This software is licensed under the terms of the Apache Licence Version 2.0 4 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 # 4 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 # 6 6 # Functionality provided: Prepare input 3D-wind fields in hybrid coordinates + surface fields for FLEXPART runs 7 7 # 8 8 # Creation: October 2014 - Anne Fouilloux - University of Oslo 9 # Extension November 2015 - Leopold Haimberger - University of Vienna for: 9 # Extension November 2015 - Leopold Haimberger - University of Vienna for: 10 10 # - using the WebAPI also for general MARS retrievals 11 11 # - job submission on ecgate and cca … … 15 15 # - conversion into GRIB2 16 16 # - conversion into .fp format for faster execution of FLEXPART 17 # 17 # 18 18 # 19 19 # Further documentation may be obtained from www.flexpart.eu 20 # 21 # Requirements: 20 # 21 # Requirements: 22 22 # in addition to a standard python 2.6 or 2.7 installation the following packages need to be installed 23 23 # ECMWF WebMARS, gribAPI with python enabled, emoslib, ecaccess web toolkit, all available from https://software.ecmwf.int/ … … 25 25 # matplotlib (optional, for debugging) 26 26 # 27 # 27 # 28 28 import calendar 29 29 import shutil … … 45 45 46 46 47 48 47 def main(): 49 50 calledfromdir=os.getcwd() 48 ''' 49 ''' 50 # calledfromdir = os.getcwd() 51 51 os.chdir(localpythonpath) 52 args, c=install_args_and_control()52 args, c = install_args_and_control() 53 53 # if c.outputdir[0]!='/': 54 # c.outputdir=os.path.join(calledfromdir,c.outputdir) 55 # c.inputdir=c.outputdir 56 if args.install_target!=None: 57 install_via_gateway(c,args.install_target) 58 54 # c.outputdir=os.path.join(calledfromdir,c.outputdir) 55 # c.inputdir=c.outputdir 56 if args.install_target is not None: 57 install_via_gateway(c, args.install_target) 59 58 else: 60 print 'Please specify installation target (local|ecgate|cca)' 61 print 'use -h or --help for help' 59 print('Please specify installation target (local|ecgate|cca)') 60 print('use -h or --help for help') 62 61 sys.exit() 63 64 def install_via_gateway(c,target): 65 66 ecd=c.ecmwfdatadir 67 template=ecd+'python/compilejob.temp' 68 job=ecd+'python/compilejob.ksh' 69 fo=open(job,'w') 62 63 def install_via_gateway(c, target): 64 65 ecd = c.ecmwfdatadir 66 template = ecd + 'python/compilejob.temp' 67 job = ecd + 'python/compilejob.ksh' 68 fo = open(job, 'w') 69 #AP could do with open(template) as f, open(job, 'w') as fo: 70 #AP or nested with statements 70 71 with open(template) as f: 71 fdata = f.read().split('\n') 72 for data in fdata: 73 if 'MAKEFILE=' in data: 74 if c.makefile is not None: 75 data='export MAKEFILE='+c.makefile 76 if 'FLEXPART_ROOT_SCRIPTS=' in data: 77 if c.flexpart_root_scripts!='../': 78 data='export FLEXPART_ROOT_SCRIPTS='+c.flexpart_root_scripts 79 else: 80 data='export FLEXPART_ROOT_SCRIPTS=$HOME' 81 if target.lower()!='local': 82 if '--workdir' in data: 83 data='#SBATCH --workdir=/scratch/ms/'+c.ecgid+'/'+c.ecuid 84 if '##PBS -o' in data: 85 data='##PBS -o /scratch/ms/'+c.ecgid+'/'+c.ecuid+'flex_ecmwf.$Jobname.$Job_ID.out' 86 if 'FLEXPART_ROOT_SCRIPTS=' in data: 87 if c.ec_flexpart_root_scripts!='../': 88 data='export FLEXPART_ROOT_SCRIPTS='+c.ec_flexpart_root_scripts 89 else: 90 data='export FLEXPART_ROOT_SCRIPTS=$HOME' 91 92 fo.write(data+'\n') 72 fdata = f.read().split('\n') 73 for data in fdata: 74 if 'MAKEFILE=' in data: 75 if c.makefile is not None: 76 data = 'export MAKEFILE=' + c.makefile 77 if 'FLEXPART_ROOT_SCRIPTS=' in data: 78 if c.flexpart_root_scripts != '../': 79 data = 'export FLEXPART_ROOT_SCRIPTS=' + \ 80 c.flexpart_root_scripts 81 else: 82 data='export FLEXPART_ROOT_SCRIPTS=$HOME' 83 if target.lower() != 'local': 84 if '--workdir' in data: 85 data = '#SBATCH --workdir=/scratch/ms/' + c.ecgid + \ 86 '/' + c.ecuid 87 if '##PBS -o' in data: 88 data = '##PBS -o /scratch/ms/' + c.ecgid + '/' + c.ecuid + \ 89 'flex_ecmwf.$Jobname.$Job_ID.out' 90 if 'FLEXPART_ROOT_SCRIPTS=' in data: 91 if c.ec_flexpart_root_scripts != '../': 92 data = 'export FLEXPART_ROOT_SCRIPTS=' + \ 93 c.ec_flexpart_root_scripts 94 else: 95 data = 'export FLEXPART_ROOT_SCRIPTS=$HOME' 96 fo.write(data + '\n') 93 97 f.close() 94 98 fo.close() 95 96 if target.lower()!='local': 97 template=ecd+'python/job.temp.o' 98 with open(template) as f: 99 fdata = f.read().split('\n') 100 f.close() 101 fo=open(template[:-2],'w') 102 for data in fdata: 103 if '--workdir' in data: 104 data='#SBATCH --workdir=/scratch/ms/'+c.ecgid+'/'+c.ecuid 105 if '##PBS -o' in data: 106 data='##PBS -o /scratch/ms/'+c.ecgid+'/'+c.ecuid+'flex_ecmwf.$Jobname.$Job_ID.out' 107 if 'export PATH=${PATH}:' in data: 108 data+=c.ec_flexpart_root_scripts+'/ECMWFDATA7.0/python' 109 110 if 'cat>>' in data or 'cat >>' in data: 111 i=data.index('>') 112 fo.write(data[:i]+data[i+1:]+'\n') 113 fo.write('GATEWAY '+c.gateway+'\n') 114 fo.write('DESTINATION '+c.destination+'\n') 115 fo.write('EOF\n') 116 117 fo.write(data+'\n') 118 fo.close() 119 120 job=ecd+'python/ECMWF_ENV' 121 fo=open(job,'w') 122 fo.write('ECUID '+c.ecuid+'\n') 123 fo.write('ECGID '+c.ecgid+'\n') 124 fo.write('GATEWAY '+c.gateway+'\n') 125 fo.write('DESTINATION '+c.destination+'\n') 126 fo.close() 127 128 129 130 if target.lower()=='local': 99 100 if target.lower() != 'local': 101 template = ecd + 'python/job.temp.o' 102 #AP hier eventuell Zeile für Zeile lesen und dann if Entscheidung 103 with open(template) as f: 104 fdata = f.read().split('\n') 105 f.close() 106 fo = open(template[:-2], 'w') 107 for data in fdata: 108 if '--workdir' in data: 109 data = '#SBATCH --workdir=/scratch/ms/' + c.ecgid + \ 110 '/' + c.ecuid 111 if '##PBS -o' in data: 112 data = '##PBS -o /scratch/ms/' + c.ecgid + '/' + \ 113 c.ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out' 114 if 'export PATH=${PATH}:' in data: 115 data += c.ec_flexpart_root_scripts + '/ECMWFDATA7.0/python' 116 if 'cat>>' in data or 'cat >>' in data: 117 i = data.index('>') 118 fo.write(data[:i] + data[i+1:] + '\n') 119 fo.write('GATEWAY ' + c.gateway + '\n') 120 fo.write('DESTINATION ' + c.destination + '\n') 121 fo.write('EOF\n') 122 123 fo.write(data + '\n') 124 fo.close() 125 126 job = ecd + 'python/ECMWF_ENV' 127 with open(job, 'w') as fo: 128 fo.write('ECUID ' + c.ecuid + '\n') 129 fo.write('ECGID ' + c.ecgid + '\n') 130 fo.write('GATEWAY ' + c.gateway + '\n') 131 fo.write('DESTINATION ' + c.destination + '\n') 132 fo.close() 133 134 135 136 if target.lower() == 'local': 131 137 # compile CONVERT2 132 if c.flexpart_root_scripts==None or c.flexpart_root_scripts=='../': 133 print 'Warning: FLEXPART_ROOT_SCRIPTS has not been specified' 134 print 'Only CONVERT2 will be compiled in '+ecd+'/../src' 135 else: 136 c.flexpart_root_scripts=os.path.expandvars(os.path.expanduser(c.flexpart_root_scripts)) 137 if os.path.abspath(ecd)!=os.path.abspath(c.flexpart_root_scripts): 138 os.chdir('/') 139 p=subprocess.check_call(['tar','-cvf',ecd+'../ECMWFDATA7.0.tar',ecd+'python',ecd+'grib_templates',ecd+'src']) 140 try: 141 os.makedirs(c.flexpart_root_scripts+'/ECMWFDATA7.0') 142 except: 143 pass 144 os.chdir(c.flexpart_root_scripts+'/ECMWFDATA7.0') 145 p=subprocess.check_call(['tar','-xvf',ecd+'../ECMWFDATA7.0.tar']) 146 os.chdir(c.flexpart_root_scripts+'/ECMWFDATA7.0/src') 147 148 os.chdir('../src') 149 print 'install ECMWFDATA7.0 software on '+target+' in directory '+os.getcwd() 150 if c.makefile==None: 151 makefile='Makefile.local.ifort' 152 else: 153 makefile=c.makefile 154 flist=glob.glob('*.mod')+glob.glob('*.o') 155 if flist: 156 p=subprocess.check_call(['rm']+flist) 138 if c.flexpart_root_scripts is None or c.flexpart_root_scripts == '../': 139 print('Warning: FLEXPART_ROOT_SCRIPTS has not been specified') 140 print('Only CONVERT2 will be compiled in ' + ecd + '/../src') 141 else: 142 c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser( 143 c.flexpart_root_scripts)) 144 if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts): 145 os.chdir('/') 146 p = subprocess.check_call(['tar', '-cvf', 147 ecd + '../ECMWFDATA7.0.tar', 148 ecd + 'python', 149 ecd + 'grib_templates', 150 ecd + 'src']) 151 try: 152 os.makedirs(c.flexpart_root_scripts + '/ECMWFDATA7.0') 153 except: 154 pass 155 os.chdir(c.flexpart_root_scripts + '/ECMWFDATA7.0') 156 p = subprocess.check_call(['tar', '-xvf', 157 ecd + '../ECMWFDATA7.0.tar']) 158 os.chdir(c.flexpart_root_scripts + '/ECMWFDATA7.0/src') 159 160 os.chdir('../src') 161 print(('install ECMWFDATA7.0 software on ' + target + ' in directory ' 162 + os.getcwd())) 163 if c.makefile is None: 164 makefile = 'Makefile.local.ifort' 165 else: 166 makefile = c.makefile 167 flist = glob.glob('*.mod') + glob.glob('*.o') 168 if flist: 169 p = subprocess.check_call(['rm'] + flist) 157 170 try: 158 print 'Using makefile: '+makefile 159 p=subprocess.check_call(['make','-f',makefile]) 160 p=subprocess.check_call(['ls','-l','CONVERT2']) 161 except: 162 print 'compile failed - please edit '+makefile+' or try another Makefile in the src directory.' 163 print 'most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB and EMOSLIB must be adapted.' 164 print 'Available Makefiles:' 165 print glob.glob('Makefile*') 166 167 elif target.lower()=='ecgate': 171 print(('Using makefile: ' + makefile)) 172 p = subprocess.check_call(['make', '-f', makefile]) 173 p = subprocess.check_call(['ls', '-l',' CONVERT2']) 174 except: 175 print('compile failed - please edit ' + makefile + 176 ' or try another Makefile in the src directory.') 177 print('most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB \ 178 and EMOSLIB must be adapted.') 179 print('Available Makefiles:') 180 print(glob.glob('Makefile*')) 181 182 elif target.lower() == 'ecgate': 168 183 os.chdir('/') 169 p=subprocess.check_call(['tar','-cvf',ecd+'../ECMWFDATA7.0.tar',ecd+'python',ecd+'grib_templates',ecd+'src']) 170 try: 171 p=subprocess.check_call(['ecaccess-file-put',ecd+'../ECMWFDATA7.0.tar','ecgate:/scratch/ms/'+c.ecgid+'/'+c.ecuid+'/ECMWFDATA7.0.tar']) 172 except: 173 print 'ecaccess-file-put failed! Probably the eccert key has expired.' 174 exit(1) 175 p=subprocess.check_call(['ecaccess-job-submit','-queueName',target,ecd+'python/compilejob.ksh']) 176 print 'compilejob.ksh has been submitted to ecgate for installation in '+c.ec_flexpart_root_scripts+'/ECMWFDATA7.0' 177 print 'You should get an email with subject flexcompile within the next few minutes' 178 elif target.lower()=='cca': 184 p = subprocess.check_call(['tar', '-cvf', 185 ecd + '../ECMWFDATA7.0.tar', 186 ecd + 'python', 187 ecd + 'grib_templates', 188 ecd + 'src']) 189 try: 190 p = subprocess.check_call(['ecaccess-file-put', 191 ecd + '../ECMWFDATA7.0.tar', 192 'ecgate:/scratch/ms/' + c.ecgid + '/' + 193 c.ecuid + '/ECMWFDATA7.0.tar']) 194 except: 195 print('ecaccess-file-put failed! Probably the eccert key has expired.') 196 exit(1) 197 p = subprocess.check_call(['ecaccess-job-submit', 198 '-queueName', 199 target, 200 ecd + 'python/compilejob.ksh']) 201 print('compilejob.ksh has been submitted to ecgate for \ 202 installation in ' + c.ec_flexpart_root_scripts + 203 '/ECMWFDATA7.0') 204 print('You should get an email with subject flexcompile within \ 205 the next few minutes') 206 207 elif target.lower() == 'cca': 179 208 os.chdir('/') 180 p=subprocess.check_call(['tar','-cvf',ecd+'../ECMWFDATA7.0.tar',ecd+'python',ecd+'grib_templates',ecd+'src']) 181 try: 182 p=subprocess.check_call(['ecaccess-file-put',ecd+'../ECMWFDATA7.0.tar','cca:/scratch/ms/'+c.ecgid+'/'+c.ecuid+'/ECMWFDATA7.0.tar']) 183 except: 184 print 'ecaccess-file-put failed! Probably the eccert key has expired.' 185 exit(1) 186 187 p=subprocess.check_call(['ecaccess-job-submit','-queueName',target,ecd+'python/compilejob.ksh']) 188 print 'compilejob.ksh has been submitted to cca for installation in '+c.ec_flexpart_root_scripts+'/ECMWFDATA7.0' 189 print 'You should get an email with subject flexcompile within the next few minutes' 209 p = subprocess.check_call(['tar', '-cvf', 210 ecd + '../ECMWFDATA7.0.tar', 211 ecd + 'python', 212 ecd + 'grib_templates', 213 ecd + 'src']) 214 try: 215 p = subprocess.check_call(['ecaccess-file-put', 216 ecd + '../ECMWFDATA7.0.tar', 217 'cca:/scratch/ms/' + c.ecgid + '/' + 218 c.ecuid + '/ECMWFDATA7.0.tar']) 219 except: 220 print('ecaccess-file-put failed! \ 221 Probably the eccert key has expired.') 222 exit(1) 223 224 p=subprocess.check_call(['ecaccess-job-submit', 225 '-queueName', 226 target, 227 ecd + 'python/compilejob.ksh'])) 228 print('compilejob.ksh has been submitted to cca for installation in ' + 229 c.ec_flexpart_root_scripts + '/ECMWFDATA7.0') 230 print('You should get an email with subject flexcompile \ 231 within the next few minutes') 190 232 else: 191 print 'ERROR: unknown installation target ',target 192 print 'Valid targets: ecgate, cca, local' 233 print('ERROR: unknown installation target ', target) 234 print('Valid targets: ecgate, cca, local') 235 193 236 return 194 237 195 238 196 239 if __name__ == "__main__": 197 240 main()
Note: See TracChangeset
for help on using the changeset viewer.