Changeset c5074d2 in flex_extract.git
- Timestamp:
- Oct 9, 2018, 8:42:16 PM (5 years ago)
- Branches:
- master, ctbto, dev
- Children:
- aa275fc
- Parents:
- 0a06607
- Files:
-
- 2 added
- 2 deleted
- 9 edited
- 2 moved
Legend:
- Unmodified
- Added
- Removed
-
source/python/_config.py
r0aaeb04 rc5074d2 38 38 TEMPFILE_INSTALL_COMPILEJOB = 'compilejob.template' 39 39 FILE_INSTALL_COMPILEJOB = 'compilejob.ksh' 40 TEMPFILE_INSTALL_JOB = 'job.temp .o'40 TEMPFILE_INSTALL_JOB = 'job.template' 41 41 TEMPFILE_JOB = 'job.temp' 42 42 FILE_JOB_OD = 'job.ksh' 43 43 FILE_JOB_OP = 'jopoper.ksh' 44 TEMPFILE_NAMELIST = 'convert.nl' 44 45 FILE_NAMELIST = 'fort.4' 45 46 FILE_GRIB_INDEX = 'date_time_stepRange.idx' -
source/python/classes/EcFlexpart.py
r0e576fc rc5074d2 658 658 659 659 660 def write_namelist(self, c , filename):660 def write_namelist(self, c): 661 661 ''' 662 662 @Description: … … 683 683 ''' 684 684 685 from genshi.template.text import NewTextTemplate 686 from genshi.template import TemplateLoader 687 688 loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False) 689 compile_template = loader.load(_config.TEMPFILE_NAMELIST, 690 cls=NewTextTemplate) 691 685 692 self.inputdir = c.inputdir 686 693 area = np.asarray(self.area.split('/')).astype(float) … … 692 699 maxb = int((area[0] - area[2]) / grid[0]) + 1 693 700 694 with open(self.inputdir + '/' + filename, 'w') as f: 695 f.write('&NAMGEN\n') 696 f.write(',\n '.join(['maxl = ' + str(maxl), 'maxb = ' + str(maxb), 697 'mlevel = ' + str(self.level), 698 'mlevelist = ' + '"' + str(self.levelist) 699 + '"', 700 'mnauf = ' + str(self.resol), 701 'metapar = ' + '77', 702 'rlo0 = ' + str(area[1]), 703 'rlo1 = ' + str(area[3]), 704 'rla0 = ' + str(area[2]), 705 'rla1 = ' + str(area[0]), 706 'momega = ' + str(c.omega), 707 'momegadiff = ' + str(c.omegadiff), 708 'mgauss = ' + str(c.gauss), 709 'msmooth = ' + str(c.smooth), 710 'meta = ' + str(c.eta), 711 'metadiff = ' + str(c.etadiff), 712 'mdpdeta = ' + str(c.dpdeta)])) 713 714 f.write('\n/\n') 701 stream = compile_template.generate( 702 maxl = str(maxl), 703 maxb = str(maxb), 704 mlevel = str(self.level), 705 mlevelist = str(self.levelist), 706 mnauf = str(self.resol), 707 metapar = '77', 708 rlo0 = str(area[1]), 709 rlo1 = str(area[3]), 710 rla0 = str(area[2]), 711 rla1 = str(area[0]), 712 momega = str(c.omega), 713 momegadiff = str(c.omegadiff), 714 mgauss = str(c.gauss), 715 msmooth = str(c.smooth), 716 meta = str(c.eta), 717 metadiff = str(c.etadiff), 718 mdpdeta = str(c.dpdeta) 719 ) 720 721 namelistfile = os.path.join(self.inputdir, _config.FILE_NAMELIST) 722 723 with open(namelistfile, 'w') as f: 724 f.write(stream.render('text')) 715 725 716 726 return -
source/python/install.py
r0aaeb04 rc5074d2 396 396 397 397 stream = compile_template.generate( 398 usergroup = ecgid, 398 399 username = ecuid, 399 usergroup = ecgid,400 400 version_number = _config._VERSION_STR, 401 401 fp_root_scripts = fp_root, … … 410 410 f.write(stream.render('text')) 411 411 412 413 # template = os.path.join(_config.PATH_REL_TEMPLATES,414 # _config.TEMPFILE_INSTALL_COMPILEJOB)415 # with open(template) as f:416 # fdata = f.read().split('\n')417 418 419 # with open(compilejob, 'w') as fo:420 # for data in fdata:421 # if 'MAKEFILE=' in data:422 # data = 'export MAKEFILE=' + makefile423 # elif 'FLEXPART_ROOT_SCRIPTS=' in data:424 # if fp_root != '../':425 # data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root426 # else:427 # data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'428 # elif target.lower() != 'local':429 # if '--workdir' in data:430 # data = '#SBATCH --workdir=/scratch/ms/' + \431 # ecgid + '/' + ecuid432 # elif '##PBS -o' in data:433 # data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \434 # 'flex_ecmwf.$Jobname.$Job_ID.out'435 # elif 'FLEXPART_ROOT_SCRIPTS=' in data:436 # if fp_root != '../':437 # data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root438 # else:439 # data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'440 # fo.write(data + '\n')441 442 412 return 443 413 … … 470 440 <nothing> 471 441 ''' 472 fp_root_path_to_python = os.path.join(fp_root, _config.FLEXEXTRACT_DIRNAME, 473 _config.PATH_REL_PYTHON) 474 475 template = os.path.join(_config.PATH_REL_TEMPLATES, 476 _config.TEMPFILE_INSTALL_JOB) 477 with open(template) as f: 478 fdata = f.read().split('\n') 479 480 jobfile_temp = os.path.join(_config.PATH_REL_TEMPLATES, 481 _config.TEMPFILE_JOB) 482 with open(jobfile_temp, 'w') as fo: 483 for data in fdata: 484 if '--workdir' in data: 485 data = '#SBATCH --workdir=/scratch/ms/' + ecgid + '/' + ecuid 486 elif '##PBS -o' in data: 487 data = '##PBS -o /scratch/ms/' + ecgid + '/' + \ 488 ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out' 489 elif 'export PATH=${PATH}:' in data: 490 data += fp_root_path_to_python 491 492 fo.write(data + '\n') 442 from genshi.template.text import NewTextTemplate 443 from genshi.template import TemplateLoader 444 445 loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False) 446 compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB, 447 cls=NewTextTemplate) 448 449 fp_root_path_to_python = os.path.join(fp_root, 450 _config.FLEXEXTRACT_DIRNAME, 451 _config.PATH_REL_PYTHON) 452 453 stream = compile_template.generate( 454 usergroup = ecgid, 455 username = ecuid, 456 version_number = _config._VERSION_STR, 457 fp_root_path = fp_root_path_to_python, 458 ) 459 460 tempjobfile = os.path.join(_config.PATH_TEMPLATES, 461 _config.TEMPFILE_JOB) 462 463 with open(tempjobfile, 'w') as f: 464 f.write(stream.render('text')) 465 493 466 return 494 467 -
source/python/mods/prepare_flexpart.py
r5bad6ec rc5074d2 160 160 # deaccumulate the flux data 161 161 flexpart = EcFlexpart(c, fluxes=True) 162 flexpart.write_namelist(c , _config.FILE_NAMELIST)162 flexpart.write_namelist(c) 163 163 flexpart.deacc_fluxes(inputfiles, c) 164 164 -
source/pythontest/TestData/CONTROL.temp
r25b14be rc5074d2 9 9 NUMBER OFF 10 10 EXPVER 1 11 GRID 500011 GRID 1000 12 12 LEFT -15000 13 13 LOWER 30000 -
source/pythontest/TestData/compilejob.test
r0aaeb04 rc5074d2 17 17 ##PBS -q ns 18 18 ##PBS -S /usr/bin/ksh 19 ##PBS -o /scratch/ms/testgroup/testuser/flex_ecmwf.$ Jobname.$Job_ID.out19 ##PBS -o /scratch/ms/testgroup/testuser/flex_ecmwf.${Jobname}.${Job_ID}.out 20 20 # job output is in .ecaccess_DO_NOT_REMOVE 21 21 ##PBS -j oe … … 26 26 set -x 27 27 export VERSION=7.1 28 case $ HOSTin28 case ${HOST} in 29 29 *ecg*) 30 30 module load python … … 43 43 echo ${GROUP} 44 44 echo ${HOME} 45 echo $ HOME| awk -F / '{print $1, $2, $3, $4}'46 export GROUP=`echo $ HOME| awk -F / '{print $4}'`45 echo ${HOME} | awk -F / '{print $1, $2, $3, $4}' 46 export GROUP=`echo ${HOME} | awk -F / '{print $4}'` 47 47 export SCRATCH=/scratch/ms/${GROUP}/${USER} 48 export FLEXPART_ROOT_SCRIPTS= p_root_test_path48 export FLEXPART_ROOT_SCRIPTS=fp_root_test_path 49 49 export MAKEFILE=Makefile.TEST 50 50 ;; 51 51 esac 52 52 53 mkdir -p $ FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION54 cd $ FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION# if FLEXPART_ROOT is not set this means cd to the home directory55 tar -xvf $ HOME/flex_extract_v$VERSION.tar53 mkdir -p ${FLEXPART_ROOT_SCRIPTS}/flex_extract_v${VERSION} 54 cd ${FLEXPART_ROOT_SCRIPTS}/flex_extract_v${VERSION} # if FLEXPART_ROOT is not set this means cd to the home directory 55 tar -xvf ${HOME}/flex_extract_v${VERSION}.tar 56 56 cd src 57 57 \rm *.o *.mod CONVERT2 58 make -f $ MAKEFILE>flexcompile 2>flexcompile58 make -f ${MAKEFILE} >flexcompile 2>flexcompile 59 59 60 60 ls -l CONVERT2 >>flexcompile 61 61 if [ $? -eq 0 ]; then 62 62 echo 'SUCCESS!' >>flexcompile 63 mail -s flexcompile.$ HOST.$$ $USER<flexcompile63 mail -s flexcompile.${HOST}.$$ ${USER} <flexcompile 64 64 else 65 65 echo Environment: >>flexcompile 66 66 env >> flexcompile 67 mail -s "ERROR! flexcompile.$ HOST.$$" $USER<flexcompile67 mail -s "ERROR! flexcompile.${HOST}.$$" ${USER} <flexcompile 68 68 fi -
source/pythontest/TestData/job.temp.test
r25b14be rc5074d2 5 5 # start with sbatch NAME_OF_THIS_FILE directly on machine 6 6 7 #SBATCH --workdir=/scratch/ms/ at/km4a7 #SBATCH --workdir=/scratch/ms/testgroup/testuser 8 8 #SBATCH --qos=normal 9 9 #SBATCH --job-name=flex_ecmwf … … 17 17 ##PBS -q np 18 18 ##PBS -S /usr/bin/ksh 19 ## -o /scratch/ms/ spatlh00/lh0/flex_ecmwf.$PBS_JOBID.out19 ## -o /scratch/ms/testgroup/testuser/flex_ecmwf.${PBS_JOBID}.out 20 20 ## job output is in .ecaccess_DO_NOT_REMOVE 21 21 ##PBS -j oe … … 26 26 set -x 27 27 export VERSION=7.1 28 case $ HOSTin28 case ${HOST} in 29 29 *ecg*) 30 30 module load python … … 33 33 module load grib_api/1.14.5 34 34 module load emos/437-r64 35 export PATH=${PATH}: ${HOME}/flex_extract_v7.1/source/python35 export PATH=${PATH}:fp_root_test_path/flex_extract_v7.1/source/python 36 36 ;; 37 37 *cca*) … … 40 40 module load emos 41 41 module load python 42 export SCRATCH=$ TMPDIR43 export PATH=${PATH}: ${HOME}/flex_extract_v7.1/source/python42 export SCRATCH=${TMPDIR} 43 export PATH=${PATH}:fp_root_test_path/flex_extract_v7.1/source/python 44 44 ;; 45 45 esac 46 46 47 cd $ SCRATCH47 cd ${SCRATCH} 48 48 mkdir -p python$$ 49 49 cd python$$ … … 51 51 export CONTROL=CONTROL 52 52 53 cat >$ CONTROL<<EOF53 cat >${CONTROL}<<EOF 54 54 EOF 55 55 56 56 57 submit.py --controlfile=$ CONTROL--inputdir=./work --outputdir=./work 1> prot 2>&157 submit.py --controlfile=${CONTROL} --inputdir=./work --outputdir=./work 1> prot 2>&1 58 58 59 59 if [ $? -eq 0 ] ; then 60 60 l=0 61 for muser in `grep -i MAILOPS $ CONTROL`; do62 if [ $ l-gt 0 ] ; then63 mail -s flex.${HOST}.$$ $ muser<prot61 for muser in `grep -i MAILOPS ${CONTROL}`; do 62 if [ ${l} -gt 0 ] ; then 63 mail -s flex.${HOST}.$$ ${muser} <prot 64 64 fi 65 l=$(($ l+1))65 l=$((${l}+1)) 66 66 done 67 67 else 68 68 l=0 69 for muser in `grep -i MAILFAIL $ CONTROL`; do70 if [ $ l-gt 0 ] ; then71 mail -s "ERROR! flex.${HOST}.$$" $ muser<prot69 for muser in `grep -i MAILFAIL ${CONTROL}`; do 70 if [ ${l} -gt 0 ] ; then 71 mail -s "ERROR! flex.${HOST}.$$" ${muser} <prot 72 72 fi 73 l=$(($ l+1))73 l=$((${l}+1)) 74 74 done 75 75 fi -
source/pythontest/TestEcFlexpart.py
r25b14be rc5074d2 4 4 5 5 import sys 6 import os 7 import inspect 6 8 import pytest 7 9 8 10 sys.path.append('../python') 11 import _config 9 12 from classes.EcFlexpart import EcFlexpart 13 from classes.ControlFile import ControlFile 14 from mods.tools import silent_remove 10 15 11 16 class TestEcFlexpart(): … … 13 18 ''' 14 19 15 def test_init(self): 16 # create an instance of EcFlexpart and get a dictionary of the 17 # class attributes, compare this dict with an expected dict! 18 assert True == True 20 def test_write_namelist(self): 21 import filecmp 19 22 20 def test_write_namelist(self): 21 # simple 22 assert True == True 23 control_file = os.path.join(_config.PATH_TEST_DIR, 24 'TestData', 25 'CONTROL.temp') 26 c = ControlFile(control_file) 27 flexpart = EcFlexpart(c) 28 29 c.inputdir = 'TestData' 30 31 # comparison file 32 testfile = os.path.join(_config.PATH_TEST_DIR, 33 'TestData', 34 'convert.nl.test') 35 36 # create 37 flexpart.write_namelist(c) 38 39 finalfile = os.path.join(c.inputdir, _config.FILE_NAMELIST) 40 assert filecmp.cmp(testfile, finalfile, shallow=False) 41 42 # delete test file 43 silent_remove(finalfile) 23 44 24 45 def test_retrieve(self): -
source/pythontest/TestInstall.py
r0aaeb04 rc5074d2 25 25 #! - un_tarball 26 26 #! - mk_env_vars 27 # - mk_compilejob28 # - mk_job_template27 #! - mk_compilejob 28 #! - mk_job_template 29 29 # - delete_convert_build 30 30 # - make_convert_build … … 155 155 # delete test file 156 156 silent_remove(finalfile) 157 158 def test_mk_job_template(self): 159 import filecmp 160 161 # comparison file 162 testfile = os.path.join(_config.PATH_TEST_DIR, 163 'TestData', 164 'job.temp.test') 165 166 # create 167 install.mk_job_template('testuser', 168 'testgroup', 169 'gateway.test.ac.at', 170 'dest@generic', 171 'fp_root_test_path') 172 173 finalfile = os.path.join(_config.PATH_TEMPLATES, 174 _config.TEMPFILE_JOB) 175 assert filecmp.cmp(testfile, finalfile, shallow=False) 176 177 # delete test file 178 silent_remove(finalfile) -
templates/compilejob.template
r0aaeb04 rc5074d2 17 17 ##PBS -q ns 18 18 ##PBS -S /usr/bin/ksh 19 ##PBS -o /scratch/ms/$usergroup/$username/flex_ecmwf.$ Jobname.$Job_ID.out19 ##PBS -o /scratch/ms/$usergroup/$username/flex_ecmwf.$${Jobname}.$${Job_ID}.out 20 20 # job output is in .ecaccess_DO_NOT_REMOVE 21 21 ##PBS -j oe … … 26 26 set -x 27 27 export VERSION=$version_number 28 case $ HOSTin28 case $${HOST} in 29 29 *ecg*) 30 30 module load python … … 41 41 module load emos 42 42 module load python 43 echo $ {GROUP}44 echo $ {HOME}45 echo $ HOME| awk -F / '{print $1, $2, $3, $4}'46 export GROUP=`echo $ HOME| awk -F / '{print $4}'`47 export SCRATCH=/scratch/ms/$ {GROUP}/${USER}43 echo $${GROUP} 44 echo $${HOME} 45 echo $${HOME} | awk -F / '{print $1, $2, $3, $4}' 46 export GROUP=`echo $${HOME} | awk -F / '{print $4}'` 47 export SCRATCH=/scratch/ms/$${GROUP}/$${USER} 48 48 export FLEXPART_ROOT_SCRIPTS=$fp_root_scripts 49 49 export MAKEFILE=$makefile … … 51 51 esac 52 52 53 mkdir -p $ FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION54 cd $ FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION# if FLEXPART_ROOT is not set this means cd to the home directory55 tar -xvf $ HOME/flex_extract_v$VERSION.tar53 mkdir -p $${FLEXPART_ROOT_SCRIPTS}/flex_extract_v$${VERSION} 54 cd $${FLEXPART_ROOT_SCRIPTS}/flex_extract_v$${VERSION} # if FLEXPART_ROOT is not set this means cd to the home directory 55 tar -xvf $${HOME}/flex_extract_v$${VERSION}.tar 56 56 cd src 57 57 \rm *.o *.mod $fortran_program 58 make -f $ MAKEFILE>flexcompile 2>flexcompile58 make -f $${MAKEFILE} >flexcompile 2>flexcompile 59 59 60 60 ls -l $fortran_program >>flexcompile 61 if [ $ ? -eq 0 ]; then61 if [ $$? -eq 0 ]; then 62 62 echo 'SUCCESS!' >>flexcompile 63 mail -s flexcompile.$ HOST.$$ $USER<flexcompile63 mail -s flexcompile.$${HOST}.$$$$ $${USER} <flexcompile 64 64 else 65 65 echo Environment: >>flexcompile 66 66 env >> flexcompile 67 mail -s "ERROR! flexcompile.$ HOST.$$" $USER<flexcompile67 mail -s "ERROR! flexcompile.$${HOST}.$$$$" $${USER} <flexcompile 68 68 fi -
templates/job.template
r82b624a rc5074d2 5 5 # start with sbatch NAME_OF_THIS_FILE directly on machine 6 6 7 #SBATCH --workdir=/scratch/ms/ spatlh00/lh07 #SBATCH --workdir=/scratch/ms/$usergroup/$username 8 8 #SBATCH --qos=normal 9 9 #SBATCH --job-name=flex_ecmwf … … 17 17 ##PBS -q np 18 18 ##PBS -S /usr/bin/ksh 19 ## -o /scratch/ms/ spatlh00/lh0/flex_ecmwf.$PBS_JOBID.out19 ## -o /scratch/ms/$usergroup/$username/flex_ecmwf.$${PBS_JOBID}.out 20 20 ## job output is in .ecaccess_DO_NOT_REMOVE 21 21 ##PBS -j oe … … 25 25 26 26 set -x 27 export VERSION= 7.128 case $ HOSTin27 export VERSION=$version_number 28 case $${HOST} in 29 29 *ecg*) 30 30 module load python … … 33 33 module load grib_api/1.14.5 34 34 module load emos/437-r64 35 export PATH=$ {PATH}:35 export PATH=$${PATH}:$fp_root_path 36 36 ;; 37 37 *cca*) … … 40 40 module load emos 41 41 module load python 42 export SCRATCH=$ TMPDIR43 export PATH=$ {PATH}:42 export SCRATCH=$${TMPDIR} 43 export PATH=$${PATH}:$fp_root_path 44 44 ;; 45 45 esac 46 46 47 cd $ SCRATCH48 mkdir -p python$$ 49 cd python$$ 47 cd $${SCRATCH} 48 mkdir -p python$$$$ 49 cd python$$$$ 50 50 51 51 export CONTROL=CONTROL 52 52 53 cat >$ CONTROL<<EOF53 cat >$${CONTROL}<<EOF 54 54 EOF 55 55 56 56 57 submit.py --controlfile=$ CONTROL--inputdir=./work --outputdir=./work 1> prot 2>&157 submit.py --controlfile=$${CONTROL} --inputdir=./work --outputdir=./work 1> prot 2>&1 58 58 59 59 if [ $? -eq 0 ] ; then 60 60 l=0 61 for muser in `grep -i MAILOPS $ CONTROL`; do62 if [ $ l-gt 0 ] ; then63 mail -s flex.$ {HOST}.$$ $muser<prot61 for muser in `grep -i MAILOPS $${CONTROL}`; do 62 if [ $${l} -gt 0 ] ; then 63 mail -s flex.$${HOST}.$$$$ $${muser} <prot 64 64 fi 65 l=$(($ l+1))65 l=$(($${l}+1)) 66 66 done 67 67 else 68 68 l=0 69 for muser in `grep -i MAILFAIL $ CONTROL`; do70 if [ $ l-gt 0 ] ; then71 mail -s "ERROR! flex.$ {HOST}.$$" $muser<prot69 for muser in `grep -i MAILFAIL $${CONTROL}`; do 70 if [ $${l} -gt 0 ] ; then 71 mail -s "ERROR! flex.$${HOST}.$$$$" $${muser} <prot 72 72 fi 73 l=$(($ l+1))73 l=$(($${l}+1)) 74 74 done 75 75 fi 76
Note: See TracChangeset
for help on using the changeset viewer.