Changeset c5074d2 in flex_extract.git


Ignore:
Timestamp:
Oct 9, 2018, 8:42:16 PM (12 months ago)
Author:
Anne Philipp <anne.philipp@…>
Branches:
dev
Children:
aa275fc
Parents:
0a06607
Message:

added making of namelist file and jobscript via genshi templates

Files:
2 added
2 deleted
9 edited
2 moved

Legend:

Unmodified
Added
Removed
  • source/python/_config.py

    r0aaeb04 rc5074d2  
    3838TEMPFILE_INSTALL_COMPILEJOB = 'compilejob.template'
    3939FILE_INSTALL_COMPILEJOB = 'compilejob.ksh'
    40 TEMPFILE_INSTALL_JOB = 'job.temp.o'
     40TEMPFILE_INSTALL_JOB = 'job.template'
    4141TEMPFILE_JOB = 'job.temp'
    4242FILE_JOB_OD = 'job.ksh'
    4343FILE_JOB_OP = 'jopoper.ksh'
     44TEMPFILE_NAMELIST = 'convert.nl'
    4445FILE_NAMELIST = 'fort.4'
    4546FILE_GRIB_INDEX = 'date_time_stepRange.idx'
  • source/python/classes/EcFlexpart.py

    r0e576fc rc5074d2  
    658658
    659659
    660     def write_namelist(self, c, filename):
     660    def write_namelist(self, c):
    661661        '''
    662662        @Description:
     
    683683        '''
    684684
     685        from genshi.template.text import NewTextTemplate
     686        from genshi.template import  TemplateLoader
     687
     688        loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
     689        compile_template = loader.load(_config.TEMPFILE_NAMELIST,
     690                                       cls=NewTextTemplate)
     691
    685692        self.inputdir = c.inputdir
    686693        area = np.asarray(self.area.split('/')).astype(float)
     
    692699        maxb = int((area[0] - area[2]) / grid[0]) + 1
    693700
    694         with open(self.inputdir + '/' + filename, 'w') as f:
    695             f.write('&NAMGEN\n')
    696             f.write(',\n  '.join(['maxl = ' + str(maxl), 'maxb = ' + str(maxb),
    697                                   'mlevel = ' + str(self.level),
    698                                   'mlevelist = ' + '"' + str(self.levelist)
    699                                                  + '"',
    700                                   'mnauf = ' + str(self.resol),
    701                                   'metapar = ' + '77',
    702                                   'rlo0 = ' + str(area[1]),
    703                                   'rlo1 = ' + str(area[3]),
    704                                   'rla0 = ' + str(area[2]),
    705                                   'rla1 = ' + str(area[0]),
    706                                   'momega = ' + str(c.omega),
    707                                   'momegadiff = ' + str(c.omegadiff),
    708                                   'mgauss = ' + str(c.gauss),
    709                                   'msmooth = ' + str(c.smooth),
    710                                   'meta = ' + str(c.eta),
    711                                   'metadiff = ' + str(c.etadiff),
    712                                   'mdpdeta = ' + str(c.dpdeta)]))
    713 
    714             f.write('\n/\n')
     701        stream = compile_template.generate(
     702            maxl = str(maxl),
     703            maxb = str(maxb),
     704            mlevel = str(self.level),
     705            mlevelist = str(self.levelist),
     706            mnauf = str(self.resol),
     707            metapar = '77',
     708            rlo0 = str(area[1]),
     709            rlo1 = str(area[3]),
     710            rla0 = str(area[2]),
     711            rla1 = str(area[0]),
     712            momega = str(c.omega),
     713            momegadiff = str(c.omegadiff),
     714            mgauss = str(c.gauss),
     715            msmooth = str(c.smooth),
     716            meta = str(c.eta),
     717            metadiff = str(c.etadiff),
     718            mdpdeta = str(c.dpdeta)
     719        )
     720
     721        namelistfile = os.path.join(self.inputdir, _config.FILE_NAMELIST)
     722
     723        with open(namelistfile, 'w') as f:
     724            f.write(stream.render('text'))
    715725
    716726        return
  • source/python/install.py

    r0aaeb04 rc5074d2  
    396396
    397397    stream = compile_template.generate(
     398        usergroup = ecgid,
    398399        username = ecuid,
    399         usergroup = ecgid,
    400400        version_number = _config._VERSION_STR,
    401401        fp_root_scripts = fp_root,
     
    410410        f.write(stream.render('text'))
    411411
    412 
    413     # template = os.path.join(_config.PATH_REL_TEMPLATES,
    414                             # _config.TEMPFILE_INSTALL_COMPILEJOB)
    415     # with open(template) as f:
    416         # fdata = f.read().split('\n')
    417 
    418 
    419     # with open(compilejob, 'w') as fo:
    420         # for data in fdata:
    421             # if 'MAKEFILE=' in data:
    422                 # data = 'export MAKEFILE=' + makefile
    423             # elif 'FLEXPART_ROOT_SCRIPTS=' in data:
    424                 # if fp_root != '../':
    425                     # data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
    426                 # else:
    427                     # data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
    428             # elif target.lower() != 'local':
    429                 # if '--workdir' in data:
    430                     # data = '#SBATCH --workdir=/scratch/ms/' + \
    431                             # ecgid + '/' + ecuid
    432                 # elif '##PBS -o' in data:
    433                     # data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \
    434                            # 'flex_ecmwf.$Jobname.$Job_ID.out'
    435                 # elif 'FLEXPART_ROOT_SCRIPTS=' in data:
    436                     # if fp_root != '../':
    437                         # data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
    438                     # else:
    439                         # data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
    440             # fo.write(data + '\n')
    441 
    442412    return
    443413
     
    470440        <nothing>
    471441    '''
    472     fp_root_path_to_python = os.path.join(fp_root, _config.FLEXEXTRACT_DIRNAME,
    473                          _config.PATH_REL_PYTHON)
    474 
    475     template = os.path.join(_config.PATH_REL_TEMPLATES,
    476                             _config.TEMPFILE_INSTALL_JOB)
    477     with open(template) as f:
    478         fdata = f.read().split('\n')
    479 
    480     jobfile_temp = os.path.join(_config.PATH_REL_TEMPLATES,
    481                                 _config.TEMPFILE_JOB)
    482     with open(jobfile_temp, 'w') as fo:
    483         for data in fdata:
    484             if '--workdir' in data:
    485                 data = '#SBATCH --workdir=/scratch/ms/' + ecgid + '/' + ecuid
    486             elif '##PBS -o' in data:
    487                 data = '##PBS -o /scratch/ms/' + ecgid + '/' + \
    488                         ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out'
    489             elif  'export PATH=${PATH}:' in data:
    490                 data += fp_root_path_to_python
    491 
    492             fo.write(data + '\n')
     442    from genshi.template.text import NewTextTemplate
     443    from genshi.template import  TemplateLoader
     444
     445    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
     446    compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
     447                                   cls=NewTextTemplate)
     448
     449    fp_root_path_to_python = os.path.join(fp_root,
     450                                          _config.FLEXEXTRACT_DIRNAME,
     451                                          _config.PATH_REL_PYTHON)
     452
     453    stream = compile_template.generate(
     454        usergroup = ecgid,
     455        username = ecuid,
     456        version_number = _config._VERSION_STR,
     457        fp_root_path = fp_root_path_to_python,
     458    )
     459
     460    tempjobfile = os.path.join(_config.PATH_TEMPLATES,
     461                               _config.TEMPFILE_JOB)
     462
     463    with open(tempjobfile, 'w') as f:
     464        f.write(stream.render('text'))
     465
    493466    return
    494467
  • source/python/mods/prepare_flexpart.py

    r5bad6ec rc5074d2  
    160160    # deaccumulate the flux data
    161161    flexpart = EcFlexpart(c, fluxes=True)
    162     flexpart.write_namelist(c, _config.FILE_NAMELIST)
     162    flexpart.write_namelist(c)
    163163    flexpart.deacc_fluxes(inputfiles, c)
    164164
  • source/pythontest/TestData/CONTROL.temp

    r25b14be rc5074d2  
    99NUMBER OFF
    1010EXPVER 1
    11 GRID 5000 
     11GRID 1000 
    1212LEFT -15000
    1313LOWER 30000
  • source/pythontest/TestData/compilejob.test

    r0aaeb04 rc5074d2  
    1717##PBS -q ns
    1818##PBS -S /usr/bin/ksh
    19 ##PBS -o /scratch/ms/testgroup/testuser/flex_ecmwf.$Jobname.$Job_ID.out
     19##PBS -o /scratch/ms/testgroup/testuser/flex_ecmwf.${Jobname}.${Job_ID}.out
    2020# job output is in .ecaccess_DO_NOT_REMOVE
    2121##PBS -j oe
     
    2626set -x
    2727export VERSION=7.1
    28 case $HOST in
     28case ${HOST} in
    2929  *ecg*)
    3030  module load python
     
    4343  echo ${GROUP}
    4444  echo ${HOME}
    45   echo $HOME | awk -F / '{print $1, $2, $3, $4}'
    46   export GROUP=`echo $HOME | awk -F / '{print $4}'`
     45  echo ${HOME} | awk -F / '{print $1, $2, $3, $4}'
     46  export GROUP=`echo ${HOME} | awk -F / '{print $4}'`
    4747  export SCRATCH=/scratch/ms/${GROUP}/${USER}
    48   export FLEXPART_ROOT_SCRIPTS=p_root_test_path
     48  export FLEXPART_ROOT_SCRIPTS=fp_root_test_path
    4949  export MAKEFILE=Makefile.TEST
    5050  ;;
    5151esac
    5252
    53 mkdir -p $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION
    54 cd $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION   # if FLEXPART_ROOT is not set this means cd to the home directory
    55 tar -xvf $HOME/flex_extract_v$VERSION.tar
     53mkdir -p ${FLEXPART_ROOT_SCRIPTS}/flex_extract_v${VERSION}
     54cd ${FLEXPART_ROOT_SCRIPTS}/flex_extract_v${VERSION}   # if FLEXPART_ROOT is not set this means cd to the home directory
     55tar -xvf ${HOME}/flex_extract_v${VERSION}.tar
    5656cd src
    5757\rm *.o *.mod CONVERT2
    58 make -f $MAKEFILE >flexcompile 2>flexcompile
     58make -f ${MAKEFILE} >flexcompile 2>flexcompile
    5959
    6060ls -l CONVERT2 >>flexcompile
    6161if [ $? -eq 0 ]; then
    6262  echo 'SUCCESS!' >>flexcompile
    63   mail -s flexcompile.$HOST.$$ $USER <flexcompile
     63  mail -s flexcompile.${HOST}.$$ ${USER} <flexcompile
    6464else
    6565  echo Environment: >>flexcompile
    6666  env >> flexcompile
    67   mail -s "ERROR! flexcompile.$HOST.$$" $USER <flexcompile
     67  mail -s "ERROR! flexcompile.${HOST}.$$" ${USER} <flexcompile
    6868fi
  • source/pythontest/TestData/job.temp.test

    r25b14be rc5074d2  
    55# start with sbatch NAME_OF_THIS_FILE directly on machine
    66
    7 #SBATCH --workdir=/scratch/ms/at/km4a
     7#SBATCH --workdir=/scratch/ms/testgroup/testuser
    88#SBATCH --qos=normal
    99#SBATCH --job-name=flex_ecmwf
     
    1717##PBS -q np
    1818##PBS -S /usr/bin/ksh
    19 ## -o /scratch/ms/spatlh00/lh0/flex_ecmwf.$PBS_JOBID.out
     19## -o /scratch/ms/testgroup/testuser/flex_ecmwf.${PBS_JOBID}.out
    2020## job output is in .ecaccess_DO_NOT_REMOVE
    2121##PBS -j oe
     
    2626set -x
    2727export VERSION=7.1
    28 case $HOST in
     28case ${HOST} in
    2929  *ecg*)
    3030  module load python
     
    3333  module load grib_api/1.14.5
    3434  module load emos/437-r64
    35   export PATH=${PATH}:${HOME}/flex_extract_v7.1/source/python
     35  export PATH=${PATH}:fp_root_test_path/flex_extract_v7.1/source/python
    3636  ;;
    3737  *cca*)
     
    4040  module load emos
    4141  module load python
    42   export SCRATCH=$TMPDIR
    43   export PATH=${PATH}:${HOME}/flex_extract_v7.1/source/python
     42  export SCRATCH=${TMPDIR}
     43  export PATH=${PATH}:fp_root_test_path/flex_extract_v7.1/source/python
    4444  ;;
    4545esac
    4646
    47 cd $SCRATCH
     47cd ${SCRATCH}
    4848mkdir -p python$$
    4949cd python$$
     
    5151export CONTROL=CONTROL
    5252
    53 cat >$CONTROL<<EOF
     53cat >${CONTROL}<<EOF
    5454EOF
    5555
    5656
    57 submit.py --controlfile=$CONTROL --inputdir=./work --outputdir=./work 1> prot 2>&1
     57submit.py --controlfile=${CONTROL} --inputdir=./work --outputdir=./work 1> prot 2>&1
    5858
    5959if [ $? -eq 0 ] ; then
    6060  l=0
    61   for muser in `grep -i MAILOPS $CONTROL`; do
    62       if [ $l -gt 0 ] ; then
    63          mail -s flex.${HOST}.$$ $muser <prot
     61  for muser in `grep -i MAILOPS ${CONTROL}`; do
     62      if [ ${l} -gt 0 ] ; then
     63         mail -s flex.${HOST}.$$ ${muser} <prot
    6464      fi
    65       l=$(($l+1))
     65      l=$((${l}+1))
    6666  done
    6767else
    6868  l=0
    69   for muser in `grep -i MAILFAIL $CONTROL`; do
    70       if [ $l -gt 0 ] ; then
    71          mail -s "ERROR! flex.${HOST}.$$" $muser <prot
     69  for muser in `grep -i MAILFAIL ${CONTROL}`; do
     70      if [ ${l} -gt 0 ] ; then
     71         mail -s "ERROR! flex.${HOST}.$$" ${muser} <prot
    7272      fi
    73       l=$(($l+1))
     73      l=$((${l}+1))
    7474  done
    7575fi
  • source/pythontest/TestEcFlexpart.py

    r25b14be rc5074d2  
    44
    55import sys
     6import os
     7import inspect
    68import pytest
    79
    810sys.path.append('../python')
     11import _config
    912from classes.EcFlexpart import EcFlexpart
     13from classes.ControlFile import ControlFile
     14from mods.tools import silent_remove
    1015
    1116class TestEcFlexpart():
     
    1318    '''
    1419
    15     def test_init(self):
    16         # create an instance of EcFlexpart and get a dictionary of the
    17         # class attributes, compare this dict with an expected dict!
    18         assert True == True
     20    def test_write_namelist(self):
     21        import filecmp
    1922
    20     def test_write_namelist(self):
    21         # simple
    22         assert True == True
     23        control_file = os.path.join(_config.PATH_TEST_DIR,
     24                                        'TestData',
     25                                        'CONTROL.temp')
     26        c = ControlFile(control_file)
     27        flexpart = EcFlexpart(c)
     28
     29        c.inputdir = 'TestData'
     30
     31        # comparison file
     32        testfile = os.path.join(_config.PATH_TEST_DIR,
     33                                'TestData',
     34                                'convert.nl.test')
     35
     36        # create
     37        flexpart.write_namelist(c)
     38
     39        finalfile = os.path.join(c.inputdir, _config.FILE_NAMELIST)
     40        assert filecmp.cmp(testfile, finalfile, shallow=False)
     41
     42        # delete test file
     43        silent_remove(finalfile)
    2344
    2445    def test_retrieve(self):
  • source/pythontest/TestInstall.py

    r0aaeb04 rc5074d2  
    2525    #!    - un_tarball
    2626    #!    - mk_env_vars
    27     #    - mk_compilejob
    28     #    - mk_job_template
     27    #!    - mk_compilejob
     28    #!    - mk_job_template
    2929    #    - delete_convert_build
    3030    #    - make_convert_build
     
    155155        # delete test file
    156156        silent_remove(finalfile)
     157
     158    def test_mk_job_template(self):
     159        import filecmp
     160
     161        # comparison file
     162        testfile = os.path.join(_config.PATH_TEST_DIR,
     163                                'TestData',
     164                                'job.temp.test')
     165
     166        # create
     167        install.mk_job_template('testuser',
     168                                'testgroup',
     169                                'gateway.test.ac.at',
     170                                'dest@generic',
     171                                'fp_root_test_path')
     172
     173        finalfile = os.path.join(_config.PATH_TEMPLATES,
     174                                 _config.TEMPFILE_JOB)
     175        assert filecmp.cmp(testfile, finalfile, shallow=False)
     176
     177        # delete test file
     178        silent_remove(finalfile)
  • templates/compilejob.template

    r0aaeb04 rc5074d2  
    1717##PBS -q ns
    1818##PBS -S /usr/bin/ksh
    19 ##PBS -o /scratch/ms/$usergroup/$username/flex_ecmwf.$Jobname.$Job_ID.out
     19##PBS -o /scratch/ms/$usergroup/$username/flex_ecmwf.$${Jobname}.$${Job_ID}.out
    2020# job output is in .ecaccess_DO_NOT_REMOVE
    2121##PBS -j oe
     
    2626set -x
    2727export VERSION=$version_number
    28 case $HOST in
     28case $${HOST} in
    2929  *ecg*)
    3030  module load python
     
    4141  module load emos
    4242  module load python
    43   echo ${GROUP}
    44   echo ${HOME}
    45   echo $HOME | awk -F / '{print $1, $2, $3, $4}'
    46   export GROUP=`echo $HOME | awk -F / '{print $4}'`
    47   export SCRATCH=/scratch/ms/${GROUP}/${USER}
     43  echo $${GROUP}
     44  echo $${HOME}
     45  echo $${HOME} | awk -F / '{print $1, $2, $3, $4}'
     46  export GROUP=`echo $${HOME} | awk -F / '{print $4}'`
     47  export SCRATCH=/scratch/ms/$${GROUP}/$${USER}
    4848  export FLEXPART_ROOT_SCRIPTS=$fp_root_scripts
    4949  export MAKEFILE=$makefile
     
    5151esac
    5252
    53 mkdir -p $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION
    54 cd $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION   # if FLEXPART_ROOT is not set this means cd to the home directory
    55 tar -xvf $HOME/flex_extract_v$VERSION.tar
     53mkdir -p $${FLEXPART_ROOT_SCRIPTS}/flex_extract_v$${VERSION}
     54cd $${FLEXPART_ROOT_SCRIPTS}/flex_extract_v$${VERSION}   # if FLEXPART_ROOT is not set this means cd to the home directory
     55tar -xvf $${HOME}/flex_extract_v$${VERSION}.tar
    5656cd src
    5757\rm *.o *.mod $fortran_program
    58 make -f $MAKEFILE >flexcompile 2>flexcompile
     58make -f $${MAKEFILE} >flexcompile 2>flexcompile
    5959
    6060ls -l $fortran_program >>flexcompile
    61 if [ $? -eq 0 ]; then
     61if [ $$? -eq 0 ]; then
    6262  echo 'SUCCESS!' >>flexcompile
    63   mail -s flexcompile.$HOST.$$ $USER <flexcompile
     63  mail -s flexcompile.$${HOST}.$$$$ $${USER} <flexcompile
    6464else
    6565  echo Environment: >>flexcompile
    6666  env >> flexcompile
    67   mail -s "ERROR! flexcompile.$HOST.$$" $USER <flexcompile
     67  mail -s "ERROR! flexcompile.$${HOST}.$$$$" $${USER} <flexcompile
    6868fi
  • templates/job.template

    r82b624a rc5074d2  
    55# start with sbatch NAME_OF_THIS_FILE directly on machine
    66
    7 #SBATCH --workdir=/scratch/ms/spatlh00/lh0
     7#SBATCH --workdir=/scratch/ms/$usergroup/$username
    88#SBATCH --qos=normal
    99#SBATCH --job-name=flex_ecmwf
     
    1717##PBS -q np
    1818##PBS -S /usr/bin/ksh
    19 ## -o /scratch/ms/spatlh00/lh0/flex_ecmwf.$PBS_JOBID.out
     19## -o /scratch/ms/$usergroup/$username/flex_ecmwf.$${PBS_JOBID}.out
    2020## job output is in .ecaccess_DO_NOT_REMOVE
    2121##PBS -j oe
     
    2525
    2626set -x
    27 export VERSION=7.1
    28 case $HOST in
     27export VERSION=$version_number
     28case $${HOST} in
    2929  *ecg*)
    3030  module load python
     
    3333  module load grib_api/1.14.5
    3434  module load emos/437-r64
    35   export PATH=${PATH}:
     35  export PATH=$${PATH}:$fp_root_path
    3636  ;;
    3737  *cca*)
     
    4040  module load emos
    4141  module load python
    42   export SCRATCH=$TMPDIR
    43   export PATH=${PATH}:
     42  export SCRATCH=$${TMPDIR}
     43  export PATH=$${PATH}:$fp_root_path
    4444  ;;
    4545esac
    4646
    47 cd $SCRATCH
    48 mkdir -p python$$
    49 cd python$$
     47cd $${SCRATCH}
     48mkdir -p python$$$$
     49cd python$$$$
    5050
    5151export CONTROL=CONTROL
    5252
    53 cat >$CONTROL<<EOF
     53cat >$${CONTROL}<<EOF
    5454EOF
    5555
    5656
    57 submit.py --controlfile=$CONTROL --inputdir=./work --outputdir=./work 1> prot 2>&1
     57submit.py --controlfile=$${CONTROL} --inputdir=./work --outputdir=./work 1> prot 2>&1
    5858
    5959if [ $? -eq 0 ] ; then
    6060  l=0
    61   for muser in `grep -i MAILOPS $CONTROL`; do
    62       if [ $l -gt 0 ] ; then
    63          mail -s flex.${HOST}.$$ $muser <prot
     61  for muser in `grep -i MAILOPS $${CONTROL}`; do
     62      if [ $${l} -gt 0 ] ; then
     63         mail -s flex.$${HOST}.$$$$ $${muser} <prot
    6464      fi
    65       l=$(($l+1))
     65      l=$(($${l}+1))
    6666  done
    6767else
    6868  l=0
    69   for muser in `grep -i MAILFAIL $CONTROL`; do
    70       if [ $l -gt 0 ] ; then
    71          mail -s "ERROR! flex.${HOST}.$$" $muser <prot
     69  for muser in `grep -i MAILFAIL $${CONTROL}`; do
     70      if [ $${l} -gt 0 ] ; then
     71         mail -s "ERROR! flex.$${HOST}.$$$$" $${muser} <prot
    7272      fi
    73       l=$(($l+1))
     73      l=$(($${l}+1))
    7474  done
    7575fi
     76
Note: See TracChangeset for help on using the changeset viewer.
hosted by ZAMG