Changeset 0aaeb04 in flex_extract.git for source/python

Timestamp:

Oct 8, 2018, 3:49:02 PM (6 years ago)

Author:

Anne Philipp <anne.philipp@…>

Branches:

master, ctbto, dev

Children:

0a06607

Parents:

0e576fc

Message:

added writing of compilejob and ECMWF_ENV via templates and added tests

Location:

source/python

Files:

• _config.py (modified) (1 diff)
• install.py (modified) (3 diffs)

source/python/_config.py

@@ -34 +34 @@
 FILE_MARS_REQUESTS = 'mars_requests.csv'
 FORTRAN_EXECUTABLE = 'CONVERT2'
+TEMPFILE_USER_ENVVARS = 'ECMWF_ENV.template'
 FILE_USER_ENVVARS = 'ECMWF_ENV'
-TEMPFILE_INSTALL_COMPILEJOB = 'compilejob.temp'
+TEMPFILE_INSTALL_COMPILEJOB = 'compilejob.template'
 FILE_INSTALL_COMPILEJOB = 'compilejob.ksh'
 TEMPFILE_INSTALL_JOB = 'job.temp.o'

source/python/install.py

@@ -320 +320 @@
         Creates a file named ECMWF_ENV which contains the
         necessary environmental variables at ECMWF servers.
+        It is based on the template ECMWF_ENV.template.
 
     @Input:
@@ -338 +339 @@
         <nothing>
     '''
-
-    with open(_config.PATH_REL_ECMWF_ENV, 'w') as fo:
-        fo.write('ECUID ' + ecuid + '\n')
-        fo.write('ECGID ' + ecgid + '\n')
-        fo.write('GATEWAY ' + gateway + '\n')
-        fo.write('DESTINATION ' + destination + '\n')
+    from genshi.template.text import NewTextTemplate
+    from genshi.template import  TemplateLoader
+
+    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
+    ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
+                                     cls=NewTextTemplate)
+
+    stream = ecmwfvars_template.generate(user_name = ecuid,
+                                       user_group = ecgid,
+                                       gateway_name = gateway,
+                                       destination_name = destination
+                                       )
+
+    with open(_config.PATH_ECMWF_ENV, 'w') as f:
+        f.write(stream.render('text'))
 
     return
@@ -375 +385 @@
         <nothing>
     '''
-
-    template = os.path.join(_config.PATH_REL_TEMPLATES,
-                            _config.TEMPFILE_INSTALL_COMPILEJOB)
-    with open(template) as f:
-        fdata = f.read().split('\n')
-
-    compilejob = os.path.join(_config.PATH_REL_JOBSCRIPTS,
+    from genshi.template.text import NewTextTemplate
+    from genshi.template import  TemplateLoader
+
+    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
+    compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
+                                   cls=NewTextTemplate)
+
+    if fp_root == '../':
+        fp_root = '$HOME'
+
+    stream = compile_template.generate(
+        username = ecuid,
+        usergroup = ecgid,
+        version_number = _config._VERSION_STR,
+        fp_root_scripts = fp_root,
+        makefile = makefile,
+        fortran_program = _config.FORTRAN_EXECUTABLE
+    )
+
+    compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
                               _config.FILE_INSTALL_COMPILEJOB)
-    with open(compilejob, 'w') as fo:
-        for data in fdata:
-            if 'MAKEFILE=' in data:
-                data = 'export MAKEFILE=' + makefile
-            elif 'FLEXPART_ROOT_SCRIPTS=' in data:
-                if fp_root != '../':
-                    data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
-                else:
-                    data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
-            elif target.lower() != 'local':
-                if '--workdir' in data:
-                    data = '#SBATCH --workdir=/scratch/ms/' + \
-                            ecgid + '/' + ecuid
-                elif '##PBS -o' in data:
-                    data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \
-                           'flex_ecmwf.$Jobname.$Job_ID.out'
-                elif 'FLEXPART_ROOT_SCRIPTS=' in data:
-                    if fp_root != '../':
-                        data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
-                    else:
-                        data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
-            fo.write(data + '\n')
+
+    with open(compilejob, 'w') as f:
+        f.write(stream.render('text'))
+
+
+    # template = os.path.join(_config.PATH_REL_TEMPLATES,
+                            # _config.TEMPFILE_INSTALL_COMPILEJOB)
+    # with open(template) as f:
+        # fdata = f.read().split('\n')
+
+
+    # with open(compilejob, 'w') as fo:
+        # for data in fdata:
+            # if 'MAKEFILE=' in data:
+                # data = 'export MAKEFILE=' + makefile
+            # elif 'FLEXPART_ROOT_SCRIPTS=' in data:
+                # if fp_root != '../':
+                    # data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
+                # else:
+                    # data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
+            # elif target.lower() != 'local':
+                # if '--workdir' in data:
+                    # data = '#SBATCH --workdir=/scratch/ms/' + \
+                            # ecgid + '/' + ecuid
+                # elif '##PBS -o' in data:
+                    # data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \
+                           # 'flex_ecmwf.$Jobname.$Job_ID.out'
+                # elif 'FLEXPART_ROOT_SCRIPTS=' in data:
+                    # if fp_root != '../':
+                        # data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
+                    # else:
+                        # data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
+            # fo.write(data + '\n')
 
     return