Changeset 47be2684 in flex_extract.git for Source

Timestamp:

Oct 28, 2020, 10:28:24 AM (4 years ago)

Author:

Leopold Haimberger <leopold.haimberger@…>

Branches:

ctbto, dev

Children:

75db9b0

Parents:

697b8d0

Message:

Adaptations to allow for a system installation with separate user and system path. Updated documentation

Location:

Source/Python

Files:

• Classes/ControlFile.py (modified) (1 diff)
• Mods/tools.py (modified) (4 diffs)
• _config.py (modified) (5 diffs)
• install.py (modified) (12 diffs)
• submit.py (modified) (1 diff)

Source/Python/Classes/ControlFile.py

@@ -404 +404 @@
         self.exedir = _config.PATH_FORTRAN_SRC
         self.installdir = None
+        self.sysinstalldir = None
         self.makefile = None
         self.destination = None

Source/Python/Mods/tools.py

@@ -249 +249 @@
                         type=none_or_str, default=None,
                         help='The name of the ECMWF server name where the'
-                        'job script is to be submitted ' 
+                        'job script is to be submitted '
                         '(e.g. ecgate | cca | ccb)')
 
@@ -624 +624 @@
     '''Creates a directory.
 
-    If the directory already exists, an information is printed and the creation 
+    If the directory already exists, an information is printed and the creation
     skipped. The program stops only if there is another problem.
 
@@ -856 +856 @@
     ----------
     cmd_list : list of str
-        A list of the components for the command line execution. 
-        They will be concatenated with blank space for the command 
+        A list of the components for the command line execution.
+        They will be concatenated with blank space for the command
         to be submitted, like ['mv', file1, file2] for mv file1 file2.
 
@@ -910 +910 @@
 
     return start_period, end_period
+
+
+def check_for_string_in_file(filepath, search_string):
+    """
+    Search for a specific string in a file and return True if
+    the string was found.
+
+    Parameters
+    ----------
+    filepath : str
+        The full file path which is to be examined.
+
+    search_string : str
+        The string which is looked up for in the file.
+
+    Return
+    ------
+    Boolean :
+        True : String was found
+        False : String was not found
+    """
+    with open(filepath, 'r') as fio:
+        for line in fio:
+            if search_string in line:
+                return True
+    return False
+
+
+def overwrite_lines_in_file(filepath, search_string, sub_string):
+    """
+    Overwrites lines which contain the given search string with the
+    substitution string.
+
+    Parameters
+    ----------
+    search_string : str
+        The string which is looked up for in the file.
+
+    sub_string : str
+        The string which overwrites the search string.
+
+    Return
+    ------
+    """
+    with open(filepath, 'r') as fio:
+        data = fio.readlines()
+
+    with open(filepath, 'w') as fio:
+        for line in data:
+            if search_string in line:
+                fio.write(sub_string)
+            else:
+                fio.write(line)
+
+    return
+

Source/Python/_config.py

@@ -38 +38 @@
 # ------------------------------------------------------------------------------
 
-_VERSION_STR = '7.1.2'
+_VERSION_STR = '7.1.2_ctbto'
 
 FLAG_ON_ECMWFSERVER = 'ecgb' in platform.node()
@@ -44 +44 @@
 QUEUES_LIST = ['ecgate', 'cca', 'ccb']
 
-INSTALL_TARGETS = ['local', 'ecgate', 'cca', 'ccb']
+INSTALL_TARGETS = ['local', 'syslocal', 'ecgate', 'cca', 'ccb']
 
 CDS_DATASET_ML = 'reanalysis-era5-complete'
@@ -70 +70 @@
 FILE_GRIB_INDEX = 'date_time_stepRange.idx'
 FILE_GRIBTABLE = 'ecmwf_grib1_table_128'
+FILE_SYS_CONFIG = '.setup.rc'
 
 # ------------------------------------------------------------------------------
@@ -79 +80 @@
 
 # ------------------------------------------------------------------------------
-#  PATHES
+#  LOAD ENVIRONMENT VARIABLES FOR SYS VERSION; IF NECESSARRY
 # ------------------------------------------------------------------------------
 
@@ -89 +90 @@
 if PATH_LOCAL_PYTHON not in sys.path:
     sys.path.append(PATH_LOCAL_PYTHON)
+
+# ------------------------------------------------------------------------------
+#  PATHES
+# ------------------------------------------------------------------------------
+
 PATH_FLEXEXTRACT_DIR = os.path.normpath(os.path.dirname(os.path.abspath(
     inspect.getfile(inspect.currentframe()))) + '/../../')
+if not os.path.isdir(os.path.join(PATH_FLEXEXTRACT_DIR,'Run')):
+    # if it does not exist, we have a system installation in place
+    # we need to have a sys and user path
+    # configure correct system path
+    PATH_SYSTEM_DIR = os.path.join(PATH_FLEXEXTRACT_DIR, FLEXEXTRACT_DIRNAME)
+    # configure correct user path
+    PATH_FLEXEXTRACT_DIR = os.environ.get('FLEXEXTRACT_USER_DIR')
+else:
+    PATH_SYSTEM_DIR = PATH_FLEXEXTRACT_DIR
+
 PATH_RUN_DIR = os.path.join(PATH_FLEXEXTRACT_DIR, 'Run')
-PATH_SOURCES = os.path.join(PATH_FLEXEXTRACT_DIR, 'Source')
+PATH_SOURCES = os.path.join(PATH_SYSTEM_DIR, 'Source')
 PATH_TEMPLATES = os.path.join(PATH_FLEXEXTRACT_DIR, 'Templates')
 PATH_ECMWF_ENV = os.path.join(PATH_RUN_DIR, FILE_USER_ENVVARS)
 PATH_GRIBTABLE = os.path.join(PATH_TEMPLATES, FILE_GRIBTABLE)
 PATH_JOBSCRIPTS = os.path.join(PATH_RUN_DIR, 'Jobscripts')
-PATH_FORTRAN_SRC = os.path.join(PATH_SOURCES, 'Fortran')
+if os.path.isdir(os.path.join(PATH_SYSTEM_DIR,'Fortran')):
+    PATH_FORTRAN_SRC = PATH_SYSTEM_DIR
+else:
+    PATH_FORTRAN_SRC = os.path.join(PATH_SOURCES, 'Fortran')
 PATH_PYTHONTEST_SRC = os.path.join(PATH_SOURCES, 'Pythontest')
 PATH_INPUT_DIR = os.path.join(PATH_RUN_DIR, INPUT_DIRNAME_DEFAULT)

Source/Python/install.py

@@ -19 +19 @@
 #    June 2020 - Anne Philipp
 #        - renamed "convert" functions to "fortran" functions
-#        - reconfigured mk_tarball to select *.template files instead 
+#        - reconfigured mk_tarball to select *.template files instead
 #          of *.nl and *.temp
 #        - added check for makefile settings
@@ -75 +75 @@
 import subprocess
 import tarfile
+import shutil
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
 
@@ -82 +83 @@
 from Classes.UioFiles import UioFiles
 from Mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
-                        silent_remove, execute_subprocess, none_or_str)
+                        silent_remove, execute_subprocess, none_or_str,
+                        overwrite_lines_in_file, check_for_string_in_file)
 
 # ------------------------------------------------------------------------------
@@ -103 +105 @@
     check_install_conditions(c)
 
-    if c.install_target.lower() != 'local': # ecgate or cca
+    if c.install_target.lower() not in ['local', 'syslocal']: # ecgate or cca
         install_via_gateway(c)
     else: # local
         install_local(c)
 
+    print("SUCCESS: INSTALLATION FINISHED!")
+
     return
 
@@ -128 +132 @@
     parser.add_argument('--target', dest='install_target',
                         type=none_or_str, default=None,
-                        help="Valid targets: local | ecgate | cca , \
+                        help="Valid targets: syslocal | local | ecgate | cca , \
                         the latter two are at ECMWF")
     parser.add_argument("--makefile", dest="makefile",
@@ -150 +154 @@
     parser.add_argument("--installdir", dest="installdir",
                         type=none_or_str, default=None,
-                        help='Root directory of the '
+                        help='Root (user) directory of the '
                         'flex_extract installation')
+    parser.add_argument("--sysinstalldir", dest="sysinstalldir",
+                        type=none_or_str, default=None,
+                        help='System installation path; where '
+                        'executables are stored.')
 
     # arguments for job submission to ECMWF, only needed by submit.py
@@ -228 +236 @@
                             _config.FLEXEXTRACT_DIRNAME + '.tar')
 
-    if c.installdir == _config.PATH_FLEXEXTRACT_DIR:
-        print('WARNING: installdir has not been specified')
-        print('flex_extract will be installed in here by compiling the ' +
-              'Fortran source in ' + _config.PATH_FORTRAN_SRC)
-        os.chdir(_config.PATH_FORTRAN_SRC)
-    else: # creates the target working directory for flex_extract
-        c.installdir = os.path.expandvars(os.path.expanduser(
-            c.installdir))
-        if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) != \
-           os.path.abspath(c.installdir):
+    c.installdir = os.path.abspath(os.path.expandvars(os.path.expanduser(
+                c.installdir)))
+    c.sysinstalldir = os.path.abspath(os.path.expandvars(os.path.expanduser(
+        c.sysinstalldir)))
+
+    # this is standard installation into a single directory
+    if c.install_target == 'local':
+
+        # installation into the current directory
+        if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.installdir:
+            print('WARNING: installdir has not been specified')
+            print('flex_extract will be installed in current dir by compiling the ' +
+                  'Fortran source in ' + _config.PATH_FORTRAN_SRC)
+            os.chdir(_config.PATH_FORTRAN_SRC)
+        # installation into a different path
+        elif os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) != c.installdir :
+
+            # creates the target working directory for flex_extract
             mk_tarball(tar_file, c.install_target)
             make_dir(os.path.join(c.installdir,
@@ -248 +264 @@
                                   _config.PATH_REL_FORTRAN_SRC))
 
-    # Create Fortran executable
-    print('Install ' +  _config.FLEXEXTRACT_DIRNAME + ' software at ' +
-          c.install_target + ' in directory ' +
-          os.path.abspath(c.installdir) + '\n')
-
-    del_fortran_build('.')
-    mk_fortran_build('.', c.makefile)
+        # Create Fortran executable
+        print('Install ' +  _config.FLEXEXTRACT_DIRNAME + ' software at ' +
+              c.install_target + ' in directory ' + c.installdir + '\n')
+
+        del_fortran_build('.')
+        mk_fortran_build('.', c.makefile)
+        os.chdir('../../')
+        # make sure that the correct calling of submit.py script is in run_local.sh
+        overwrite_lines_in_file('Run/run_local.sh',
+                                'pyscript=', 'pyscript=../Source/Python/submit.py\n')
+
+    # this is system installation were executables and user files are separated
+    elif c.install_target == 'syslocal':
+        if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.sysinstalldir :
+            sys.exit('ERROR: System installation path is equal to user '
+                     'installation path.\n Please change either the system '
+                     'installation path or use installation target "local".')
+        if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.installdir :
+            print('Flex_extract will be installed in current directory!')
+        else: # install user part in different dir
+            print('Flex_extract will be installed in ' + c.installdir )
+
+            c.installdir = os.path.join(c.installdir,_config.FLEXEXTRACT_DIRNAME)
+            if os.path.isdir(c.installdir):
+                shutil.rmtree(c.installdir)
+
+            # copy all files except Python and Fortranfiles to this dir
+            shutil.copytree(_config.PATH_FLEXEXTRACT_DIR,
+                            c.installdir, symlinks=True)
+            shutil.rmtree(os.path.join(c.installdir,'Source'))
+            shutil.rmtree(os.path.join(c.installdir,'.git'))
+            for x in UioFiles(c.installdir, '*~').files:
+                silent_remove(x)
+
+            os.remove(os.path.join(c.installdir,'setup.sh'))
+            os.remove(os.path.join(c.installdir,'setup_local.sh'))
+
+        # configure run_local script correctly
+        # check if source of system config file is already in run_local.sh,
+        # if not, add it
+        if not check_for_string_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
+                                 'source .setup.rc'):
+            overwrite_lines_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
+                                    '# PATH TO SUBMISSION SCRIPT',
+                                    '# PATH TO SUBMISSION SCRIPT\nsource '+_config.FILE_SYS_CONFIG+'\n')
+        # make sure that the correct calling of submit.py script is in run_local.sh
+        overwrite_lines_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
+                                'pyscript=', 'pyscript=submit.py\n')
+
+        # change permission for file to executable
+        execute_subprocess(['chmod', '0775',
+                            os.path.join(os.path.abspath(c.installdir),'Run/run_local.sh')])
+
+
+        # create systemdir
+        c.sysinstalldir = os.path.join(c.sysinstalldir,_config.FLEXEXTRACT_DIRNAME)
+        if os.path.isdir(c.sysinstalldir):
+            shutil.rmtree(c.sysinstalldir)
+
+        # create setup file for running flex_extract with system installation
+        with open(os.path.join(os.path.abspath(c.installdir),'Run/.setup.rc'),'w') as fio:
+            fio.write('#!/bin/bash \n')
+            fio.write('export FLEXEXTRACT_USER_DIR='+os.path.abspath(c.installdir)+'\n')
+            fio.write('export PATH='+os.path.abspath(c.sysinstalldir)+'/Python:${PATH}\n')
+            fio.write('export PATH='+os.path.abspath(c.sysinstalldir)+':${PATH}\n')
+
+        # copy all Python and Fortranfiles to this dir
+        shutil.copytree(_config.PATH_SOURCES, c.sysinstalldir, symlinks=True)
+
+        os.chdir(os.path.join(c.sysinstalldir,'Fortran'))
+        # Create Fortran executable
+        print('Install ' +  _config.FLEXEXTRACT_DIRNAME + ' software as ' +
+              c.install_target + ' in directory ' +
+              os.path.abspath(c.sysinstalldir) + '\n')
+
+        del_fortran_build('.')
+        mk_fortran_build('.', c.makefile)
+
+        outfile = [x for x in UioFiles('.','*.out').files]
+        test=os.path.join(c.sysinstalldir,'calc_etadot')
+        if len(outfile) != 1:
+            print('WARNING: Multiple executables for Fortran code are available!')
+        # move executable one dir up and delete Fortran dir
+        os.chdir('..')
+        shutil.move(outfile[0], os.path.join(c.sysinstalldir,'calc_etadot'))
+        shutil.rmtree(os.path.join(os.path.abspath(c.sysinstalldir),'Fortran'))
 
     os.chdir(_config.PATH_FLEXEXTRACT_DIR)
@@ -289 +384 @@
         sys.exit(1)
 
-    if c.install_target and c.install_target != 'local':
+    if c.install_target and c.install_target not in ['local', 'syslocal']:
         if not c.ecgid or not c.ecuid:
             print('Please enter your ECMWF user id and group id '
@@ -305 +400 @@
         if not c.installdir:
             c.installdir = '${HOME}'
-    else: # local
+    elif c.install_target == 'local':
         if not c.installdir:
             c.installdir = _config.PATH_FLEXEXTRACT_DIR
+    elif c.install_target == 'syslocal':
+        if not c.installdir:
+            c.installdir = _config.PATH_FLEXEXTRACT_DIR
+        if not c.sysinstalldir:
+            print('ERROR: System installation was selected but '
+                  'no system installation path was defined.')
+            sys.exit()
 
     if not c.makefile:
@@ -323 +425 @@
         else:
             pass
-        
+
     return
 
@@ -715 +817 @@
         print(e)
     else:
-        execute_subprocess(['ls', '-l', 
+        execute_subprocess(['ls', '-l',
                             os.path.join(src_path, _config.FORTRAN_EXECUTABLE)],
                            error_msg='FORTRAN EXECUTABLE COULD NOT BE FOUND!')

Source/Python/submit.py

@@ -114 +114 @@
     else:
         submit(job_template, c, queue)
-        exit_message = 'FLEX_EXTRACT JOB SCRIPT IS SUBMITED!'
+        exit_message = 'FLEX_EXTRACT JOB SCRIPT IS SUBMITTED!'
 
     normal_exit(exit_message)