Changeset 81c180c in flex_extract.git for Source

Timestamp:

Jul 20, 2022, 3:55:51 PM (22 months ago)

Author:

Anne Tipka <anne.tipka@…>

Branches:

dev

Children:

4f24798

Parents:

6857073 (diff), 7896304 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch current master branch from CTBTO's gitlab repo into dev branch from flexpart.eu repo

Location:

Source/Python

Files:

• Classes/ControlFile.py (modified) (2 diffs)
• Mods/tools.py (modified) (5 diffs)
• _config.py (modified) (6 diffs)
• install.py (modified) (12 diffs)
• submit.py (modified) (1 diff)
• Classes/MarsRetrieval.py (modified) (2 diffs)

Source/Python/Classes/ControlFile.py

@@ -21 +21 @@
 #   June 2020 - Anne Philipp
 #        - update default makefile to None
+#   August 2020 - Leopold Haimberger
+#        - added a class parameter for system installation path
 #
 # @License:
@@ -404 +406 @@
         self.exedir = _config.PATH_FORTRAN_SRC
         self.installdir = None
+        self.sysinstalldir = None
         self.makefile = None
         self.destination = None

Source/Python/Mods/tools.py

@@ -22 +22 @@
 #          put_file_to_ecserver, submit_job_to_ecserver, get_informations,
 #          get_dimensions, execute_subprocess, none_or_int, none_or_str
+#
+#    August 2020 - Leopold Haimberger (University of Vienna)
+#        - added function to check if a specific string is in a file
+#        - added function to overwrite lines in a file which contain specific string
 #
 # @License:
@@ -249 +253 @@
                         type=none_or_str, default=None,
                         help='The name of the ECMWF server name where the'
-                        'job script is to be submitted ' 
+                        'job script is to be submitted '
                         '(e.g. ecgate | cca | ccb)')
 
@@ -624 +628 @@
     '''Creates a directory.
 
-    If the directory already exists, an information is printed and the creation 
+    If the directory already exists, an information is printed and the creation
     skipped. The program stops only if there is another problem.
 
@@ -856 +860 @@
     ----------
     cmd_list : list of str
-        A list of the components for the command line execution. 
-        They will be concatenated with blank space for the command 
+        A list of the components for the command line execution.
+        They will be concatenated with blank space for the command
         to be submitted, like ['mv', file1, file2] for mv file1 file2.
 
@@ -910 +914 @@
 
     return start_period, end_period
+
+
+def check_for_string_in_file(filepath, search_string):
+    """
+    Search for a specific string in a file and return True if
+    the string was found.
+
+    Parameters
+    ----------
+    filepath : str
+        The full file path which is to be examined.
+
+    search_string : str
+        The string which is looked up for in the file.
+
+    Return
+    ------
+    Boolean :
+        True : String was found
+        False : String was not found
+    """
+    with open(filepath, 'r') as fio:
+        for line in fio:
+            if search_string in line:
+                return True
+    return False
+
+
+def overwrite_lines_in_file(filepath, search_string, sub_string):
+    """
+    Overwrites lines which contain the given search string with the
+    substitution string.
+
+    Parameters
+    ----------
+    search_string : str
+        The string which is looked up for in the file.
+
+    sub_string : str
+        The string which overwrites the search string.
+
+    Return
+    ------
+    """
+    with open(filepath, 'r') as fio:
+        data = fio.readlines()
+
+    with open(filepath, 'w') as fio:
+        for line in data:
+            if search_string in line:
+                fio.write(sub_string)
+            else:
+                fio.write(line)
+
+    return
+

Source/Python/_config.py

@@ -9 +9 @@
 #      June 2020 - Anne Philipp
 #         - changed template filenames to .template
+#      August 2020 - Leopold Haimberger
+#         - added another target for installation
+#         - added filename which will contain paths for system version     
+#         - checks if software runs in normal local mode or system local mode
+#           and defines paths to user directory and executable paths    
 #
 # @License:
@@ -38 +43 @@
 # ------------------------------------------------------------------------------
 
-_VERSION_STR = '7.1.2'
+_VERSION_STR = '7.1.2_ctbto'
 
 FLAG_ON_ECMWFSERVER = 'ecgb' in platform.node()
@@ -44 +49 @@
 QUEUES_LIST = ['ecgate', 'cca', 'ccb']
 
-INSTALL_TARGETS = ['local', 'ecgate', 'cca', 'ccb']
+INSTALL_TARGETS = ['local', 'syslocal', 'ecgate', 'cca', 'ccb']
 
 CDS_DATASET_ML = 'reanalysis-era5-complete'
@@ -70 +75 @@
 FILE_GRIB_INDEX = 'date_time_stepRange.idx'
 FILE_GRIBTABLE = 'ecmwf_grib1_table_128'
+FILE_SYS_CONFIG = '.setup.rc'
 
 # ------------------------------------------------------------------------------
@@ -79 +85 @@
 
 # ------------------------------------------------------------------------------
-#  PATHES
+#  LOAD ENVIRONMENT VARIABLES FOR SYS VERSION; IF NECESSARRY
 # ------------------------------------------------------------------------------
 
@@ -89 +95 @@
 if PATH_LOCAL_PYTHON not in sys.path:
     sys.path.append(PATH_LOCAL_PYTHON)
+
+# ------------------------------------------------------------------------------
+#  PATHES
+# ------------------------------------------------------------------------------
+
 PATH_FLEXEXTRACT_DIR = os.path.normpath(os.path.dirname(os.path.abspath(
     inspect.getfile(inspect.currentframe()))) + '/../../')
+if not os.path.isdir(os.path.join(PATH_FLEXEXTRACT_DIR,'Run')):
+    # if it does not exist, we have a system installation in place
+    # we need to have a sys and user path
+    # configure correct system path
+    PATH_SYSTEM_DIR = os.path.join(PATH_FLEXEXTRACT_DIR, FLEXEXTRACT_DIRNAME)
+    # configure correct user path
+    PATH_FLEXEXTRACT_DIR = os.environ.get('FLEXEXTRACT_USER_DIR')
+else:
+    PATH_SYSTEM_DIR = PATH_FLEXEXTRACT_DIR
+
 PATH_RUN_DIR = os.path.join(PATH_FLEXEXTRACT_DIR, 'Run')
-PATH_SOURCES = os.path.join(PATH_FLEXEXTRACT_DIR, 'Source')
+PATH_SOURCES = os.path.join(PATH_SYSTEM_DIR, 'Source')
 PATH_TEMPLATES = os.path.join(PATH_FLEXEXTRACT_DIR, 'Templates')
 PATH_ECMWF_ENV = os.path.join(PATH_RUN_DIR, FILE_USER_ENVVARS)
 PATH_GRIBTABLE = os.path.join(PATH_TEMPLATES, FILE_GRIBTABLE)
 PATH_JOBSCRIPTS = os.path.join(PATH_RUN_DIR, 'Jobscripts')
-PATH_FORTRAN_SRC = os.path.join(PATH_SOURCES, 'Fortran')
+if os.path.isdir(os.path.join(PATH_SYSTEM_DIR,'Fortran')):
+    PATH_FORTRAN_SRC = PATH_SYSTEM_DIR
+else:
+    PATH_FORTRAN_SRC = os.path.join(PATH_SOURCES, 'Fortran')
 PATH_PYTHONTEST_SRC = os.path.join(PATH_SOURCES, 'Pythontest')
 PATH_INPUT_DIR = os.path.join(PATH_RUN_DIR, INPUT_DIRNAME_DEFAULT)

Source/Python/install.py

@@ -19 +19 @@
 #    June 2020 - Anne Philipp
 #        - renamed "convert" functions to "fortran" functions
-#        - reconfigured mk_tarball to select *.template files instead 
+#        - reconfigured mk_tarball to select *.template files instead
 #          of *.nl and *.temp
 #        - added check for makefile settings
+#    August 2020 - Leopold Haimberger
+#        - added a new installation section for system installation (if-else block)
+#        - read new argument from command line
+#        - write .setup.rc for a system installation into Run directory
+#        - copy executables to system path and user files to user path
 #
 # @License:
@@ -75 +80 @@
 import subprocess
 import tarfile
+import shutil
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
 
@@ -82 +88 @@
 from Classes.UioFiles import UioFiles
 from Mods.tools import (make_dir, put_file_to_ecserver, submit_job_to_ecserver,
-                        silent_remove, execute_subprocess, none_or_str)
+                        silent_remove, execute_subprocess, none_or_str,
+                        overwrite_lines_in_file, check_for_string_in_file)
 
 # ------------------------------------------------------------------------------
@@ -103 +110 @@
     check_install_conditions(c)
 
-    if c.install_target.lower() != 'local': # ecgate or cca
+    if c.install_target.lower() not in ['local', 'syslocal']: # ecgate or cca
         install_via_gateway(c)
     else: # local
         install_local(c)
 
+    print("SUCCESS: INSTALLATION FINISHED!")
+
     return
 
@@ -128 +137 @@
     parser.add_argument('--target', dest='install_target',
                         type=none_or_str, default=None,
-                        help="Valid targets: local | ecgate | cca , \
+                        help="Valid targets: syslocal | local | ecgate | cca , \
                         the latter two are at ECMWF")
     parser.add_argument("--makefile", dest="makefile",
@@ -150 +159 @@
     parser.add_argument("--installdir", dest="installdir",
                         type=none_or_str, default=None,
-                        help='Root directory of the '
+                        help='Root (user) directory of the '
                         'flex_extract installation')
+    parser.add_argument("--sysinstalldir", dest="sysinstalldir",
+                        type=none_or_str, default=None,
+                        help='System installation path; where '
+                        'executables are stored.')
 
     # arguments for job submission to ECMWF, only needed by submit.py
@@ -228 +241 @@
                             _config.FLEXEXTRACT_DIRNAME + '.tar')
 
-    if c.installdir == _config.PATH_FLEXEXTRACT_DIR:
-        print('WARNING: installdir has not been specified')
-        print('flex_extract will be installed in here by compiling the ' +
-              'Fortran source in ' + _config.PATH_FORTRAN_SRC)
-        os.chdir(_config.PATH_FORTRAN_SRC)
-    else: # creates the target working directory for flex_extract
-        c.installdir = os.path.expandvars(os.path.expanduser(
-            c.installdir))
-        if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) != \
-           os.path.abspath(c.installdir):
+    c.installdir = os.path.abspath(os.path.expandvars(os.path.expanduser(
+                c.installdir)))
+    c.sysinstalldir = os.path.abspath(os.path.expandvars(os.path.expanduser(
+        c.sysinstalldir)))
+
+    # this is standard installation into a single directory
+    if c.install_target == 'local':
+
+        # installation into the current directory
+        if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.installdir:
+            print('WARNING: installdir has not been specified')
+            print('flex_extract will be installed in current dir by compiling the ' +
+                  'Fortran source in ' + _config.PATH_FORTRAN_SRC)
+            os.chdir(_config.PATH_FORTRAN_SRC)
+        # installation into a different path
+        elif os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) != c.installdir :
+
+            # creates the target working directory for flex_extract
             mk_tarball(tar_file, c.install_target)
             make_dir(os.path.join(c.installdir,
@@ -248 +269 @@
                                   _config.PATH_REL_FORTRAN_SRC))
 
-    # Create Fortran executable
-    print('Install ' +  _config.FLEXEXTRACT_DIRNAME + ' software at ' +
-          c.install_target + ' in directory ' +
-          os.path.abspath(c.installdir) + '\n')
-
-    del_fortran_build('.')
-    mk_fortran_build('.', c.makefile)
+        # Create Fortran executable
+        print('Install ' +  _config.FLEXEXTRACT_DIRNAME + ' software at ' +
+              c.install_target + ' in directory ' + c.installdir + '\n')
+
+        del_fortran_build('.')
+        mk_fortran_build('.', c.makefile)
+        os.chdir('../../')
+        # make sure that the correct calling of submit.py script is in run_local.sh
+        overwrite_lines_in_file('Run/run_local.sh',
+                                'pyscript=', 'pyscript=../Source/Python/submit.py\n')
+
+    # this is system installation were executables and user files are separated
+    elif c.install_target == 'syslocal':
+        if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.sysinstalldir :
+            sys.exit('ERROR: System installation path is equal to user '
+                     'installation path.\n Please change either the system '
+                     'installation path or use installation target "local".')
+        if os.path.abspath(_config.PATH_FLEXEXTRACT_DIR) == c.installdir :
+            print('Flex_extract will be installed in current directory!')
+        else: # install user part in different dir
+            print('Flex_extract will be installed in ' + c.installdir )
+
+            c.installdir = os.path.join(c.installdir,_config.FLEXEXTRACT_DIRNAME)
+            if os.path.isdir(c.installdir):
+                shutil.rmtree(c.installdir)
+
+            # copy all files except Python and Fortranfiles to this dir
+            shutil.copytree(_config.PATH_FLEXEXTRACT_DIR,
+                            c.installdir, symlinks=True)
+            shutil.rmtree(os.path.join(c.installdir,'Source'))
+            shutil.rmtree(os.path.join(c.installdir,'.git'))
+            for x in UioFiles(c.installdir, '*~').files:
+                silent_remove(x)
+
+            os.remove(os.path.join(c.installdir,'setup.sh'))
+            os.remove(os.path.join(c.installdir,'setup_local.sh'))
+
+        # configure run_local script correctly
+        # check if source of system config file is already in run_local.sh,
+        # if not, add it
+        if not check_for_string_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
+                                 'source .setup.rc'):
+            overwrite_lines_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
+                                    '# PATH TO SUBMISSION SCRIPT',
+                                    '# PATH TO SUBMISSION SCRIPT\nsource '+_config.FILE_SYS_CONFIG+'\n')
+        # make sure that the correct calling of submit.py script is in run_local.sh
+        overwrite_lines_in_file(os.path.join(c.installdir,'Run/run_local.sh'),
+                                'pyscript=', 'pyscript=submit.py\n')
+
+        # change permission for file to executable
+        execute_subprocess(['chmod', '0775',
+                            os.path.join(os.path.abspath(c.installdir),'Run/run_local.sh')])
+
+
+        # create systemdir
+        c.sysinstalldir = os.path.join(c.sysinstalldir,_config.FLEXEXTRACT_DIRNAME)
+        if os.path.isdir(c.sysinstalldir):
+            shutil.rmtree(c.sysinstalldir)
+
+        # create setup file for running flex_extract with system installation
+        with open(os.path.join(os.path.abspath(c.installdir),'Run/.setup.rc'),'w') as fio:
+            fio.write('#!/bin/bash \n')
+            fio.write('export FLEXEXTRACT_USER_DIR='+os.path.abspath(c.installdir)+'\n')
+            fio.write('export PATH='+os.path.abspath(c.sysinstalldir)+'/Python:${PATH}\n')
+            fio.write('export PATH='+os.path.abspath(c.sysinstalldir)+':${PATH}\n')
+
+        # copy all Python and Fortranfiles to this dir
+        shutil.copytree(_config.PATH_SOURCES, c.sysinstalldir, symlinks=True)
+
+        os.chdir(os.path.join(c.sysinstalldir,'Fortran'))
+        # Create Fortran executable
+        print('Install ' +  _config.FLEXEXTRACT_DIRNAME + ' software as ' +
+              c.install_target + ' in directory ' +
+              os.path.abspath(c.sysinstalldir) + '\n')
+
+        del_fortran_build('.')
+        mk_fortran_build('.', c.makefile)
+
+        outfile = [x for x in UioFiles('.','*.out').files]
+        test=os.path.join(c.sysinstalldir,'calc_etadot')
+        if len(outfile) != 1:
+            print('WARNING: Multiple executables for Fortran code are available!')
+        # move executable one dir up and delete Fortran dir
+        os.chdir('..')
+        shutil.move(outfile[0], os.path.join(c.sysinstalldir,'calc_etadot'))
+        shutil.rmtree(os.path.join(os.path.abspath(c.sysinstalldir),'Fortran'))
 
     os.chdir(_config.PATH_FLEXEXTRACT_DIR)
@@ -289 +389 @@
         sys.exit(1)
 
-    if c.install_target and c.install_target != 'local':
+    if c.install_target and c.install_target not in ['local', 'syslocal']:
         if not c.ecgid or not c.ecuid:
             print('Please enter your ECMWF user id and group id '
@@ -305 +405 @@
         if not c.installdir:
             c.installdir = '${HOME}'
-    else: # local
+    elif c.install_target == 'local':
         if not c.installdir:
             c.installdir = _config.PATH_FLEXEXTRACT_DIR
+    elif c.install_target == 'syslocal':
+        if not c.installdir:
+            c.installdir = _config.PATH_FLEXEXTRACT_DIR
+        if not c.sysinstalldir:
+            print('ERROR: System installation was selected but '
+                  'no system installation path was defined.')
+            sys.exit()
 
     if not c.makefile:
@@ -323 +430 @@
         else:
             pass
-        
+
     return
 
@@ -715 +822 @@
         print(e)
     else:
-        execute_subprocess(['ls', '-l', 
+        execute_subprocess(['ls', '-l',
                             os.path.join(src_path, _config.FORTRAN_EXECUTABLE)],
                            error_msg='FORTRAN EXECUTABLE COULD NOT BE FOUND!')

Source/Python/submit.py

@@ -114 +114 @@
     else:
         submit(job_template, c, queue)
-        exit_message = 'FLEX_EXTRACT JOB SCRIPT IS SUBMITED!'
+        exit_message = 'FLEX_EXTRACT JOB SCRIPT IS SUBMITTED!'
 
     normal_exit(exit_message)

Source/Python/Classes/MarsRetrieval.py

@@ -505 +505 @@
             newattrs['year'] = date.year
             newattrs['month'] = date.month
-            newattrs['day'] =  date.day                  
+            newattrs['day'] =  date.day          
+  
+        # need to correct the time appearance for CDS surface field retrievals 
+        if attrs['type'] == 'FC': # for EA5 only flux fields are retrieved as FC type
+            # need to convert fc start times 06/18 to usual AN times
+            # since the surface fields can only be accessed through their validity time
+            start, end, step = map(int,attrs['step'].split('/')[::2])
+            newattrs['time'] = [ "{0:0=2d}".format(s) for s in range(0,24,step) ]
+        elif '/' in attrs['time']: # we expect a list of times separated by /
+            newattrs['time'] = attrs['time'].split('/')
+        elif isinstance(attrs['time'], str): # we expect a single time here
+            newattrs['time'] = [ attrs['time'] ]
         
         newattrs['product_type'] = 'reanalysis'
@@ -511 +522 @@
         newattrs['grid'] = list(map(float,attrs['grid'].split('/')))
         newattrs['param'] = attrs['param'].split('/')        
-        newattrs['time'] = list(map(str,range(0,24,3)))
+        if '/' in attrs['step']:
+            sstep = int(attrs['step'].split('/')[4])
+            newattrs['time'] = list(map(str,range(0,24,sstep)))              
+        elif '160.128' in attrs['param']: 
+            newattrs['time'] = attrs['time']
         newattrs['format'] = 'grib'