Changeset 25b14be in flex_extract.git for source/python

Timestamp:

Sep 23, 2018, 11:40:28 AM (6 years ago)

Author:

Anne Philipp <anne.philipp@…>

Branches:

master, ctbto, dev

Children:

4971f63

Parents:

5d42acd

Message:

changed whole tree structure of flex_extract to have better overview

Location:

source/python

Files:

• __init__.py (added)
• _config.py (added)
• classes/ControlFile.py (moved) (moved from python/ControlFile.py) (3 diffs)
• classes/EcFlexpart.py (moved) (moved from python/EcFlexpart.py) (3 diffs)
• classes/GribTools.py (moved) (moved from python/GribTools.py)
• classes/MarsRetrieval.py (moved) (moved from python/MarsRetrieval.py) (3 diffs)
• classes/UioFiles.py (moved) (moved from python/pythontest/TestInstallTar/flex_extract_v7.1/python/UioFiles.py) (5 diffs)
• classes/__init__.py (added)
• install.py (moved) (moved from python/install.py) (11 diffs)
• mods/__init__.py (added)
• mods/disaggregation.py (moved) (moved from python/disaggregation.py)
• mods/get_mars_data.py (moved) (moved from python/get_mars_data.py) (1 diff)
• mods/plot_retrieved.py (moved) (moved from python/pythontest/TestInstallTar/test_untar/python/plot_retrieved.py) (1 diff)
• mods/prepare_flexpart.py (moved) (moved from python/prepare_flexpart.py) (1 diff)
• mods/profiling.py (moved) (moved from python/profiling.py)
• mods/tools.py (moved) (moved from python/tools.py)
• submit.py (moved) (moved from python/submit.py) (1 diff)

source/python/classes/ControlFile.py

@@ -143 +143 @@
         self.ecstorage = 0
         self.ectrans = 0
-        self.inputdir = '../work'
+        self.inputdir = _config.PATH_INPUT_DIR
         self.outputdir = self.inputdir
         self.ecmwfdatadir = _config.PATH_FLEXEXTRACT_DIR
@@ -177 +177 @@
             <nothing>
         '''
-        from tools import my_error
+        from mods.tools import my_error
 
         # read whole CONTROL file
@@ -331 +331 @@
             <nothing>
         '''
-        from tools import my_error
+        from mods.tools import my_error
         import numpy as np
 

source/python/classes/EcFlexpart.py

@@ -61 +61 @@
 #
 # @Class Attributes:
-#    - dtime
-#    - basetime
-#    - server
-#    - marsclass
-#    - stream
-#    - resol
-#    - accuracy
-#    - number
-#    - expver
-#    - glevelist
-#    - area
-#    - grid
-#    - level
-#    - levelist
-#    - types
-#    - dates
-#    - area
-#    - gaussian
-#    - params
-#    - inputdir
-#    - outputfilelist
+#
+#  TODO
 #
 #*******************************************************************************
 #pylint: disable=unsupported-assignment-operation
-# this is disabled because its an error in pylint for this specific case
+# this is disabled because for this specific case its an error in pylint
 #pylint: disable=consider-using-enumerate
 # this is not useful in this case
@@ -104 +85 @@
 import _config
 from GribTools import GribTools
-from tools import init128, to_param_id, silent_remove, product, my_error
+from mods.tools import init128, to_param_id, silent_remove, product, my_error
 from MarsRetrieval import MarsRetrieval
-import disaggregation
+import mods.disaggregation
 
 # ------------------------------------------------------------------------------
@@ -471 +452 @@
                         MR.data_retrieve()
                     elif request == 1:
-                        MR.print_info()
+                        MR.print_info(self.inputdir)
                     elif request == 2:
-                        MR.print_info()
+                        MR.print_info(self.inputdir)
                         MR.display_info()
                         MR.data_retrieve()

source/python/classes/MarsRetrieval.py

@@ -335 +335 @@
 
 
-    def print_info(self):
+    def print_info(self, inputdir):
         '''
         @Description:
@@ -345 +345 @@
                 For description see class documentation.
 
+            inputdir: string
+                The path where all data from the retrievals are stored.
+
         @Return:
             <nothing>
@@ -352 +355 @@
 
         # open a file to store all requests to
-        with open(os.path.join(_config.PATH_RUN_DIR,
+        with open(os.path.join(inputdir,
                                _config.FILE_MARS_REQUESTS), 'a') as f:
             f.write('mars\n')

source/python/classes/UioFiles.py

@@ -52 +52 @@
 # software specific module from flex_extract
 #import profiling
-from tools import silent_remove, get_list_as_string
+from mods.tools import silent_remove, get_list_as_string
 
 # ------------------------------------------------------------------------------
@@ -60 +60 @@
 class UioFiles(object):
     '''
-    Class to manipulate files. At initialisation it has the attribute
-    pattern which stores a regular expression pattern for the files associated
-    with the instance of the class.
+    Class to manipulate files. At initialisation it has the pattern
+    which stores a regular expression pattern for the files, the path
+    to the files and the files already.
     '''
     # --------------------------------------------------------------------------
@@ -88 +88 @@
         self.path = path
         self.pattern = pattern
-        self.files = None
+        self.files = []
 
         self.__list_files__(self.path)
@@ -95 +95 @@
 
     #@profiling.timefn
-    def __list_files__(self, path, callid=0):
+    def __list_files__(self, path):
         '''
         @Description:
@@ -108 +108 @@
                 Path to the files.
 
-            callid: integer
-                Id which tells the function if its the first call
-                or a recursive call. Default and first call is 0.
-                Everything different from 0 is ment to be a recursive case.
-
         @Return:
             <nothing>
         '''
-
-        # initialize variable in first function call
-        if callid == 0:
-            self.files = []
-
         # Get the absolute path
         path = os.path.abspath(path)
 
-        # get the file list of the path if its not a directory and
-        # if it contains the pattern
-        self.files.extend([os.path.join(path, k) for k in os.listdir(path)
-                           if fnmatch.fnmatch(k, self.pattern)])
-
-        # find possible sub-directories in the path
-        subdirs = [s for s in os.listdir(path)
-                   if os.path.isdir(os.path.join(path, s))]
-
-        # do recursive calls for sub-direcorties
-        if subdirs:
-            for subdir in subdirs:
-                self.__list_files__(os.path.join(path, subdir), callid=1)
+        # get all files in the dir and subdir as absolut path
+        for root, dirnames, filenames in os.walk(path):
+            for filename in fnmatch.filter(filenames, self.pattern):
+                self.files.append(os.path.join(root, filename))
 
         return

source/python/install.py

@@ -58 +58 @@
 # software specific classes and modules from flex_extract
 import _config
-from ControlFile import ControlFile
-from UioFiles import UioFiles
-from tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
+from classes.ControlFile import ControlFile
+from classes.UioFiles import UioFiles
+from mods.tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
 
 # ------------------------------------------------------------------------------
@@ -189 +189 @@
         mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
 
-        mk_tarball(tar_file)
+        mk_tarball(tar_file, c.install_target)
 
         put_file_to_ecserver(ecd, tarball_name, c.install_target,
@@ -195 +195 @@
 
         submit_job_to_ecserver(c.install_target,
-                               os.path.join(_config.PATH_RELATIVE_JOBSCRIPTS,
+                               os.path.join(_config.PATH_REL_JOBSCRIPTS,
                                             _config.FILE_INSTALL_COMPILEJOB))
 
@@ -214 +214 @@
                                         c.flexpart_root_scripts))
             if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
-                mk_tarball(tar_file)
+                mk_tarball(tar_file, c.install_target)
                 make_dir(os.path.join(c.flexpart_root_scripts,
                                       target_dirname))
@@ -222 +222 @@
                 os.chdir(os.path.join(c.flexpart_root_scripts,
                                       target_dirname,
-                                      _config.PATH_RELATIVE_FORTRAN_SRC))
+                                      _config.PATH_REL_FORTRAN_SRC))
 
         # Create Fortran executable - CONVERT2
@@ -238 +238 @@
     return
 
-def mk_tarball(tarball_path):
+def mk_tarball(tarball_path, target):
     '''
     @Description:
@@ -253 +253 @@
             relevant data for flex_extract.
 
+        target: string
+            The queue where the job is submitted to.
+
     @Return:
         <nothing>
@@ -267 +270 @@
 
     # get lists of the files to be added to the tar file
-    ECMWF_ENV_FILE = [_config.PATH_RELATIVE_ECMWF_ENV]
+    if target == 'local':
+        ECMWF_ENV_FILE = []
+    else:
+        ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
+
     pyfiles = [os.path.relpath(x, ecd)
-               for x in glob(_config.PATH_LOCAL_PYTHON +
-                             os.path.sep + '*py')]
+               for x in UioFiles(_config.PATH_LOCAL_PYTHON, '*py').files]
     controlfiles = [os.path.relpath(x, ecd)
-                    for x in glob(_config.PATH_CONTROLFILES +
-                                  os.path.sep + 'CONTROL*')]
+                    for x in UioFiles(_config.PATH_CONTROLFILES,
+                                      'CONTROL*').files]
     tempfiles = [os.path.relpath(x, ecd)
-                 for x in glob(_config.PATH_TEMPLATES +
-                               os.path.sep + '*')]
+                 for x in UioFiles(_config.PATH_TEMPLATES , '*').files]
     ffiles = [os.path.relpath(x, ecd)
-              for x in glob(_config.PATH_FORTRAN_SRC +
-                            os.path.sep + '*.f*')]
+              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.f*').files]
     hfiles = [os.path.relpath(x, ecd)
-              for x in glob(_config.PATH_FORTRAN_SRC +
-                            os.path.sep + '*.h')]
+              for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.h').files]
     makefiles = [os.path.relpath(x, ecd)
-                 for x in glob(_config.PATH_FORTRAN_SRC +
-                               os.path.sep + 'Makefile*')]
+                 for x in UioFiles(_config.PATH_FORTRAN_SRC, 'Makefile*').files]
 
     # concatenate single lists to one for a better looping
@@ -352 +354 @@
     '''
 
-    with open(_config.PATH_RELATIVE_ECMWF_ENV, 'w') as fo:
+    with open(_config.PATH_REL_ECMWF_ENV, 'w') as fo:
         fo.write('ECUID ' + ecuid + '\n')
         fo.write('ECGID ' + ecgid + '\n')
@@ -389 +391 @@
     '''
 
-    template = os.path.join(_config.PATH_RELATIVE_TEMPLATES,
+    template = os.path.join(_config.PATH_REL_TEMPLATES,
                             _config.TEMPFILE_INSTALL_COMPILEJOB)
     with open(template) as f:
         fdata = f.read().split('\n')
 
-    compilejob = os.path.join(_config.PATH_RELATIVE_JOBSCRIPTS,
+    compilejob = os.path.join(_config.PATH_REL_JOBSCRIPTS,
                               _config.FILE_INSTALL_COMPILEJOB)
     with open(compilejob, 'w') as fo:
@@ -450 +452 @@
     '''
     fp_root_path_to_python = os.path.join(fp_root, _config.FLEXEXTRACT_DIRNAME,
-                         _config.PATH_RELATIVE_PYTHON)
-
-    template = os.path.join(_config.PATH_RELATIVE_TEMPLATES,
+                         _config.PATH_REL_PYTHON)
+
+    template = os.path.join(_config.PATH_REL_TEMPLATES,
                             _config.TEMPFILE_INSTALL_JOB)
     with open(template) as f:
         fdata = f.read().split('\n')
 
-    jobfile_temp = os.path.join(_config.PATH_RELATIVE_TEMPLATES,
+    jobfile_temp = os.path.join(_config.PATH_REL_TEMPLATES,
                                 _config.TEMPFILE_JOB)
     with open(jobfile_temp, 'w') as fo:

source/python/mods/get_mars_data.py

@@ -59 +59 @@
 import _config
 from tools import my_error, normal_exit, get_cmdline_arguments, read_ecenv
-from EcFlexpart import EcFlexpart
-from UioFiles import UioFiles
+from classes.EcFlexpart import EcFlexpart
+from classes.UioFiles import UioFiles
 # ------------------------------------------------------------------------------
 # FUNCTION

source/python/mods/plot_retrieved.py

@@ -53 +53 @@
 
 # software specific classes and modules from flex_extract
-from ControlFile import ControlFile
-from UioFiles import UioFiles
-
-# add path to pythonpath so that python finds its buddies
-LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
-    inspect.getfile(inspect.currentframe())))
-if LOCAL_PYTHON_PATH not in sys.path:
-    sys.path.append(LOCAL_PYTHON_PATH)
+import _config
+from classes.ControlFile import ControlFile
+from classes.UioFiles import UioFiles
 
 font = {'family': 'monospace', 'size': 12}

source/python/mods/prepare_flexpart.py

@@ -60 +60 @@
 # software specific classes and modules from flex_extract
 import _config
-from UioFiles import UioFiles
+from classes.UioFiles import UioFiles
 from tools import clean_up, get_cmdline_arguments, read_ecenv
-from EcFlexpart import EcFlexpart
+from classes.EcFlexpart import EcFlexpart
 
 ecapi = 'ecmwf' not in socket.gethostname()

source/python/submit.py

@@ -51 +51 @@
 # software specific classes and modules from flex_extract
 import _config
-from tools import normal_exit, get_cmdline_arguments, submit_job_to_ecserver, \
-                  read_ecenv
-from get_mars_data import get_mars_data
-from prepare_flexpart import prepare_flexpart
-from ControlFile import ControlFile
+from mods.tools import (normal_exit, get_cmdline_arguments,
+                        submit_job_to_ecserver, read_ecenv)
+from mods.get_mars_data import get_mars_data
+from mods.prepare_flexpart import prepare_flexpart
+from classes.ControlFile import ControlFile
 
 # ------------------------------------------------------------------------------