Changeset efdb01a in flex_extract.git for python/UIOFiles.py

Timestamp:

May 9, 2018, 12:15:00 PM (6 years ago)

Author:

Anne Philipp <anne.philipp@…>

Branches:

master, ctbto, dev

Children:

991df6a

Parents:

02c8c50

Message:

whole bunch of modifications due to new structure of ECMWFDATA, added basics of documentation, minor programming corrections

File:

• python/UIOFiles.py (moved) (moved from python/UIOTools.py) (7 diffs)

python/UIOFiles.py

@@ -4 +4 @@
 # TODO AP
 #AP
-# - File name und Klassenname gleichsetzen?
 # - checken welche regelmässigen methoden auf diese Files noch angewendet werden
 # und dann hier implementieren
-# - löschen?
+# - add description of file!
 #************************************************************************
 """
@@ -15 +14 @@
 
 @ChangeHistory:
-   February 2018 - Anne Philipp (University of Vienna):
+    November 2015 - Leopold Haimberger (University of Vienna):
+        - modified method listFiles to work with glob instead of listdir
+        - added pattern search in method listFiles
+
+    February 2018 - Anne Philipp (University of Vienna):
         - applied PEP8 style guide
         - added documentation
+        - optimisation of method listFiles since it didn't work correctly
+          for sub directories
+        - additional speed up of method listFiles
 
 @License:
-    (C) Copyright 2014 UIO.
+    (C) Copyright 2014-2018.
 
     This software is licensed under the terms of the Apache Licence Version 2.0
@@ -26 +32 @@
 
 @Requirements:
-    - A standard python 2.6 or 2.7 installation
-    - dateutils
-    - matplotlib (optional, for debugging)
-    - ECMWF specific packages, all available from https://software.ecmwf.int/
-        ECMWF WebMARS, gribAPI with python enabled, emoslib and
-        ecaccess web toolkit
+    A standard python 2.6 or 2.7 installation
 
 @Description:
-    Further documentation may be obtained from www.flexpart.eu.
-
-    Functionality provided:
-        Prepare input 3D-wind fields in hybrid coordinates +
-        surface fields for FLEXPART runs
+    ...
 """
 # ------------------------------------------------------------------------------
@@ -45 +42 @@
 import os
 import glob
-
+import fnmatch
+import time
+import profiling
 # ------------------------------------------------------------------------------
-# Class
+# CLASS
 # ------------------------------------------------------------------------------
 class UIOFiles:
@@ -56 +55 @@
     '''
     # --------------------------------------------------------------------------
-    # FUNCTIONS
+    # CLASS FUNCTIONS
     # --------------------------------------------------------------------------
     def __init__(self, suffix):
@@ -80 +79 @@
         return
 
-    def listFiles(self, path, pattern):
+    #@profiling.timefn
+    def listFiles(self, path, pattern, callid=0):
         '''
         @Description:
@@ -98 +98 @@
                 '*OG_acc_SL*.'+c.ppid+'.*'
 
+            callid: integer
+                Id which tells the function if its the first call
+                or a recursive call. Default and first call is 0.
+                Everything different from 0 is ment to be a recursive case.
+
         @Return:
             <nothing>
         '''
 
+        # initialize variable in first function call
+        if callid == 0:
+            self.files = []
+
         # Get the absolute path
         path = os.path.abspath(path)
 
-        # Get a list of files in pathname
-        filesInCurDir0 = glob.glob(path + '/' + pattern)
-        filesInCurDir = []
-        for f in filesInCurDir0:
-            filesInCurDir.append(f.split('/')[-1])
+        # get the file list of the path if its not a directory and
+        # if it contains one of the suffixes
+        self.files.extend([os.path.join(path, k) for k in os.listdir(path)
+                           if fnmatch.fnmatch(k, pattern) and
+                           os.path.splitext(k)[-1] in self.suffix])
 
-        self.counter = 0
-        self.files = []
-        # Traverse through all files
-        for file in filesInCurDir:
-            curFile = os.path.join(path, file)
+        # find possible sub-directories in the path
+        subdirs = [s for s in os.listdir(path)
+                   if os.path.isdir(os.path.join(path, s))]
 
-            # Check if it's a normal file or directory
-            if os.path.isfile(curFile):
-                # Get the file extension
-                fileNoExt, curFileExtension = os.path.splitext(curFile)
-                # Check if the file has an extension of typical video files
-                if curFileExtension in self.suffix:
-                    # We have got a file file! Increment the counter
-                    self.counter += 1
-                    # add this filename in the list
-                    self.files.append(curFile)
-            else:
-                # We got a directory, enter into it for further processing
-                self.listFiles(curFile)
+        # do recursive calls for sub-direcorties
+        if subdirs:
+            for subdir in subdirs:
+                self.listFiles(os.path.join(path, subdir), pattern, callid=1)
 
         return