Optimized filesystem access for spack compiler add

lib/spack/llnl/util/filesystem.py

@@ -21,7 +21,7 @@
 
 import six
 from llnl.util import tty
-from llnl.util.lang import dedupe
+from llnl.util.lang import dedupe, memoized
 from spack.util.executable import Executable
 
 __all__ = [
@@ -1351,3 +1351,37 @@ def find_libraries(libraries, root, shared=True, recursive=False):
     libraries = ['{0}.{1}'.format(lib, suffix) for lib in libraries]
 
     return LibraryList(find(root, libraries, recursive))
+
+
+@memoized
+def is_accessible_dir(path):
+    """Returns True if the argument is an accessible directory.
+
+    Args:
+        path: path to be tested
+
+    Returns:
+        True if ``path`` is an accessible directory, else False
+    """
+    return os.path.isdir(path) and os.access(path, os.R_OK | os.X_OK)
+
+
+@memoized
+def files_in(*search_paths):
+    """Returns all the files in paths passed as arguments.
+
+    Caller must ensure that each path in ``search_paths`` is a directory.
+
+    Args:
+        *search_paths: directories to be searched
+
+    Returns:
+        List of (file, full_path) tuples with all the files found.
+    """
+    files = []
+    for d in filter(is_accessible_dir, search_paths):
+        files.extend(filter(
+            lambda x: os.path.isfile(x[1]),
+            [(f, os.path.join(d, f)) for f in os.listdir(d)]
+        ))
+    return files

lib/spack/spack/compiler.py

@@ -13,6 +13,7 @@
 
 import llnl.util.lang
 import llnl.util.multiproc
+import llnl.util.filesystem
 import llnl.util.tty as tty
 
 import spack.error
@@ -268,12 +269,7 @@ def search_compiler_commands(cls, operating_system, *search_paths):
                 is a list of commands that, when executed, will detect the
                 version of the corresponding compiler.
         """
-        def is_accessible_dir(x):
-            """Returns True if the argument is an accessible directory."""
-            return os.path.isdir(x) and os.access(x, os.R_OK | os.X_OK)
-
-        # Select accessible directories
-        search_directories = list(filter(is_accessible_dir, search_paths))
+        files_to_be_tested = llnl.util.filesystem.files_in(*search_paths)
 
         tags, commands = [], []
         for language in ('cc', 'cxx', 'f77', 'fc'):
@@ -295,23 +291,18 @@ def is_accessible_dir(x):
             ]
 
             # Select only the files matching a regexp
-            for d in search_directories:
-                # Only select actual files, use the full path
-                files = filter(
-                    os.path.isfile, [os.path.join(d, f) for f in os.listdir(d)]
-                )
-                for full_path in files:
-                    file = os.path.basename(full_path)
-                    for regexp in search_regexps:
-                        match = regexp.match(file)
-                        if match:
-                            tags.append(
-                                (_CompilerID(operating_system, cls, None),
-                                 _NameVariation(*match.groups()), language)
-                            )
-                            commands.append(
-                                detect_version_command(callback, full_path)
-                            )
+            for (file, full_path), regexp in itertools.product(
+                    files_to_be_tested, search_regexps
+            ):
+                match = regexp.match(file)
+                if match:
+                    tags.append(
+                        (_CompilerID(operating_system, cls, None),
+                         _NameVariation(*match.groups()), language)
+                    )
+                    commands.append(
+                        detect_version_command(callback, full_path)
+                    )
 
         # Reverse it here so that the dict creation (last insert wins)
         # does not spoil the intended precedence.