diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py
index f5b2922273a..145421130a2 100644
--- a/lib/spack/spack/directory_layout.py
+++ b/lib/spack/spack/directory_layout.py
@@ -7,6 +7,7 @@
 import shutil
 import glob
 import tempfile
+import errno
 from contextlib import contextmanager
 
 import ruamel.yaml as yaml
@@ -119,9 +120,17 @@ def remove_install_directory(self, spec, deprecated=False):
         path = os.path.dirname(path)
         while path != self.root:
             if os.path.isdir(path):
-                if os.listdir(path):
-                    return
-                os.rmdir(path)
+                try:
+                    os.rmdir(path)
+                except OSError as e:
+                    if e.errno == errno.ENOENT:
+                        # already deleted, continue with parent
+                        pass
+                    elif e.errno == errno.ENOTEMPTY:
+                        # directory wasn't empty, done
+                        return
+                    else:
+                        raise e
             path = os.path.dirname(path)