Windows: Symlink support

To provide Windows-compatible functionality, spack code should use
llnl.util.symlink instead of os.symlink. On non-Windows platforms
and on Windows where supported, os.symlink will still be used.

Use junctions when symlinks aren't supported on Windows (#22583)

Support islink for junctions (#24182)

Windows: Update llnl/util/filesystem

* Use '/' as path separator on Windows.
* Recognizing that Windows paths start with '<Letter>:/' instead of '/'

Co-authored-by: lou.lawrence@kitware.com <lou.lawrence@kitware.com>
Co-authored-by: John Parent <john.parent@kitware.com>

lib/spack/llnl/util/filesystem.py

@@ -24,6 +24,7 @@
 from llnl.util import tty
 from llnl.util.compat import Sequence
 from llnl.util.lang import dedupe, memoized
+from llnl.util.symlink import symlink
 
 from spack.util.executable import Executable
 
@@ -508,7 +509,7 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False):
                                       .format(target, new_target))
                             target = new_target
 
-                    os.symlink(target, d)
+                    symlink(target, d)
                 elif os.path.isdir(link_target):
                     mkdirp(d)
                 else:
@@ -806,10 +807,10 @@ def touchp(path):
 
 def force_symlink(src, dest):
     try:
-        os.symlink(src, dest)
+        symlink(src, dest)
     except OSError:
         os.remove(dest)
-        os.symlink(src, dest)
+        symlink(src, dest)
 
 
 def join_path(prefix, *args):

lib/spack/llnl/util/link_tree.py

@@ -13,6 +13,7 @@
 
 import llnl.util.tty as tty
 from llnl.util.filesystem import mkdirp, touch, traverse_tree
+from llnl.util.symlink import islink, symlink
 
 __all__ = ['LinkTree']
 
@@ -20,7 +21,7 @@
 
 
 def remove_link(src, dest):
-    if not os.path.islink(dest):
+    if not islink(dest):
         raise ValueError("%s is not a link tree!" % dest)
     # remove if dest is a hardlink/symlink to src; this will only
     # be false if two packages are merged into a prefix and have a
@@ -113,7 +114,7 @@ def unmerge_directories(self, dest_root, ignore):
                     os.remove(marker)
 
     def merge(self, dest_root, ignore_conflicts=False, ignore=None,
-              link=os.symlink, relative=False):
+              link=symlink, relative=False):
         """Link all files in src into dest, creating directories
            if necessary.
 
@@ -125,7 +126,7 @@ def merge(self, dest_root, ignore_conflicts=False, ignore=None,
         ignore (callable): callable that returns True if a file is to be
             ignored in the merge (by default ignore nothing)
 
-        link (callable): function to create links with (defaults to os.symlink)
+        link (callable): function to create links with (defaults to llnl.util.symlink)
 
         relative (bool): create all symlinks relative to the target
             (default False)

lib/spack/llnl/util/symlink.py

@@ -0,0 +1,139 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import errno
+import os
+import shutil
+import tempfile
+from os.path import exists, join
+from sys import platform as _platform
+
+is_windows = _platform == 'win32'
+
+__win32_can_symlink__ = None
+
+
+def symlink(real_path, link_path):
+    """
+    Create a symbolic link.
+
+    On Windows, use junctions if os.symlink fails.
+    """
+    if not is_windows or _win32_can_symlink():
+        os.symlink(real_path, link_path)
+    else:
+        try:
+            # Try to use junctions
+            _win32_junction(real_path, link_path)
+        except OSError:
+            # If all else fails, fall back to copying files
+            shutil.copyfile(real_path, link_path)
+
+
+def islink(path):
+    return os.path.islink(path) or _win32_is_junction(path)
+
+
+# '_win32' functions based on
+# https://github.com/Erotemic/ubelt/blob/master/ubelt/util_links.py
+def _win32_junction(path, link):
+    # junctions require absolute paths
+    if not os.path.isabs(link):
+        link = os.path.abspath(link)
+
+    # os.symlink will fail if link exists, emulate the behavior here
+    if exists(link):
+        raise OSError(errno.EEXIST, 'File  exists: %s -> %s' % (link, path))
+
+    if not os.path.isabs(path):
+        parent = os.path.join(link, os.pardir)
+        path = os.path.join(parent, path)
+        path = os.path.abspath(path)
+
+    if os.path.isdir(path):
+        # try using a junction
+        command = 'mklink /J "%s" "%s"' % (link, path)
+    else:
+        # try using a hard link
+        command = 'mklink /H "%s" "%s"' % (link, path)
+
+    _cmd(command)
+
+
+def _win32_can_symlink():
+    global __win32_can_symlink__
+    if __win32_can_symlink__ is not None:
+        return __win32_can_symlink__
+
+    tempdir = tempfile.mkdtemp()
+
+    dpath = join(tempdir, 'dpath')
+    fpath = join(tempdir, 'fpath.txt')
+
+    dlink = join(tempdir, 'dlink')
+    flink = join(tempdir, 'flink.txt')
+
+    import llnl.util.filesystem as fs
+    fs.touchp(fpath)
+
+    try:
+        os.symlink(dpath, dlink)
+        can_symlink_directories = os.path.islink(dlink)
+    except OSError:
+        can_symlink_directories = False
+
+    try:
+        os.symlink(fpath, flink)
+        can_symlink_files = os.path.islink(flink)
+    except OSError:
+        can_symlink_files = False
+
+    # Cleanup the test directory
+    shutil.rmtree(tempdir)
+
+    __win32_can_symlink__ = can_symlink_directories and can_symlink_files
+
+    return __win32_can_symlink__
+
+
+def _win32_is_junction(path):
+    """
+    Determines if a path is a win32 junction
+    """
+    if os.path.islink(path):
+        return False
+
+    if is_windows:
+        import ctypes.wintypes
+
+        GetFileAttributes = ctypes.windll.kernel32.GetFileAttributesW
+        GetFileAttributes.argtypes = (ctypes.wintypes.LPWSTR,)
+        GetFileAttributes.restype = ctypes.wintypes.DWORD
+
+        INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF
+        FILE_ATTRIBUTE_REPARSE_POINT = 0x400
+
+        res = GetFileAttributes(path)
+        return res != INVALID_FILE_ATTRIBUTES and \
+            bool(res & FILE_ATTRIBUTE_REPARSE_POINT)
+
+    return False
+
+
+# Based on https://github.com/Erotemic/ubelt/blob/master/ubelt/util_cmd.py
+def _cmd(command):
+    import subprocess
+
+    # Create a new process to execute the command
+    def make_proc():
+        # delay the creation of the process until we validate all args
+        proc = subprocess.Popen(command, stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE, shell=True,
+                                universal_newlines=True, cwd=None, env=None)
+        return proc
+
+    proc = make_proc()
+    (out, err) = proc.communicate()
+    if proc.wait() != 0:
+        raise OSError(str(err))