Bootstrap GnuPG and file on Windows (#41810)

Spack can now bootstrap two new dependencies on Windows: GnuPG, and file. 

These dependencies are modeled as a separate package, and they install a cross-compiled binary.
Details on how they binaries are built are in https://github.com/spack/windows-bootstrap-resources

.github/workflows/bootstrap.yml

@@ -112,10 +112,10 @@ jobs:
     runs-on: ${{ matrix.runner }}
     strategy:
       matrix:
-        runner: ['macos-13', 'macos-14', "ubuntu-latest"]
+        runner: ['macos-13', 'macos-14', "ubuntu-latest", "windows-latest"]
     steps:
       - name: Setup macOS
-        if: ${{ matrix.runner != 'ubuntu-latest' }}
+        if: ${{ matrix.runner != 'ubuntu-latest' && matrix.runner != 'windows-latest'}}
         run: |
           brew install tree
           # Remove GnuPG since we want to bootstrap it
@@ -124,6 +124,11 @@ jobs:
         if: ${{ matrix.runner == 'ubuntu-latest' }}
         run: |
           sudo rm -rf $(which gpg) $(which gpg2) $(which patchelf)
+      - name: Setup Windows
+        if: ${{ matrix.runner == 'windows-latest' }}
+        run: |
+          Remove-Item -Path (Get-Command gpg).Path
+          Remove-Item -Path (Get-Command file).Path
       - name: Checkout
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         with:
@@ -137,11 +142,20 @@ jobs:
             3.11
             3.12
       - name: Set bootstrap sources
+        env:
+          SETUP_SCRIPT_EXT: ${{ matrix.runner == 'windows-latest' && 'ps1' || 'sh' }}
+          SETUP_SCRIPT_SOURCE: ${{ matrix.runner == 'windows-latest' && './' || 'source ' }}
+        run: |
+          ${{ env.SETUP_SCRIPT_SOURCE }}share/spack/setup-env.${{ env.SETUP_SCRIPT_EXT }}
+          spack bootstrap disable github-actions-v0.4
+      - name: Disable from source bootstrap
+        if: ${{ matrix.runner != 'windows-latest' }}
         run: |
           source share/spack/setup-env.sh
-          spack bootstrap disable github-actions-v0.4
           spack bootstrap disable spack-install
       - name: Bootstrap clingo
+        # No binary clingo on Windows yet
+        if: ${{ matrix.runner != 'windows-latest' }}
         run: |
           set -e
           for ver in '3.8' '3.9' '3.10' '3.11' '3.12' ; do
@@ -164,7 +178,24 @@ jobs:
             fi
           done
       - name: Bootstrap GnuPG
+        env:
+          SETUP_SCRIPT_EXT: ${{ matrix.runner == 'windows-latest' && 'ps1' || 'sh' }}
+          SETUP_SCRIPT_SOURCE: ${{ matrix.runner == 'windows-latest' && './' || 'source ' }}
+          USER_SCOPE_PARENT_DIR: ${{ matrix.runner == 'windows-latest' && '$env:userprofile' || '$HOME' }}
+          VALIDATE_LAST_EXIT: ${{ matrix.runner == 'windows-latest' && './share/spack/qa/validate_last_exit.ps1' || '' }}
         run: |
-          source share/spack/setup-env.sh
+          ${{ env.SETUP_SCRIPT_SOURCE }}share/spack/setup-env.${{ env.SETUP_SCRIPT_EXT }}
           spack -d gpg list
-          tree ~/.spack/bootstrap/store/
+          ${{ env.VALIDATE_LAST_EXIT }}
+          tree ${{ env.USER_SCOPE_PARENT_DIR }}/.spack/bootstrap/store/
+      - name: Bootstrap File
+        env:
+          SETUP_SCRIPT_EXT: ${{ matrix.runner == 'windows-latest' && 'ps1' || 'sh' }}
+          SETUP_SCRIPT_SOURCE: ${{ matrix.runner == 'windows-latest' && './' || 'source ' }}
+          USER_SCOPE_PARENT_DIR: ${{ matrix.runner == 'windows-latest' && '$env:userprofile' || '$HOME' }}
+          VALIDATE_LAST_EXIT: ${{ matrix.runner == 'windows-latest' && './share/spack/qa/validate_last_exit.ps1' || '' }}
+        run: |
+          ${{ env.SETUP_SCRIPT_SOURCE }}share/spack/setup-env.${{ env.SETUP_SCRIPT_EXT }}
+          spack -d python share/spack/qa/bootstrap-file.py
+          ${{ env.VALIDATE_LAST_EXIT }}
+          tree ${{ env.USER_SCOPE_PARENT_DIR }}/.spack/bootstrap/store/

lib/spack/llnl/util/filesystem.py

@@ -27,8 +27,6 @@
 from llnl.util.lang import dedupe, memoized
 from llnl.util.symlink import islink, readlink, resolve_link_target_relative_to_the_link, symlink
 
-from spack.util.executable import Executable, which
-
 from ..path import path_to_os_path, system_path_filter
 
 if sys.platform != "win32":
@@ -53,7 +51,6 @@
     "find_all_headers",
     "find_libraries",
     "find_system_libraries",
-    "fix_darwin_install_name",
     "force_remove",
     "force_symlink",
     "getuid",
@@ -248,42 +245,6 @@ def path_contains_subdirectory(path, root):
     return norm_path.startswith(norm_root)
 
 
-@memoized
-def file_command(*args):
-    """Creates entry point to `file` system command with provided arguments"""
-    file_cmd = which("file", required=True)
-    for arg in args:
-        file_cmd.add_default_arg(arg)
-    return file_cmd
-
-
-@memoized
-def _get_mime_type():
-    """Generate method to call `file` system command to aquire mime type
-    for a specified path
-    """
-    if sys.platform == "win32":
-        # -h option (no-dereference) does not exist in Windows
-        return file_command("-b", "--mime-type")
-    else:
-        return file_command("-b", "-h", "--mime-type")
-
-
-def mime_type(filename):
-    """Returns the mime type and subtype of a file.
-
-    Args:
-        filename: file to be analyzed
-
-    Returns:
-        Tuple containing the MIME type and subtype
-    """
-    output = _get_mime_type()(filename, output=str, error=str).strip()
-    tty.debug("==> " + output)
-    type, _, subtype = output.partition("/")
-    return type, subtype
-
-
 #: This generates the library filenames that may appear on any OS.
 library_extensions = ["a", "la", "so", "tbd", "dylib"]
 
@@ -1679,41 +1640,6 @@ def safe_remove(*files_or_dirs):
         raise
 
 
-@system_path_filter
-def fix_darwin_install_name(path):
-    """Fix install name of dynamic libraries on Darwin to have full path.
-
-    There are two parts of this task:
-
-    1. Use ``install_name('-id', ...)`` to change install name of a single lib
-    2. Use ``install_name('-change', ...)`` to change the cross linking between
-       libs. The function assumes that all libraries are in one folder and
-       currently won't follow subfolders.
-
-    Parameters:
-        path (str): directory in which .dylib files are located
-    """
-    libs = glob.glob(join_path(path, "*.dylib"))
-    for lib in libs:
-        # fix install name first:
-        install_name_tool = Executable("install_name_tool")
-        install_name_tool("-id", lib, lib)
-        otool = Executable("otool")
-        long_deps = otool("-L", lib, output=str).split("\n")
-        deps = [dep.partition(" ")[0][1::] for dep in long_deps[2:-1]]
-        # fix all dependencies:
-        for dep in deps:
-            for loc in libs:
-                # We really want to check for either
-                #     dep == os.path.basename(loc)   or
-                #     dep == join_path(builddir, os.path.basename(loc)),
-                # but we don't know builddir (nor how symbolic links look
-                # in builddir). We thus only compare the basenames.
-                if os.path.basename(dep) == os.path.basename(loc):
-                    install_name_tool("-change", dep, loc, lib)
-                    break
-
-
 def find_first(root: str, files: Union[Iterable[str], str], bfs_depth: int = 2) -> Optional[str]:
     """Find the first file matching a pattern.
 

lib/spack/spack/binary_distribution.py

@@ -54,6 +54,7 @@
 import spack.util.archive
 import spack.util.crypto
 import spack.util.file_cache as file_cache
+import spack.util.filesystem as ssys
 import spack.util.gpg
 import spack.util.parallel
 import spack.util.path
@@ -687,7 +688,7 @@ def get_buildfile_manifest(spec):
     # Non-symlinks.
     for rel_path in visitor.files:
         abs_path = os.path.join(root, rel_path)
-        m_type, m_subtype = fsys.mime_type(abs_path)
+        m_type, m_subtype = ssys.mime_type(abs_path)
 
         if relocate.needs_binary_relocation(m_type, m_subtype):
             # Why is this branch not part of needs_binary_relocation? :(

lib/spack/spack/bootstrap/__init__.py

@@ -9,6 +9,7 @@
     all_core_root_specs,
     ensure_clingo_importable_or_raise,
     ensure_core_dependencies,
+    ensure_file_in_path_or_raise,
     ensure_gpg_in_path_or_raise,
     ensure_patchelf_in_path_or_raise,
 )
@@ -19,6 +20,7 @@
     "is_bootstrapping",
     "ensure_bootstrap_configuration",
     "ensure_core_dependencies",
+    "ensure_file_in_path_or_raise",
     "ensure_gpg_in_path_or_raise",
     "ensure_clingo_importable_or_raise",
     "ensure_patchelf_in_path_or_raise",

lib/spack/spack/bootstrap/core.py

@@ -472,7 +472,8 @@ def ensure_clingo_importable_or_raise() -> None:
 
 def gnupg_root_spec() -> str:
     """Return the root spec used to bootstrap GnuPG"""
-    return _root_spec("gnupg@2.3:")
+    root_spec_name = "win-gpg" if IS_WINDOWS else "gnupg"
+    return _root_spec(f"{root_spec_name}@2.3:")
 
 
 def ensure_gpg_in_path_or_raise() -> None:
@@ -482,6 +483,19 @@ def ensure_gpg_in_path_or_raise() -> None:
     )
 
 
+def file_root_spec() -> str:
+    """Return the root spec used to bootstrap file"""
+    root_spec_name = "win-file" if IS_WINDOWS else "file"
+    return _root_spec(root_spec_name)
+
+
+def ensure_file_in_path_or_raise() -> None:
+    """Ensure file is in the PATH or raise"""
+    return ensure_executables_in_path_or_raise(
+        executables=["file"], abstract_spec=file_root_spec()
+    )
+
+
 def patchelf_root_spec() -> str:
     """Return the root spec used to bootstrap patchelf"""
     # 0.13.1 is the last version not to require C++17.
@@ -565,14 +579,15 @@ def ensure_core_dependencies() -> None:
     """Ensure the presence of all the core dependencies."""
     if sys.platform.lower() == "linux":
         ensure_patchelf_in_path_or_raise()
-    if not IS_WINDOWS:
+    elif sys.platform == "win32":
+        ensure_file_in_path_or_raise()
     ensure_gpg_in_path_or_raise()
     ensure_clingo_importable_or_raise()
 
 
 def all_core_root_specs() -> List[str]:
     """Return a list of all the core root specs that may be used to bootstrap Spack"""
-    return [clingo_root_spec(), gnupg_root_spec(), patchelf_root_spec()]
+    return [clingo_root_spec(), gnupg_root_spec(), patchelf_root_spec(), file_root_spec()]
 
 
 def bootstrapping_sources(scope: Optional[str] = None):

lib/spack/spack/bootstrap/status.py

@@ -88,7 +88,7 @@ def _core_requirements() -> List[RequiredResponseType]:
 
 def _buildcache_requirements() -> List[RequiredResponseType]:
     _buildcache_exes = {
-        "file": _missing("file", "required to analyze files for buildcaches"),
+        "file": _missing("file", "required to analyze files for buildcaches", system_only=False),
         ("gpg2", "gpg"): _missing("gpg2", "required to sign/verify buildcaches", False),
     }
     if platform.system().lower() == "darwin":

lib/spack/spack/package.py

@@ -104,6 +104,7 @@
 from spack.spec import InvalidSpecDetected, Spec
 from spack.util.cpus import determine_number_of_jobs
 from spack.util.executable import *
+from spack.util.filesystem import file_command, fix_darwin_install_name, mime_type
 from spack.variant import (
     any_combination_of,
     auto_or_any_combination_of,

lib/spack/spack/relocate.py

@@ -12,7 +12,6 @@
 import macholib.mach_o
 import macholib.MachO
 
-import llnl.util.filesystem as fs
 import llnl.util.lang
 import llnl.util.tty as tty
 from llnl.util.lang import memoized
@@ -25,6 +24,7 @@
 import spack.store
 import spack.util.elf as elf
 import spack.util.executable as executable
+import spack.util.filesystem as ssys
 import spack.util.path
 
 from .relocate_text import BinaryFilePrefixReplacer, TextFilePrefixReplacer
@@ -664,7 +664,7 @@ def is_binary(filename):
     Returns:
         True or False
     """
-    m_type, _ = fs.mime_type(filename)
+    m_type, _ = ssys.mime_type(filename)
 
     msg = "[{0}] -> ".format(filename)
     if m_type == "application":
@@ -692,7 +692,7 @@ def fixup_macos_rpath(root, filename):
         True if fixups were applied, else False
     """
     abspath = os.path.join(root, filename)
-    if fs.mime_type(abspath) != ("application", "x-mach-binary"):
+    if ssys.mime_type(abspath) != ("application", "x-mach-binary"):
         return False
 
     # Get Mach-O header commands

lib/spack/spack/util/filesystem.py

@@ -0,0 +1,102 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+"""
+Utilities for interacting with files,
+like those in llnl.util.filesystem, but which require logic from spack.util
+"""
+
+import glob
+import os
+import sys
+
+from llnl.util import tty
+from llnl.util.filesystem import join_path
+from llnl.util.lang import memoized
+
+from spack.util.executable import Executable, which
+
+
+def _ensure_file_on_win():
+    """Ensures the file command is available on Windows
+    If not, it is bootstrapped.
+    No-op on all other platforms"""
+    if sys.platform != "win32":
+        return
+    import spack.bootstrap
+
+    with spack.bootstrap.ensure_bootstrap_configuration():
+        spack.bootstrap.ensure_file_in_path_or_raise()
+
+
+@memoized
+def file_command(*args):
+    """Creates entry point to `file` system command with provided arguments"""
+    _ensure_file_on_win()
+    file_cmd = which("file", required=True)
+    for arg in args:
+        file_cmd.add_default_arg(arg)
+    return file_cmd
+
+
+@memoized
+def _get_mime_type():
+    """Generate method to call `file` system command to aquire mime type
+    for a specified path
+    """
+    if sys.platform == "win32":
+        # -h option (no-dereference) does not exist in Windows
+        return file_command("-b", "--mime-type")
+    else:
+        return file_command("-b", "-h", "--mime-type")
+
+
+def mime_type(filename):
+    """Returns the mime type and subtype of a file.
+
+    Args:
+        filename: file to be analyzed
+
+    Returns:
+        Tuple containing the MIME type and subtype
+    """
+    output = _get_mime_type()(filename, output=str, error=str).strip()
+    tty.debug("==> " + output)
+    type, _, subtype = output.partition("/")
+    return type, subtype
+
+
+def fix_darwin_install_name(path):
+    """Fix install name of dynamic libraries on Darwin to have full path.
+
+    There are two parts of this task:
+
+    1. Use ``install_name('-id', ...)`` to change install name of a single lib
+    2. Use ``install_name('-change', ...)`` to change the cross linking between
+       libs. The function assumes that all libraries are in one folder and
+       currently won't follow subfolders.
+
+    Parameters:
+        path (str): directory in which .dylib files are located
+    """
+    libs = glob.glob(join_path(path, "*.dylib"))
+    for lib in libs:
+        # fix install name first:
+        install_name_tool = Executable("install_name_tool")
+        install_name_tool("-id", lib, lib)
+        otool = Executable("otool")
+        long_deps = otool("-L", lib, output=str).split("\n")
+        deps = [dep.partition(" ")[0][1::] for dep in long_deps[2:-1]]
+        # fix all dependencies:
+        for dep in deps:
+            for loc in libs:
+                # We really want to check for either
+                #     dep == os.path.basename(loc)   or
+                #     dep == join_path(builddir, os.path.basename(loc)),
+                # but we don't know builddir (nor how symbolic links look
+                # in builddir). We thus only compare the basenames.
+                if os.path.basename(dep) == os.path.basename(loc):
+                    install_name_tool("-change", dep, loc, lib)
+                    break

share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml

@@ -218,7 +218,7 @@ default:
     - $ErrorActionPreference=$ErrorActionOld
 
   tags: ["spack", "public", "medium", "x86_64-win"]
-  image: "ghcr.io/johnwparent/windows-server21h2:sha-c749cf3"
+  image: "ghcr.io/johnwparent/windows-server21h2:sha-1c12b61"
 
 .generate-deprecated:
   extends: [ ".base-job" ]

share/spack/gitlab/cloud_pipelines/configs/win64/ci.yaml

@@ -15,4 +15,4 @@ ci:
       - spack.ps1 config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{hash}'"
       - mkdir ${SPACK_ARTIFACTS_ROOT}/user_data
       - spack.ps1 --backtrace ci rebuild | Tee-Object -FilePath "${env:SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt" 2>&1 | Tee-Object -FilePath "${env:SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt"
-      image: "ghcr.io/johnwparent/windows-server21h2:sha-c749cf3"
+      image: "ghcr.io/johnwparent/windows-server21h2:sha-1c12b61"

share/spack/qa/bootstrap-file.py

@@ -0,0 +1,4 @@
+from spack.util.filesystem import file_command
+
+if __name__ == "__main__":
+    file_command()

var/spack/repos/builtin/packages/win-file/package.py

@@ -0,0 +1,35 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import os
+import re
+import shutil
+
+from spack.package import *
+
+
+class WinFile(Package):
+    """File "file type guesser" system utility cross compiled for x86_64 Windows
+    systems via the Mingw-w64 cross compiler and a custom Spack repository
+    """
+
+    homepage = "https://spack.github.io/windows-bootstrap-resources"
+    url = (
+        "https://spack.github.io/windows-bootstrap-resources/resources/file/5.45/file_5.45.tar.gz"
+    )
+
+    executables = ["^file$"]
+
+    version("5.45", sha256="11b8f3abf647c711bc50ef8451c8d6e955f11c4afd8b0a98f2ac65e9b6e10d5e")
+
+    @classmethod
+    def determine_version(cls, exe):
+        output = Executable(exe)("--version", output=str, error=str)
+        match = re.search(r"file-(\S+)", output)
+        return match.group(1) if match else None
+
+    def install(self, spec, prefix):
+        mkdirp(prefix)
+        for subdir in os.listdir(self.stage.source_path):
+            shutil.move(subdir, prefix)

var/spack/repos/builtin/packages/win-gpg/package.py

@@ -0,0 +1,36 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import os
+import re
+import shutil
+
+from spack.package import *
+
+
+class WinGpg(Package):
+    """GnuPG is a complete and free implementation of the OpenPGP
+    standard as defined by RFC4880 (also known as PGP).
+
+    This utility was cross compiled for x86_64 Windows
+    systems via the Mingw-w64 cross compiler and a custom Spack repository
+    """
+
+    homepage = "https://spack.github.io/windows-bootstrap-resources/"
+    url = "https://spack.github.io/windows-bootstrap-resources/resources/gpg/2.4.5/gpg4win_2.4.5.tar.gz"
+
+    executables = ["^gpg$"]
+
+    version("2.4.5", sha256="249ab87bd06abea3140054089bad44d9a5d1531413590576da609142db2673ec")
+
+    @classmethod
+    def determine_version(cls, exe):
+        output = Executable(exe)("--version", output=str, error=str)
+        match = re.search(r"gpg (\S+)", output)
+        return match.group(1) if match else None
+
+    def install(self, spec, prefix):
+        mkdirp(prefix)
+        for subdir in os.listdir(self.stage.source_path):
+            shutil.move(subdir, prefix)