zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Windows: add registry query and SDK/WDK packages (#33021)

Browse Source 
* Add a WindowsRegistryView class, which can query for existing
  package installations on Windows. This is particularly important
  because some Windows packages (including those added here)
  do not allow two simultaneous installs, and this can be
  queried in order to provide a clear error message.
* Consolidate external path detection logic for Windows into
  WindowsKitExternalPaths and WindowsCompilerExternalPaths objects.
* Add external-only packages win-sdk and wgl
* Add win-wdk (including external detection) which depends on
  win-sdk
* Replace prior msmpi implementation with a source-based install
  (depends on win-wdk). This install can control the install
  destination (unlike the binary installation).
* Update MSVC compiler to choose vcvars based on win-sdk dependency
* Provide "msbuild" module-level variable to packages during build
* When creating symlinks on Windows, need to explicitly specify when
  a symlink target is a directory
* executables_in_path no-longer defaults to using PATH (this is
  now expected to be taken care of by the caller)
This commit is contained in:
John W. Parent 2022-11-22 03:27:42 -05:00 committed by GitHub
parent 376afd631c
commit 793a7bc6a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 961 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/spack/llnl/util/symlink.py
View File

@ -24,7 +24,7 @@ def symlink(real_path, link_path):
On Windows, use junctions if os.symlink fails.
"""
if not is_windows or _win32_can_symlink():
os.symlink(real_path, link_path)
os.symlink(real_path, link_path, target_is_directory=os.path.isdir(real_path))
else:
try:
# Try to use junctions

1
lib/spack/spack/build_environment.py
View File

@ -566,6 +566,7 @@ def _set_variables_for_single_module(pkg, module):
if sys.platform == "win32":
m.nmake = Executable("nmake")
m.msbuild = Executable("msbuild")
# Standard CMake arguments
m.std_cmake_args = spack.build_systems.cmake.CMakeBuilder.std_args(pkg)
m.std_meson_args = spack.build_systems.meson.MesonBuilder.std_args(pkg)

41
lib/spack/spack/compilers/msvc.py
View File

@ -6,13 +6,17 @@
import os
import re
import subprocess
import sys
from distutils.version import StrictVersion
from typing import Dict, List, Set # novm
import spack.compiler
import spack.operating_systems.windows_os
import spack.platforms
import spack.util.executable
from spack.compiler import Compiler
from spack.error import SpackError
from spack.version import Version
avail_fc_version = set() # type: Set[str]
fc_path = dict() # type: Dict[str, str]
@ -38,10 +42,10 @@ def get_valid_fortran_pth(comp_ver):
class Msvc(Compiler):
# Subclasses use possible names of C compiler
cc_names = ["cl.exe"]
cc_names = ["cl.exe"] # type: List[str]
# Subclasses use possible names of C++ compiler
cxx_names = ["cl.exe"]
cxx_names = ["cl.exe"] # type: List[str]
# Subclasses use possible names of Fortran 77 compiler
f77_names = ["ifx.exe"] # type: List[str]
@ -90,10 +94,25 @@ def __init__(self, *args, **kwargs):
@property
def msvc_version(self):
ver = re.search(Msvc.version_regex, self.cc).group(1)
ver = "".join(ver.split(".")[:2])[:-1]
"""This is the VCToolset version *NOT* the actual version of the cl compiler
For CL version, query `Msvc.cl_version`"""
return Version(re.search(Msvc.version_regex, self.cc).group(1))
@property
def short_msvc_version(self):
"""
This is the shorthand VCToolset version of form
MSVC<short-ver> *NOT* the full version, for that see
Msvc.msvc_version
"""
ver = self.msvc_version[:2].joined.string[:3]
return "MSVC" + ver
@property
def cl_version(self):
"""Cl toolset version"""
return spack.compiler.get_compiler_version_output(self.cc)
def setup_custom_environment(self, pkg, env):
"""Set environment variables for MSVC using the
Microsoft-provided script."""
@ -103,10 +122,22 @@ def setup_custom_environment(self, pkg, env):
# once the process terminates. So go the long way around: examine
# output, sort into dictionary, use that to make the build
# environment.
# get current platform architecture and format for vcvars argument
arch = spack.platforms.real_host().default.lower()
arch = arch.replace("-", "_")
# vcvars can target specific sdk versions, force it to pick up concretized sdk
# version, if needed by spec
sdk_ver = "" if "win-sdk" not in pkg.spec else pkg.spec["win-sdk"].version.string + ".0"
# provide vcvars with msvc version selected by concretization,
# not whatever it happens to pick up on the system (highest available version)
out = subprocess.check_output( # novermin
'cmd /u /c "{}" {} && set'.format(self.setvarsfile, "amd64"),
'cmd /u /c "{}" {} {} {} && set'.format(
self.setvarsfile, arch, sdk_ver, "-vcvars_ver=%s" % self.msvc_version
),
stderr=subprocess.STDOUT,
)
if sys.version_info[0] >= 3:
out = out.decode("utf-16le", errors="replace") # novermin
int_env = dict(

171
lib/spack/spack/detection/common.py
View File

@ -14,6 +14,7 @@
detection mechanisms.
"""
import collections
import glob
import itertools
import os
import os.path
@ -23,8 +24,10 @@
import llnl.util.tty
import spack.config
import spack.operating_systems.windows_os as winOs
import spack.spec
import spack.util.spack_yaml
import spack.util.windows_registry
is_windows = sys.platform == "win32"
#: Information on a package that has been detected
@ -104,6 +107,19 @@ def _spec_is_valid(spec):
return True
def path_to_dict(search_paths):
"""Return dictionary[fullpath]: basename from list of paths"""
path_to_lib = {}
# Reverse order of search directories so that a lib in the first
# entry overrides later entries
for search_path in reversed(search_paths):
for lib in os.listdir(search_path):
lib_path = os.path.join(search_path, lib)
if llnl.util.filesystem.is_readable_file(lib_path):
path_to_lib[lib_path] = lib
return path_to_lib
def is_executable(file_path):
"""Return True if the path passed as argument is that of an executable"""
return os.path.isfile(file_path) and os.access(file_path, os.X_OK)
@ -139,9 +155,11 @@ def executable_prefix(executable_dir):
assert os.path.isdir(executable_dir)
components = executable_dir.split(os.sep)
if "bin" not in components:
# convert to lower to match Bin, BIN, bin
lowered_components = executable_dir.lower().split(os.sep)
if "bin" not in lowered_components:
return executable_dir
idx = components.index("bin")
idx = lowered_components.index("bin")
return os.sep.join(components[:idx])
@ -158,11 +176,16 @@ def library_prefix(library_dir):
assert os.path.isdir(library_dir)
components = library_dir.split(os.sep)
if "lib64" in components:
idx = components.index("lib64")
# covert to lowercase to match lib, LIB, Lib, etc.
lowered_components = library_dir.lower().split(os.sep)
if "lib64" in lowered_components:
idx = lowered_components.index("lib64")
return os.sep.join(components[:idx])
elif "lib" in components:
idx = components.index("lib")
elif "lib" in lowered_components:
idx = lowered_components.index("lib")
return os.sep.join(components[:idx])
elif is_windows and "bin" in lowered_components:
idx = lowered_components.index("bin")
return os.sep.join(components[:idx])
else:
return library_dir
@ -195,10 +218,117 @@ def update_configuration(detected_packages, scope=None, buildable=True):
return all_new_specs
def _windows_drive():
"""Return Windows drive string"""
return os.environ["HOMEDRIVE"]
class WindowsCompilerExternalPaths(object):
@staticmethod
def find_windows_compiler_root_paths():
"""Helper for Windows compiler installation root discovery
At the moment simply returns location of VS install paths from VSWhere
But should be extended to include more information as relevant"""
return list(winOs.WindowsOs.vs_install_paths)
@staticmethod
def find_windows_compiler_cmake_paths():
"""Semi hard-coded search path for cmake bundled with MSVC"""
return [
os.path.join(
path, "Common7", "IDE", "CommonExtensions", "Microsoft", "CMake", "CMake", "bin"
)
for path in WindowsCompilerExternalPaths.find_windows_compiler_root_paths()
]
@staticmethod
def find_windows_compiler_ninja_paths():
"""Semi hard-coded search heuristic for locating ninja bundled with MSVC"""
return [
os.path.join(path, "Common7", "IDE", "CommonExtensions", "Microsoft", "CMake", "Ninja")
for path in WindowsCompilerExternalPaths.find_windows_compiler_root_paths()
]
@staticmethod
def find_windows_compiler_bundled_packages():
"""Return all MSVC compiler bundled packages"""
return (
WindowsCompilerExternalPaths.find_windows_compiler_cmake_paths()
+ WindowsCompilerExternalPaths.find_windows_compiler_ninja_paths()
)
class WindowsKitExternalPaths(object):
if is_windows:
plat_major_ver = str(winOs.windows_version()[0])
@staticmethod
def find_windows_kit_roots():
"""Return Windows kit root, typically %programfiles%\\Windows Kits\\10|11\\"""
if not is_windows:
return []
program_files = os.environ["PROGRAMFILES(x86)"]
kit_base = os.path.join(
program_files, "Windows Kits", WindowsKitExternalPaths.plat_major_ver
)
return kit_base
@staticmethod
def find_windows_kit_bin_paths(kit_base=None):
"""Returns Windows kit bin directory per version"""
kit_base = WindowsKitExternalPaths.find_windows_kit_roots() if not kit_base else kit_base
kit_bin = os.path.join(kit_base, "bin")
return glob.glob(os.path.join(kit_bin, "[0-9]*", "*\\"))
@staticmethod
def find_windows_kit_lib_paths(kit_base=None):
"""Returns Windows kit lib directory per version"""
kit_base = WindowsKitExternalPaths.find_windows_kit_roots() if not kit_base else kit_base
kit_lib = os.path.join(kit_base, "Lib")
return glob.glob(os.path.join(kit_lib, "[0-9]*", "*", "*\\"))
@staticmethod
def find_windows_driver_development_kit_paths():
"""Provides a list of all installation paths
for the WDK by version and architecture
"""
wdk_content_root = os.getenv("WDKContentRoot")
return WindowsKitExternalPaths.find_windows_kit_lib_paths(wdk_content_root)
@staticmethod
def find_windows_kit_reg_installed_roots_paths():
reg = spack.util.windows_registry.WindowsRegistryView(
"SOFTWARE\\Microsoft\\Windows Kits\\Installed Roots",
root_key=spack.util.windows_registry.HKEY.HKEY_LOCAL_MACHINE,
)
if not reg:
# couldn't find key, return empty list
return []
return WindowsKitExternalPaths.find_windows_kit_lib_paths(
reg.get_value("KitsRoot%s" % WindowsKitExternalPaths.plat_major_ver).value
)
@staticmethod
def find_windows_kit_reg_sdk_paths():
reg = spack.util.windows_registry.WindowsRegistryView(
"SOFTWARE\\WOW6432Node\\Microsoft\\Microsoft SDKs\\Windows\\v%s.0"
% WindowsKitExternalPaths.plat_major_ver,
root_key=spack.util.windows_registry.HKEY.HKEY_LOCAL_MACHINE,
)
if not reg:
# couldn't find key, return empty list
return []
return WindowsKitExternalPaths.find_windows_kit_lib_paths(
reg.get_value("InstallationFolder").value
)
def find_win32_additional_install_paths():
"""Not all programs on Windows live on the PATH
Return a list of other potential install locations.
"""
drive_letter = _windows_drive()
windows_search_ext = []
cuda_re = r"CUDA_PATH[a-zA-Z1-9_]*"
# The list below should be expanded with other
@ -211,7 +341,7 @@ def find_win32_additional_install_paths():
# to interact with Windows
# Add search path for default Chocolatey (https://github.com/chocolatey/choco)
# install directory
windows_search_ext.append("C:\\ProgramData\\chocolatey\\bin")
windows_search_ext.append("%s\\ProgramData\\chocolatey\\bin" % drive_letter)
# Add search path for NuGet package manager default install location
windows_search_ext.append(os.path.join(user, ".nuget", "packages"))
windows_search_ext.extend(
@ -233,9 +363,32 @@ def compute_windows_program_path_for_package(pkg):
return []
# note windows paths are fine here as this method should only ever be invoked
# to interact with Windows
program_files = "C:\\Program Files{}\\{}"
program_files = "{}\\Program Files{}\\{}"
drive_letter = _windows_drive()
return [
program_files.format(arch, name)
program_files.format(drive_letter, arch, name)
for arch, name in itertools.product(("", " (x86)"), (pkg.name, pkg.name.capitalize()))
]
def compute_windows_user_path_for_package(pkg):
"""Given a package attempt to compute its user scoped
install location, return list of potential locations based
on common heuristics. For more info on Windows user specific
installs see:
https://learn.microsoft.com/en-us/dotnet/api/system.environment.specialfolder?view=netframework-4.8"""
if not is_windows:
return []
# Current user directory
user = os.environ["USERPROFILE"]
app_data = "AppData"
app_data_locations = ["Local", "Roaming"]
user_appdata_install_stubs = [os.path.join(app_data, x) for x in app_data_locations]
return [
os.path.join(user, app_data, name)
for app_data, name in list(
itertools.product(user_appdata_install_stubs, (pkg.name, pkg.name.capitalize()))
)
] + [os.path.join(user, name) for name in (pkg.name, pkg.name.capitalize())]

94
lib/spack/spack/detection/path.py
View File

@ -15,22 +15,35 @@
import llnl.util.filesystem
import llnl.util.tty
import spack.operating_systems.windows_os as winOs
import spack.util.environment
import spack.util.ld_so_conf
from .common import (
from .common import ( # find_windows_compiler_bundled_packages,
DetectedPackage,
WindowsCompilerExternalPaths,
WindowsKitExternalPaths,
_convert_to_iterable,
compute_windows_program_path_for_package,
compute_windows_user_path_for_package,
executable_prefix,
find_win32_additional_install_paths,
is_executable,
library_prefix,
path_to_dict,
)
is_windows = sys.platform == "win32"
def executables_in_path(path_hints=None):
def common_windows_package_paths():
paths = WindowsCompilerExternalPaths.find_windows_compiler_bundled_packages()
paths.extend(find_win32_additional_install_paths())
paths.extend(WindowsKitExternalPaths.find_windows_kit_bin_paths())
paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_installed_roots_paths())
paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_sdk_paths())
return paths
def executables_in_path(path_hints):
"""Get the paths of all executables available from the current PATH.
For convenience, this is constructed as a dictionary where the keys are
@ -44,36 +57,10 @@ def executables_in_path(path_hints=None):
path_hints (list): list of paths to be searched. If None the list will be
constructed based on the PATH environment variable.
"""
# If we're on a Windows box, run vswhere,
# steal the installationPath using windows_os.py logic,
# construct paths to CMake and Ninja, add to PATH
path_hints = path_hints or spack.util.environment.get_path("PATH")
if sys.platform == "win32":
msvc_paths = list(winOs.WindowsOs.vs_install_paths)
msvc_cmake_paths = [
os.path.join(
path, "Common7", "IDE", "CommonExtensions", "Microsoft", "CMake", "CMake", "bin"
)
for path in msvc_paths
]
path_hints = msvc_cmake_paths + path_hints
msvc_ninja_paths = [
os.path.join(path, "Common7", "IDE", "CommonExtensions", "Microsoft", "CMake", "Ninja")
for path in msvc_paths
]
path_hints = msvc_ninja_paths + path_hints
path_hints.extend(find_win32_additional_install_paths())
if is_windows:
path_hints.extend(common_windows_package_paths())
search_paths = llnl.util.filesystem.search_paths_for_executables(*path_hints)
path_to_exe = {}
# Reverse order of search directories so that an exe in the first PATH
# entry overrides later entries
for search_path in reversed(search_paths):
for exe in os.listdir(search_path):
exe_path = os.path.join(search_path, exe)
if is_executable(exe_path):
path_to_exe[exe_path] = exe
return path_to_exe
return path_to_dict(search_paths)
def libraries_in_ld_and_system_library_path(path_hints=None):
@ -102,16 +89,23 @@ def libraries_in_ld_and_system_library_path(path_hints=None):
+ spack.util.ld_so_conf.host_dynamic_linker_search_paths()
)
search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints)
return path_to_dict(search_paths)
path_to_lib = {}
# Reverse order of search directories so that a lib in the first
# LD_LIBRARY_PATH entry overrides later entries
for search_path in reversed(search_paths):
for lib in os.listdir(search_path):
lib_path = os.path.join(search_path, lib)
if llnl.util.filesystem.is_readable_file(lib_path):
path_to_lib[lib_path] = lib
return path_to_lib
def libraries_in_windows_paths(path_hints):
path_hints.extend(spack.util.environment.get_path("PATH"))
search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints)
# on Windows, some libraries (.dlls) are found in the bin directory or sometimes
# at the search root. Add both of those options to the search scheme
search_paths.extend(llnl.util.filesystem.search_paths_for_executables(*path_hints))
search_paths.extend(WindowsKitExternalPaths.find_windows_kit_lib_paths())
search_paths.extend(WindowsKitExternalPaths.find_windows_kit_bin_paths())
search_paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_installed_roots_paths())
search_paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_sdk_paths())
# SDK and WGL should be handled by above, however on occasion the WDK is in an atypical
# location, so we handle that case specifically.
search_paths.extend(WindowsKitExternalPaths.find_windows_driver_development_kit_paths())
return path_to_dict(search_paths)
def _group_by_prefix(paths):
@ -141,12 +135,23 @@ def by_library(packages_to_check, path_hints=None):
DYLD_LIBRARY_PATH, DYLD_FALLBACK_LIBRARY_PATH environment variables
and standard system library paths.
"""
path_to_lib_name = libraries_in_ld_and_system_library_path(path_hints=path_hints)
# If no path hints from command line, intialize to empty list so
# we can add default hints on a per package basis
path_hints = [] if path_hints is None else path_hints
lib_pattern_to_pkgs = collections.defaultdict(list)
for pkg in packages_to_check:
if hasattr(pkg, "libraries"):
for lib in pkg.libraries:
lib_pattern_to_pkgs[lib].append(pkg)
path_hints.extend(compute_windows_user_path_for_package(pkg))
path_hints.extend(compute_windows_program_path_for_package(pkg))
path_to_lib_name = (
libraries_in_ld_and_system_library_path(path_hints=path_hints)
if not is_windows
else libraries_in_windows_paths(path_hints)
)
pkg_to_found_libs = collections.defaultdict(set)
for lib_pattern, pkgs in lib_pattern_to_pkgs.items():
@ -231,13 +236,14 @@ def by_executable(packages_to_check, path_hints=None):
path_hints (list): list of paths to be searched. If None the list will be
constructed based on the PATH environment variable.
"""
path_hints = [] if path_hints is None else path_hints
path_hints = spack.util.environment.get_path("PATH") if path_hints is None else path_hints
exe_pattern_to_pkgs = collections.defaultdict(list)
for pkg in packages_to_check:
if hasattr(pkg, "executables"):
for exe in pkg.platform_executables():
exe_pattern_to_pkgs[exe].append(pkg)
# Add Windows specific, package related paths to the search paths
path_hints.extend(compute_windows_user_path_for_package(pkg))
path_hints.extend(compute_windows_program_path_for_package(pkg))
path_to_exe_name = executables_in_path(path_hints=path_hints)

14
lib/spack/spack/operating_systems/windows_os.py
View File

@ -1,4 +1,4 @@
# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
@ -15,8 +15,12 @@
def windows_version():
"""temporary workaround to return a Windows version as a Version object"""
return Version(platform.release())
"""Windows version as a Version object"""
# include the build number as this provides important information
# for low lever packages and components like the SDK and WDK
# The build number is the version component that would otherwise
# be the patch version in sematic versioning, i.e. z of x.y.z
return Version(platform.version())
class WindowsOs(OperatingSystem):
@ -65,8 +69,8 @@ class WindowsOs(OperatingSystem):
compiler_search_paths = comp_search_paths
def __init__(self):
plat_ver = platform.release()
if Version(plat_ver) < Version("10"):
plat_ver = windows_version()
if plat_ver < Version("10"):
raise SpackError("Spack is not supported on Windows versions older than 10")
super(WindowsOs, self).__init__("windows{}".format(plat_ver), plat_ver)

4
lib/spack/spack/test/cmd/external.py
View File

@ -107,9 +107,7 @@ def test_find_external_update_config(mutable_config):
def test_get_executables(working_env, mock_executable):
cmake_path1 = mock_executable("cmake", output="echo cmake version 1.foo")
os.environ["PATH"] = os.pathsep.join([os.path.dirname(cmake_path1)])
path_to_exe = spack.detection.executables_in_path()
path_to_exe = spack.detection.executables_in_path([os.path.dirname(cmake_path1)])
cmake_exe = define_plat_exe("cmake")
assert path_to_exe[cmake_path1] == cmake_exe

291
lib/spack/spack/util/windows_registry.py Normal file
View File

@ -0,0 +1,291 @@
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""
Utility module for dealing with Windows Registry.
"""
import os
import sys
from contextlib import contextmanager
from llnl.util import tty
is_windows = sys.platform == "win32"
if is_windows:
import winreg
class RegistryValue(object):
"""
Class defining a Windows registry entry
"""
def __init__(self, name, value, parent_key):
self.path = name
self.value = value
self.key = parent_key
class RegistryKey(object):
"""
Class wrapping a Windows registry key
"""
def __init__(self, name, handle):
self.path = name
self.name = os.path.split(name)[-1]
self._handle = handle
self._keys = []
self._values = {}
@property
def values(self):
"""Returns all subvalues of this key as RegistryValue objects in dictionary
of value name : RegistryValue object
"""
self._gather_value_info()
return self._values
@property
def subkeys(self):
"""Returns list of all subkeys of this key as RegistryKey objects"""
self._gather_subkey_info()
return self._keys
@property
def hkey(self):
return self._handle
def __str__(self):
return self.name
def _gather_subkey_info(self):
"""Composes all subkeys into a list for access"""
if self._keys:
return
sub_keys, _, _ = winreg.QueryInfoKey(self.hkey)
for i in range(sub_keys):
sub_name = winreg.EnumKey(self.hkey, i)
sub_handle = winreg.OpenKeyEx(self.hkey, sub_name, access=winreg.KEY_READ)
self._keys.append(RegistryKey(os.path.join(self.path, sub_name), sub_handle))
def _gather_value_info(self):
"""Compose all values for this key into a dict of form value name: RegistryValue Object"""
if self._values:
return
_, values, _ = winreg.QueryInfoKey(self.hkey)
for i in range(values):
value_name, value_data, _ = winreg.EnumValue(self.hkey, i)
self._values[value_name] = RegistryValue(value_name, value_data, self.hkey)
def get_subkey(self, sub_key):
"""Returns subkey of name sub_key in a RegistryKey objects"""
return RegistryKey(
os.path.join(self.path, sub_key),
winreg.OpenKeyEx(self.hkey, sub_key, access=winreg.KEY_READ),
)
def get_value(self, val_name):
"""Returns value associated with this key in RegistryValue object"""
return RegistryValue(val_name, winreg.QueryValueEx(self.hkey, val_name)[0], self.hkey)
class _HKEY_CONSTANT(RegistryKey):
"""Subclass of RegistryKey to represent the prebaked, always open registry HKEY constants"""
def __init__(self, hkey_constant):
hkey_name = hkey_constant
# This class is instantiated at module import time
# on non Windows platforms, winreg would not have been
# imported. For this reason we can't reference winreg yet,
# so handle is none for now to avoid invalid references to a module.
# _handle provides a workaround to prevent null references to self.handle
# when coupled with the handle property.
super(_HKEY_CONSTANT, self).__init__(hkey_name, None)
def _get_hkey(self, key):
return getattr(winreg, key)
@property
def hkey(self):
if not self._handle:
self._handle = self._get_hkey(self.path)
return self._handle
class HKEY(object):
"""
Predefined, open registry HKEYs
From the Microsoft docs:
An application must open a key before it can read data from the registry.
To open a key, an application must supply a handle to another key in
the registry that is already open. The system defines predefined keys
that are always open. Predefined keys help an application navigate in
the registry."""
HKEY_CLASSES_ROOT = _HKEY_CONSTANT("HKEY_CLASSES_ROOT")
HKEY_CURRENT_USER = _HKEY_CONSTANT("HKEY_CURRENT_USER")
HKEY_USERS = _HKEY_CONSTANT("HKEY_USERS")
HKEY_LOCAL_MACHINE = _HKEY_CONSTANT("HKEY_LOCAL_MACHINE")
HKEY_CURRENT_CONFIG = _HKEY_CONSTANT("HKEY_CURRENT_CONFIG")
HKEY_PERFORMANCE_DATA = _HKEY_CONSTANT("HKEY_PERFORMANCE_DATA")
class WindowsRegistryView(object):
"""
Interface to provide access, querying, and searching to Windows registry entries.
This class represents a single key entrypoint into the Windows registry
and provides an interface to this key's values, its subkeys, and those subkey's values.
This class cannot be used to move freely about the registry, only subkeys/values of
the root key used to instantiate this class.
"""
def __init__(self, key, root_key=HKEY.HKEY_CURRENT_USER):
"""Constructs a Windows Registry entrypoint to key provided
root_key should be an already open root key or an hkey constant if provided
Args:
key (str): registry key to provide root for registry key for this clas
root_key: Already open registry key or HKEY constant to provide access into
the Windows registry. Registry access requires an already open key
to get an entrypoint, the HKEY constants are always open, or an already
open key can be used instead.
"""
if not is_windows:
raise RuntimeError(
"Cannot instantiate Windows Registry class on non Windows platforms"
)
self.key = key
self.root = root_key
self._reg = None
@contextmanager
def invalid_reg_ref_error_handler(self):
try:
yield
except FileNotFoundError as e:
if e.winerror == 2:
tty.debug("Key %s at position %s does not exist" % (self.key, str(self.root)))
else:
raise e
def __bool__(self):
return self.reg != -1
def _load_key(self):
try:
self._reg = RegistryKey(
os.path.join(str(self.root), self.key),
winreg.OpenKeyEx(self.root.hkey, self.key, access=winreg.KEY_READ),
)
except FileNotFoundError as e:
if e.winerror == 2:
self._reg = -1
tty.debug("Key %s at position %s does not exist" % (self.key, str(self.root)))
else:
raise e
def _valid_reg_check(self):
if self.reg == -1:
tty.debug("Cannot perform operation for nonexistent key %s" % self.key)
return False
return True
@property
def reg(self):
if not self._reg:
self._load_key()
return self._reg
def get_value(self, value_name):
"""Return registry value corresponding to provided argument (if it exists)"""
if not self._valid_reg_check():
raise RegistryError("Cannot query value from invalid key %s" % self.key)
with self.invalid_reg_ref_error_handler():
return self.reg.get_value(value_name)
def get_subkey(self, subkey_name):
if not self._valid_reg_check():
raise RegistryError("Cannot query subkey from invalid key %s" % self.key)
with self.invalid_reg_ref_error_handler():
return self.reg.get_subkey(subkey_name)
def get_subkeys(self):
if not self._valid_reg_check():
raise RegistryError("Cannot query subkeys from invalid key %s" % self.key)
with self.invalid_reg_ref_error_handler():
return self.reg.subkeys
def get_values(self):
if not self._valid_reg_check():
raise RegistryError("Cannot query values from invalid key %s" % self.key)
with self.invalid_reg_ref_error_handler():
return self.reg.values
def _traverse_subkeys(self, stop_condition):
"""Perform simple BFS of subkeys, returning the key
that successfully triggers the stop condition.
Args:
stop_condition: lambda or function pointer that takes a single argument
a key and returns a boolean value based on that key
Return:
the key if stop_condition is triggered, or None if not
"""
if not self._valid_reg_check():
raise RegistryError("Cannot query values from invalid key %s" % self.key)
with self.invalid_reg_ref_error_handler():
queue = self.reg.subkeys
for key in queue:
if stop_condition(key):
return key
queue.extend(key.subkeys)
return None
def find_subkey(self, subkey_name, recursive=True):
"""If non recursive, this method is the same as get subkey with error handling
Otherwise perform a BFS of subkeys until desired key is found
Returns None or RegistryKey object corresponding to requested key name
Args:
subkey_name (str): string representing subkey to be searched for
recursive (bool): optional argument, if True, subkey need not be a direct
sub key of this registry entry, and this method will
search all subkeys recursively.
Default is True
Return:
the desired subkey as a RegistryKey object, or none
"""
if not recursive:
return self.get_subkey(subkey_name)
else:
return self._traverse_subkeys(lambda x: x.name == subkey_name)
def find_value(self, val_name, recursive=True):
"""
If non recursive, return RegistryValue object corresponding to name
Args:
val_name (str): name of value desired from registry
recursive (bool): optional argument, if True, the registry is searched recursively
for the value of name val_name, else only the current key is searched
Return:
The desired registry value as a RegistryValue object if it exists, otherwise, None
"""
if not recursive:
return self.get_value(val_name)
else:
key = self._traverse_subkeys(lambda x: val_name in x.values)
if not key:
return None
else:
return key.values[val_name]
class RegistryError(RuntimeError):
"""Runtime Error describing issue with invalid key access to Windows registry"""

34
var/spack/repos/builtin/packages/msmpi/ifort_compat.patch Normal file
View File

@ -0,0 +1,34 @@
diff --git a/src/mpi/msmpi/dll/msmpi.vcxproj b/src/mpi/msmpi/dll/msmpi.vcxproj
index 255b9f5..cc4f096 100644
--- a/src/mpi/msmpi/dll/msmpi.vcxproj
+++ b/src/mpi/msmpi/dll/msmpi.vcxproj
@@ -57,6 +57,9 @@
$(OutDir)\..\mpi_debugger\mpi_debugger.lib;
$(CRT_Libs);
</AdditionalDependencies>
+ <AdditionalLibraryDirectories>
+ $(SPACK_IFORT)compiler\lib\intel64
+ </AdditionalLibraryDirectories>
<ModuleDefinitionFile>.\msmpi.def</ModuleDefinitionFile>
</Link>
</ItemDefinitionGroup>
diff --git a/src/mpi/msmpi/fortran/lib/mpifort.vcxproj b/src/mpi/msmpi/fortran/lib/mpifort.vcxproj
index 24bd29d..57d0292 100644
--- a/src/mpi/msmpi/fortran/lib/mpifort.vcxproj
+++ b/src/mpi/msmpi/fortran/lib/mpifort.vcxproj
@@ -8,12 +8,12 @@
</PropertyGroup>
<Target Name="CompileFortran" AfterTargets="ClCompile" Inputs="@(ForCompile)" Outputs="@(ForCompile->'$(O)%(FileName).obj')">
<PropertyGroup Condition="'$(BuildArchitecture)'=='i386'">
- <Fort_Flags>-fno-underscoring -D_X86_=1 -Di386=1 -march=x86-64 -m32</Fort_Flags>
+ <Fort_Flags>/D _X86_=1 /D i386=1 -march=x86-64 -m32</Fort_Flags>
</PropertyGroup>
<PropertyGroup Condition="'$(BuildArchitecture)'=='amd64'">
- <Fort_Flags>-fno-underscoring -D_WIN64 -D_AMD64_ -DAMD64</Fort_Flags>
+ <Fort_Flags>/D _WIN64=1 /D _AMD64_=1 /D AMD64=1</Fort_Flags>
</PropertyGroup>
- <Exec Command="$(GFORTRAN_BIN)\gfortran.exe -I$(MPI_INC_ROOT) -c %(ForCompile.Identity) $(Fort_Flags) -o $(O)\%(ForCompile.FileName).obj" />
+ <Exec Command="$(IFORT_BIN)\ifort.exe /I$(MPI_INC_ROOT) /c %(ForCompile.Identity) $(Fort_Flags) /names:lowercase /assume:nounderscore /o $(O)\%(ForCompile.FileName).obj" />
<ItemGroup>
<Lib Condition="'$(ConfigurationType)'=='StaticLibrary'" Include="@(ForCompile->'$(O)\%(Filename).obj')" />
</ItemGroup>

61
var/spack/repos/builtin/packages/msmpi/package.py
View File

@ -3,40 +3,61 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import platform
import re
from spack.build_systems.generic import GenericBuilder
from spack.package import *
class Msmpi(Package):
"""A Windows-specced build of MPICH provided directly by
Microsoft Support Team
"""
"""MSMPI is a Windows port of MPICH provided by the Windows team"""
homepage = "https://www.microsoft.com/en-us/download/default.aspx"
maintainers = ["jpopelar"]
homepage = "https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi"
url = "https://github.com/microsoft/Microsoft-MPI/archive/refs/tags/v10.1.1.tar.gz"
git = "https://github.com/microsoft/Microsoft-MPI.git"
executable = ["mpiexec.exe"]
version(
"10.0",
sha256="7dae13797627726f67fab9c1d251aec2df9ecd25939984645ec05748bdffd396",
extension="exe",
expand=False,
)
version("10.1.1", sha256="63c7da941fc4ffb05a0f97bd54a67968c71f63389a0d162d3182eabba1beab3d")
version("10.0.0", sha256="cfb53cf53c3cf0d4935ab58be13f013a0f7ccb1189109a5b8eea0fcfdcaef8c1")
provides("mpi")
conflicts("platform=linux")
conflicts("platform=darwin")
conflicts("platform=cray")
depends_on("win-wdk")
def url_for_version(self, version):
return "https://download.microsoft.com/download/A/E/0/AE002626-9D9D-448D-8197-1EA510E297CE/msmpisetup.exe"
patch("ifort_compat.patch")
def determine_version(self, exe):
output = Executable("mpiexec.exe")
@classmethod
def determine_version(cls, exe):
output = Executable(exe)()
ver_str = re.search("[Version ([0-9.]+)]", output)
return Version(ver_str.group(0)) if ver_str else None
class GenericBuilder(GenericBuilder):
def setup_build_environment(self, env):
ifort_root = os.path.join(*self.compiler.fc.split(os.path.sep)[:-2])
env.set("SPACK_IFORT", ifort_root)
def is_64bit(self):
return platform.machine().endswith("64")
def build_command_line(self):
args = ["-noLogo"]
ifort_bin = self.compiler.fc
if not ifort_bin:
raise InstallError(
"Cannot install MSMPI without fortran"
"please select a compiler with fortran support."
)
args.append("/p:IFORT_BIN=%s" % os.path.dirname(ifort_bin))
args.append("/p:VCToolsVersion=%s" % self.compiler.msvc_version)
args.append("/p:WindowsTargetPlatformVersion=%s" % str(self.spec["wdk"].version))
args.append("/p:PlatformToolset=%s" % self.compiler.cc_version)
return args
def install(self, spec, prefix):
installer = Executable("msmpisetup.exe")
installer("-unattend")
with working_dir(self.stage.build_directory, create=True):
msbuild(*self.build_command_line())

2
var/spack/repos/builtin/packages/perl/package.py
View File

@ -229,7 +229,7 @@ def do_stage(self, mirror_only=False):
def nmake_arguments(self):
args = []
if self.spec.satisfies("%msvc"):
args.append("CCTYPE=%s" % self.compiler.msvc_version)
args.append("CCTYPE=%s" % self.compiler.short_msvc_version)
else:
raise RuntimeError("Perl unsupported for non MSVC compilers on Windows")
args.append("INST_TOP=%s" % self.prefix.replace("/", "\\"))

96
var/spack/repos/builtin/packages/wgl/package.py Normal file
View File

@ -0,0 +1,96 @@
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import re
from spack.package import *
class Wgl(Package):
"""External WGl and Windows OpenGL emulation representation in Spack"""
homepage = "https://learn.microsoft.com/en-us/windows/win32/opengl/wgl-and-windows-reference"
has_code = False
# hard code the extension as shared lib
libraries = ["OpenGL32.Lib"]
# versions here are in no way related to actual WGL versions
# (there is only one on a system at a time)
# but instead reflects the Windows Kit version that a particular WGL library file is found in
# Windows Kits are intended to be more or less contained environments so this allows us to
# marshall our SDK and WDK to their respective WGLs. The purpose here is to better reflect
# the concept of an MS build toolchain version W.R.T. to MSVC
version("10.0.22621")
version("10.0.19041")
version("10.0.18362")
version("10.0.17763")
version("10.0.17134")
version("10.0.16299")
version("10.0.15063")
version("10.0.14393")
version("10.0.10586")
version("10.0.26639")
# As per https://github.com/spack/spack/pull/31748 this provisory version represents
# an arbitrary openGL version designed for maximum compatibility with calling packages
# this current version simply reflects the latest OpenGL vesion available at the time of
# package creation and is set in a way that all specs currently depending on GL are
# satisfied appropriately
provides("gl@4.6")
variant("plat", values=("x64", "x86", "arm", "arm64"), default="x64")
# WGL exists on all Windows systems post win 98, however the headers
# needed to use OpenGL are found in the SDK (GL/gl.h)
# Dep is needed to consolidate sdk version to locate header files for
# version of SDK being used
depends_on("win-sdk@10.0.19041", when="@10.0.19041")
depends_on("win-sdk@10.0.18362", when="@10.0.18362")
depends_on("win-sdk@10.0.17763", when="@10.0.17763")
depends_on("win-sdk@10.0.17134", when="@10.0.17134")
depends_on("win-sdk@10.0.16299", when="@10.0.16299")
depends_on("win-sdk@10.0.15063", when="@10.0.15063")
depends_on("win-sdk@10.0.14393", when="@10.0.14393")
# WGL has no meaning on other platforms, should not be able to spec
for plat in ["linux", "darwin", "cray"]:
conflicts("platform=%s" % plat)
@classmethod
def determine_version(cls, lib):
"""Allow for WGL to be externally detectable"""
version_match_pat = re.compile(r"[0-9][0-9].[0-9]+.[0-9][0-9][0-9][0-9][0-9]")
ver_str = re.search(version_match_pat, lib)
return ver_str if not ver_str else Version(ver_str.group())
@classmethod
def determine_variants(cls, libs, ver_str):
"""Allow for determination of toolchain arch for detected WGL"""
variants = []
for lib in libs:
base, lib_name = os.path.split(lib)
_, arch = os.path.split(base)
variants.append("plat=%s" % arch)
return variants
# As noted above, the headers neccesary to include
@property
def headers(self):
return find_headers("GL/gl.h", root=self.spec["win-sdk"].prefix.includes, recursive=True)
@property
def libs(self):
return find_libraries("opengl32", shared=False, root=self.prefix, recursive=True)
def install(self, spec, prefix):
raise RuntimeError(
"This package is not installable from Spack\
and should be installed on the system prior to Spack use.\
If not installed this package should be installed via\
the Visual Studio installer in order to use the \
MSVC compiler on Windows."
)

90
var/spack/repos/builtin/packages/win-sdk/package.py Normal file
View File

@ -0,0 +1,90 @@
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import re
from spack.package import *
class WinSdk(Package):
"""
Windows Desktop C++ development SDK
Spack packaged used to define search heuristics
to locate the SDK on a filesystem
"""
homepage = "https://developer.microsoft.com/en-us/windows/downloads/windows-sdk/"
has_code = False
# The sdk has many libraries and executables. Record one for detection purposes
libraries = ["rcdll.dll"]
version("10.0.22621")
version("10.0.19041")
version("10.0.18362")
version("10.0.17763")
version("10.0.17134")
version("10.0.16299")
version("10.0.15063")
version("10.0.14393")
version("10.0.10586")
version("10.0.26639")
variant("plat", values=("x64", "x86", "arm", "arm64"), default="x64")
# WinSDK versions depend on compatible compilers
# WDK versions do as well, but due to their one to one dep on the SDK
# we can ensure that requirment here
# WinSDK is very backwards compatible, however older
# MSVC editions may have problems with newer SDKs
conflicts("%msvc@:19.16.00000", when="@10.0.19041")
conflicts("%msvc@:19.16.00000", when="@10.0.18362")
conflicts("%msvc@:19.15.00000", when="@10.0.17763")
conflicts("%msvc@:19.14.00000", when="@10.0.17134")
conflicts("%msvc@:19.11.00000", when="@10.0.16299")
conflicts("%msvc@:19.10.00000", when="@10.0.15063")
conflicts("%msvc@:19.10.00000", when="@10.0.14393")
conflicts("%msvc@:19.00.00000", when="@10.0.10586")
# For now we don't support Windows development env
# on other platforms
for plat in ["linux", "darwin", "cray"]:
conflicts("platform=%s" % plat)
@classmethod
def determine_version(cls, lib):
"""
WinSDK that we would like to
be discoverable externally by Spack.
"""
# This version is found in the package's path
# not by calling an exe or a libraries name
version_match_pat = re.compile(r"[0-9][0-9].[0-9]+.[0-9][0-9][0-9][0-9][0-9]")
ver_str = re.search(version_match_pat, lib)
return ver_str if not ver_str else Version(ver_str.group())
@classmethod
def determine_variants(cls, libs, ver_str):
"""Allow for determination of toolchain arch for detected WGL"""
variants = []
for lib in libs:
base, lib_name = os.path.split(lib)
_, arch = os.path.split(base)
variants.append("plat=%s" % arch)
return variants
def install(self, spec, prefix):
raise RuntimeError(
"This package is not installable from Spack\
and should be installed on the system prior to Spack use.\
If not installed this package should be installed via\
the Visual Studio installer in order to use the \
MSVC compiler on Windows."
"If absolutely neccesary this SDK can be installed directly from Microsoft\
but this approach is not recommended unless you know what you're doing \
or if you're on Windows 11 you have no choice for the moment."
)

147
var/spack/repos/builtin/packages/win-wdk/package.py Normal file
View File

@ -0,0 +1,147 @@
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import glob
import os
import re
import spack.util.windows_registry as winreg
from spack.package import *
class WinWdk(Package):
"""
Windows Driver Kit development package
"""
homepage = "https://learn.microsoft.com/en-us/windows-hardware/drivers/"
# The wdk has many libraries and executables. Record one for detection purposes
libraries = ["mmos.lib"]
version(
"10.0.19041",
sha256="5f4ea0c55af099f97cb569a927c3a290c211f17edcfc65009f5b9253b9827925",
url="https://go.microsoft.com/fwlink/?linkid=2128854",
expand=False,
)
version(
"10.0.18362",
sha256="c35057cb294096c63bbea093e5024a5fb4120103b20c13fa755c92f227b644e5",
url="https://go.microsoft.com/fwlink/?linkid=2085767",
expand=False,
)
version(
"10.0.17763",
sha256="e6e5a57bf0a58242363cd6ca4762f44739f19351efc06cad382cca944b097235",
url="https://go.microsoft.com/fwlink/?linkid=2026156",
expand=False,
)
version(
"10.0.17134",
sha256="48e636117bb7bfe66b1ade793cc8e885c42c880fadaee471782d31b5c4d13e9b",
url="https://go.microsoft.com/fwlink/?linkid=873060",
expand=False,
)
version(
"10.0.16299",
sha256="14efbcc849e5977417e962f1cd68357d21abf27393110b9d95983ad03fc22ef4",
url="https://go.microsoft.com/fwlink/p/?linkid=859232",
expand=False,
)
version(
"10.0.15063",
sha256="489b497111bc791d9021b3573bfd93086a28b598c7325ab255e81c6f5d80a820",
url="https://go.microsoft.com/fwlink/p/?LinkID=845980",
expand=False,
)
version(
"10.0.14393",
sha256="0bfb2ac9db446e0d98c29ef7341a8c8e8e7aa24bc72b00c5704a88b13f48b3cb",
url="https://go.microsoft.com/fwlink/p/?LinkId=526733",
expand=False,
)
variant("plat", values=("x64", "x86", "arm", "arm64"), default="x64")
# need one to one dep on SDK per https://github.com/MicrosoftDocs/windows-driver-docs/issues/1550
# additionally, the WDK needs to be paired with a version of the Windows SDK
# as per https://learn.microsoft.com/en-us/windows-hardware/drivers/download-the-wdk#download-icon-step-2-install-windows-11-version-22h2-sdk
depends_on("win-sdk@10.0.19041", when="@10.0.19041")
depends_on("win-sdk@10.0.18362", when="@10.0.18362")
depends_on("win-sdk@10.0.17763", when="@10.0.17763")
depends_on("win-sdk@10.0.17134", when="@10.0.17134")
depends_on("win-sdk@10.0.16299", when="@10.0.16299")
depends_on("win-sdk@10.0.15063", when="@10.0.15063")
depends_on("win-sdk@10.0.14393", when="@10.0.14393")
for plat in ["linux", "darwin", "cray"]:
conflicts("platform=%s" % plat)
@classmethod
def determine_version(cls, lib):
"""
WDK is a set of drivers that we would like to
be discoverable externally by Spack.
The lib does not provide the WDK
version so we derive from the lib path
"""
version_match_pat = re.compile(r"[0-9][0-9].[0-9]+.[0-9][0-9][0-9][0-9][0-9]")
ver_str = re.search(version_match_pat, lib)
return ver_str if not ver_str else Version(ver_str.group())
@classmethod
def determine_variants(cls, libs, ver_str):
"""Allow for determination of toolchain arch for detected WGL"""
variants = []
for lib in libs:
base, lib_name = os.path.split(lib)
_, arch = os.path.split(base)
variants.append("plat=%s" % arch)
return variants
def setup_dependent_environment(self):
# This points to all core build extensions needed to build
# drivers on Windows
os.environ["WDKContentRoot"] = self.prefix
@run_before("install")
def rename_downloaded_executable(self):
"""WGL download is named by fetch based on name derived from Link redirection
This name is not properly formated so that Windows understands it as an executable
We rename so as to allow Windows to run the WGL installer"""
installer = glob.glob(os.path.join(self.stage.source_path, "linkid=**"))
if len(installer) > 1:
raise RuntimeError(
"Fetch has failed, unable to determine installer path from:\n%s"
% "\n".join(installer)
)
installer = installer[0]
os.rename(installer, os.path.join(self.stage.source_path, "wdksetup.exe"))
def install(self, spec, prefix):
install_args = ["/features", "+", "/quiet", "/installpath", self.prefix]
with working_dir(self.stage.source_path):
try:
Executable("wdksetup.exe")(*install_args)
except ProcessError as pe:
reg = winreg.WindowsRegistryView(
"SOFTWARE\\Microsoft\\Windows Kits\\Installed Roots",
root_key=spack.util.windows_registry.HKEY_LOCAL_MACHINE,
)
if not reg:
# No Kits are available, failure was genuine
raise pe
else:
versions = [str(subkey) for subkey in reg.get_subkeys()]
versions = ",".join(versions) if len(versions) > 1 else versions[0]
plural = "s" if len(versions) > 1 else ""
raise InstallError(
"Cannot install WDK version %s. "
"Version%s %s already present on system."
"Please run `spack external find win-wdk` to use the WDK"
% (self.version, plural, versions)
)