From 31daf0f2b6bd769135e088294eff93a94342c3d2 Mon Sep 17 00:00:00 2001 From: John Parent Date: Fri, 22 Oct 2021 12:25:46 -0400 Subject: [PATCH] Add Windows to platform and target changes Add compiler hint to the root spec for Windows Reporters on Windows (#26038) Reporters use Jinja2 as the templating engine, and Jinja2 indexes templates by Unix separators, even on Windows, so search using Unix paths on all systems. Support patching on win via git (#25871) Handle GRP on windows --- lib/spack/spack/bootstrap.py | 4 +++- lib/spack/spack/cmd/modules/lmod.py | 1 - lib/spack/spack/hooks/sbang.py | 8 +++++++- lib/spack/spack/operating_systems/__init__.py | 6 ++++-- lib/spack/spack/operating_systems/windows_os.py | 7 ++++--- lib/spack/spack/patch.py | 11 +++++++++-- lib/spack/spack/platforms/__init__.py | 2 ++ lib/spack/spack/platforms/_functions.py | 3 ++- lib/spack/spack/platforms/windows.py | 6 ++++-- lib/spack/spack/reporters/cdash.py | 13 +++++++------ lib/spack/spack/reporters/junit.py | 5 ++--- lib/spack/spack/test/sbang.py | 4 +++- 12 files changed, 47 insertions(+), 23 deletions(-) diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py index c44f217ccd7..335999a120d 100644 --- a/lib/spack/spack/bootstrap.py +++ b/lib/spack/spack/bootstrap.py @@ -727,9 +727,11 @@ def _root_spec(spec_str): spec_str (str): spec to be bootstrapped. Must be without compiler and target. """ # Add a proper compiler hint to the root spec. We use GCC for - # everything but MacOS. + # everything but MacOS and Windows. if str(spack.platforms.host()) == 'darwin': spec_str += ' %apple-clang' + elif str(spack.platforms.host()) == 'windows': + spec_str += ' %msvc' else: spec_str += ' %gcc' diff --git a/lib/spack/spack/cmd/modules/lmod.py b/lib/spack/spack/cmd/modules/lmod.py index abd89b3a184..fec663e53e1 100644 --- a/lib/spack/spack/cmd/modules/lmod.py +++ b/lib/spack/spack/cmd/modules/lmod.py @@ -61,7 +61,6 @@ def setdefault(module_type, specs, args): writer = spack.modules.module_types['lmod'](spec, args.module_set_name) writer.update_module_defaults() - module_folder = os.path.dirname(writer.layout.filename) module_basename = os.path.basename(writer.layout.filename) with llnl.util.filesystem.working_dir(module_folder): diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 75eff662814..2793906ebd6 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import filecmp -import grp import os import re import shutil @@ -15,6 +14,7 @@ import llnl.util.filesystem as fs import llnl.util.tty as tty +import spack.error import spack.package_prefs import spack.paths import spack.spec @@ -28,6 +28,12 @@ else: system_shebang_limit = 127 +#: Groupdb does not exist on Windows, prevent imports +#: on supported systems +is_windows = str(spack.platforms.host()) == 'windows' +if not is_windows: + import grp + #: Spack itself also limits the shebang line to at most 4KB, which should be plenty. spack_shebang_limit = 4096 diff --git a/lib/spack/spack/operating_systems/__init__.py b/lib/spack/spack/operating_systems/__init__.py index bd2377d80ca..5a029ae5a8b 100644 --- a/lib/spack/spack/operating_systems/__init__.py +++ b/lib/spack/spack/operating_systems/__init__.py @@ -7,14 +7,16 @@ from .cray_frontend import CrayFrontend from .linux_distro import LinuxDistro from .mac_os import MacOs +from .windows_os import WindowsOs __all__ = [ 'OperatingSystem', 'LinuxDistro', 'MacOs', 'CrayFrontend', - 'CrayBackend' + 'CrayBackend', + 'WindowsOs' ] #: List of all the Operating Systems known to Spack -operating_systems = [LinuxDistro, MacOs, CrayFrontend, CrayBackend] +operating_systems = [LinuxDistro, MacOs, CrayFrontend, CrayBackend, WindowsOs] diff --git a/lib/spack/spack/operating_systems/windows_os.py b/lib/spack/spack/operating_systems/windows_os.py index 7d5937c4a93..06113a44343 100755 --- a/lib/spack/spack/operating_systems/windows_os.py +++ b/lib/spack/spack/operating_systems/windows_os.py @@ -8,9 +8,10 @@ import subprocess import sys -from spack.architecture import OperatingSystem from spack.version import Version +from ._operating_system import OperatingSystem + # FIXME: To get the actual Windows version, we need a python that runs # natively on Windows, not Cygwin. @@ -37,14 +38,14 @@ class WindowsOs(OperatingSystem): extra_args = {} if sys.version_info[:3] >= (3, 6, 0): extra_args = {'encoding': 'mbcs', 'errors': 'strict'} - paths = subprocess.check_output([ # novermin + paths = subprocess.check_output([ # type: ignore[call-overload] # novermin os.path.join(root, "Microsoft Visual Studio", "Installer", "vswhere.exe"), "-prerelease", "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "-property", "installationPath", "-products", "*", - ], **extra_args).strip() # type: ignore[call-overload] + ], **extra_args).strip() if (3, 0) <= sys.version_info[:2] <= (3, 5): paths = paths.decode() vs_install_paths = paths.split('\n') diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index d248966eded..64f39190297 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -19,7 +19,7 @@ import spack.util.spack_json as sjson from spack.util.compression import allowed_archive from spack.util.crypto import Checker, checksum -from spack.util.executable import which +from spack.util.executable import which, which_string def apply_patch(stage, patch_path, level=1, working_dir='.'): @@ -32,7 +32,14 @@ def apply_patch(stage, patch_path, level=1, working_dir='.'): working_dir (str): relative path *within* the stage to change to (default '.') """ - patch = which("patch", required=True) + git_utils_path = os.environ.get('PATH', '') + if os.name == 'nt': + git = which_string('git', required=True) + git_root = os.path.dirname(git).split('/')[:-1] + git_root.extend(['usr', 'bin']) + git_utils_path = os.sep.join(git_root) + + patch = which("patch", required=True, path=git_utils_path) with llnl.util.filesystem.working_dir(stage.source_path): patch('-s', '-p', str(level), diff --git a/lib/spack/spack/platforms/__init__.py b/lib/spack/spack/platforms/__init__.py index 83de03de121..50acc3ec423 100644 --- a/lib/spack/spack/platforms/__init__.py +++ b/lib/spack/spack/platforms/__init__.py @@ -10,6 +10,7 @@ from .darwin import Darwin from .linux import Linux from .test import Test +from .windows import Windows __all__ = [ 'Platform', @@ -17,6 +18,7 @@ 'Darwin', 'Linux', 'Test', + 'Windows', 'platforms', 'host', 'by_name', diff --git a/lib/spack/spack/platforms/_functions.py b/lib/spack/spack/platforms/_functions.py index 7d3a440a3a1..503be5afcb9 100644 --- a/lib/spack/spack/platforms/_functions.py +++ b/lib/spack/spack/platforms/_functions.py @@ -12,9 +12,10 @@ from .darwin import Darwin from .linux import Linux from .test import Test +from .windows import Windows #: List of all the platform classes known to Spack -platforms = [Cray, Darwin, Linux, Test] +platforms = [Cray, Darwin, Linux, Windows, Test] @llnl.util.lang.memoized diff --git a/lib/spack/spack/platforms/windows.py b/lib/spack/spack/platforms/windows.py index 7c9ed1443dd..552e520193b 100755 --- a/lib/spack/spack/platforms/windows.py +++ b/lib/spack/spack/platforms/windows.py @@ -7,9 +7,11 @@ import archspec.cpu -from spack.architecture import Platform, Target +import spack.target from spack.operating_systems.windows_os import WindowsOs +from ._platform import Platform + class Windows(Platform): priority = 101 @@ -20,7 +22,7 @@ def __init__(self): super(Windows, self).__init__('windows') for name in archspec.cpu.TARGETS: - self.add_target(name, Target(name)) + self.add_target(name, spack.target.Target(name)) self.default = archspec.cpu.host().name self.front_end = self.default diff --git a/lib/spack/spack/reporters/cdash.py b/lib/spack/spack/reporters/cdash.py index 212283f39e2..06f4479b14f 100644 --- a/lib/spack/spack/reporters/cdash.py +++ b/lib/spack/spack/reporters/cdash.py @@ -7,6 +7,7 @@ import hashlib import os.path import platform +import posixpath import re import socket import time @@ -61,7 +62,7 @@ class CDash(Reporter): def __init__(self, args): Reporter.__init__(self, args) self.success = True - self.template_dir = os.path.join('reports', 'cdash') + self.template_dir = posixpath.join('reports', 'cdash') self.cdash_upload_url = args.cdash_upload_url if self.cdash_upload_url: @@ -219,11 +220,11 @@ def clean_log_event(event): if phase != 'update': # Update.xml stores site information differently # than the rest of the CTest XML files. - site_template = os.path.join(self.template_dir, 'Site.xml') + site_template = posixpath.join(self.template_dir, 'Site.xml') t = env.get_template(site_template) f.write(t.render(report_data)) - phase_template = os.path.join(self.template_dir, report_name) + phase_template = posixpath.join(self.template_dir, report_name) t = env.get_template(phase_template) f.write(t.render(report_data)) self.upload(phase_report) @@ -346,11 +347,11 @@ def clean_log_event(event): if phase != 'update': # Update.xml stores site information differently # than the rest of the CTest XML files. - site_template = os.path.join(self.template_dir, 'Site.xml') + site_template = posixpath.join(self.template_dir, 'Site.xml') t = env.get_template(site_template) f.write(t.render(report_data)) - phase_template = os.path.join(self.template_dir, report_name) + phase_template = posixpath.join(self.template_dir, report_name) t = env.get_template(phase_template) f.write(t.render(report_data)) self.upload(phase_report) @@ -376,7 +377,7 @@ def concretization_report(self, directory_name, msg): report_data['update']['log'] = msg env = spack.tengine.make_environment() - update_template = os.path.join(self.template_dir, 'Update.xml') + update_template = posixpath.join(self.template_dir, 'Update.xml') t = env.get_template(update_template) output_filename = os.path.join(directory_name, 'Update.xml') with open(output_filename, 'w') as f: diff --git a/lib/spack/spack/reporters/junit.py b/lib/spack/spack/reporters/junit.py index 323010c46f5..bcd777e5327 100644 --- a/lib/spack/spack/reporters/junit.py +++ b/lib/spack/spack/reporters/junit.py @@ -3,8 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import os -import os.path +import posixpath import spack.build_environment import spack.fetch_strategy @@ -19,7 +18,7 @@ class JUnit(Reporter): def __init__(self, args): Reporter.__init__(self, args) - self.template_file = os.path.join('reports', 'junit.xml') + self.template_file = posixpath.join('reports', 'junit.xml') def build_report(self, filename, report_data): # Write the report diff --git a/lib/spack/spack/test/sbang.py b/lib/spack/spack/test/sbang.py index f0cc1835d9f..4488d8845a5 100644 --- a/lib/spack/spack/test/sbang.py +++ b/lib/spack/spack/test/sbang.py @@ -7,7 +7,6 @@ Test that Spack's shebang filtering works correctly. """ import filecmp -import grp import os import shutil import stat @@ -24,6 +23,9 @@ import spack.util.spack_yaml as syaml from spack.util.executable import which +if sys.platform != 'win32': + import grp + too_long = sbang.system_shebang_limit + 1