zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Spack on Windows package ports

Browse Source 
CMake - Windows Bootstrap (#25825)

Remove hardcoded cmake compiler (#26410)

Revert breaking cmake changes
Ensure no autotools on Windows

Perl on Windows (#26612)

Python source build windows (#26313)

Reconfigure sysconf for Windows

Python2.6 compatibility

Fxixup new sbang tests for windows

Ruby support (#28287)

Add NASM support (#28319)

Add mock Ninja package for testing
This commit is contained in:
John Parent
2021-10-22 14:00:02 -04:00
committed by Peter Scheibel
parent 90c773488c
commit 3a994032f8
27 changed files with 646 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
lib/spack/docs/getting_started.rst
View File

@@ -1587,14 +1587,6 @@ When given the option of adjusting your ``PATH``, choose the ``Git from the
command line and also from 3rd-party software`` option. This will automatically
update your ``PATH`` variable to include the ``git`` command.
""""
Perl
""""
Perl is a flexible and feature-rich programming language that comes built-in
on Unix boxes but needs to be installed externally for Windows users. Fortunately,
you can find the most recent release at https://www.perl.org/get.html.
""""
NASM
""""

2
lib/spack/llnl/util/tty/__init__.py
View File

@@ -146,7 +146,7 @@ def process_stacktrace(countback):
file_list = []
for frame in st:
# Check that the file is a spack file
if frame[0].find("/spack") >= 0:
if frame[0].find(os.path.sep + "spack") >= 0:
file_list.append(frame[0])
# We use commonprefix to find what the spack 'root' directory is.
root_dir = os.path.commonprefix(file_list)

4
lib/spack/spack/build_environment.py
View File

@@ -528,7 +528,9 @@ def _set_variables_for_single_module(pkg, module):
m.cmake = Executable('cmake')
m.ctest = MakeExecutable('ctest', jobs)
# Standard build system arguments
if os.name == 'nt':
m.nmake = Executable('nmake')
# Standard CMake arguments
m.std_cmake_args = spack.build_systems.cmake.CMakePackage._std_args(pkg)
m.std_meson_args = spack.build_systems.meson.MesonPackage._std_args(pkg)
m.std_pip_args = spack.build_systems.python.PythonPackage._std_args(pkg)

3
lib/spack/spack/build_systems/autotools.py
View File

@@ -14,7 +14,7 @@
from llnl.util.filesystem import force_remove, working_dir
from spack.build_environment import InstallError
from spack.directives import depends_on
from spack.directives import conflicts, depends_on
from spack.operating_systems.mac_os import macos_version
from spack.package import PackageBase, run_after, run_before
from spack.util.executable import Executable
@@ -104,6 +104,7 @@ def patch_config_files(self):
depends_on('gnuconfig', type='build', when='target=ppc64le:')
depends_on('gnuconfig', type='build', when='target=aarch64:')
depends_on('gnuconfig', type='build', when='target=riscv64:')
conflicts('platform=windows')
@property
def _removed_la_files_log(self):

30
lib/spack/spack/build_systems/cmake.py
View File

@@ -93,15 +93,11 @@ class CMakePackage(PackageBase):
#: See https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html
#: for more information.
variant('generator',
default='Make' if sys.platform != 'win32' else 'Ninja',
description='Build system to generate',
values=('Make', 'Ninja'))
generator = "Unix Makefiles"
depends_on('ninja', when='generator=Ninja')
generatorMap = {'Make': 'Unix Makefiles',
'Ninja': 'Ninja'}
if sys.platform == 'win32':
generator = "Ninja"
depends_on('ninja')
# https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html
variant('build_type', default='RelWithDebInfo',
@@ -150,10 +146,10 @@ def _std_args(pkg):
"""Computes the standard cmake arguments for a generic package"""
try:
pkg.generator = pkg.spec.variants['generator'].value
except KeyError:
pkg.generator = 'Make' if sys.platform != 'win32' else 'Ninja'
primary_generator = CMakePackage.generatorMap[pkg.generator]
if not pkg.generator:
raise AttributeError
except AttributeError:
pkg.generator = CMakePackage.generator
try:
build_type = pkg.spec.variants['build_type'].value
@@ -167,22 +163,16 @@ def _std_args(pkg):
define = CMakePackage.define
args = [
'-G', primary_generator,
'-G', pkg.generator,
define('CMAKE_INSTALL_PREFIX', pkg.prefix.replace('\\', '/')),
define('CMAKE_BUILD_TYPE', build_type),
define('CMAKE_C_COMPILER:FILEPATH', pkg.compiler.cc.replace('\\', '/')),
define('CMAKE_CXX_COMPILER:FILEPATH', pkg.compiler.cxx.replace('\\', '/'))
]
if pkg.compiler.fc is not None:
args.append(define('CMAKE_Fortran_COMPILER:FILEPATH',
pkg.compiler.fc.replace('\\', '/')))
# CMAKE_INTERPROCEDURAL_OPTIMIZATION only exists for CMake >= 3.9
if pkg.spec.satisfies('^cmake@3.9:'):
args.append(define('CMAKE_INTERPROCEDURAL_OPTIMIZATION', ipo))
if primary_generator == 'Unix Makefiles':
if pkg.generator == 'Unix Makefiles':
args.append(define('CMAKE_VERBOSE_MAKEFILE', True))
if platform.mac_ver()[0]:

2
lib/spack/spack/cmd/unit_test.py
View File

@@ -130,7 +130,7 @@ def colorize(c, prefix):
# in the future - so this manipulation might be fragile
if nodetype.lower() == 'function':
name_parts.append(item)
key_end = os.path.join(*[x[1] for x in key_parts])
key_end = os.path.join(*key_parts[-1][1].split('/'))
key = next(f for f in files if f.endswith(key_end))
tests[key].add(tuple(x[1] for x in name_parts))
elif nodetype.lower() == 'class':

70
lib/spack/spack/compilers/msvc.py
View File

@@ -4,13 +4,37 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import re
import subprocess
import sys
from typing import List # novm
from distutils.version import StrictVersion
from typing import Dict, List, Set # novm
import spack.operating_systems.windows_os
import spack.util.executable
from spack.compiler import Compiler
from spack.error import SpackError
avail_fc_version = set() # type: Set[str]
fc_path = dict() # type: Dict[str, str]
fortran_mapping = {
'2021.3.0': '19.29.30133',
'2021.2.1': '19.28.29913',
'2021.2.0': '19.28.29334',
'2021.1.0': '19.28.29333',
}
def get_valid_fortran_pth(comp_ver):
cl_ver = str(comp_ver).split('@')[1]
sort_fn = lambda fc_ver: StrictVersion(fc_ver)
sort_fc_ver = sorted(list(avail_fc_version), key=sort_fn)
for ver in sort_fc_ver:
if ver in fortran_mapping:
if StrictVersion(cl_ver) <= StrictVersion(fortran_mapping[ver]):
return fc_path[ver]
return None
class Msvc(Compiler):
@@ -46,6 +70,8 @@ class Msvc(Compiler):
# file based on compiler executable path.
def __init__(self, *args, **kwargs):
new_pth = [pth if pth else get_valid_fortran_pth(args[0]) for pth in args[3]]
args[3][:] = new_pth
super(Msvc, self).__init__(*args, **kwargs)
if os.getenv("ONEAPI_ROOT"):
# If this found, it sets all the vars
@@ -65,6 +91,12 @@ def verbose_flag(self):
def pic_flag(self):
return ""
@property
def msvc_version(self):
ver = re.search(Msvc.version_regex, self.cc).group(1)
ver = "".join(ver.split('.')[:2])[:-1]
return "MSVC" + ver
def setup_custom_environment(self, pkg, env):
"""Set environment variables for MSVC using the
Microsoft-provided script."""
@@ -81,44 +113,40 @@ def setup_custom_environment(self, pkg, env):
if sys.version_info[0] >= 3:
out = out.decode('utf-16le', errors='replace') # novermin
int_env = { # novermin
key.lower(): value
for key, _, value in
(line.partition('=') for line in out.splitlines())
if key and value
}
int_env = dict((key.lower(), value) for key, _, value in
(line.partition('=') for line in out.splitlines())
if key and value)
if 'path' in int_env:
env.set_path('PATH', int_env['path'].split(';'))
env.set_path('INCLUDE', int_env.get('include', '').split(';'))
env.set_path('LIB', int_env.get('lib', '').split(';'))
env.set('CC', self.cc)
env.set('CXX', self.cxx)
env.set('FC', self.fc)
env.set('F77', self.f77)
else:
# Should not this be an exception?
print("Cannot pull msvc compiler information in Python 2.6 or below")
# fc_version only loads the ifx compiler into the first MSVC stanza;
# if there are other versions of Microsoft VS installed and detected, they
# will only have cl.exe as the C/C++ compiler
@classmethod
def fc_version(cls, fc):
# We're using intel for the Fortran compilers, which exist if
# ONEAPI_ROOT is a meaningful variable
fc_ver = cls.default_version(fc)
avail_fc_version.add(fc_ver)
fc_path[fc_ver] = fc
if os.getenv("ONEAPI_ROOT"):
try:
sps = spack.operating_systems.windows_os.WindowsOs.compiler_search_paths
except Exception:
print("sps not found.")
raise
try:
clp = spack.util.executable.which_string("cl", path=sps)
except Exception:
print("cl not found.")
raise
except AttributeError:
raise SpackError("Windows compiler search paths not established")
clp = spack.util.executable.which_string("cl", path=sps)
ver = cls.default_version(clp)
return ver
else:
return cls.default_version(fc)
ver = fc_ver
return ver
@classmethod
def f77_version(cls, f77):

1
lib/spack/spack/detection/path.py
View File

@@ -12,7 +12,6 @@
import sys
import warnings
import llnl.util.filesystem
import llnl.util.tty

5
lib/spack/spack/fetch_strategy.py
View File

@@ -353,8 +353,9 @@ def _existing_url(self, url):
# Telling urllib to check if url is accessible
try:
url, headers, response = spack.util.web.read_from_url(url)
except spack.util.web.SpackWebError:
msg = "Urllib fetch failed to verify url {0}".format(url)
except spack.util.web.SpackWebError as werr:
msg = "Urllib fetch failed to verify url\
{0}\n with error {1}".format(url, werr)
raise FailedDownloadError(url, msg)
return (response.getcode() is None or response.getcode() == 200)

5
lib/spack/spack/platforms/_platform.py
View File

@@ -2,6 +2,8 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import platform
import llnl.util.lang
import spack.error
@@ -143,3 +145,6 @@ def oses():
for o in sorted(self.operating_sys.values()):
yield o._cmp_iter
yield oses
def is_64bit(self):
return platform.machine().endswith('64')

2
lib/spack/spack/test/bootstrap.py
View File

@@ -131,7 +131,7 @@ def test_custom_store_in_environment(mutable_config, tmpdir):
# Don't trigger evaluation here
with spack.bootstrap.ensure_bootstrap_configuration():
pass
assert str(spack.store.root) == '/tmp/store'
assert str(spack.store.root) == sep + os.path.join('tmp', 'store')
def test_nested_use_of_context_manager(mutable_config):

4
lib/spack/spack/test/cmd/unit_test.py
View File

@@ -3,10 +3,12 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
from spack.main import SpackCommand
spack_test = SpackCommand('unit-test')
cmd_test_py = 'lib/spack/spack/test/cmd/unit_test.py'
cmd_test_py = os.path.join('lib', 'spack', 'spack', 'test', 'cmd', 'unit_test.py')
def test_list():

1
lib/spack/spack/test/cmd/url.py
View File

@@ -3,6 +3,7 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import re
import sys
import pytest

1
lib/spack/spack/test/relocate.py
View File

@@ -2,6 +2,7 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import os.path
import re
import shutil

13
lib/spack/spack/test/sbang.py
View File

@@ -11,6 +11,7 @@
import os
import shutil
import stat
import sys
import tempfile
import pytest
@@ -193,6 +194,8 @@ def test_shebang_interpreter_regex(shebang, interpreter):
sbang.get_interpreter(shebang) == interpreter
@pytest.mark.skipif(sys.platform == 'win32',
reason="Not supported on Windows (yet)")
def test_shebang_handling(script_dir, sbang_line):
sbang.filter_shebangs_in_directory(script_dir.tempdir)
@@ -339,6 +342,8 @@ def test_install_user_sbang(install_mockery, configure_user_perms):
run_test_install_sbang(False)
@pytest.mark.skipif(sys.platform == 'win32',
reason="Not supported on Windows (yet)")
def test_install_sbang_too_long(tmpdir):
root = str(tmpdir)
num_extend = sbang.system_shebang_limit - len(root) - len('/bin/sbang')
@@ -357,6 +362,8 @@ def test_install_sbang_too_long(tmpdir):
assert 'cannot patch' in err
@pytest.mark.skipif(sys.platform == 'win32',
reason="Not supported on Windows (yet)")
def test_sbang_hook_skips_nonexecutable_blobs(tmpdir):
# Write a binary blob to non-executable.sh, with a long interpreter "path"
# consisting of invalid UTF-8. The latter is technically not really necessary for
@@ -374,6 +381,8 @@ def test_sbang_hook_skips_nonexecutable_blobs(tmpdir):
assert b'sbang' not in f.readline()
@pytest.mark.skipif(sys.platform == 'win32',
reason="Not supported on Windows (yet)")
def test_sbang_handles_non_utf8_files(tmpdir):
# We have an executable with a copyright sign as filename
contents = (b'#!' + b'\xa9' * sbang.system_shebang_limit +
@@ -408,6 +417,8 @@ def shebang_limits_system_8_spack_16():
sbang.spack_shebang_limit = spack_limit
@pytest.mark.skipif(sys.platform == 'win32',
reason="Not supported on Windows (yet)")
def test_shebang_exceeds_spack_shebang_limit(shebang_limits_system_8_spack_16, tmpdir):
"""Tests whether shebangs longer than Spack's limit are skipped"""
file = str(tmpdir.join('longer_than_spack_limit.sh'))
@@ -421,6 +432,8 @@ def test_shebang_exceeds_spack_shebang_limit(shebang_limits_system_8_spack_16, t
assert b'sbang' not in f.read()
@pytest.mark.skipif(sys.platform == 'win32',
reason="Not supported on Windows (yet)")
def test_sbang_hook_handles_non_writable_files_preserving_permissions(tmpdir):
path = str(tmpdir.join('file.sh'))
with open(path, 'w') as f: