zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Microarchitecture specific optimizations for LLVM (#13250)

Browse Source 
* Added architecture specific optimization flags for Clang / LLVM
* Disallow compiler optimizations for mixed toolchains
    * We emit a warning when building for a mixed toolchain
* Fixed issues with suffixed versions of compilers; Apple's Clang will, 
    for the time being, fall back on x86-64 for every compilation.
This commit is contained in:
Massimiliano Culpo
2019-10-19 22:19:29 +02:00
committed by Todd Gamblin
parent 0fb563f3d9
commit 41fb0395a6
9 changed files with 292 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
lib/spack/llnl/util/cpu/microarchitecture.py
View File

@@ -224,7 +224,8 @@ def satisfies_constraint(entry, version):
version, _, suffix = version.partition('-')
# If the suffixes are not all equal there's no match
if suffix != min_suffix or suffix != max_suffix:
if ((suffix != min_suffix and min_version) or
(suffix != max_suffix and max_version)):
return False
# Assume compiler versions fit into semver

187
lib/spack/llnl/util/cpu/microarchitectures.json
View File

@@ -60,7 +60,12 @@
"name": "x86-64",
"flags": "-march={name} -mtune={name}"
}
]
],
"clang": {
"versions": ":",
"family": "x86-64",
"flags": "-march={family} -mcpu=generic"
}
}
},
"nocona": {
@@ -76,6 +81,11 @@
"gcc": {
"versions": "4:",
"flags": "-march={name} -mtune={name}"
},
"clang": {
"versions": "3.9:",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -92,6 +102,11 @@
"gcc": {
"versions": "4:",
"flags": "-march={name} -mtune={name}"
},
"clang": {
"versions": "3.9:",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -118,7 +133,12 @@
"name": "corei7",
"flags": "-march={name} -mtune={name}"
}
]
],
"clang": {
"versions": "3.9:",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
"westmere": {
@@ -139,6 +159,11 @@
"gcc": {
"versions": "4.9:",
"flags": "-march={name} -mtune={name}"
},
"clang": {
"versions": "3.9:",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -168,7 +193,12 @@
"name": "corei7-avx",
"flags": "-march={name} -mtune={name}"
}
]
],
"clang": {
"versions": "3.9:",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
"ivybridge": {
@@ -199,7 +229,12 @@
"name": "core-avx-i",
"flags": "-march={name} -mtune={name}"
}
]
],
"clang": {
"versions": "3.9:",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
"haswell": {
@@ -235,7 +270,12 @@
"name": "core-avx2",
"flags": "-march={name} -mtune={name}"
}
]
],
"clang": {
"versions": "3.9:",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
"broadwell": {
@@ -266,6 +306,11 @@
"gcc": {
"versions": "4.9:",
"flags": "-march={name} -mtune={name}"
},
"clang": {
"versions": "3.9:",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -300,6 +345,11 @@
"gcc": {
"versions": "5.3:",
"flags": "-march={name} -mtune={name}"
},
"clang": {
"versions": "3.9:",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -337,6 +387,12 @@
"versions": "5.1:",
"name": "knl",
"flags": "-march={name} -mtune={name}"
},
"clang": {
"versions": "3.9:",
"name": "knl",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -378,6 +434,12 @@
"name": "skylake-avx512",
"versions": "5.3:",
"flags": "-march={name} -mtune={name}"
},
"clang": {
"versions": "3.9:",
"name": "skylake-avx512",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -421,6 +483,11 @@
"gcc": {
"versions": "8:",
"flags": "-march={name} -mtune={name}"
},
"clang": {
"versions": "3.9:",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -462,6 +529,11 @@
"gcc": {
"versions": "9:",
"flags": "-march={name} -mtune={name}"
},
"clang": {
"versions": "8.0:",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -518,7 +590,20 @@
"name": "icelake-client",
"versions": "8:",
"flags": "-march={name} -mtune={name}"
}
},
"clang": [
{
"versions": "7.0:",
"name": "icelake-client",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
},
{
"versions": "6.0:6.9",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
]
}
},
"bulldozer": {
@@ -545,6 +630,12 @@
"name": "bdver1",
"versions": "4.6:",
"flags": "-march={name} -mtune={name}"
},
"clang": {
"versions": "3.9:",
"name": "bdver1",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -576,6 +667,12 @@
"name": "bdver2",
"versions": "4.7:",
"flags": "-march={name} -mtune={name}"
},
"clang": {
"versions": "3.9:",
"name": "bdver2",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -608,6 +705,12 @@
"name": "bdver3",
"versions": "4.8:",
"flags": "-march={name} -mtune={name}"
},
"clang": {
"versions": "3.9:",
"name": "bdver3",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -643,6 +746,12 @@
"name": "bdver4",
"versions": "4.9:",
"flags": "-march={name} -mtune={name}"
},
"clang": {
"versions": "3.9:",
"name": "bdver4",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -681,6 +790,12 @@
"name": "znver1",
"versions": "6:",
"flags": "-march={name} -mtune={name}"
},
"clang": {
"versions": "4.0:",
"name": "znver1",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -720,6 +835,12 @@
"name": "znver2",
"versions": "9:",
"flags": "-march={name} -mtune={name}"
},
"clang": {
"versions": "9.0:",
"name": "znver2",
"family": "x86-64",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -732,6 +853,11 @@
"name": "powerpc64",
"versions": "4:",
"flags": "-mcpu={name} -mtune={name}"
},
"clang": {
"versions": ":",
"family": "ppc64",
"flags": "-march={family} -mcpu=generic"
}
}
},
@@ -744,6 +870,12 @@
"gcc": {
"versions": "4.5:",
"flags": "-mcpu={name} -mtune={name}"
},
"clang": {
"versions": "3.9:",
"family": "ppc64",
"name": "pwr7",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -763,7 +895,13 @@
"warnings": "Using GCC 4.8 to optimize for Power 8 might not work if you are not on Red Hat Enterprise Linux 7, where a custom backport of the feature has been done. Upstream support from GCC starts in version 4.9",
"flags": "-mcpu={name} -mtune={name}"
}
]
],
"clang": {
"versions": "3.9:",
"family": "ppc64",
"name": "pwr8",
"flags": "-march={family} -mcpu={name}"
}
}
},
"power9": {
@@ -775,6 +913,12 @@
"gcc": {
"versions": "6:",
"flags": "-mcpu={name} -mtune={name}"
},
"clang": {
"versions": "3.9:",
"family": "ppc64",
"name": "pwr9",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -787,6 +931,11 @@
"name": "powerpc64le",
"versions": "4:",
"flags": "-mcpu={name} -mtune={name}"
},
"clang": {
"versions": ":",
"family": "ppc64le",
"flags": "-march={family} -mcpu=generic"
}
}
},
@@ -808,7 +957,13 @@
"name": "power8",
"flags": "-mcpu={name} -mtune={name}"
}
]
],
"clang": {
"versions": "3.9:",
"family": "ppc64le",
"name": "pwr8",
"flags": "-march={family} -mcpu={name}"
}
}
},
"power9le": {
@@ -821,6 +976,12 @@
"name": "power9",
"versions": "6:",
"flags": "-mcpu={name} -mtune={name}"
},
"clang": {
"versions": "3.9:",
"family": "ppc64le",
"name": "pwr9",
"flags": "-march={family} -mcpu={name}"
}
}
},
@@ -832,6 +993,11 @@
"gcc": {
"versions": "4:",
"flags": "-march=armv8-a -mtune=generic"
},
"clang": {
"versions": ":",
"family": "aarch64",
"flags": "-march={family} -mcpu=generic"
}
}
},
@@ -840,6 +1006,11 @@
"vendor": "generic",
"features": [],
"compilers": {
"clang": {
"versions": ":",
"family": "arm",
"flags": "-march={family} -mcpu=generic"
}
}
},
"ppc": {

16
lib/spack/spack/architecture.py
View File

@@ -58,6 +58,7 @@
"""
import functools
import inspect
import warnings
import six
@@ -184,6 +185,21 @@ def __contains__(self, cpu_flag):
return cpu_flag in self.microarchitecture
def optimization_flags(self, compiler):
"""Returns the flags needed to optimize for this target using
the compiler passed as argument.
Args:
compiler (CompilerSpec or Compiler): object that contains both the
name and the version of the compiler we want to use
"""
if isinstance(compiler, spack.compiler.Compiler):
if spack.compilers.is_mixed_toolchain(compiler):
msg = ('microarchitecture specific optimizations are not '
'supported yet on mixed compiler toolchains [check'
' {0.name}@{0.version} for further details]')
warnings.warn(msg.format(compiler))
return ''
return self.microarchitecture.optimization_flags(
compiler.name, str(compiler.version)
)

38
lib/spack/spack/compilers/__init__.py
View File

@@ -676,6 +676,44 @@ def _default(cmp_id, paths):
return compilers
def is_mixed_toolchain(compiler):
"""Returns True if the current compiler is a mixed toolchain,
False otherwise.
Args:
compiler (Compiler): a valid compiler object
"""
cc = os.path.basename(compiler.cc or '')
cxx = os.path.basename(compiler.cxx or '')
f77 = os.path.basename(compiler.f77 or '')
fc = os.path.basename(compiler.fc or '')
toolchains = set()
for compiler_cls in all_compiler_types():
# Inspect all the compiler toolchain we know. If a compiler is the
# only compiler supported there it belongs to that toolchain.
def name_matches(name, name_list):
# This is such that 'gcc' matches variations
# like 'ggc-9' etc that are found in distros
name, _, _ = name.partition('-')
return len(name_list) == 1 and name and name in name_list
if any([
name_matches(cc, compiler_cls.cc_names),
name_matches(cxx, compiler_cls.cxx_names),
name_matches(f77, compiler_cls.f77_names),
name_matches(fc, compiler_cls.fc_names)
]):
tty.debug("[TOOLCHAIN] MATCH {0}".format(compiler_cls.__name__))
toolchains.add(compiler_cls.__name__)
if len(toolchains) > 1:
tty.debug("[TOOLCHAINS] {0}".format(toolchains))
return True
return False
class InvalidCompilerConfigurationError(spack.error.SpackError):
def __init__(self, compiler_spec):

18
lib/spack/spack/test/architecture.py
View File

@@ -169,3 +169,21 @@ def test_target_container_semantic(cpu_flag, target_name):
def test_arch_spec_container_semantic(item, architecture_str):
architecture = spack.spec.ArchSpec(architecture_str)
assert item in architecture
@pytest.mark.parametrize('compiler_spec,target_name,expected_flags', [
# Check compilers with version numbers from a single toolchain
('gcc@4.7.2', 'haswell', '-march=core-avx2 -mtune=core-avx2'),
# Check mixed toolchains
('clang@8.0.0', 'broadwell', ''),
# Check clang compilers with 'apple' suffix
('clang@9.1.0-apple', 'x86_64', '-march=x86-64 -mcpu=generic')
])
@pytest.mark.filterwarnings("ignore:microarchitecture specific")
def test_optimization_flags(
compiler_spec, target_name, expected_flags, config
):
target = spack.architecture.Target(target_name)
compiler = spack.compilers.compilers_for_spec(compiler_spec).pop()
opt_flags = target.optimization_flags(compiler)
assert opt_flags == expected_flags

8
lib/spack/spack/test/compilers.py
View File

@@ -469,3 +469,11 @@ def test_cce_version_detection(version_str, expected_version):
def test_fj_version_detection(version_str, expected_version):
version = spack.compilers.fj.Fj.extract_version_from_output(version_str)
assert version == expected_version
@pytest.mark.parametrize('compiler_spec,expected_result', [
('gcc@4.7.2', False), ('clang@3.3', False), ('clang@8.0.0', True)
])
def test_detecting_mixed_toolchains(compiler_spec, expected_result, config):
compiler = spack.compilers.compilers_for_spec(compiler_spec).pop()
assert spack.compilers.is_mixed_toolchain(compiler) is expected_result

4
lib/spack/spack/test/concretize.py
View File

@@ -585,7 +585,9 @@ def test_noversion_pkg(self, spec):
@pytest.mark.parametrize('spec, best_achievable', [
('mpileaks%gcc@4.8', 'haswell'),
('mpileaks%gcc@5.3.0', 'skylake_avx512')
('mpileaks%gcc@5.3.0', 'skylake_avx512'),
# Apple's clang always falls back to x86-64 for now
('mpileaks%clang@9.1.0-apple', 'x86_64')
])
def test_adjusting_default_target_based_on_compiler(
self, spec, best_achievable, current_host

21
lib/spack/spack/test/data/compilers.yaml
View File

@@ -114,3 +114,24 @@ compilers:
cflags: -O3
cxxflags: -O3
modules: 'None'
- compiler:
spec: clang@8.0.0
operating_system: redhat7
paths:
cc: /path/to/clang-8
cxx: /path/to/clang++-8
f77: /path/to/gfortran-9
fc: /path/to/gfortran-9
flags:
cflags: -O3
cxxflags: -O3
modules: 'None'
- compiler:
spec: clang@9.1.0-apple
operating_system: elcapitan
paths:
cc: /path/to/clang
cxx: /path/to/clang++
f77: None
fc: None
modules: 'None'

7
lib/spack/spack/test/llnl/util/cpu.py
View File

@@ -191,6 +191,7 @@ def test_target_json_schema():
@pytest.mark.parametrize('target_name,compiler,version,expected_flags', [
# Test GCC
('x86_64', 'gcc', '4.9.3', '-march=x86-64 -mtune=generic'),
('x86_64', 'gcc', '4.2.0', '-march=x86-64 -mtune=generic'),
('x86_64', 'gcc', '4.1.1', '-march=x86-64 -mtune=x86-64'),
@@ -198,6 +199,12 @@ def test_target_json_schema():
('nehalem', 'gcc', '4.9.3', '-march=nehalem -mtune=nehalem'),
('nehalem', 'gcc', '4.8.5', '-march=corei7 -mtune=corei7'),
('sandybridge', 'gcc', '4.8.5', '-march=corei7-avx -mtune=corei7-avx'),
# Test Clang / LLVM
('sandybridge', 'clang', '3.9.0', '-march=x86-64 -mcpu=sandybridge'),
('icelake', 'clang', '6.0.0', '-march=x86-64 -mcpu=icelake'),
('icelake', 'clang', '8.0.0', '-march=x86-64 -mcpu=icelake-client'),
('zen2', 'clang', '9.0.0', '-march=x86-64 -mcpu=znver2'),
('power9le', 'clang', '8.0.0', '-march=ppc64le -mcpu=pwr9'),
# Test that an unknown compiler returns an empty string
('sandybridge', 'unknown', '4.8.5', ''),
])