Fix hipcc once more (#20095)

lib/spack/spack/build_systems/rocm.py

@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-# Troubleshooting advice for +hip builds:
+# Troubleshooting advice for +rocm builds:
 #
 # 1. When building with clang, go your compilers.yaml,
 #    add an entry for the amd version of clang, as below.
@@ -73,9 +73,11 @@
 from spack.package import PackageBase
 from spack.directives import depends_on, variant, conflicts
 
+import spack.variant
 
-class HipPackage(PackageBase):
+
-    """Auxiliary class which contains HIP variant, dependencies and conflicts
+class ROCmPackage(PackageBase):
+    """Auxiliary class which contains ROCm variant, dependencies and conflicts
     and is meant to unify and facilitate its usage. Closely mimics CudaPackage.
 
     Maintainers: dtaller
@@ -86,24 +88,26 @@ class HipPackage(PackageBase):
     amdgpu_targets = (
         'gfx701', 'gfx801', 'gfx802', 'gfx803',
         'gfx900', 'gfx906', 'gfx908', 'gfx1010',
-        'gfx1011', 'gfx1012', 'none'
+        'gfx1011', 'gfx1012'
     )
 
-    variant('hip', default=False, description='Enable HIP support')
+    variant('rocm', default=False, description='Enable ROCm support')
 
-    # possible amd gpu targets for hip builds
+    # possible amd gpu targets for rocm builds
-    variant('amdgpu_target', default='none', values=amdgpu_targets)
+    variant('amdgpu_target',
+            description='AMD GPU architecture',
+            values=spack.variant.any_combination_of(*amdgpu_targets))
 
-    depends_on('llvm-amdgpu', when='+hip')
+    depends_on('llvm-amdgpu', when='+rocm')
-    depends_on('hsa-rocr-dev', when='+hip')
+    depends_on('hsa-rocr-dev', when='+rocm')
-    depends_on('hip', when='+hip')
+    depends_on('hip', when='+rocm')
 
-    # need amd gpu type for hip builds
+    # need amd gpu type for rocm builds
-    conflicts('amdgpu_target=none', when='+hip')
+    conflicts('amdgpu_target=none', when='+rocm')
 
-    # Make sure non-'none' amdgpu_targets cannot be used without +hip
+    # Make sure amdgpu_targets cannot be used without +rocm
-    for value in amdgpu_targets[:-1]:
+    for value in amdgpu_targets:
-        conflicts('~hip', when='amdgpu_target=' + value)
+        conflicts('~rocm', when='amdgpu_target=' + value)
 
     # https://github.com/ROCm-Developer-Tools/HIP/blob/master/bin/hipcc
     # It seems that hip-clang does not (yet?) accept this flag, in which case
@@ -111,17 +115,8 @@ class HipPackage(PackageBase):
     # hip package file. But I will leave this here for future development.
     @staticmethod
     def hip_flags(amdgpu_target):
-        return '--amdgpu-target={0}'.format(amdgpu_target)
+        archs = ",".join(amdgpu_target)
-
+        return '--amdgpu-target={0}'.format(archs)
-    # https://llvm.org/docs/AMDGPUUsage.html
-    # Possible architectures (not including 'none' option)
-    @staticmethod
-    def amd_gputargets_list():
-        return (
-            'gfx701', 'gfx801', 'gfx802', 'gfx803',
-            'gfx900', 'gfx906', 'gfx908', 'gfx1010',
-            'gfx1011', 'gfx1012'
-        )
 
     # HIP version vs Architecture
 

lib/spack/spack/pkgkit.py

@@ -20,7 +20,7 @@
 from spack.build_systems.autotools import AutotoolsPackage
 from spack.build_systems.cmake import CMakePackage
 from spack.build_systems.cuda import CudaPackage
-from spack.build_systems.hip import HipPackage
+from spack.build_systems.rocm import ROCmPackage
 from spack.build_systems.qmake import QMakePackage
 from spack.build_systems.maven import MavenPackage
 from spack.build_systems.scons import SConsPackage

var/spack/repos/builtin/packages/camp/package.py

@@ -6,7 +6,7 @@
 from spack import *
 
 
-class Camp(CMakePackage, CudaPackage, HipPackage):
+class Camp(CMakePackage, CudaPackage, ROCmPackage):
     """
     Compiler agnostic metaprogramming library providing concepts,
     type operations and tuples for C++ and cuda
@@ -40,12 +40,17 @@ def cmake_args(self):
         else:
             options.append('-DENABLE_CUDA=OFF')
 
-        if '+hip' in spec:
+        if '+rocm' in spec:
-            arch = self.spec.variants['amdgpu_target'].value
             options.extend([
                 '-DENABLE_HIP=ON',
-                '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix),
+                '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix)
-                '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch)])
+            ])
+            archs = self.spec.variants['amdgpu_target'].value
+            if archs != 'none':
+                arch_str = ",".join(archs)
+                options.append(
+                    '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch_str)
+                )
         else:
             options.append('-DENABLE_HIP=OFF')
 

var/spack/repos/builtin/packages/chai/package.py

@@ -6,7 +6,7 @@
 from spack import *
 
 
-class Chai(CMakePackage, CudaPackage, HipPackage):
+class Chai(CMakePackage, CudaPackage, ROCmPackage):
     """
     Copy-hiding array interface for data migration between memory spaces
     """
@@ -36,12 +36,11 @@ class Chai(CMakePackage, CudaPackage, HipPackage):
     depends_on('umpire+cuda', when="+cuda")
     depends_on('raja+cuda', when="+raja+cuda")
 
-    # variants +hip and amdgpu_targets are not automatically passed to
+    # variants +rocm and amdgpu_targets are not automatically passed to
     # dependencies, so do it manually.
-    amdgpu_targets = HipPackage.amd_gputargets_list()
+    depends_on('umpire+rocm', when='+rocm')
-    depends_on('umpire+hip', when='+hip')
+    depends_on('raja+rocm', when="+raja+rocm")
-    depends_on('raja+hip', when="+raja+hip")
+    for val in ROCmPackage.amdgpu_targets:
-    for val in amdgpu_targets:
         depends_on('umpire amdgpu_target=%s' % val, when='amdgpu_target=%s' % val)
         depends_on('raja amdgpu_target=%s' % val, when='+raja amdgpu_target=%s' % val)
 
@@ -63,12 +62,17 @@ def cmake_args(self):
         else:
             options.append('-DENABLE_CUDA=OFF')
 
-        if '+hip' in spec:
+        if '+rocm' in spec:
-            arch = self.spec.variants['amdgpu_target'].value
             options.extend([
                 '-DENABLE_HIP=ON',
-                '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix),
+                '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix)
-                '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch)])
+            ])
+            archs = self.spec.variants['amdgpu_target'].value
+            if archs != 'none':
+                arch_str = ",".join(archs)
+                options.append(
+                    '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch_str)
+                )
         else:
             options.append('-DENABLE_HIP=OFF')
 

var/spack/repos/builtin/packages/hip/package.py

@@ -49,30 +49,6 @@ class Hip(CMakePackage):
     # See https://github.com/ROCm-Developer-Tools/HIP/pull/2141
     patch('0002-Fix-detection-of-HIP_CLANG_ROOT.patch', when='@3.5.0:')
 
-    def setup_run_environment(self, env):
-        # NOTE: DO NOT PUT LOGIC LIKE self.spec[name] in this function!!!!!
-        # It DOES NOT WORK FOR EXTERNAL PACKAGES!!!! See get_rocm_prefix_info
-        rocm_prefixes = self.get_rocm_prefix_info()
-
-        env.set('ROCM_PATH', rocm_prefixes['rocm-path'])
-        env.set('HIP_COMPILER', 'clang')
-        env.set('HIP_PLATFORM', 'hcc')
-        env.set('HIP_CLANG_PATH', rocm_prefixes['llvm-amdgpu'].bin)
-        env.set('HSA_PATH', rocm_prefixes['hsa-rocr-dev'])
-        env.set('ROCMINFO_PATH', rocm_prefixes['rocminfo'])
-        env.set('DEVICE_LIB_PATH', rocm_prefixes['rocm-device-libs'].lib)
-        env.set('HIP_PATH', rocm_prefixes['rocm-path'])
-        env.set('HIPCC_COMPILE_FLAGS_APPEND',
-                '--rocm-path={0}'.format(rocm_prefixes['rocm-path']))
-
-        if 'amdgpu_target' in self.spec.variants:
-            arch = self.spec.variants['amdgpu_target'].value
-            if arch != 'none':
-                env.set('HCC_AMDGPU_TARGET', arch)
-
-    def setup_dependent_run_environment(self, env, dependent_spec):
-        self.setup_run_environment(env)
-
     def get_rocm_prefix_info(self):
         # External packages in Spack do not currently contain dependency
         # information. External installations of hip therefore must compute
@@ -98,15 +74,18 @@ def get_rocm_prefix_info(self):
                 'hsa-rocr-dev': fallback_prefix.hsa,
                 'rocminfo': fallback_prefix.bin,
                 'rocm-device-libs': fallback_prefix,
+                'device_lib_path': fallback_prefix
             }
         else:
             mydict = dict((name, self.spec[name].prefix)
                           for name in ('llvm-amdgpu', 'hsa-rocr-dev',
                                        'rocminfo', 'rocm-device-libs'))
-            mydict['rocm-path'] = os.path.dirname(self.spec.prefix)
+            mydict['rocm-path'] = self.spec.prefix
+            device_lib_path = mydict['rocm-device-libs'].amdgcn.bitcode
+            mydict['device_lib_path'] = device_lib_path
             return mydict
 
-    def setup_dependent_build_environment(self, env, dependent_spec):
+    def set_variables(self, env):
         # Indirection for dependency paths because hip may be an external in
         # Spack. See block comment on get_rocm_prefix_info .
 
@@ -120,15 +99,24 @@ def setup_dependent_build_environment(self, env, dependent_spec):
         env.set('HIP_CLANG_PATH', rocm_prefixes['llvm-amdgpu'].bin)
         env.set('HSA_PATH', rocm_prefixes['hsa-rocr-dev'])
         env.set('ROCMINFO_PATH', rocm_prefixes['rocminfo'])
-        env.set('DEVICE_LIB_PATH', rocm_prefixes['rocm-device-libs'].lib)
+        env.set('DEVICE_LIB_PATH', rocm_prefixes['device_lib_path'])
         env.set('HIP_PATH', rocm_prefixes['rocm-path'])
         env.set('HIPCC_COMPILE_FLAGS_APPEND',
-                '--rocm-path={0}'.format(rocm_prefixes['rocm-path']))
+                '--rocm-path={0}'.format(rocm_prefixes['device_lib_path']))
+
+    def setup_run_environment(self, env):
+        self.set_variables(env)
+
+    def setup_dependent_build_environment(self, env, dependent_spec):
+        self.set_variables(env)
 
         if 'amdgpu_target' in dependent_spec.variants:
             arch = dependent_spec.variants['amdgpu_target'].value
             if arch != 'none':
-                env.set('HCC_AMDGPU_TARGET', arch)
+                env.set('HCC_AMDGPU_TARGET', ','.join(arch))
+
+    def setup_dependent_run_environment(self, env, dependent_spec):
+        self.setup_dependent_build_environment(env, dependent_spec)
 
     def setup_dependent_package(self, module, dependent_spec):
         self.spec.hipcc = join_path(self.prefix.bin, 'hipcc')

var/spack/repos/builtin/packages/raja/package.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 
-class Raja(CMakePackage, CudaPackage, HipPackage):
+class Raja(CMakePackage, CudaPackage, ROCmPackage):
     """RAJA Parallel Framework."""
 
     homepage = "http://software.llnl.gov/RAJA/"
@@ -33,7 +33,7 @@ class Raja(CMakePackage, CudaPackage, HipPackage):
     variant('examples', default=True, description='Build examples.')
     variant('exercises', default=True, description='Build exercises.')
 
-    conflicts('+openmp', when='+hip')
+    conflicts('+openmp', when='+rocm')
 
     depends_on('cmake@3.8:', type='build')
     depends_on('cmake@3.9:', when='+cuda', type='build')
@@ -56,12 +56,16 @@ def cmake_args(self):
         else:
             options.append('-DENABLE_CUDA=OFF')
 
-        if '+hip' in spec:
+        if '+rocm' in spec:
-            arch = self.spec.variants['amdgpu_target'].value
             options.extend([
                 '-DENABLE_HIP=ON',
-                '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix),
+                '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix)])
-                '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch)])
+            archs = self.spec.variants['amdgpu_target'].value
+            if archs != 'none':
+                arch_str = ",".join(archs)
+                options.append(
+                    '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch_str)
+                )
         else:
             options.append('-DENABLE_HIP=OFF')
 

var/spack/repos/builtin/packages/rocblas/package.py

@@ -20,9 +20,9 @@ class Rocblas(CMakePackage):
     version('3.7.0', sha256='9425db5f8e8b6f7fb172d09e2a360025b63a4e54414607709efc5acb28819642')
     version('3.5.0', sha256='8560fabef7f13e8d67da997de2295399f6ec595edfd77e452978c140d5f936f0')
 
-    amdgpu_targets = ('all', 'gfx803', 'gfx900', 'gfx906', 'gfx908')
+    tensile_architecture = ('all', 'gfx803', 'gfx900', 'gfx906', 'gfx908')
 
-    variant('amdgpu_target', default='all', multi=True, values=amdgpu_targets)
+    variant('tensile_architecture', default='all', values=tensile_architecture, multi=False)
 
     depends_on('cmake@3:', type='build')
 
@@ -73,7 +73,7 @@ def setup_build_environment(self, env):
         env.set('CXX', self.spec['hip'].hipcc)
 
     def cmake_args(self):
-        archs = ",".join(self.spec.variants['amdgpu_target'].value)
+        arch = self.spec.variants['tensile_architecture'].value
 
         tensile = join_path(self.stage.source_path, 'Tensile')
 
@@ -86,7 +86,7 @@ def cmake_args(self):
             '-DBUILD_WITH_TENSILE=ON',
             '-DTensile_TEST_LOCAL_PATH={0}'.format(tensile),
             '-DTensile_COMPILER=hipcc',
-            '-DTensile_ARCHITECTURE={0}'.format(archs),
+            '-DTensile_ARCHITECTURE={0}'.format(arch),
             '-DTensile_LOGIC=asm_full',
             '-DTensile_CODE_OBJECT_VERSION=V3',
             '-DBUILD_WITH_TENSILE_HOST={0}'.format(

var/spack/repos/builtin/packages/umpire/package.py

@@ -7,7 +7,7 @@
 import llnl.util.tty as tty
 
 
-class Umpire(CMakePackage, CudaPackage, HipPackage):
+class Umpire(CMakePackage, CudaPackage, ROCmPackage):
     """An application-focused API for memory management on NUMA & GPU
     architectures"""
 
@@ -62,11 +62,10 @@ class Umpire(CMakePackage, CudaPackage, HipPackage):
 
     depends_on('blt', type='build')
 
-    # variants +hip and amdgpu_targets are not automatically passed to
+    # variants +rocm and amdgpu_targets are not automatically passed to
     # dependencies, so do it manually.
-    depends_on('camp+hip', when='+hip')
+    depends_on('camp+rocm', when='+rocm')
-    amdgpu_targets = HipPackage.amd_gputargets_list()
+    for val in ROCmPackage.amdgpu_targets:
-    for val in amdgpu_targets:
         depends_on('camp amdgpu_target=%s' % val, when='amdgpu_target=%s' % val)
 
     depends_on('camp')
@@ -97,12 +96,17 @@ def cmake_args(self):
         else:
             options.append('-DENABLE_CUDA=Off')
 
-        if '+hip' in spec:
+        if '+rocm' in spec:
-            arch = self.spec.variants['amdgpu_target'].value
             options.extend([
                 '-DENABLE_HIP=ON',
-                '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix),
+                '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix)
-                '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch)])
+            ])
+            archs = self.spec.variants['amdgpu_target'].value
+            if archs != 'none':
+                arch_str = ",".join(archs)
+                options.append(
+                    '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch_str)
+                )
         else:
             options.append('-DENABLE_HIP=OFF')