From aa55b196809072456f6b2e185543dcaf200ac1da Mon Sep 17 00:00:00 2001 From: afzpatel <122491982+afzpatel@users.noreply.github.com> Date: Mon, 3 Mar 2025 20:57:34 -0500 Subject: [PATCH] fix +asan in ROCm packages (#48745) * fix asan for hsa-rocr-dev * add libclang_rt.asan-x86_64.so to LD_LIBRARY_PATH * fix +asan for hipsparselt * fix rocm-openmp-extras asan and add rccl +asan support * add missing comgr build env variables * add missing rocm-smi-lib build env variables * minor dependency change * fix style --- var/spack/repos/builtin/packages/comgr/package.py | 9 +++++++++ .../repos/builtin/packages/hipsparselt/package.py | 2 ++ .../builtin/packages/hsa-rocr-dev/package.py | 11 +++++++++++ .../repos/builtin/packages/llvm-amdgpu/package.py | 5 +++++ var/spack/repos/builtin/packages/rccl/package.py | 6 ++++++ .../packages/rocm-openmp-extras/package.py | 15 +++++++-------- .../builtin/packages/rocm-smi-lib/package.py | 12 ++++++++++++ 7 files changed, 52 insertions(+), 8 deletions(-) diff --git a/var/spack/repos/builtin/packages/comgr/package.py b/var/spack/repos/builtin/packages/comgr/package.py index df11eb0cdb1..464a30dec28 100644 --- a/var/spack/repos/builtin/packages/comgr/package.py +++ b/var/spack/repos/builtin/packages/comgr/package.py @@ -136,6 +136,15 @@ def cmake_args(self): args.append(self.define_from_variant("ADDRESS_SANITIZER", "asan")) return args + def setup_build_environment(self, env): + if self.spec.satisfies("@5.7: +asan"): + env.set("CC", f"{self.spec['llvm-amdgpu'].prefix}/bin/clang") + env.set("CXX", f"{self.spec['llvm-amdgpu'].prefix}/bin/clang++") + env.set("ASAN_OPTIONS", "detect_leaks=0") + env.set("CFLAGS", "-fsanitize=address -shared-libasan") + env.set("CXXFLAGS", "-fsanitize=address -shared-libasan") + env.set("LDFLAGS", "-fuse-ld=lld") + @classmethod def determine_version(cls, lib): match = re.search(r"lib\S*\.so\.\d+\.\d+\.(\d)(\d\d)(\d\d)", lib) diff --git a/var/spack/repos/builtin/packages/hipsparselt/package.py b/var/spack/repos/builtin/packages/hipsparselt/package.py index 1bc6cf5a3bb..f39e6fe8f1f 100644 --- a/var/spack/repos/builtin/packages/hipsparselt/package.py +++ b/var/spack/repos/builtin/packages/hipsparselt/package.py @@ -90,6 +90,8 @@ class Hipsparselt(CMakePackage, ROCmPackage): def setup_build_environment(self, env): env.set("CXX", self.spec["hip"].hipcc) + if self.spec.satisfies("+asan"): + env.set("CC", f"{self.spec['llvm-amdgpu'].prefix}/bin/clang") env.set("TENSILE_ROCM_ASSEMBLER_PATH", f"{self.spec['llvm-amdgpu'].prefix}/bin/clang++") env.set( "TENSILE_ROCM_OFFLOAD_BUNDLER_PATH", diff --git a/var/spack/repos/builtin/packages/hsa-rocr-dev/package.py b/var/spack/repos/builtin/packages/hsa-rocr-dev/package.py index 8c08496b5fa..00978026516 100644 --- a/var/spack/repos/builtin/packages/hsa-rocr-dev/package.py +++ b/var/spack/repos/builtin/packages/hsa-rocr-dev/package.py @@ -155,6 +155,17 @@ def determine_version(cls, lib): ver = None return ver + def setup_build_environment(self, env): + if self.spec.satisfies("@5.7: +asan"): + numa_inc = self.spec["numactl"].prefix.include + numa_lib = self.spec["numactl"].prefix.lib + env.set("CC", f"{self.spec['llvm-amdgpu'].prefix}/bin/clang") + env.set("CXX", f"{self.spec['llvm-amdgpu'].prefix}/bin/clang++") + env.set("ASAN_OPTIONS", "detect_leaks=0") + env.set("CFLAGS", f"-fsanitize=address -shared-libasan -I{numa_inc} -L{numa_lib}") + env.set("CXXFLAGS", f"-fsanitize=address -shared-libasan -I{numa_inc} -L{numa_lib}") + env.set("LDFLAGS", "-fuse-ld=lld") + def cmake_args(self): spec = self.spec diff --git a/var/spack/repos/builtin/packages/llvm-amdgpu/package.py b/var/spack/repos/builtin/packages/llvm-amdgpu/package.py index 1b887702295..7f05da37a34 100644 --- a/var/spack/repos/builtin/packages/llvm-amdgpu/package.py +++ b/var/spack/repos/builtin/packages/llvm-amdgpu/package.py @@ -315,6 +315,11 @@ def setup_run_environment(self, env): def setup_dependent_run_environment(self, env, dependent_spec): llvm_amdgpu_home = self.spec["llvm-amdgpu"].prefix env.prepend_path("LD_LIBRARY_PATH", llvm_amdgpu_home + "/lib") + # Required for enabling asan on dependent packages + for root, _, files in os.walk(self.spec["llvm-amdgpu"].prefix): + if "libclang_rt.asan-x86_64.so" in files: + env.prepend_path("LD_LIBRARY_PATH", root) + env.prune_duplicate_paths("LD_LIBRARY_PATH") @run_after("install") def post_install(self): diff --git a/var/spack/repos/builtin/packages/rccl/package.py b/var/spack/repos/builtin/packages/rccl/package.py index e39ddca453f..0ff63ed5a75 100644 --- a/var/spack/repos/builtin/packages/rccl/package.py +++ b/var/spack/repos/builtin/packages/rccl/package.py @@ -69,6 +69,7 @@ class Rccl(CMakePackage): values=auto_or_any_combination_of(*amdgpu_targets), sticky=True, ) + variant("asan", default=False, description="Build with address-sanitizer enabled or disabled") patch("0003-Fix-numactl-rocm-smi-path-issue.patch", when="@5.2.3:5.6") patch("0004-Set-rocm-core-path-for-version-file.patch", when="@6.0:6.2") @@ -143,6 +144,11 @@ def determine_version(cls, lib): def setup_build_environment(self, env): env.set("CXX", self.spec["hip"].hipcc) env.set("ROCMCORE_PATH", self.spec["rocm-core"].prefix) + if self.spec.satisfies("+asan"): + env.set("ASAN_OPTIONS", "detect_leaks=0") + env.set("CFLAGS", "-fsanitize=address -shared-libasan") + env.set("CXXFLAGS", "-fsanitize=address -shared-libasan") + env.set("LDFLAGS", "-fuse-ld=lld") def cmake_args(self): args = [ diff --git a/var/spack/repos/builtin/packages/rocm-openmp-extras/package.py b/var/spack/repos/builtin/packages/rocm-openmp-extras/package.py index 8c0f7d49ed2..4798a1765cf 100644 --- a/var/spack/repos/builtin/packages/rocm-openmp-extras/package.py +++ b/var/spack/repos/builtin/packages/rocm-openmp-extras/package.py @@ -364,14 +364,6 @@ def setup_build_environment(self, env): env.set("FC", "{0}/bin/flang".format(openmp_extras_prefix)) if self.spec.satisfies("@6.1:"): env.prepend_path("LD_LIBRARY_PATH", self.spec["hsa-rocr-dev"].prefix.lib) - if self.spec.satisfies("+asan"): - env.set("SANITIZER", 1) - env.set("VERBOSE", 1) - env.set( - "LDSHARED", - self.spec["llvm-amdgpu"].prefix.bin.clang - + " -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fwrapv -O2", - ) gfx_list = "gfx700 gfx701 gfx801 gfx803 gfx900 gfx902 gfx906 gfx908" if self.spec.version >= Version("4.3.1"): @@ -611,6 +603,13 @@ def install(self, spec, prefix): "-DHSAKMT_LIB={0}/lib".format(hsakmt_prefix), "-DHSAKMT_LIB64={0}/lib64".format(hsakmt_prefix), ] + if self.spec.satisfies("+asan"): + openmp_common_args += [ + "-DASAN_OPTIONS=detect_leaks=0", + "-DCMAKE_C_FLAGS=-fsanitize=address -shared-libasan", + "-DCMAKE_CXX_FLAGS=-fsanitize=address -shared-libasan", + "-DCMAKE_LD_FLAGS=-fuse-ld=lld", + ] components["openmp"] = ["../rocm-openmp-extras/llvm-project/openmp"] components["openmp"] += openmp_common_args diff --git a/var/spack/repos/builtin/packages/rocm-smi-lib/package.py b/var/spack/repos/builtin/packages/rocm-smi-lib/package.py index ac3f16bb74e..e9ec49988f6 100644 --- a/var/spack/repos/builtin/packages/rocm-smi-lib/package.py +++ b/var/spack/repos/builtin/packages/rocm-smi-lib/package.py @@ -76,6 +76,9 @@ class RocmSmiLib(CMakePackage): ]: depends_on(f"rocm-core@{ver}", when=f"@{ver}") + for ver in ["6.1.0", "6.1.1", "6.1.2", "6.2.0", "6.2.1", "6.2.4", "6.3.0", "6.3.1", "6.3.2"]: + depends_on("llvm-amdgpu", when=f"@{ver}+asan") + patch( "https://github.com/ROCm/rocm_smi_lib/commit/11f12b86517d0e9868f4d16d74d4e8504c3ba7da.patch?full_index=1", sha256="62be7262f6e1e71bf82a03f500a424a536638f04e913d0f4b477f60e8e1190fd", @@ -94,6 +97,15 @@ def cmake_args(self): args.append(self.define_from_variant("ADDRESS_SANITIZER", "asan")) return args + def setup_build_environment(self, env): + if self.spec.satisfies("@6.1: +asan"): + env.set("CC", f"{self.spec['llvm-amdgpu'].prefix}/bin/clang") + env.set("CXX", f"{self.spec['llvm-amdgpu'].prefix}/bin/clang++") + env.set("ASAN_OPTIONS", "detect_leaks=0") + env.set("CFLAGS", "-fsanitize=address -shared-libasan") + env.set("CXXFLAGS", "-fsanitize=address -shared-libasan") + env.set("LDFLAGS", "-fuse-ld=lld") + @classmethod def determine_version(cls, lib): match = re.search(r"lib\S*\.so\.\d+\.\d+\.(\d)(\d\d)(\d\d)", lib)