[AMD][rocm-openmp-extras] Update for ROCm 4.3. (#25482)

- Added new checksums for 4.3.
- Now using llvm-amdgpu ~openmp in order to use the rocm-device-libs
  build as external project in llvm-amdgpu package. We still need
  to pull device-libs in using resource for the build as some headers
  are not installed.
- Updated symlink creation to now remove an existing link if  present
  to avoid issues on partial reinstalls when debugging.
- Adjusted the flang_warning to be a part of Cmake options instead of
  a filter_file for better compatability.
- The dependency on hsa-rocr-dev created some problems as type was changed
  to the default build/link. This issue was because ROCr uses libelf and
  the build of openmp expects elfutils. When link is specified libelf
  was being found in the path first, causing errors. This was
  introduced with the llvm-amdgpu external project build of device-libs.
- On a bare bone installation of sles15 it was noted that libquadmath0 was
  needed as a dependency. On 18.04 gcc-multilib was also needed.

* Workaround for libelf headers being used instead of elfutils.

var/spack/repos/builtin/packages/rocm-openmp-extras/package.py

@@ -19,7 +19,8 @@
     "808fca9bdefb109d5bcbbc9f5b59c564a6d422488869e986516f2a7233eda235",
     "aa75455cf1d333419e5310117678e5789c5222f7cb05b05e3dfacef855c55d84",
     "9e6ed2c7bdc3b4af069751b5d3e92913fd5ac318ae844f68bd78c5def990a8f7",
-    "c368d39ba9c1bc8b0edbe66edaa3f2a4ff5649c2bd16f499ac19dfd1591dec5a"
+    "c368d39ba9c1bc8b0edbe66edaa3f2a4ff5649c2bd16f499ac19dfd1591dec5a",
+    "c2b1a61a15fdf8d50c7c7a1ad75512f059c53a7bd5afe85f69e984f1174aa74a"
 ]
 
 devlib = [
@@ -27,7 +28,8 @@
     "bca9291385d6bdc91a8b39a46f0fd816157d38abb1725ff5222e6a0daa0834cc",
     "d0aa495f9b63f6d8cf8ac668f4dc61831d996e9ae3f15280052a37b9d7670d2a",
     "f5f5aa6bfbd83ff80a968fa332f80220256447c4ccb71c36f1fbd2b4a8e9fc1b",
-    "34a2ac39b9bb7cfa8175cbab05d30e7f3c06aaffce99eed5f79c616d0f910f5f"
+    "34a2ac39b9bb7cfa8175cbab05d30e7f3c06aaffce99eed5f79c616d0f910f5f",
+    "055a67e63da6491c84cd45865500043553fb33c44d538313dd87040a6f3826f2"
 ]
 
 llvm = [
@@ -35,7 +37,8 @@
     "8262aff88c1ff6c4deb4da5a4f8cda1bf90668950e2b911f93f73edaee53b370",
     "aa1f80f429fded465e86bcfaef72255da1af1c5c52d58a4c979bc2f6c2da5a69",
     "244e38d824fa7dfa8d0edf3c036b3c84e9c17a16791828e4b745a8d31eb374ae",
-    "751eca1d18595b565cfafa01c3cb43efb9107874865a60c80d6760ba83edb661"
+    "751eca1d18595b565cfafa01c3cb43efb9107874865a60c80d6760ba83edb661",
+    "1567d349cd3bcd2c217b3ecec2f70abccd5e9248bd2c3c9f21d4cdb44897fc87"
 ]
 
 flang = [
@@ -43,7 +46,8 @@
     "3990d39ff1c908b150f464f0653a123d94be30802f9cad6af18fbb560c4b412e",
     "f3e19699ce4ac404f41ffe08ef4546e31e2e741d8deb403b5477659e054275d5",
     "f41f661425534b5cfb20e2c0efd9d0800609dc3876ee9c3f76f026d36abbfa35",
-    "d6c3f3aaa289251a433d99d1cffe432812093089ae876a6863295a15066c1eaf"
+    "d6c3f3aaa289251a433d99d1cffe432812093089ae876a6863295a15066c1eaf",
+    "13d3525078fd1c569f7c8ea7fce439b04f6b03814bbe88600c08f95c788e7802"
 ]
 
 extras = [
@@ -51,10 +55,11 @@
     "5d98d34aff97416d8b5b9e16e7cf474580f8de8a73bd0e549c4440a3c5df4ef5",
     "51cc8a7c5943e1d9bc657fc9b9797f45e3ce6a4e544d3d3a967c7cd0185a0510",
     "91fdfadb94aa6afc1942124d0953ddc80c297fa75de1897fb42ac8e7dea51ab9",
-    "31bbe70b51c259a54370d021ae63528a1740b5477a22412685afd14150fff6f4"
+    "31bbe70b51c259a54370d021ae63528a1740b5477a22412685afd14150fff6f4",
+    "ec6cc4a9c24f098496de3206714dafe9a714f06afacfe21d53a4e6344f9cb4c9"
 ]
 
-versions = ['3.9.0', '3.10.0', '4.0.0', '4.1.0', '4.2.0']
+versions = ['3.9.0', '3.10.0', '4.0.0', '4.1.0', '4.2.0', '4.3.0']
 versions_dict = dict()
 components = ['aomp', 'devlib', 'llvm', 'flang', 'extras']
 component_hashes = [aomp, devlib, llvm, flang, extras]
@@ -70,9 +75,10 @@ class RocmOpenmpExtras(Package):
     """OpenMP support for ROCm LLVM."""
 
     homepage = tools_url + "/aomp"
-    url = tools_url + "/aomp/archive/rocm-4.2.0.tar.gz"
+    url = tools_url + "/aomp/archive/rocm-4.3.0.tar.gz"
 
     maintainers = ['srekolam', 'arjun-raj-kuppala', 'estewart08']
+    version('4.3.0', sha256=versions_dict['4.3.0']['aomp'])
     version('4.2.0', sha256=versions_dict['4.2.0']['aomp'])
     version('4.1.0', sha256=versions_dict['4.1.0']['aomp'])
     version('4.0.0', sha256=versions_dict['4.0.0']['aomp'])
@@ -88,13 +94,11 @@ class RocmOpenmpExtras(Package):
     depends_on('elfutils', type=('build', 'link'))
     depends_on('libffi', type=('build', 'link'))
 
-    for ver in ['3.9.0', '3.10.0', '4.0.0', '4.1.0', '4.2.0']:
+    for ver in ['3.9.0', '3.10.0', '4.0.0', '4.1.0', '4.2.0', '4.3.0']:
         depends_on('hsakmt-roct@' + ver, when='@' + ver)
         depends_on('comgr@' + ver, when='@' + ver)
         depends_on('hsa-rocr-dev@' + ver, when='@' + ver)
-        # standalone rocm-device-libs
+        depends_on('llvm-amdgpu@{0} ~openmp'.format(ver),
-        depends_on('rocm-device-libs@' + ver, when='@' + ver)
-        depends_on('llvm-amdgpu@{0} ~rocm-device-libs ~openmp'.format(ver),
                    when='@' + ver)
 
         # tag changed to 'rocm-' in 4.0.0
@@ -141,7 +145,7 @@ class RocmOpenmpExtras(Package):
             when='@' + ver)
 
     def setup_run_environment(self, env):
-        devlibs_prefix = self.spec['rocm-device-libs'].prefix
+        devlibs_prefix = self.spec['llvm-amdgpu'].prefix
         openmp_extras_prefix = self.spec['rocm-openmp-extras'].prefix
         llvm_prefix = self.spec['llvm-amdgpu'].prefix
         env.set('AOMP', '{0}'.format(llvm_prefix))
@@ -169,22 +173,22 @@ def setup_build_environment(self, env):
 
     def patch(self):
         src = self.stage.source_path
-        flang_warning = '-Wno-incompatible-pointer-types-discards-qualifiers)'
         aomp_extras = '{0}/rocm-openmp-extras/aomp-extras/aomp-device-libs'
         libomptarget = \
             '{0}/rocm-openmp-extras/llvm-project/openmp/libomptarget'
         flang = '{0}/rocm-openmp-extras/flang/'
 
+        # If not in a git repo the STRIP command will have an empty
+        # argument. This is fixed in later versions.
+        if self.spec.version == Version('4.3.0'):
+            filter_file('STRIP ${FLANG_SHA}', 'STRIP 0',
+                        flang.format(src) + 'CMakeLists.txt', string=True)
+
         if self.spec.version < Version('4.1.0'):
             plugin = '/plugins/hsa/CMakeLists.txt'
         else:
-            # Spack thinks some warnings from the flang build are errors.
-            # Disable those warnings.
-            filter_file('PRIVATE -fPIC)',
-                        'PRIVATE -fPIC PRIVATE ' + flang_warning,
-                        flang.format(src) + 'runtime/flang/CMakeLists.txt',
-                        string=True)
             plugin = '/plugins/amdgpu/CMakeLists.txt'
+
         filter_file(
             '{ROCM_DIR}/amdgcn/bitcode', '{DEVICE_LIBS_DIR}',
             aomp_extras.format(src) + '/aompextras/CMakeLists.txt',
@@ -257,7 +261,7 @@ def install(self, spec, prefix):
         src = self.stage.source_path
         gfx_list = "gfx700;gfx701;gfx801;gfx803;gfx900;gfx902;gfx906;gfx908"
         openmp_extras_prefix = self.spec['rocm-openmp-extras'].prefix
-        devlibs_prefix = self.spec['rocm-device-libs'].prefix
+        devlibs_prefix = self.spec['llvm-amdgpu'].prefix
         devlibs_src = '{0}/rocm-openmp-extras/rocm-device-libs'.format(src)
         hsa_prefix = self.spec['hsa-rocr-dev'].prefix
         hsakmt_prefix = self.spec['hsakmt-roct'].prefix
@@ -268,22 +272,30 @@ def install(self, spec, prefix):
         omp_lib_dir = '{0}/lib'.format(openmp_extras_prefix)
         bin_dir = '{0}/bin'.format(llvm_prefix)
         lib_dir = '{0}/lib'.format(llvm_prefix)
+        flang_warning = '-Wno-incompatible-pointer-types-discards-qualifiers'
+        libpgmath = '/rocm-openmp-extras/flang/runtime/libpgmath/lib/common'
+        elfutils_inc = spec['elfutils'].prefix.include
 
         # flang1 and flang2 symlink needed for build of flang-runtime
         # libdevice symlink to rocm-openmp-extras for runtime
         # libdebug symlink to rocm-openmp-extras for runtime
-        if not (os.path.islink((os.path.join(bin_dir, 'flang1')))):
+        if (os.path.islink((os.path.join(bin_dir, 'flang1')))):
-            os.symlink(os.path.join(omp_bin_dir, 'flang1'),
+            os.unlink(os.path.join(bin_dir, 'flang1'))
-                       os.path.join(bin_dir, 'flang1'))
+        if (os.path.islink((os.path.join(bin_dir, 'flang2')))):
-        if not (os.path.islink((os.path.join(bin_dir, 'flang2')))):
+            os.unlink(os.path.join(bin_dir, 'flang2'))
-            os.symlink(os.path.join(omp_bin_dir, 'flang2'),
+        if (os.path.islink((os.path.join(lib_dir, 'libdevice')))):
-                       os.path.join(bin_dir, 'flang2'))
+            os.unlink(os.path.join(lib_dir, 'libdevice'))
-        if not (os.path.islink((os.path.join(lib_dir, 'libdevice')))):
+        if (os.path.islink((os.path.join(llvm_prefix, 'lib-debug')))):
-            os.symlink(os.path.join(omp_lib_dir, 'libdevice'),
+            os.unlink(os.path.join(llvm_prefix, 'lib-debug'))
-                       os.path.join(lib_dir, 'libdevice'))
+
-        if not (os.path.islink((os.path.join(llvm_prefix, 'lib-debug')))):
+        os.symlink(os.path.join(omp_bin_dir, 'flang1'),
-            os.symlink(os.path.join(openmp_extras_prefix, 'lib-debug'),
+                   os.path.join(bin_dir, 'flang1'))
-                       os.path.join(llvm_prefix, 'lib-debug'))
+        os.symlink(os.path.join(omp_bin_dir, 'flang2'),
+                   os.path.join(bin_dir, 'flang2'))
+        os.symlink(os.path.join(omp_lib_dir, 'libdevice'),
+                   os.path.join(lib_dir, 'libdevice'))
+        os.symlink(os.path.join(openmp_extras_prefix, 'lib-debug'),
+                   os.path.join(llvm_prefix, 'lib-debug'))
 
         # Set cmake args
         components = dict()
@@ -299,6 +311,11 @@ def install(self, spec, prefix):
         ]
 
         # Shared cmake configuration for openmp, openmp-debug
+        # Due to hsa-rocr-dev using libelf instead of elfutils
+        # the build of openmp fails because the include path
+        # for libelf is placed before elfutils in SPACK_INCLUDE_DIRS.
+        # Passing the elfutils include path via cmake options is a
+        # workaround until hsa-rocr-dev switches to elfutils.
         openmp_common_args = [
             '-DROCM_DIR={0}'.format(hsa_prefix),
             '-DDEVICE_LIBS_DIR={0}/amdgcn/bitcode'.format(devlibs_prefix),
@@ -316,7 +333,9 @@ def install(self, spec, prefix):
             '-DOPENMP_ENABLE_LIBOMPTARGET=1',
             '-DOPENMP_ENABLE_LIBOMPTARGET_HSA=1',
             '-DLLVM_MAIN_INCLUDE_DIR={0}{1}'.format(src, llvm_inc),
-            '-DLLVM_INSTALL_PREFIX={0}'.format(llvm_prefix)
+            '-DLLVM_INSTALL_PREFIX={0}'.format(llvm_prefix),
+            '-DCMAKE_C_FLAGS=-isystem{0}'.format(elfutils_inc),
+            '-DCMAKE_CXX_FLAGS=-isystem{0}'.format(elfutils_inc)
         ]
 
         if self.spec.version < Version('4.1.0'):
@@ -349,6 +368,15 @@ def install(self, spec, prefix):
             '-DCMAKE_Fortran_COMPILER={0}/flang'.format(bin_dir),
             '-DLLVM_TARGETS_TO_BUILD=AMDGPU;x86'
         ]
+        if self.spec.version >= Version('4.2.0'):
+            # Spack thinks some warnings from the flang build are errors.
+            # Disable those warnings in C and CXX flags.
+            flang_common_args += [
+                '-DCMAKE_CXX_FLAGS={0}'.format(flang_warning) +
+                ' -I{0}{1}'.format(src, libpgmath),
+                '-DCMAKE_C_FLAGS={0}'.format(flang_warning) +
+                ' -I{0}{1}'.format(src, libpgmath)
+            ]
 
         components['pgmath'] = [
             '../rocm-openmp-extras/flang/runtime/libpgmath'