Run config audits in CI, add a new audit to detect wrongly named external specs (#42289)

2024-01-26 10:21:43 +01:00 · 2024-01-26 10:21:43 +01:00 · e77128dfa2
commit e77128dfa2
parent 19df8e45ec
12 changed files with 50 additions and 20 deletions
--- a/lib/spack/spack/audit.py
+++ b/lib/spack/spack/audit.py
@ -244,7 +244,7 @@ def _search_duplicate_specs_in_externals(error_cls):
                + lines
                + ["as they might result in non-deterministic hashes"]
            )
-        except TypeError:
+        except (TypeError, AttributeError):
            details = []

        errors.append(error_cls(summary=error_msg, details=details))
@ -292,12 +292,6 @@ def _avoid_mismatched_variants(error_cls):
    errors = []
    packages_yaml = spack.config.CONFIG.get_config("packages")

-    def make_error(config_data, summary):
-        s = io.StringIO()
-        s.write("Occurring in the following file:\n")
-        syaml.dump_config(config_data, stream=s, blame=True)
-        return error_cls(summary=summary, details=[s.getvalue()])
-
    for pkg_name in packages_yaml:
        # 'all:' must be more forgiving, since it is setting defaults for everything
        if pkg_name == "all" or "variants" not in packages_yaml[pkg_name]:
@ -317,7 +311,7 @@ def make_error(config_data, summary):
                        f"Setting a preference for the '{pkg_name}' package to the "
                        f"non-existing variant '{variant.name}'"
                    )
-                    errors.append(make_error(preferences, summary))
+                    errors.append(_make_config_error(preferences, summary, error_cls=error_cls))
                    continue

                # Variant cannot accept this value
@ -329,11 +323,41 @@ def make_error(config_data, summary):
                        f"Setting the variant '{variant.name}' of the '{pkg_name}' package "
                        f"to the invalid value '{str(variant)}'"
                    )
-                    errors.append(make_error(preferences, summary))
+                    errors.append(_make_config_error(preferences, summary, error_cls=error_cls))

    return errors


+@config_packages
+def _wrongly_named_spec(error_cls):
+    """Warns if the wrong name is used for an external spec"""
+    errors = []
+    packages_yaml = spack.config.CONFIG.get_config("packages")
+    for pkg_name in packages_yaml:
+        if pkg_name == "all":
+            continue
+
+        externals = packages_yaml[pkg_name].get("externals", [])
+        is_virtual = spack.repo.PATH.is_virtual(pkg_name)
+        for entry in externals:
+            spec = spack.spec.Spec(entry["spec"])
+            regular_pkg_is_wrong = not is_virtual and pkg_name != spec.name
+            virtual_pkg_is_wrong = is_virtual and not any(
+                p.name == spec.name for p in spack.repo.PATH.providers_for(pkg_name)
+            )
+            if regular_pkg_is_wrong or virtual_pkg_is_wrong:
+                summary = f"Wrong external spec detected for '{pkg_name}': {spec}"
+                errors.append(_make_config_error(entry, summary, error_cls=error_cls))
+    return errors
+
+
+def _make_config_error(config_data, summary, error_cls):
+    s = io.StringIO()
+    s.write("Occurring in the following file:\n")
+    syaml.dump_config(config_data, stream=s, blame=True)
+    return error_cls(summary=summary, details=[s.getvalue()])
+
+
 #: Sanity checks on package directives
 package_directives = AuditClass(
    group="packages",
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@ -154,6 +154,12 @@ default:
      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
      ${CI_STACK_CONFIG_SCOPES}
      compiler find
+    - spack
+      --config-scope "${SPACK_CI_CONFIG_ROOT}"
+      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
+      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
+      ${CI_STACK_CONFIG_SCOPES}
+      audit configs
    - spack python -c "import os,sys; print(os.path.expandvars(sys.stdin.read()))"
      < "${SPACK_CI_CONFIG_ROOT}/${PIPELINE_MIRROR_TEMPLATE}" > "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
    - spack config add -f "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml
@ -21,7 +21,7 @@ spack:
      version:
        - 1.8.20
    elfutils:
-      variants: +bzip2 ~nls +xz
+      variants: ~nls
    hdf5:
      variants: +fortran +hl +shared
    libfabric:
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml
@ -21,7 +21,7 @@ spack:
      version:
        - 1.8.20
    elfutils:
-      variants: +bzip2 ~nls +xz
+      variants: ~nls
    hdf5:
      variants: +fortran +hl +shared
    libfabric:
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
@ -31,7 +31,7 @@ spack:
    cuda:
      version: [11.7.0]
    elfutils:
-      variants: +bzip2 ~nls +xz
+      variants: ~nls
      require: "%gcc"
    gcc-runtime:
      require: "%gcc"
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
@ -29,7 +29,7 @@ spack:
    cuda:
      version: [11.7.0]
    elfutils:
-      variants: +bzip2 ~nls +xz
+      variants: ~nls
      require: "%gcc"
    hdf5:
      variants: +fortran +hl +shared
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse-v2/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse-v2/spack.yaml
@ -15,7 +15,7 @@ spack:
    binutils:
      variants: +ld +gold +headers +libiberty ~nls
    elfutils:
-      variants: +bzip2 ~nls +xz
+      variants: ~nls
    hdf5:
      variants: +fortran +hl +shared
    libfabric:
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml
@ -15,7 +15,7 @@ spack:
    binutils:
      variants: +ld +gold +headers +libiberty ~nls
    elfutils:
-      variants: +bzip2 ~nls +xz
+      variants: ~nls
    hdf5:
      variants: +fortran +hl +shared
    libfabric:
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
@ -17,7 +17,7 @@ spack:
        tbb: [intel-tbb]
      variants: +mpi
    elfutils:
-      variants: +bzip2 ~nls +xz
+      variants: ~nls
    gcc-runtime:
      require: "%gcc"
    hdf5:
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
@ -16,7 +16,7 @@ spack:
    binutils:
      variants: +ld +gold +headers +libiberty ~nls
    elfutils:
-      variants: +bzip2 ~nls +xz
+      variants: ~nls
    hdf5:
      variants: +fortran +hl +shared
    libfabric:
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml
@ -15,7 +15,7 @@ spack:
    binutils:
      variants: +ld +gold +headers +libiberty ~nls
    elfutils:
-      variants: +bzip2 ~nls +xz
+      variants: ~nls
    hdf5:
      variants: +fortran +hl +shared
    libfabric:
@ -91,7 +91,7 @@ spack:
    miopen-hip:
      buildable: false
      externals:
-      - spec: hip-rocclr@5.4.3
+      - spec: miopen-hip@5.4.3
        prefix: /opt/rocm-5.4.3/
    miopengemm:
      buildable: false
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
@ -15,7 +15,7 @@ spack:
    binutils:
      variants: +ld +gold +headers +libiberty ~nls
    elfutils:
-      variants: +bzip2 ~nls +xz
+      variants: ~nls
    hdf5:
      variants: +fortran +hl +shared
    libfabric: