From 22deed708e8168a9fafee27380774e13273ec1b7 Mon Sep 17 00:00:00 2001
From: "Adam J. Stewart" <ajstewart426@gmail.com>
Date: Wed, 30 Aug 2023 18:15:44 -0500
Subject: [PATCH 01/10] py-llvmlite: add Python version requirements (#39711)

---
 .../builtin/packages/py-llvmlite/package.py   | 31 +++++++++----------
 1 file changed, 15 insertions(+), 16 deletions(-)

diff --git a/var/spack/repos/builtin/packages/py-llvmlite/package.py b/var/spack/repos/builtin/packages/py-llvmlite/package.py
index b4a5ad65736..a6f7cd8a4db 100644
--- a/var/spack/repos/builtin/packages/py-llvmlite/package.py
+++ b/var/spack/repos/builtin/packages/py-llvmlite/package.py
@@ -26,19 +26,18 @@ class PyLlvmlite(PythonPackage):
     version("0.27.1", sha256="48a1c3ae69fd8920cba153bfed8a46ac46474bc706a2100226df4abffe0000ab")
     version("0.26.0", sha256="13e84fe6ebb0667233074b429fd44955f309dead3161ec89d9169145dbad2ebf")
     version("0.25.0", sha256="fd64def9a51dd7dc61913a7a08eeba5b9785522740bec5a7c5995b2a90525025")
-    version("0.23.0", sha256="bc8b1b46274d05b578fe9e980a6d98fa71c8727f6f9ed31d4d8468dce7aa5762")
 
     depends_on("py-setuptools", type="build")
-    depends_on("python@3.8:3.11", type=("build", "run"), when="@0.40.0:")
-    depends_on("python@3.7:3.10", type=("build", "run"), when="@0.38.0:0.39")
-    depends_on("python@3.7:3.9", type=("build", "run"), when="@0.37")
-    depends_on("python@3.6:", type=("build", "run"), when="@0.33:")
-    depends_on("python@2.6:2.8,3.4:", type=("build", "run"))
+    depends_on("python@3.8:3.11", when="@0.40:", type=("build", "run"))
+    depends_on("python@:3.10", when="@0.38:0.39", type=("build", "run"))
+    depends_on("python@:3.9", when="@0.36:0.37", type=("build", "run"))
+    depends_on("python@:3.8", when="@0.31:0.35", type=("build", "run"))
+    depends_on("python@:3.7", when="@:0.30", type=("build", "run"))
 
-    # llvmlite compatibility information taken from https://github.com/numba/llvmlite#compatibility
-    depends_on("llvm@14:~flang", when="@0.41:")
-    depends_on("llvm@11:14~flang", when="@0.40")
-    depends_on("llvm@11~flang", when="@0.37.0:0.39")
+    # https://github.com/numba/llvmlite#compatibility
+    depends_on("llvm@14", when="@0.41:")
+    depends_on("llvm@11:14", when="@0.40")
+    depends_on("llvm@11", when="@0.37:0.39")
     for t in [
         "arm:",
         "ppc:",
@@ -50,12 +49,12 @@ class PyLlvmlite(PythonPackage):
         "x86:",
         "x86_64:",
     ]:
-        depends_on("llvm@10.0.0:10.0~flang", when="@0.34.0:0.36 target={0}".format(t))
-    depends_on("llvm@9.0.0:9.0~flang", when="@0.34.0:0.36 target=aarch64:")
-    depends_on("llvm@9.0.0:9.0~flang", when="@0.33.0:0.33")
-    depends_on("llvm@7.0.0:8.0~flang", when="@0.29.0:0.32")
-    depends_on("llvm@7.0.0:7.0~flang", when="@0.27.0:0.28")
-    depends_on("llvm@6.0.0:6.0~flang", when="@0.23.0:0.26")
+        depends_on("llvm@10.0", when=f"@0.34:0.36 target={t}")
+    depends_on("llvm@9.0", when="@0.34:0.36 target=aarch64:")
+    depends_on("llvm@9.0", when="@0.33")
+    depends_on("llvm@7.0:7.1,8.0", when="@0.29:0.32")
+    depends_on("llvm@7.0", when="@0.27:0.28")
+    depends_on("llvm@6.0", when="@0.23:0.26")
     depends_on("binutils", type="build")
 
     def setup_build_environment(self, env):

From 1ee7c735ec98c3a810d0280a33bb05ca6bc79987 Mon Sep 17 00:00:00 2001
From: "John W. Parent" <45471568+johnwparent@users.noreply.github.com>
Date: Wed, 30 Aug 2023 19:19:38 -0400
Subject: [PATCH 02/10] Windows: oneapi/msvc consistency (#39180)

Currently, OneAPI's setvars scripts effectively disregard any arguments
we're passing to the MSVC vcvars env setup script, and additionally,
completely ignore the requested version of OneAPI, defaulting to whatever
the latest installed on the system is.

This leads to a scenario where we have improperly constructed Windows
native development environments, with potentially multiple versions of
MSVC and OneAPI being loaded or called in the same env. Obviously this is
far from ideal and leads to some fairly inscrutable errors such as
overlapping header files between MSVC and OneAPI and a different version
of OneAPI being called than the env was setup for.

This PR solves this issue by creating a structured invocation of each
relevant script in an order that ensures the correct values are set in
the resultant build env.

The order needs to be:

1. MSVC vcvarsall
2. The compiler specific env.bat script for the relevant version of
   the oneapi compiler we're looking for. The root setvars scripts seems
   to respect this as well, although it is less explicit
3. The root oneapi setvars script, which sets up everything else the
   oneapi env needs and seems to respect previous env invocations.
---
 lib/spack/spack/compilers/msvc.py | 181 ++++++++++++++++++++++++------
 1 file changed, 145 insertions(+), 36 deletions(-)

diff --git a/lib/spack/spack/compilers/msvc.py b/lib/spack/spack/compilers/msvc.py
index c9efa264eea..7e91e8eca2e 100644
--- a/lib/spack/spack/compilers/msvc.py
+++ b/lib/spack/spack/compilers/msvc.py
@@ -29,6 +29,90 @@
 }
 
 
+class CmdCall:
+    """Compose a call to `cmd` for an ordered series of cmd commands/scripts"""
+
+    def __init__(self, *cmds):
+        if not cmds:
+            raise RuntimeError(
+                """Attempting to run commands from CMD without specifying commands.
+                Please add commands to be run."""
+            )
+        self._cmds = cmds
+
+    def __call__(self):
+        out = subprocess.check_output(self.cmd_line, stderr=subprocess.STDOUT)  # novermin
+        return out.decode("utf-16le", errors="replace")  # novermin
+
+    @property
+    def cmd_line(self):
+        base_call = "cmd /u /c "
+        commands = " && ".join([x.command_str() for x in self._cmds])
+        # If multiple commands are being invoked by a single subshell
+        # they must be encapsulated by a double quote. Always double
+        # quote to be sure of proper handling
+        # cmd will properly resolve nested double quotes as needed
+        #
+        # `set`` writes out the active env to the subshell stdout,
+        # and in this context we are always trying to obtain env
+        # state so it should always be appended
+        return base_call + f'"{commands} && set"'
+
+
+class VarsInvocation:
+    def __init__(self, script):
+        self._script = script
+
+    def command_str(self):
+        return f'"{self._script}"'
+
+    @property
+    def script(self):
+        return self._script
+
+
+class VCVarsInvocation(VarsInvocation):
+    def __init__(self, script, arch, msvc_version):
+        super(VCVarsInvocation, self).__init__(script)
+        self._arch = arch
+        self._msvc_version = msvc_version
+
+    @property
+    def sdk_ver(self):
+        """Accessor for Windows SDK version property
+
+        Note: This property may not be set by
+        the calling context and as such this property will
+        return an empty string
+
+        This property will ONLY be set if the SDK package
+        is a dependency somewhere in the Spack DAG of the package
+        for which we are constructing an MSVC compiler env.
+        Otherwise this property should be unset to allow the VCVARS
+        script to use its internal heuristics to determine appropriate
+        SDK version
+        """
+        if getattr(self, "_sdk_ver", None):
+            return self._sdk_ver + ".0"
+        return ""
+
+    @sdk_ver.setter
+    def sdk_ver(self, val):
+        self._sdk_ver = val
+
+    @property
+    def arch(self):
+        return self._arch
+
+    @property
+    def vcvars_ver(self):
+        return f"-vcvars_ver={self._msvc_version}"
+
+    def command_str(self):
+        script = super(VCVarsInvocation, self).command_str()
+        return f"{script} {self.arch} {self.sdk_ver} {self.vcvars_ver}"
+
+
 def get_valid_fortran_pth(comp_ver):
     cl_ver = str(comp_ver)
     sort_fn = lambda fc_ver: StrictVersion(fc_ver)
@@ -75,22 +159,48 @@ class Msvc(Compiler):
     # file based on compiler executable path.
 
     def __init__(self, *args, **kwargs):
-        new_pth = [pth if pth else get_valid_fortran_pth(args[0].version) for pth in args[3]]
-        args[3][:] = new_pth
+        # This positional argument "paths" is later parsed and process by the base class
+        # via the call to `super` later in this method
+        paths = args[3]
+        # This positional argument "cspec" is also parsed and handled by the base class
+        # constructor
+        cspec = args[0]
+        new_pth = [pth if pth else get_valid_fortran_pth(cspec.version) for pth in paths]
+        paths[:] = new_pth
         super().__init__(*args, **kwargs)
-        if os.getenv("ONEAPI_ROOT"):
+        # To use the MSVC compilers, VCVARS must be invoked
+        # VCVARS is located at a fixed location, referencable
+        # idiomatically by the following relative path from the
+        # compiler.
+        # Spack first finds the compilers via VSWHERE
+        # and stores their path, but their respective VCVARS
+        # file must be invoked before useage.
+        env_cmds = []
+        compiler_root = os.path.join(self.cc, "../../../../../../..")
+        vcvars_script_path = os.path.join(compiler_root, "Auxiliary", "Build", "vcvars64.bat")
+        # get current platform architecture and format for vcvars argument
+        arch = spack.platforms.real_host().default.lower()
+        arch = arch.replace("-", "_")
+        self.vcvars_call = VCVarsInvocation(vcvars_script_path, arch, self.msvc_version)
+        env_cmds.append(self.vcvars_call)
+        # Below is a check for a valid fortran path
+        # paths has c, cxx, fc, and f77 paths in that order
+        # paths[2] refers to the fc path and is a generic check
+        # for a fortran compiler
+        if paths[2]:
             # If this found, it sets all the vars
-            self.setvarsfile = os.path.join(os.getenv("ONEAPI_ROOT"), "setvars.bat")
-        else:
-            # To use the MSVC compilers, VCVARS must be invoked
-            # VCVARS is located at a fixed location, referencable
-            # idiomatically by the following relative path from the
-            # compiler.
-            # Spack first finds the compilers via VSWHERE
-            # and stores their path, but their respective VCVARS
-            # file must be invoked before useage.
-            self.setvarsfile = os.path.abspath(os.path.join(self.cc, "../../../../../../.."))
-            self.setvarsfile = os.path.join(self.setvarsfile, "Auxiliary", "Build", "vcvars64.bat")
+            oneapi_root = os.getenv("ONEAPI_ROOT")
+            oneapi_root_setvars = os.path.join(oneapi_root, "setvars.bat")
+            oneapi_version_setvars = os.path.join(
+                oneapi_root, "compiler", str(self.ifx_version), "env", "vars.bat"
+            )
+            # order matters here, the specific version env must be invoked first,
+            # otherwise it will be ignored if the root setvars sets up the oneapi
+            # env first
+            env_cmds.extend(
+                [VarsInvocation(oneapi_version_setvars), VarsInvocation(oneapi_root_setvars)]
+            )
+        self.msvc_compiler_environment = CmdCall(*env_cmds)
 
     @property
     def msvc_version(self):
@@ -119,16 +229,30 @@ def platform_toolset_ver(self):
         """
         return self.msvc_version[:2].joined.string[:3]
 
-    @property
-    def cl_version(self):
-        """Cl toolset version"""
+    def _compiler_version(self, compiler):
+        """Returns version object for given compiler"""
+        # ignore_errors below is true here due to ifx's
+        # non zero return code if it is not provided
+        # and input file
         return Version(
             re.search(
                 Msvc.version_regex,
-                spack.compiler.get_compiler_version_output(self.cc, version_arg=None),
+                spack.compiler.get_compiler_version_output(
+                    compiler, version_arg=None, ignore_errors=True
+                ),
             ).group(1)
         )
 
+    @property
+    def cl_version(self):
+        """Cl toolset version"""
+        return self._compiler_version(self.cc)
+
+    @property
+    def ifx_version(self):
+        """Ifx compiler version associated with this version of MSVC"""
+        return self._compiler_version(self.fc)
+
     @property
     def vs_root(self):
         # The MSVC install root is located at a fix level above the compiler
@@ -146,27 +270,12 @@ def setup_custom_environment(self, pkg, env):
         # output, sort into dictionary, use that to make the build
         # environment.
 
-        # get current platform architecture and format for vcvars argument
-        arch = spack.platforms.real_host().default.lower()
-        arch = arch.replace("-", "_")
         # vcvars can target specific sdk versions, force it to pick up concretized sdk
         # version, if needed by spec
-        sdk_ver = (
-            ""
-            if "win-sdk" not in pkg.spec or pkg.name == "win-sdk"
-            else pkg.spec["win-sdk"].version.string + ".0"
-        )
-        # provide vcvars with msvc version selected by concretization,
-        # not whatever it happens to pick up on the system (highest available version)
-        out = subprocess.check_output(  # novermin
-            'cmd /u /c "{}" {} {} {} && set'.format(
-                self.setvarsfile, arch, sdk_ver, "-vcvars_ver=%s" % self.msvc_version
-            ),
-            stderr=subprocess.STDOUT,
-        )
-        if sys.version_info[0] >= 3:
-            out = out.decode("utf-16le", errors="replace")  # novermin
+        if pkg.name != "win-sdk" and "win-sdk" in pkg.spec:
+            self.vcvars_call.sdk_ver = pkg.spec["win-sdk"].version.string
 
+        out = self.msvc_compiler_environment()
         int_env = dict(
             (key, value)
             for key, _, value in (line.partition("=") for line in out.splitlines())

From c1756257c241334719f0f08a80c6e88eaac180e6 Mon Sep 17 00:00:00 2001
From: Pat McCormick <651611+pmccormick@users.noreply.github.com>
Date: Wed, 30 Aug 2023 22:22:33 -0600
Subject: [PATCH 03/10] llvm: fix for Flang variant due to exception handling
 (#36171)

* The flang project does not support exceptions enabled in the core llvm
library (and developer guidelines explicitly state they should not be
used).  For this reason, when the flang variant is selected,
LLVM_ENABLE_EH needs to be disabled.  In the current main branch of
llvm (and thus future releases), enabling flang and setting
LLVM_ENABLE_EH will cause the overall build to fail.

* Update var/spack/repos/builtin/packages/llvm/package.py

Co-authored-by: Harmen Stoppels <harmenstoppels@gmail.com>

* fix syntax

---------

Co-authored-by: Harmen Stoppels <harmenstoppels@gmail.com>
Co-authored-by: Satish Balay <balay@mcs.anl.gov>
---
 var/spack/repos/builtin/packages/llvm/package.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py
index 16ab6177f71..a4a89f17bc6 100644
--- a/var/spack/repos/builtin/packages/llvm/package.py
+++ b/var/spack/repos/builtin/packages/llvm/package.py
@@ -756,7 +756,6 @@ def cmake_args(self):
         cmake_args = [
             define("LLVM_REQUIRES_RTTI", True),
             define("LLVM_ENABLE_RTTI", True),
-            define("LLVM_ENABLE_EH", True),
             define("LLVM_ENABLE_LIBXML2", False),
             define("CLANG_DEFAULT_OPENMP_RUNTIME", "libomp"),
             define("PYTHON_EXECUTABLE", python.command.path),
@@ -765,6 +764,16 @@ def cmake_args(self):
             from_variant("LLVM_ENABLE_ZSTD", "zstd"),
         ]
 
+        # Flang does not support exceptions from core llvm.
+        # LLVM_ENABLE_EH=True when building flang will soon
+        # fail (with changes at the llvm-project level).
+        # Only enable exceptions in LLVM if we are *not*
+        # building flang.  FYI: LLVM <= 16.x will build flang
+        # successfully but the executable will suffer from
+        # link errors looking for C++ EH support.
+        if "+flang" not in spec:
+            cmake_args.append(define("LLVM_ENABLE_EH", True))
+
         version_suffix = spec.variants["version_suffix"].value
         if version_suffix != "none":
             cmake_args.append(define("LLVM_VERSION_SUFFIX", version_suffix))

From acb02326aa6a0c03193ea6213657d7921fb1cd95 Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <harmenstoppels@gmail.com>
Date: Thu, 31 Aug 2023 10:09:37 +0200
Subject: [PATCH 04/10] ASP-based solver: add hidden mode to ignore versions
 that are moving targets, use that in CI (#39611)

Setting the undocumented variable SPACK_CONCRETIZER_REQUIRE_CHECKSUM
now causes the solver to avoid accounting for versions that are not checksummed.

This feature is used in CI to avoid spurious concretization against e.g. develop branches.
---
 lib/spack/spack/directives.py                 |   6 +-
 lib/spack/spack/solver/asp.py                 | 264 +++++++++---------
 lib/spack/spack/test/concretize.py            |  47 +++-
 .../spack/test/concretize_requirements.py     |   8 +-
 lib/spack/spack/util/crypto.py                |   3 +-
 .../gitlab/cloud_pipelines/.gitlab-ci.yml     |   2 +
 6 files changed, 196 insertions(+), 134 deletions(-)

diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py
index d6d7d81298c..ccc913a1fec 100644
--- a/lib/spack/spack/directives.py
+++ b/lib/spack/spack/directives.py
@@ -42,6 +42,7 @@ class OpenMpi(Package):
 import spack.patch
 import spack.spec
 import spack.url
+import spack.util.crypto
 import spack.variant
 from spack.dependency import Dependency, canonical_deptype, default_deptype
 from spack.fetch_strategy import from_kwargs
@@ -407,10 +408,7 @@ def version(
 
 def _execute_version(pkg, ver, **kwargs):
     if (
-        any(
-            s in kwargs
-            for s in ("sha256", "sha384", "sha512", "md5", "sha1", "sha224", "checksum")
-        )
+        (any(s in kwargs for s in spack.util.crypto.hashes) or "checksum" in kwargs)
         and hasattr(pkg, "has_code")
         and not pkg.has_code
     ):
diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py
index c596b81464a..7b13b4baa38 100644
--- a/lib/spack/spack/solver/asp.py
+++ b/lib/spack/spack/solver/asp.py
@@ -13,7 +13,7 @@
 import re
 import types
 import warnings
-from typing import List, NamedTuple
+from typing import List, NamedTuple, Tuple, Union
 
 import archspec.cpu
 
@@ -44,15 +44,18 @@
 import spack.repo
 import spack.spec
 import spack.store
-import spack.traverse
+import spack.util.crypto
 import spack.util.path
 import spack.util.timer
 import spack.variant
 import spack.version as vn
 import spack.version.git_ref_lookup
+from spack import traverse
 
 from .counter import FullDuplicatesCounter, MinimalDuplicatesCounter, NoDuplicatesCounter
 
+GitOrStandardVersion = Union[spack.version.GitVersion, spack.version.StandardVersion]
+
 # these are from clingo.ast and bootstrapped later
 ASTType = None
 parse_files = None
@@ -569,6 +572,41 @@ def keyfn(x):
     return normalized_yaml
 
 
+def _is_checksummed_git_version(v):
+    return isinstance(v, vn.GitVersion) and v.is_commit
+
+
+def _is_checksummed_version(version_info: Tuple[GitOrStandardVersion, dict]):
+    """Returns true iff the version is not a moving target"""
+    version, info = version_info
+    if isinstance(version, spack.version.StandardVersion):
+        if any(h in info for h in spack.util.crypto.hashes.keys()) or "checksum" in info:
+            return True
+        return "commit" in info and len(info["commit"]) == 40
+    return _is_checksummed_git_version(version)
+
+
+def _concretization_version_order(version_info: Tuple[GitOrStandardVersion, dict]):
+    """Version order key for concretization, where preferred > not preferred,
+    not deprecated > deprecated, finite > any infinite component; only if all are
+    the same, do we use default version ordering."""
+    version, info = version_info
+    return (
+        info.get("preferred", False),
+        not info.get("deprecated", False),
+        not version.isdevelop(),
+        version,
+    )
+
+
+def _spec_with_default_name(spec_str, name):
+    """Return a spec with a default name if none is provided, used for requirement specs"""
+    spec = spack.spec.Spec(spec_str)
+    if not spec.name:
+        spec.name = name
+    return spec
+
+
 def bootstrap_clingo():
     global clingo, ASTType, parse_files
 
@@ -1855,30 +1893,27 @@ class Body:
 
         return clauses
 
-    def build_version_dict(self, possible_pkgs):
+    def define_package_versions_and_validate_preferences(
+        self, possible_pkgs, require_checksum: bool
+    ):
         """Declare any versions in specs not declared in packages."""
         packages_yaml = spack.config.get("packages")
-        packages_yaml = _normalize_packages_yaml(packages_yaml)
         for pkg_name in possible_pkgs:
             pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
 
             # All the versions from the corresponding package.py file. Since concepts
             # like being a "develop" version or being preferred exist only at a
             # package.py level, sort them in this partial list here
-            def key_fn(item):
-                version, info = item
-                # When COMPARING VERSIONS, the '@develop' version is always
-                # larger than other versions. BUT when CONCRETIZING, the largest
-                # NON-develop version is selected by default.
-                return (
-                    info.get("preferred", False),
-                    not info.get("deprecated", False),
-                    not version.isdevelop(),
-                    version,
-                )
+            package_py_versions = sorted(
+                pkg_cls.versions.items(), key=_concretization_version_order, reverse=True
+            )
 
-            for idx, item in enumerate(sorted(pkg_cls.versions.items(), key=key_fn, reverse=True)):
-                v, version_info = item
+            if require_checksum and pkg_cls.has_code:
+                package_py_versions = [
+                    x for x in package_py_versions if _is_checksummed_version(x)
+                ]
+
+            for idx, (v, version_info) in enumerate(package_py_versions):
                 self.possible_versions[pkg_name].add(v)
                 self.declared_versions[pkg_name].append(
                     DeclaredVersion(version=v, idx=idx, origin=Provenance.PACKAGE_PY)
@@ -1887,22 +1922,26 @@ def key_fn(item):
                 if deprecated:
                     self.deprecated_versions[pkg_name].add(v)
 
-            # All the preferred version from packages.yaml, versions in external
-            # specs will be computed later
-            version_preferences = packages_yaml.get(pkg_name, {}).get("version", [])
+            if pkg_name not in packages_yaml or "version" not in packages_yaml[pkg_name]:
+                continue
+
             version_defs = []
-            pkg_class = spack.repo.PATH.get_pkg_class(pkg_name)
-            for vstr in version_preferences:
+
+            for vstr in packages_yaml[pkg_name]["version"]:
                 v = vn.ver(vstr)
+
                 if isinstance(v, vn.GitVersion):
-                    version_defs.append(v)
+                    if not require_checksum or v.is_commit:
+                        version_defs.append(v)
                 else:
-                    satisfying_versions = self._check_for_defined_matching_versions(pkg_class, v)
-                    # Amongst all defined versions satisfying this specific
-                    # preference, the highest-numbered version is the
-                    # most-preferred: therefore sort satisfying versions
-                    # from greatest to least
-                    version_defs.extend(sorted(satisfying_versions, reverse=True))
+                    matches = [x for x in self.possible_versions[pkg_name] if x.satisfies(v)]
+                    matches.sort(reverse=True)
+                    if not matches:
+                        raise spack.config.ConfigError(
+                            f"Preference for version {v} does not match any known "
+                            f"version of {pkg_name} (in its package.py or any external)"
+                        )
+                    version_defs.extend(matches)
 
             for weight, vdef in enumerate(llnl.util.lang.dedupe(version_defs)):
                 self.declared_versions[pkg_name].append(
@@ -1910,31 +1949,9 @@ def key_fn(item):
                 )
                 self.possible_versions[pkg_name].add(vdef)
 
-    def _check_for_defined_matching_versions(self, pkg_class, v):
-        """Given a version specification (which may be a concrete version,
-        range, etc.), determine if any package.py version declarations
-        or externals define a version which satisfies it.
-
-        This is primarily for determining whether a version request (e.g.
-        version preferences, which should not themselves define versions)
-        refers to a defined version.
-
-        This function raises an exception if no satisfying versions are
-        found.
-        """
-        pkg_name = pkg_class.name
-        satisfying_versions = list(x for x in pkg_class.versions if x.satisfies(v))
-        satisfying_versions.extend(x for x in self.possible_versions[pkg_name] if x.satisfies(v))
-        if not satisfying_versions:
-            raise spack.config.ConfigError(
-                "Preference for version {0} does not match any version"
-                " defined for {1} (in its package.py or any external)".format(str(v), pkg_name)
-            )
-        return satisfying_versions
-
-    def add_concrete_versions_from_specs(self, specs, origin):
+    def define_ad_hoc_versions_from_specs(self, specs, origin, require_checksum: bool):
         """Add concrete versions to possible versions from lists of CLI/dev specs."""
-        for s in spack.traverse.traverse_nodes(specs):
+        for s in traverse.traverse_nodes(specs):
             # If there is a concrete version on the CLI *that we know nothing
             # about*, add it to the known versions. Use idx=0, which is the
             # best possible, so they're guaranteed to be used preferentially.
@@ -1943,9 +1960,13 @@ def add_concrete_versions_from_specs(self, specs, origin):
             if version is None or any(v == version for v in self.possible_versions[s.name]):
                 continue
 
-            self.declared_versions[s.name].append(
-                DeclaredVersion(version=version, idx=0, origin=origin)
-            )
+            if require_checksum and not _is_checksummed_git_version(version):
+                raise UnsatisfiableSpecError(
+                    s.format("No matching version for constraint {name}{@versions}")
+                )
+
+            declared = DeclaredVersion(version=version, idx=0, origin=origin)
+            self.declared_versions[s.name].append(declared)
             self.possible_versions[s.name].add(version)
 
     def _supported_targets(self, compiler_name, compiler_version, targets):
@@ -2142,7 +2163,7 @@ def generate_possible_compilers(self, specs):
         # add compiler specs from the input line to possibilities if we
         # don't require compilers to exist.
         strict = spack.concretize.Concretizer().check_for_compiler_existence
-        for s in spack.traverse.traverse_nodes(specs):
+        for s in traverse.traverse_nodes(specs):
             # we don't need to validate compilers for already-built specs
             if s.concrete or not s.compiler:
                 continue
@@ -2392,13 +2413,12 @@ def setup(self, driver, specs, reuse=None):
         self.provider_requirements()
         self.external_packages()
 
-        # traverse all specs and packages to build dict of possible versions
-        self.build_version_dict(self.pkgs)
-        self.add_concrete_versions_from_specs(specs, Provenance.SPEC)
-        self.add_concrete_versions_from_specs(dev_specs, Provenance.DEV_SPEC)
-
-        req_version_specs = self._get_versioned_specs_from_pkg_requirements()
-        self.add_concrete_versions_from_specs(req_version_specs, Provenance.PACKAGE_REQUIREMENT)
+        # TODO: make a config option for this undocumented feature
+        require_checksum = "SPACK_CONCRETIZER_REQUIRE_CHECKSUM" in os.environ
+        self.define_package_versions_and_validate_preferences(self.pkgs, require_checksum)
+        self.define_ad_hoc_versions_from_specs(specs, Provenance.SPEC, require_checksum)
+        self.define_ad_hoc_versions_from_specs(dev_specs, Provenance.DEV_SPEC, require_checksum)
+        self.validate_and_define_versions_from_requirements(require_checksum)
 
         self.gen.h1("Package Constraints")
         for pkg in sorted(self.pkgs):
@@ -2447,78 +2467,68 @@ def literal_specs(self, specs):
             if self.concretize_everything:
                 self.gen.fact(fn.solve_literal(idx))
 
-    def _get_versioned_specs_from_pkg_requirements(self):
-        """If package requirements mention versions that are not mentioned
+    def validate_and_define_versions_from_requirements(self, require_checksum: bool):
+        """If package requirements mention concrete versions that are not mentioned
         elsewhere, then we need to collect those to mark them as possible
-        versions.
-        """
-        req_version_specs = list()
-        config = spack.config.get("packages")
-        for pkg_name, d in config.items():
-            if pkg_name == "all":
+        versions. If they are abstract and statically have no match, then we
+        need to throw an error. This function assumes all possible versions are already
+        registered in self.possible_versions."""
+        for pkg_name, d in spack.config.get("packages").items():
+            if pkg_name == "all" or "require" not in d:
                 continue
-            if "require" in d:
-                req_version_specs.extend(self._specs_from_requires(pkg_name, d["require"]))
-        return req_version_specs
+
+            for s in traverse.traverse_nodes(self._specs_from_requires(pkg_name, d["require"])):
+                name, versions = s.name, s.versions
+
+                if name not in self.pkgs or versions == spack.version.any_version:
+                    continue
+
+                s.attach_git_version_lookup()
+                v = versions.concrete
+
+                if not v:
+                    # If the version is not concrete, check it's statically concretizable. If
+                    # not throw an error, which is just so that users know they need to change
+                    # their config, instead of getting a hard to decipher concretization error.
+                    if not any(x for x in self.possible_versions[name] if x.satisfies(versions)):
+                        raise spack.config.ConfigError(
+                            f"Version requirement {versions} on {pkg_name} for {name} "
+                            f"cannot match any known version from package.py or externals"
+                        )
+                    continue
+
+                if v in self.possible_versions[name]:
+                    continue
+
+                # If concrete an not yet defined, conditionally define it, like we do for specs
+                # from the command line.
+                if not require_checksum or _is_checksummed_git_version(v):
+                    self.declared_versions[name].append(
+                        DeclaredVersion(version=v, idx=0, origin=Provenance.PACKAGE_REQUIREMENT)
+                    )
+                    self.possible_versions[name].add(v)
 
     def _specs_from_requires(self, pkg_name, section):
-        """Collect specs from requirements which define versions (i.e. those that
-        have a concrete version). Requirements can define *new* versions if
-        they are included as part of an equivalence (hash=number) but not
-        otherwise.
-        """
+        """Collect specs from a requirement rule"""
         if isinstance(section, str):
-            spec = spack.spec.Spec(section)
-            if not spec.name:
-                spec.name = pkg_name
-            extracted_specs = [spec]
-        else:
-            spec_strs = []
-            for spec_group in section:
-                if isinstance(spec_group, str):
-                    spec_strs.append(spec_group)
-                else:
-                    # Otherwise it is an object. The object can contain a single
-                    # "spec" constraint, or a list of them with "any_of" or
-                    # "one_of" policy.
-                    if "spec" in spec_group:
-                        new_constraints = [spec_group["spec"]]
-                    else:
-                        key = "one_of" if "one_of" in spec_group else "any_of"
-                        new_constraints = spec_group[key]
-                    spec_strs.extend(new_constraints)
+            yield _spec_with_default_name(section, pkg_name)
+            return
 
-            extracted_specs = []
-            for spec_str in spec_strs:
-                spec = spack.spec.Spec(spec_str)
-                if not spec.name:
-                    spec.name = pkg_name
-                extracted_specs.append(spec)
-
-        version_specs = []
-        for spec in extracted_specs:
-            if spec.versions.concrete:
-                # Note: this includes git versions
-                version_specs.append(spec)
+        for spec_group in section:
+            if isinstance(spec_group, str):
+                yield _spec_with_default_name(spec_group, pkg_name)
                 continue
 
-            # Prefer spec's name if it exists, in case the spec is
-            # requiring a specific implementation inside of a virtual section
-            # e.g. packages:mpi:require:openmpi@4.0.1
-            pkg_class = spack.repo.PATH.get_pkg_class(spec.name or pkg_name)
-            satisfying_versions = self._check_for_defined_matching_versions(
-                pkg_class, spec.versions
-            )
+            # Otherwise it is an object. The object can contain a single
+            # "spec" constraint, or a list of them with "any_of" or
+            # "one_of" policy.
+            if "spec" in spec_group:
+                yield _spec_with_default_name(spec_group["spec"], pkg_name)
+                continue
 
-            # Version ranges ("@1.3" without the "=", "@1.2:1.4") and lists
-            # will end up here
-            ordered_satisfying_versions = sorted(satisfying_versions, reverse=True)
-            vspecs = list(spack.spec.Spec("@{0}".format(x)) for x in ordered_satisfying_versions)
-            version_specs.extend(vspecs)
-
-        for spec in version_specs:
-            spec.attach_git_version_lookup()
-        return version_specs
+            key = "one_of" if "one_of" in spec_group else "any_of"
+            for s in spec_group[key]:
+                yield _spec_with_default_name(s, pkg_name)
 
 
 class SpecBuilder:
diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py
index 4cd22249ca2..95c9fe87521 100644
--- a/lib/spack/spack/test/concretize.py
+++ b/lib/spack/spack/test/concretize.py
@@ -21,10 +21,11 @@
 import spack.hash_types as ht
 import spack.platforms
 import spack.repo
+import spack.solver.asp
 import spack.variant as vt
 from spack.concretize import find_spec
 from spack.spec import CompilerSpec, Spec
-from spack.version import ver
+from spack.version import Version, ver
 
 
 def check_spec(abstract, concrete):
@@ -2114,6 +2115,14 @@ def test_virtuals_are_reconstructed_on_reuse(self, spec_str, mpi_name, database)
             assert len(mpi_edges) == 1
             assert "mpi" in mpi_edges[0].virtuals
 
+    @pytest.mark.only_clingo("Use case not supported by the original concretizer")
+    def test_dont_define_new_version_from_input_if_checksum_required(self, working_env):
+        os.environ["SPACK_CONCRETIZER_REQUIRE_CHECKSUM"] = "yes"
+        with pytest.raises(spack.error.UnsatisfiableSpecError):
+            # normally spack concretizes to @=3.0 if it's not defined in package.py, except
+            # when checksums are required
+            Spec("a@=3.0").concretized()
+
 
 @pytest.fixture()
 def duplicates_test_repository():
@@ -2208,3 +2217,39 @@ def test_solution_without_cycles(self):
         s = Spec("cycle-b").concretized()
         assert s["cycle-a"].satisfies("~cycle")
         assert s["cycle-b"].satisfies("+cycle")
+
+
+@pytest.mark.parametrize(
+    "v_str,v_opts,checksummed",
+    [
+        ("1.2.3", {"sha256": f"{1:064x}"}, True),
+        # it's not about the version being "infinite",
+        # but whether it has a digest
+        ("develop", {"sha256": f"{1:064x}"}, True),
+        # other hash types
+        ("1.2.3", {"checksum": f"{1:064x}"}, True),
+        ("1.2.3", {"md5": f"{1:032x}"}, True),
+        ("1.2.3", {"sha1": f"{1:040x}"}, True),
+        ("1.2.3", {"sha224": f"{1:056x}"}, True),
+        ("1.2.3", {"sha384": f"{1:096x}"}, True),
+        ("1.2.3", {"sha512": f"{1:0128x}"}, True),
+        # no digest key
+        ("1.2.3", {"bogus": f"{1:064x}"}, False),
+        # git version with full commit sha
+        ("1.2.3", {"commit": f"{1:040x}"}, True),
+        (f"{1:040x}=1.2.3", {}, True),
+        # git version with short commit sha
+        ("1.2.3", {"commit": f"{1:07x}"}, False),
+        (f"{1:07x}=1.2.3", {}, False),
+        # git tag is a moving target
+        ("1.2.3", {"tag": "v1.2.3"}, False),
+        ("1.2.3", {"tag": "v1.2.3", "commit": f"{1:07x}"}, False),
+        # git branch is a moving target
+        ("1.2.3", {"branch": "releases/1.2"}, False),
+        # git ref is a moving target
+        ("git.branch=1.2.3", {}, False),
+    ],
+)
+def test_drop_moving_targets(v_str, v_opts, checksummed):
+    v = Version(v_str)
+    assert spack.solver.asp._is_checksummed_version((v, v_opts)) == checksummed
diff --git a/lib/spack/spack/test/concretize_requirements.py b/lib/spack/spack/test/concretize_requirements.py
index 20b89d9e5f3..d7c1c883597 100644
--- a/lib/spack/spack/test/concretize_requirements.py
+++ b/lib/spack/spack/test/concretize_requirements.py
@@ -2,6 +2,7 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import os
 import pathlib
 
 import pytest
@@ -299,9 +300,14 @@ def test_requirement_adds_version_satisfies(
     assert s1.satisfies("@2.2")
 
 
+@pytest.mark.parametrize("require_checksum", (True, False))
 def test_requirement_adds_git_hash_version(
-    concretize_scope, test_repo, mock_git_version_info, monkeypatch
+    require_checksum, concretize_scope, test_repo, mock_git_version_info, monkeypatch, working_env
 ):
+    # A full commit sha is a checksummed version, so this test should pass in both cases
+    if require_checksum:
+        os.environ["SPACK_CONCRETIZER_REQUIRE_CHECKSUM"] = "yes"
+
     repo_path, filename, commits = mock_git_version_info
     monkeypatch.setattr(
         spack.package_base.PackageBase, "git", path_to_file_url(repo_path), raising=False
diff --git a/lib/spack/spack/util/crypto.py b/lib/spack/spack/util/crypto.py
index df8102352e9..8eebcc92bc3 100644
--- a/lib/spack/spack/util/crypto.py
+++ b/lib/spack/spack/util/crypto.py
@@ -9,7 +9,8 @@
 import llnl.util.tty as tty
 
 #: Set of hash algorithms that Spack can use, mapped to digest size in bytes
-hashes = {"md5": 16, "sha1": 20, "sha224": 28, "sha256": 32, "sha384": 48, "sha512": 64}
+hashes = {"sha256": 32, "md5": 16, "sha1": 20, "sha224": 28, "sha384": 48, "sha512": 64}
+# Note: keys are ordered by popularity for earliest return in ``hash_key in version_dict`` checks.
 
 
 #: size of hash digests in bytes, mapped to algoritm names
diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
index 358d3af1215..45300eb7378 100644
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -136,6 +136,8 @@ default:
   variables:
     KUBERNETES_CPU_REQUEST: 4000m
     KUBERNETES_MEMORY_REQUEST: 16G
+    # avoid moving targets like branches and tags
+    SPACK_CONCRETIZER_REQUIRE_CHECKSUM: 1
   interruptible: true
   timeout: 60 minutes
   retry:

From 34e4c62e8ce019a84424f93102068dcea0b21dfa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Raffaele=20Solc=C3=A0?= <rasolca@cscs.ch>
Date: Thu, 31 Aug 2023 10:39:22 +0200
Subject: [PATCH 05/10] dla-future: add v0.2.0 (#39705)

---
 .../builtin/packages/dla-future/package.py    | 54 ++++++++++++++++---
 1 file changed, 47 insertions(+), 7 deletions(-)

diff --git a/var/spack/repos/builtin/packages/dla-future/package.py b/var/spack/repos/builtin/packages/dla-future/package.py
index 57d0befe32b..3aeda901030 100644
--- a/var/spack/repos/builtin/packages/dla-future/package.py
+++ b/var/spack/repos/builtin/packages/dla-future/package.py
@@ -14,20 +14,36 @@ class DlaFuture(CMakePackage, CudaPackage, ROCmPackage):
     git = "https://github.com/eth-cscs/DLA-Future.git"
     maintainers = ["rasolca", "albestro", "msimberg", "aurianer"]
 
+    version("0.2.0", sha256="da73cbd1b88287c86d84b1045a05406b742be924e65c52588bbff200abd81a10")
     version("0.1.0", sha256="f7ffcde22edabb3dc24a624e2888f98829ee526da384cd752b2b271c731ca9b1")
     version("master", branch="master")
 
     variant("shared", default=True, description="Build shared libraries.")
 
+    variant(
+        "hdf5",
+        default=False,
+        when="@0.2.0:",
+        description="HDF5 support for dealing with matrices on disk.",
+    )
+
     variant("doc", default=False, description="Build documentation.")
 
     variant("miniapps", default=False, description="Build miniapps.")
 
+    variant(
+        "scalapack",
+        default=False,
+        when="@0.2.0:",
+        description="Build C API compatible with ScaLAPACK",
+    )
+
     depends_on("cmake@3.22:", type="build")
     depends_on("doxygen", type="build", when="+doc")
     depends_on("mpi")
     depends_on("blaspp@2022.05.00:")
     depends_on("lapackpp@2022.05.00:")
+    depends_on("scalapack", when="+scalapack")
 
     depends_on("umpire~examples")
     depends_on("umpire~cuda", when="~cuda")
@@ -36,7 +52,8 @@ class DlaFuture(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("umpire+rocm~shared", when="+rocm")
     depends_on("umpire@4.1.0:")
 
-    depends_on("pika@0.15.1:")
+    depends_on("pika@0.15.1:", when="@0.1")
+    depends_on("pika@0.16:", when="@0.2.0:")
     depends_on("pika-algorithms@0.1:")
     depends_on("pika +mpi")
     depends_on("pika +cuda", when="+cuda")
@@ -52,6 +69,8 @@ class DlaFuture(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("rocsolver", when="+rocm")
     depends_on("rocthrust", when="+rocm")
 
+    depends_on("hdf5 +cxx+mpi+threadsafe+shared", when="+hdf5")
+
     conflicts("+cuda", when="+rocm")
 
     with when("+rocm"):
@@ -92,16 +111,29 @@ def cmake_args(self):
                 "openmp": "omp",
                 "tbb": "tbb",
             }  # Map MKL variants to LAPACK target name
+            mkl_threads = vmap[spec["intel-mkl"].variants["threads"].value]
             # TODO: Generalise for intel-oneapi-mkl
             args += [
                 self.define("DLAF_WITH_MKL", True),
-                self.define(
-                    "MKL_LAPACK_TARGET",
-                    "mkl::mkl_intel_32bit_{0}_dyn".format(
-                        vmap[spec["intel-mkl"].variants["threads"].value]
-                    ),
-                ),
+                self.define("MKL_LAPACK_TARGET", f"mkl::mkl_intel_32bit_{mkl_threads}_dyn"),
             ]
+            if "+scalapack" in spec:
+                if (
+                    "^mpich" in spec
+                    or "^cray-mpich" in spec
+                    or "^intel-mpi" in spec
+                    or "^mvapich" in spec
+                    or "^mvapich2" in spec
+                ):
+                    mkl_mpi = "mpich"
+                elif "^openmpi" in spec:
+                    mkl_mpi = "ompi"
+                args.append(
+                    self.define(
+                        "MKL_SCALAPACK_TARGET",
+                        f"mkl::scalapack_{mkl_mpi}_intel_32bit_{mkl_threads}_dyn",
+                    )
+                )
         else:
             args.append(self.define("DLAF_WITH_MKL", False))
             args.append(
@@ -110,6 +142,11 @@ def cmake_args(self):
                     " ".join([spec[dep].libs.ld_flags for dep in ["blas", "lapack"]]),
                 )
             )
+            if "+scalapack" in spec:
+                args.append(self.define("SCALAPACK_LIBRARY", spec["scalapack"].libs.ld_flags))
+
+        if "+scalapack" in spec:
+            args.append(self.define_from_variant("DLAF_WITH_SCALAPACK", "scalapack"))
 
         # CUDA/HIP
         args.append(self.define_from_variant("DLAF_WITH_CUDA", "cuda"))
@@ -125,6 +162,9 @@ def cmake_args(self):
                 arch_str = ";".join(archs)
                 args.append(self.define("CMAKE_CUDA_ARCHITECTURES", arch_str))
 
+        # HDF5 support
+        args.append(self.define_from_variant("DLAF_WITH_HDF5", "hdf5"))
+
         # DOC
         args.append(self.define_from_variant("DLAF_BUILD_DOC", "doc"))
 

From fb1e81657c45f3f0140e1c5ae74b7eb41764260d Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <harmenstoppels@gmail.com>
Date: Thu, 31 Aug 2023 10:40:02 +0200
Subject: [PATCH 06/10] Remove a few local imports in tests (#39719)

---
 lib/spack/spack/test/concretize.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py
index 95c9fe87521..d72860a31d2 100644
--- a/lib/spack/spack/test/concretize.py
+++ b/lib/spack/spack/test/concretize.py
@@ -1599,8 +1599,6 @@ def test_reuse_with_unknown_package_dont_raise(self, tmpdir, monkeypatch):
     )
     @pytest.mark.only_clingo("Original concretizer cannot concretize in rounds")
     def test_best_effort_coconcretize(self, specs, expected):
-        import spack.solver.asp
-
         specs = [Spec(s) for s in specs]
         solver = spack.solver.asp.Solver()
         solver.reuse = False
@@ -1644,8 +1642,6 @@ def test_best_effort_coconcretize(self, specs, expected):
     @pytest.mark.only_clingo("Original concretizer cannot concretize in rounds")
     def test_best_effort_coconcretize_preferences(self, specs, expected_spec, occurances):
         """Test package preferences during coconcretization."""
-        import spack.solver.asp
-
         specs = [Spec(s) for s in specs]
         solver = spack.solver.asp.Solver()
         solver.reuse = False
@@ -1661,8 +1657,6 @@ def test_best_effort_coconcretize_preferences(self, specs, expected_spec, occura
 
     @pytest.mark.only_clingo("Use case not supported by the original concretizer")
     def test_coconcretize_reuse_and_virtuals(self):
-        import spack.solver.asp
-
         reusable_specs = []
         for s in ["mpileaks ^mpich", "zmpi"]:
             reusable_specs.extend(Spec(s).concretized().traverse(root=True))
@@ -1683,8 +1677,6 @@ def test_misleading_error_message_on_version(self, mutable_database):
         # For this bug to be triggered we need a reusable dependency
         # that is not optimal in terms of optimization scores.
         # We pick an old version of "b"
-        import spack.solver.asp
-
         reusable_specs = [Spec("non-existing-conditional-dep@1.0").concretized()]
         root_spec = Spec("non-existing-conditional-dep@2.0")
 
@@ -1700,8 +1692,6 @@ def test_misleading_error_message_on_version(self, mutable_database):
     @pytest.mark.only_clingo("Use case not supported by the original concretizer")
     def test_version_weight_and_provenance(self):
         """Test package preferences during coconcretization."""
-        import spack.solver.asp
-
         reusable_specs = [Spec(spec_str).concretized() for spec_str in ("b@0.9", "b@1.0")]
         root_spec = Spec("a foobar=bar")
 
@@ -1733,8 +1723,6 @@ def test_version_weight_and_provenance(self):
     @pytest.mark.regression("31169")
     @pytest.mark.only_clingo("Use case not supported by the original concretizer")
     def test_not_reusing_incompatible_os_or_compiler(self):
-        import spack.solver.asp
-
         root_spec = Spec("b")
         s = root_spec.concretized()
         wrong_compiler, wrong_os = s.copy(), s.copy()

From ecb7ad493fe8d3599cbd9fd4b3dbcb2407c8e6e6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 31 Aug 2023 10:40:33 +0200
Subject: [PATCH 07/10] build(deps): bump sphinx from 7.2.4 to 7.2.5 in
 /lib/spack/docs (#39716)

Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 7.2.4 to 7.2.5.
- [Release notes](https://github.com/sphinx-doc/sphinx/releases)
- [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES)
- [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.2.4...v7.2.5)

---
updated-dependencies:
- dependency-name: sphinx
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 lib/spack/docs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/spack/docs/requirements.txt b/lib/spack/docs/requirements.txt
index 02bf9824037..adcf16f3798 100644
--- a/lib/spack/docs/requirements.txt
+++ b/lib/spack/docs/requirements.txt
@@ -1,4 +1,4 @@
-sphinx==7.2.4
+sphinx==7.2.5
 sphinxcontrib-programoutput==0.17
 sphinx_design==0.5.0
 sphinx-rtd-theme==1.3.0

From 86216cc36ed8ea2771a71f1ea01c18a10f368afa Mon Sep 17 00:00:00 2001
From: Massimiliano Culpo <massimiliano.culpo@gmail.com>
Date: Thu, 31 Aug 2023 11:28:52 +0200
Subject: [PATCH 08/10] environment: improve spack remove matching (#39390)

search for equivalent specs, not for equal strings when selecting a spec to remove.
---
 lib/spack/spack/environment/environment.py | 25 +++++++-
 lib/spack/spack/spec_list.py               |  6 +-
 lib/spack/spack/test/env.py                | 69 +++++++++++++++++++++-
 3 files changed, 95 insertions(+), 5 deletions(-)

diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py
index ea4ac49a3ac..8eb7edab409 100644
--- a/lib/spack/spack/environment/environment.py
+++ b/lib/spack/spack/environment/environment.py
@@ -2664,6 +2664,26 @@ def __init__(self, manifest_dir: Union[pathlib.Path, str]) -> None:
         self.yaml_content = with_defaults_added
         self.changed = False
 
+    def _all_matches(self, user_spec: str) -> List[str]:
+        """Maps the input string to the first equivalent user spec in the manifest,
+        and returns it.
+
+        Args:
+            user_spec: user spec to be found
+
+        Raises:
+            ValueError: if no equivalent match is found
+        """
+        result = []
+        for yaml_spec_str in self.pristine_configuration["specs"]:
+            if Spec(yaml_spec_str) == Spec(user_spec):
+                result.append(yaml_spec_str)
+
+        if not result:
+            raise ValueError(f"cannot find a spec equivalent to {user_spec}")
+
+        return result
+
     def add_user_spec(self, user_spec: str) -> None:
         """Appends the user spec passed as input to the list of root specs.
 
@@ -2684,8 +2704,9 @@ def remove_user_spec(self, user_spec: str) -> None:
             SpackEnvironmentError: when the user spec is not in the list
         """
         try:
-            self.pristine_configuration["specs"].remove(user_spec)
-            self.configuration["specs"].remove(user_spec)
+            for key in self._all_matches(user_spec):
+                self.pristine_configuration["specs"].remove(key)
+                self.configuration["specs"].remove(key)
         except ValueError as e:
             msg = f"cannot remove {user_spec} from {self}, no such spec exists"
             raise SpackEnvironmentError(msg) from e
diff --git a/lib/spack/spack/spec_list.py b/lib/spack/spack/spec_list.py
index 6cad15e14ea..3f60d572492 100644
--- a/lib/spack/spack/spec_list.py
+++ b/lib/spack/spack/spec_list.py
@@ -97,8 +97,10 @@ def remove(self, spec):
             msg += "Either %s is not in %s or %s is " % (spec, self.name, spec)
             msg += "expanded from a matrix and cannot be removed directly."
             raise SpecListError(msg)
-        assert len(remove) == 1
-        self.yaml_list.remove(remove[0])
+
+        # Remove may contain more than one string representation of the same spec
+        for item in remove:
+            self.yaml_list.remove(item)
 
         # invalidate cache variables when we change the list
         self._expanded_list = None
diff --git a/lib/spack/spack/test/env.py b/lib/spack/spack/test/env.py
index 4396557f8a3..e88af087619 100644
--- a/lib/spack/spack/test/env.py
+++ b/lib/spack/spack/test/env.py
@@ -13,7 +13,11 @@
 
 import spack.environment as ev
 import spack.spec
-from spack.environment.environment import SpackEnvironmentViewError, _error_on_nonempty_view_dir
+from spack.environment.environment import (
+    EnvironmentManifestFile,
+    SpackEnvironmentViewError,
+    _error_on_nonempty_view_dir,
+)
 
 pytestmark = pytest.mark.not_on_windows("Envs are not supported on windows")
 
@@ -623,3 +627,66 @@ def test_requires_on_virtual_and_potential_providers(
         assert mpileaks.satisfies("^mpich2")
         assert mpileaks["mpi"].satisfies("mpich2")
         assert not mpileaks.satisfies(f"^{possible_mpi_spec}")
+
+
+@pytest.mark.regression("39387")
+@pytest.mark.parametrize(
+    "spec_str", ["mpileaks +opt", "mpileaks  +opt   ~shared", "mpileaks  ~shared   +opt"]
+)
+def test_manifest_file_removal_works_if_spec_is_not_normalized(tmp_path, spec_str):
+    """Tests that we can remove a spec from a manifest file even if its string
+    representation is not normalized.
+    """
+    manifest = tmp_path / "spack.yaml"
+    manifest.write_text(
+        f"""\
+spack:
+  specs:
+  - {spec_str}
+"""
+    )
+    s = spack.spec.Spec(spec_str)
+    spack_yaml = EnvironmentManifestFile(tmp_path)
+    # Doing a round trip str -> Spec -> str normalizes the representation
+    spack_yaml.remove_user_spec(str(s))
+    spack_yaml.flush()
+
+    assert spec_str not in manifest.read_text()
+
+
+@pytest.mark.regression("39387")
+@pytest.mark.parametrize(
+    "duplicate_specs,expected_number",
+    [
+        # Swap variants, versions, etc. add spaces
+        (["foo +bar ~baz", "foo ~baz    +bar"], 3),
+        (["foo @1.0 ~baz %gcc", "foo ~baz @1.0%gcc"], 3),
+        # Item 1 and 3 are exactly the same
+        (["zlib +shared", "zlib      +shared", "zlib +shared"], 4),
+    ],
+)
+def test_removing_spec_from_manifest_with_exact_duplicates(
+    duplicate_specs, expected_number, tmp_path
+):
+    """Tests that we can remove exact duplicates from a manifest file.
+
+    Note that we can't get in a state with duplicates using only CLI, but this might happen
+    on user edited spack.yaml files.
+    """
+    manifest = tmp_path / "spack.yaml"
+    manifest.write_text(
+        f"""\
+    spack:
+      specs: [{", ".join(duplicate_specs)} , "zlib"]
+    """
+    )
+
+    with ev.Environment(tmp_path) as env:
+        assert len(env.user_specs) == expected_number
+        env.remove(duplicate_specs[0])
+        env.write()
+
+    assert "+shared" not in manifest.read_text()
+    assert "zlib" in manifest.read_text()
+    with ev.Environment(tmp_path) as env:
+        assert len(env.user_specs) == 1

From 679d41ea669c1a435c65447dee2b764623d6c318 Mon Sep 17 00:00:00 2001
From: Anton Kozhevnikov <toxa81@gmail.com>
Date: Thu, 31 Aug 2023 16:16:31 +0200
Subject: [PATCH 09/10] [NVHPC] add a possibility to control default CUDA
 version (#38909)

* add a possibility to control default cuda version

* fix stype

* style fix

* resolve comment

* resolve comment

* Fix style in nvhpc package.py

---------

Co-authored-by: antonk <antonk@cscs.ch>
Co-authored-by: Mikael Simberg <mikael.simberg@iki.fi>
---
 var/spack/repos/builtin/packages/nvhpc/package.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/var/spack/repos/builtin/packages/nvhpc/package.py b/var/spack/repos/builtin/packages/nvhpc/package.py
index cd52b8a6346..7579afd8734 100644
--- a/var/spack/repos/builtin/packages/nvhpc/package.py
+++ b/var/spack/repos/builtin/packages/nvhpc/package.py
@@ -355,6 +355,9 @@ class Nvhpc(Package):
     )
     variant("lapack", default=True, description="Enable LAPACK")
     variant("mpi", default=False, description="Enable MPI")
+    variant(
+        "default_cuda", default="default", description="Default CUDA version, for example 11.8"
+    )
 
     provides("blas", when="+blas")
     provides("lapack", when="+lapack")
@@ -373,6 +376,8 @@ def setup_build_environment(self, env):
         env.set("NVHPC_SILENT", "true")
         env.set("NVHPC_ACCEPT_EULA", "accept")
         env.set("NVHPC_INSTALL_DIR", self.prefix)
+        if self.spec.variants["default_cuda"].value != "default":
+            env.set("NVHPC_DEFAULT_CUDA", self.spec.variants["default_cuda"].value)
 
         if self.spec.variants["install_type"].value == "network":
             local_dir = join_path(self._version_prefix(), "share_objects")

From 818195a3bd19a916c73f66695fb3bd7845f94a7d Mon Sep 17 00:00:00 2001
From: "Adam J. Stewart" <ajstewart426@gmail.com>
Date: Thu, 31 Aug 2023 11:08:44 -0500
Subject: [PATCH 10/10] py-pandas: add v2.1.0 (#39707)

---
 .../builtin/packages/py-pandas/package.py     | 99 ++++++++++---------
 1 file changed, 52 insertions(+), 47 deletions(-)

diff --git a/var/spack/repos/builtin/packages/py-pandas/package.py b/var/spack/repos/builtin/packages/py-pandas/package.py
index 1a46b9a9c66..7af6d95e82f 100644
--- a/var/spack/repos/builtin/packages/py-pandas/package.py
+++ b/var/spack/repos/builtin/packages/py-pandas/package.py
@@ -17,6 +17,7 @@ class PyPandas(PythonPackage):
 
     maintainers("adamjstewart")
 
+    version("2.1.0", sha256="62c24c7fc59e42b775ce0679cfa7b14a5f9bfb7643cfbe708c960699e05fb918")
     version("2.0.3", sha256="c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c")
     version("2.0.2", sha256="dd5476b6c3fe410ee95926873f377b856dbc4e81a9c605a0dc05aaccc6a7c6c6")
     version("2.0.1", sha256="19b8e5270da32b41ebf12f0e7165efa7024492e9513fb46fb631c5022ae5709d")
@@ -61,70 +62,74 @@ class PyPandas(PythonPackage):
     version("0.24.2", sha256="4f919f409c433577a501e023943e582c57355d50a724c589e78bc1d551a535a2")
     version("0.24.1", sha256="435821cb2501eabbcee7e83614bd710940dc0cf28b5afbc4bdb816c31cec71af")
     version("0.23.4", sha256="5b24ca47acf69222e82530e89111dd9d14f9b970ab2cd3a1c2c78f0c4fbba4f4")
-    version("0.21.1", sha256="c5f5cba88bf0659554c41c909e1f78139f6fce8fa9315a29a23692b38ff9788a")
-    version("0.20.0", sha256="54f7a2bb2a7832c0446ad51d779806f07ec4ea2bb7c9aea4b83669fa97e778c4")
-    version("0.19.2", sha256="6f0f4f598c2b16746803c8bafef7c721c57e4844da752d36240c0acf97658014")
-    version("0.19.0", sha256="4697606cdf023c6b7fcb74e48aaf25cf282a1a00e339d2d274cf1b663748805b")
-    version("0.18.0", sha256="c975710ce8154b50f39a46aa3ea88d95b680191d1d9d4b5dd91eae7215e01814")
-    version("0.16.1", sha256="570d243f8cb068bf780461b9225d2e7bef7c90aa10d43cf908fe541fc92df8b6")
-    version("0.16.0", sha256="4013de6f8796ca9d2871218861823bd9878a8dfacd26e08ccf9afdd01bbad9f1")
 
     # Required dependencies
     # https://pandas.pydata.org/pandas-docs/stable/getting_started/install.html#python-version-support
-    depends_on("python@3.8:", type=("build", "run"), when="@1.4:")
-    depends_on("python@3.7.1:", type=("build", "run"), when="@1.2:")
-    depends_on("python@3.6.1:", type=("build", "run"), when="@1:")
-    depends_on("python@3.5.3:", type=("build", "run"), when="@0.25:")
+    depends_on("python@3.9:3.11", when="@2.1:", type=("build", "run"))
+    depends_on("python@3.8:3.11", when="@1.5:2.0", type=("build", "run"))
+    depends_on("python@3.8:3.10", when="@1.4", type=("build", "run"))
+    depends_on("python@:3.10", when="@1.3.3:1.3", type=("build", "run"))
+    depends_on("python@:3.9", when="@1.1.3:1.3.2", type=("build", "run"))
+    depends_on("python@:3.8", when="@0.25.2:1.1.2", type=("build", "run"))
+    depends_on("python@:3.7", when="@:0.25.1", type=("build", "run"))
 
     # pyproject.toml
-    depends_on("py-setuptools@61:", type="build", when="@2:")
-    depends_on("py-setuptools@51:", type="build", when="@1.3.2:")
-    depends_on("py-setuptools@38.6:", type="build", when="@1.3:")
-    depends_on("py-setuptools@24.2:", type="build")
-    depends_on("py-cython@0.29.33:2", type="build", when="@2:")
-    depends_on("py-cython@0.29.32:2", type="build", when="@1.4.4:")
-    depends_on("py-cython@0.29.30:2", type="build", when="@1.4.3:")
-    depends_on("py-cython@0.29.24:2", type="build", when="@1.3.4:")
-    depends_on("py-cython@0.29.21:2", type="build", when="@1.1.3:")
-    depends_on("py-cython@0.29.16:2", type="build", when="@1.1:")
-    depends_on("py-cython@0.29.13:2", type="build", when="@1:")
-    depends_on("py-versioneer+toml", type="build", when="@2:")
+    depends_on("py-meson-python@0.13.1", when="@2.1:", type="build")
+    depends_on("meson@1.0.1", when="@2.1:", type="build")
+    depends_on("py-cython@0.29.33:2", when="@2:", type="build")
+    depends_on("py-cython@0.29.32:2", when="@1.4.4:", type="build")
+    depends_on("py-cython@0.29.30:2", when="@1.4.3:", type="build")
+    depends_on("py-cython@0.29.24:2", when="@1.3.4:", type="build")
+    depends_on("py-cython@0.29.21:2", when="@1.1.3:", type="build")
+    depends_on("py-cython@0.29.16:2", when="@1.1:", type="build")
+    depends_on("py-cython@0.29.13:2", when="@1:", type="build")
+    depends_on("py-versioneer+toml", when="@2:", type="build")
 
     # https://pandas.pydata.org/pandas-docs/stable/getting_started/install.html#dependencies
-    depends_on("py-numpy@1.20.3:", type=("build", "run"), when="@1.5:")
-    depends_on("py-numpy@1.18.5:", type=("build", "run"), when="@1.4:")
-    depends_on("py-numpy@1.17.3:", type=("build", "run"), when="@1.3:")
-    depends_on("py-numpy@1.16.5:", type=("build", "run"), when="@1.2:")
-    depends_on("py-numpy@1.15.4:", type=("build", "run"), when="@1.1:")
-    depends_on("py-numpy@1.13.3:", type=("build", "run"), when="@0.25:")
+    depends_on("py-numpy@1.22.4:", when="@2.1:", type=("build", "run"))
+    depends_on("py-numpy@1.20.3:", when="@1.5:", type=("build", "run"))
+    depends_on("py-numpy@1.18.5:", when="@1.4:", type=("build", "run"))
+    depends_on("py-numpy@1.17.3:", when="@1.3:", type=("build", "run"))
+    depends_on("py-numpy@1.16.5:", when="@1.2:", type=("build", "run"))
+    depends_on("py-numpy@1.15.4:", when="@1.1:", type=("build", "run"))
+    depends_on("py-numpy@1.13.3:", when="@0.25:", type=("build", "run"))
     depends_on("py-numpy", type=("build", "run"))
     # 'NUMPY_IMPORT_ARRAY_RETVAL' was removed in numpy@1.19
-    depends_on("py-numpy@:1.18", type=("build", "run"), when="@:0.25")
-    depends_on("py-python-dateutil@2.8.2:", type=("build", "run"), when="@2:")
-    depends_on("py-python-dateutil@2.8.1:", type=("build", "run"), when="@1.4:")
-    depends_on("py-python-dateutil@2.7.3:", type=("build", "run"), when="@1.1:")
-    depends_on("py-python-dateutil@2.6.1:", type=("build", "run"), when="@0.25:")
+    depends_on("py-numpy@:1.18", when="@:0.25", type=("build", "run"))
+    depends_on("py-python-dateutil@2.8.2:", when="@2:", type=("build", "run"))
+    depends_on("py-python-dateutil@2.8.1:", when="@1.4:", type=("build", "run"))
+    depends_on("py-python-dateutil@2.7.3:", when="@1.1:", type=("build", "run"))
+    depends_on("py-python-dateutil@2.6.1:", when="@0.25:", type=("build", "run"))
     depends_on("py-python-dateutil", type=("build", "run"))
-    depends_on("py-pytz@2020.1:", type=("build", "run"), when="@1.4:")
-    depends_on("py-pytz@2017.3:", type=("build", "run"), when="@1.2:")
+    depends_on("py-pytz@2020.1:", when="@1.4:", type=("build", "run"))
+    depends_on("py-pytz@2017.3:", when="@1.2:", type=("build", "run"))
     depends_on("py-pytz@2017.2:", type=("build", "run"))
-    depends_on("py-tzdata@2022.1:", type=("build", "run"), when="@2:")
+    depends_on("py-tzdata@2022.1:", when="@2:", type=("build", "run"))
 
     # Recommended dependencies
-    # https://pandas.pydata.org/pandas-docs/stable/getting_started/install.html#recommended-dependencies
-    depends_on("py-numexpr@2.7.3:", type=("build", "run"), when="@1.5:")
-    depends_on("py-numexpr@2.7.1:", type=("build", "run"), when="@1.4:")
-    depends_on("py-numexpr@2.7.0:", type=("build", "run"), when="@1.3:")
-    depends_on("py-numexpr@2.6.8:", type=("build", "run"), when="@1.2:")
-    depends_on("py-numexpr@2.6.2:", type=("build", "run"), when="@0.25:")
+    # https://pandas.pydata.org/pandas-docs/stable/getting_started/install.html#performance-dependencies-recommended
+    depends_on("py-numexpr@2.8.0:", when="@2.1:", type=("build", "run"))
+    depends_on("py-numexpr@2.7.3:", when="@1.5:", type=("build", "run"))
+    depends_on("py-numexpr@2.7.1:", when="@1.4:", type=("build", "run"))
+    depends_on("py-numexpr@2.7.0:", when="@1.3:", type=("build", "run"))
+    depends_on("py-numexpr@2.6.8:", when="@1.2:", type=("build", "run"))
+    depends_on("py-numexpr@2.6.2:", when="@0.25:", type=("build", "run"))
     depends_on("py-numexpr", type=("build", "run"))
-    depends_on("py-bottleneck@1.3.2:", type=("build", "run"), when="@1.5:")
-    depends_on("py-bottleneck@1.3.1:", type=("build", "run"), when="@1.4:")
-    depends_on("py-bottleneck@1.2.1:", type=("build", "run"), when="@0.25:")
+    depends_on("py-bottleneck@1.3.4:", when="@2.1:", type=("build", "run"))
+    depends_on("py-bottleneck@1.3.2:", when="@1.5:", type=("build", "run"))
+    depends_on("py-bottleneck@1.3.1:", when="@1.4:", type=("build", "run"))
+    depends_on("py-bottleneck@1.2.1:", when="@0.25:", type=("build", "run"))
     depends_on("py-bottleneck", type=("build", "run"))
-    depends_on("py-numba@0.53.1:", type=("build", "run"), when="@2:")
+    depends_on("py-numba@0.55.2:", when="@2.1:", type=("build", "run"))
+    depends_on("py-numba@0.53.1:", when="@2.0:", type=("build", "run"))
 
     # Optional dependencies
     # https://pandas.pydata.org/pandas-docs/stable/getting_started/install.html#optional-dependencies
 
+    # Historical dependencies
+    depends_on("py-setuptools@61:", when="@2.0", type="build")
+    depends_on("py-setuptools@51:", when="@1.3.2:1", type="build")
+    depends_on("py-setuptools@38.6:", when="@1.3.0:1.3.1", type="build")
+    depends_on("py-setuptools@24.2:", when="@:1.2", type="build")
+
     skip_modules = ["pandas.tests", "pandas.plotting._matplotlib", "pandas.core._numba.kernels"]