more thread safety

fixes #30997

Instead of giving a penalty of 30 to all nodes when preferences
are not package specific, give a penalty of 100 to all targets
of a node where we have package specific preferences, if the target
is not explicitly preferred.

.github/workflows/bootstrap.yml

@@ -12,6 +12,7 @@ on:
       # built-in repository or documentation
       - 'var/spack/repos/builtin/**'
       - '!var/spack/repos/builtin/packages/clingo-bootstrap/**'
+      - '!var/spack/repos/builtin/packages/clingo/**'
       - '!var/spack/repos/builtin/packages/python/**'
       - '!var/spack/repos/builtin/packages/re2c/**'
       - 'lib/spack/docs/**'
@@ -19,6 +20,10 @@ on:
     # nightly at 2:16 AM
     - cron: '16 2 * * *'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  cancel-in-progress: true
+
 jobs:
 
   fedora-clingo-sources:
@@ -175,10 +180,11 @@ jobs:
           tree ~/.spack/bootstrap/store/
 
   macos-clingo-binaries:
-    runs-on: macos-latest
+    runs-on: ${{ matrix.macos-version }}
     strategy:
       matrix:
         python-version: ['3.5', '3.6', '3.7', '3.8', '3.9', '3.10']
+        macos-version: ['macos-10.15', 'macos-11', 'macos-12']
     if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
@@ -186,7 +192,7 @@ jobs:
           brew install tree
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
-      - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6
+      - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb
         with:
           python-version: ${{ matrix.python-version }}
       - name: Bootstrap clingo
@@ -205,7 +211,7 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
-      - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6
+      - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb
         with:
           python-version: ${{ matrix.python-version }}
       - name: Setup repo

.github/workflows/build-containers.yml

@@ -19,6 +19,10 @@ on:
   release:
     types: [published]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  cancel-in-progress: true
+
 jobs:
   deploy-images:
     runs-on: ubuntu-latest

.github/workflows/macos_python.yml

@@ -16,6 +16,10 @@ on:
       - '.github/workflows/macos_python.yml'
       # TODO: run if we touch any of the recipes involved in this
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  cancel-in-progress: true
+
 # GitHub Action Limits
 # https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions
 
@@ -26,7 +30,7 @@ jobs:
     runs-on: macos-latest
     steps:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6 # @v2
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb # @v2
       with:
         python-version: 3.9
     - name: spack install
@@ -42,7 +46,7 @@ jobs:
     timeout-minutes: 700
     steps:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6 # @v2
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb # @v2
       with:
         python-version: 3.9
     - name: spack install
@@ -56,7 +60,7 @@ jobs:
     runs-on: macos-latest
     steps:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6 # @v2
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb # @v2
       with:
         python-version: 3.9
     - name: spack install

.github/workflows/unit_tests.yaml

@@ -9,6 +9,11 @@ on:
     branches:
       - develop
       - releases/**
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  cancel-in-progress: true
+
 jobs:
   # Validate that the code can be run on all the Python versions
   # supported by Spack
@@ -16,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6 # @v2
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb # @v2
       with:
         python-version: '3.10'
     - name: Install Python Packages
@@ -34,12 +39,12 @@ jobs:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6 # @v2
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb # @v2
       with:
         python-version: '3.10'
     - name: Install Python packages
       run: |
-        pip install --upgrade pip six setuptools==62.3.4 types-six
+        pip install --upgrade pip six setuptools types-six
     - name: Setup git configuration
       run: |
         # Need this for the git tests to succeed.
@@ -109,7 +114,7 @@ jobs:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6 # @v2
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb # @v2
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install System packages
@@ -174,7 +179,7 @@ jobs:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6 # @v2
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb # @v2
       with:
         python-version: '3.10'
     - name: Install System packages
@@ -240,7 +245,7 @@ jobs:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6 # @v2
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb # @v2
       with:
         python-version: '3.10'
     - name: Install System packages
@@ -289,7 +294,7 @@ jobs:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6 # @v2
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb # @v2
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install Python packages
@@ -332,7 +337,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6 # @v2
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb # @v2
       with:
         python-version: '3.10'
     - name: Install Python packages
@@ -345,7 +350,7 @@ jobs:
           coverage run $(which spack) audit packages
           coverage combine
           coverage xml
-    - name: Package audits (wwithout coverage)
+    - name: Package audits (without coverage)
       if: ${{ needs.changes.outputs.with_coverage == 'false' }}
       run: |
           . share/spack/setup-env.sh

.github/workflows/windows_python.yml

@@ -9,6 +9,11 @@ on:
     branches:
       - develop
       - releases/**
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  cancel-in-progress: true
+
 defaults:
   run:
     shell:
@@ -18,7 +23,7 @@ jobs:
     runs-on: windows-latest
     steps:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb
       with:
         python-version: 3.9
     - name: Install Python Packages
@@ -36,12 +41,12 @@ jobs:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb
       with:
         python-version: 3.9
     - name: Install Python packages
       run: |
-        python -m pip install --upgrade pip six setuptools==62.3.4 flake8 isort>=4.3.5 mypy>=0.800 black pywin32 types-python-dateutil
+        python -m pip install --upgrade pip six setuptools flake8 isort>=4.3.5 mypy>=0.800 black pywin32 types-python-dateutil
     - name: Create local develop
       run: |
         .\spack\.github\workflows\setup_git.ps1
@@ -58,7 +63,7 @@ jobs:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb
       with:
         python-version: 3.9
     - name: Install Python packages
@@ -78,7 +83,7 @@ jobs:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb
       with:
         python-version: 3.9
     - name: Install Python packages
@@ -98,7 +103,7 @@ jobs:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb
       with:
         python-version: 3.9
     - name: Install Python packages
@@ -123,7 +128,7 @@ jobs:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb
       with:
         python-version: 3.9
     - name: Install Python packages
@@ -154,7 +159,7 @@ jobs:
       run:
         shell: pwsh
     steps:
-    - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6
+    - uses: actions/setup-python@d09bd5e6005b175076f227b13d9730d56e9dcfcb
       with:
         python-version: 3.9
     - name: Install Python packages

CHANGELOG.md

@@ -1,21 +1,3 @@
-# v0.18.1 (2022-07-19)
-
-### Spack Bugfixes
-* Fix several bugs related to bootstrapping (#30834,#31042,#31180)
-* Fix a regression that was causing spec hashes to differ between 
-  Python 2 and Python 3 (#31092)
-* Fixed compiler flags for oneAPI and DPC++ (#30856)
-* Fixed several issues related to concretization (#31142,#31153,#31170,#31226)
-* Improved support for Cray manifest file and `spack external find` (#31144,#31201,#31173.#31186)
-* Assign a version to openSUSE Tumbleweed according to the GLIBC version
-  in the system (#19895) 
-* Improved Dockerfile generation for `spack containerize` (#29741,#31321)
-* Fixed a few bugs related to concurrent execution of commands (#31509,#31493,#31477)  
-
-### Package updates
-* WarpX: add v22.06, fixed libs property (#30866,#31102)
-* openPMD: add v0.14.5, update recipe for @develop (#29484,#31023)
-
 # v0.18.0 (2022-05-28)
 
 `v0.18.0` is a major feature release.

lib/spack/docs/build_systems/inteloneapipackage.rst

@@ -84,8 +84,8 @@ build ``hdf5`` with Intel oneAPI MPI do::
 
   spack install hdf5 +mpi ^intel-oneapi-mpi
 
-Using an Externally Installed oneAPI
-====================================
+Using Externally Installed oneAPI Tools
+=======================================
 
 Spack can also use oneAPI tools that are manually installed with
 `Intel Installers`_.  The procedures for configuring Spack to use
@@ -110,7 +110,7 @@ Another option is to manually add the configuration to
 Libraries
 ---------
 
-If you want Spack to use MKL that you have installed without Spack in
+If you want Spack to use oneMKL that you have installed without Spack in
 the default location, then add the following to
 ``~/.spack/packages.yaml``, adjusting the version as appropriate::
 
@@ -139,7 +139,7 @@ You can also use Spack-installed libraries. For example::
   spack load intel-oneapi-mkl
 
 Will update your environment CPATH, LIBRARY_PATH, and other
-environment variables for building an application with MKL.
+environment variables for building an application with oneMKL.
 
 More information
 ================

lib/spack/docs/build_systems/intelpackage.rst

@@ -15,6 +15,9 @@ IntelPackage
 Intel packages in Spack
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
+This is an earlier version of Intel software development tools and has
+now been replaced by Intel oneAPI Toolkits.
+
 Spack can install and use several software development products offered by Intel.
 Some of these are available under no-cost terms, others require a paid license.
 All share the same basic steps for configuration, installation, and, where

lib/spack/docs/containers.rst

@@ -109,9 +109,10 @@ Spack Images on Docker Hub
 --------------------------
 
 Docker images with Spack preinstalled and ready to be used are
-built on `Docker Hub <https://hub.docker.com/u/spack>`_
-at every push to ``develop`` or to a release branch. The OS that
-are currently supported are summarized in the table below:
+built when a release is tagged, or nightly on ``develop``. The images
+are then pushed both to `Docker Hub <https://hub.docker.com/u/spack>`_
+and to `GitHub Container Registry <https://github.com/orgs/spack/packages?repo_name=spack>`_.
+The OS that are currently supported are summarized in the table below:
 
 .. _containers-supported-os:
 
@@ -121,22 +122,31 @@ are currently supported are summarized in the table below:
    * - Operating System
      - Base Image
      - Spack Image
-   * - Ubuntu 16.04
-     - ``ubuntu:16.04``
-     - ``spack/ubuntu-xenial``
    * - Ubuntu 18.04
      - ``ubuntu:18.04``
      - ``spack/ubuntu-bionic``
+   * - Ubuntu 20.04
+     - ``ubuntu:20.04``
+     - ``spack/ubuntu-focal``
+   * - Ubuntu 22.04
+     - ``ubuntu:22.04``
+     - ``spack/ubuntu-jammy``
    * - CentOS 7
      - ``centos:7``
      - ``spack/centos7``
+   * - CentOS Stream
+     - ``quay.io/centos/centos:stream``
+     - ``spack/centos-stream``
    * - openSUSE Leap
      - ``opensuse/leap``
      - ``spack/leap15``
+   * - Amazon Linux 2
+     - ``amazonlinux:2``
+     - ``spack/amazon-linux``
 
 All the images are tagged with the corresponding release of Spack:
 
-.. image:: dockerhub_spack.png
+.. image:: images/ghcr_spack.png
 
 with the exception of the ``latest`` tag that points to the HEAD
 of the ``develop`` branch. These images are available for anyone

lib/spack/docs/developer_guide.rst

@@ -151,7 +151,7 @@ Package-related modules
 ^^^^^^^^^^^^^^^^^^^^^^^
 
 :mod:`spack.package`
-  Contains the :class:`~spack.package.Package` class, which
+  Contains the :class:`~spack.package_base.Package` class, which
   is the superclass for all packages in Spack.  Methods on ``Package``
   implement all phases of the :ref:`package lifecycle
   <package-lifecycle>` and manage the build process.

lib/spack/docs/packaging_guide.rst

@@ -1070,13 +1070,32 @@ Commits
 
 Submodules
   You can supply ``submodules=True`` to cause Spack to fetch submodules
-  recursively along with the repository at fetch time. For more information
-  about git submodules see the manpage of git: ``man git-submodule``.
+  recursively along with the repository at fetch time.
 
   .. code-block:: python
 
      version('1.0.1', tag='v1.0.1', submodules=True)
 
+  If a package has needs more fine-grained control over submodules, define
+  ``submodules`` to be a callable function that takes the package instance as
+  its only argument.  The function should return a list of submodules to be fetched.
+
+  .. code-block:: python
+
+     def submodules(package):
+         submodules = []
+         if "+variant-1" in package.spec:
+             submodules.append("submodule_for_variant_1")
+         if "+variant-2" in package.spec:
+             submodules.append("submodule_for_variant_2")
+         return submodules
+
+
+      class MyPackage(Package):
+          version("0.1.0", submodules=submodules)
+
+  For more information about git submodules see the manpage of git: ``man
+  git-submodule``.
 
 .. _github-fetch:
 
@@ -2393,9 +2412,9 @@ Influence how dependents are built or run
 
 Spack provides a mechanism for dependencies to influence the
 environment of their dependents by overriding  the
-:meth:`setup_dependent_run_environment <spack.package.PackageBase.setup_dependent_run_environment>`
+:meth:`setup_dependent_run_environment <spack.package_base.PackageBase.setup_dependent_run_environment>`
 or the
-:meth:`setup_dependent_build_environment <spack.package.PackageBase.setup_dependent_build_environment>`
+:meth:`setup_dependent_build_environment <spack.package_base.PackageBase.setup_dependent_build_environment>`
 methods.
 The Qt package, for instance, uses this call:
 
@@ -2417,7 +2436,7 @@ will have the ``PYTHONPATH``, ``PYTHONHOME`` and ``PATH`` environment
 variables set appropriately before starting the installation. To make things
 even simpler the ``python setup.py`` command is also inserted into the module
 scope of dependents by overriding a third method called
-:meth:`setup_dependent_package <spack.package.PackageBase.setup_dependent_package>`
+:meth:`setup_dependent_package <spack.package_base.PackageBase.setup_dependent_package>`
 :
 
 .. literalinclude:: _spack_root/var/spack/repos/builtin/packages/python/package.py
@@ -2775,6 +2794,256 @@ Suppose a user invokes ``spack install`` like this:
 Spack will fail with a constraint violation, because the version of
 MPICH requested is too low for the ``mpi`` requirement in ``foo``.
 
+.. _custom-attributes:
+
+------------------
+Custom attributes
+------------------
+
+Often a package will need to provide attributes for dependents to query
+various details about what it provides.  While any number of custom defined
+attributes can be implemented by a package, the four specific attributes
+described below are always available on every package with default
+implementations and the ability to customize with alternate implementations
+in the case of virtual packages provided:
+
+=========== =========================================== =====================
+Attribute   Purpose                                     Default
+=========== =========================================== =====================
+``home``    The installation path for the package       ``spec.prefix``
+``command`` An executable command for the package       | ``spec.name`` found
+                                                          in
+                                                        | ``.home.bin``
+``headers`` A list of headers provided by the package   | All headers
+                                                          searched
+                                                        | recursively in
+                                                          ``.home.include``
+``libs``    A list of libraries provided by the package | ``lib{spec.name}``
+                                                          searched
+                                                        | recursively in
+                                                          ``.home`` starting
+                                                        | with ``lib``,
+                                                          ``lib64``, then the
+                                                        | rest of ``.home``
+=========== =========================================== =====================
+
+Each of these can be customized by implementing the relevant attribute
+as a ``@property`` in the package's class:
+
+.. code-block:: python
+   :linenos:
+
+   class Foo(Package):
+       ...
+       @property
+       def libs(self):
+           # The library provided by Foo is libMyFoo.so
+           return find_libraries('libMyFoo', root=self.home, recursive=True)
+
+A package may also provide a custom implementation of each attribute
+for the virtual packages it provides by implementing the
+``virtualpackagename_attributename`` property in the package's class.
+The implementation used is the first one found from:
+
+#. Specialized virtual: ``Package.virtualpackagename_attributename``
+#. Generic package: ``Package.attributename``
+#. Default
+
+The use of customized attributes is demonstrated in the next example.
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Example: Customized attributes for virtual packages
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Consider a package ``foo`` that can optionally provide two virtual
+packages ``bar`` and ``baz``.  When both are enabled the installation tree
+appears as follows:
+
+.. code-block:: console
+
+   include/foo.h
+   include/bar/bar.h
+   lib64/libFoo.so
+   lib64/libFooBar.so
+   baz/include/baz/baz.h
+   baz/lib/libFooBaz.so
+
+The install tree shows that ``foo`` is providing the header ``include/foo.h``
+and library ``lib64/libFoo.so`` in it's install prefix.  The virtual
+package ``bar`` is providing ``include/bar/bar.h`` and library
+``lib64/libFooBar.so``, also in ``foo``'s install prefix.  The ``baz``
+package, however, is provided in the ``baz`` subdirectory of ``foo``'s
+prefix with the ``include/baz/baz.h`` header and ``lib/libFooBaz.so``
+library.  Such a package could implement the optional attributes as
+follows:
+
+.. code-block:: python
+   :linenos:
+
+   class Foo(Package):
+       ...
+       variant('bar', default=False, description='Enable the Foo implementation of bar')
+       variant('baz', default=False, description='Enable the Foo implementation of baz')
+       ...
+       provides('bar', when='+bar')
+       provides('baz', when='+baz')
+       ....
+
+       # Just the foo headers
+       @property
+       def headers(self):
+           return find_headers('foo', root=self.home.include, recursive=False)
+
+       # Just the foo libraries
+       @property
+       def libs(self):
+           return find_libraries('libFoo', root=self.home, recursive=True)
+
+       # The header provided by the bar virutal package
+       @property
+       def bar_headers(self):
+           return find_headers('bar/bar.h', root=self.home.include, recursive=False)
+
+       # The libary provided by the bar virtual package
+       @property
+       def bar_libs(self):
+           return find_libraries('libFooBar', root=sef.home, recursive=True)
+
+       # The baz virtual package home
+       @property
+       def baz_home(self):
+           return self.prefix.baz
+
+       # The header provided by the baz virtual package
+       @property
+       def baz_headers(self):
+           return find_headers('baz/baz', root=self.baz_home.include, recursive=False)
+
+       # The library provided by the baz virtual package
+       @property
+       def baz_libs(self):
+           return find_libraries('libFooBaz', root=self.baz_home, recursive=True)
+
+Now consider another package, ``foo-app``, depending on all three:
+
+.. code-block:: python
+   :linenos:
+
+   class FooApp(CMakePackage):
+       ...
+       depends_on('foo')
+       depends_on('bar')
+       depends_on('baz')
+
+The resulting spec objects for it's dependencies shows the result of
+the above attribute implementations:
+
+.. code-block:: python
+
+   # The core headers and libraries of the foo package
+
+   >>> spec['foo']
+   foo@1.0%gcc@11.3.1+bar+baz arch=linux-fedora35-haswell
+   >>> spec['foo'].prefix
+   '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6'
+
+   # home defaults to the package install prefix without an explicit implementation
+   >>> spec['foo'].home
+   '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6'
+
+   # foo headers from the foo prefix
+   >>> spec['foo'].headers
+   HeaderList([
+       '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/include/foo.h',
+   ])
+
+   # foo include directories from the foo prefix
+   >>> spec['foo'].headers.directories
+   ['/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/include']
+
+   # foo libraries from the foo prefix
+   >>> spec['foo'].libs
+   LibraryList([
+       '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/lib64/libFoo.so',
+   ])
+
+   # foo library directories from the foo prefix
+   >>> spec['foo'].libs.directories
+   ['/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/lib64']
+
+.. code-block:: python
+
+   # The virtual bar package in the same prefix as foo
+
+   # bar resolves to the foo package
+   >>> spec['bar']
+   foo@1.0%gcc@11.3.1+bar+baz arch=linux-fedora35-haswell
+   >>> spec['bar'].prefix
+   '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6'
+
+   # home defaults to the foo prefix without either a Foo.bar_home
+   # or Foo.home implementation
+   >>> spec['bar'].home
+   '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6'
+
+   # bar header in the foo prefix
+   >>> spec['bar'].headers
+   HeaderList([
+       '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/include/bar/bar.h'
+   ])
+
+   # bar include dirs from the foo prefix
+   >>> spec['bar'].headers.directories
+   ['/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/include']
+
+   # bar library from the foo prefix
+   >>> spec['bar'].libs
+   LibraryList([
+       '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/lib64/libFooBar.so'
+   ])
+
+   # bar library directories from the foo prefix
+   >>> spec['bar'].libs.directories
+   ['/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/lib64']
+
+.. code-block:: python
+
+   # The virtual baz package in a subdirectory of foo's prefix
+
+   # baz resolves to the foo package
+   >>> spec['baz']
+   foo@1.0%gcc@11.3.1+bar+baz arch=linux-fedora35-haswell
+   >>> spec['baz'].prefix
+   '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6'
+
+   # baz_home implementation provides the subdirectory inside the foo prefix
+   >>> spec['baz'].home
+   '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz'
+
+   # baz headers in the baz subdirectory of the foo prefix
+   >>> spec['baz'].headers
+   HeaderList([
+       '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz/include/baz/baz.h'
+   ])
+
+   # baz include directories in the baz subdirectory of the foo prefix
+   >>> spec['baz'].headers.directories
+   [
+       '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz/include'
+   ]
+
+   # baz libraries in the baz subdirectory of the foo prefix
+   >>> spec['baz'].libs
+   LibraryList([
+       '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz/lib/libFooBaz.so'
+   ])
+
+   # baz library directories in the baz subdirectory of the foo porefix
+   >>> spec['baz'].libs.directories
+   [
+       '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz/lib'
+   ]
+    
 .. _abstract-and-concrete:
 
 -------------------------
@@ -3022,7 +3291,7 @@ The classes that are currently provided by Spack are:
 +----------------------------------------------------------+----------------------------------+
 |     **Base Class**                                       |           **Purpose**            |
 +==========================================================+==================================+
-| :class:`~spack.package.Package`                          | General base class not           |
+| :class:`~spack.package_base.Package`                     | General base class not           |
 |                                                          | specialized for any build system |
 +----------------------------------------------------------+----------------------------------+
 | :class:`~spack.build_systems.makefile.MakefilePackage`   | Specialized class for packages   |
@@ -3153,7 +3422,7 @@ for the install phase is:
     For those not used to Python instance methods, this is the
     package itself.  In this case it's an instance of ``Foo``, which
     extends ``Package``.  For API docs on Package objects, see
-    :py:class:`Package <spack.package.Package>`.
+    :py:class:`Package <spack.package_base.Package>`.
 
 ``spec``
     This is the concrete spec object created by Spack from an
@@ -5476,6 +5745,24 @@ Version Lists
 
 Spack packages should list supported versions with the newest first.
 
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Using ``home`` vs ``prefix``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``home`` and ``prefix`` are both attributes that can be queried on a
+package's dependencies, often when passing configure arguments pointing to the
+location of a dependency.  The difference is that while ``prefix`` is the
+location on disk where a concrete package resides, ``home`` is the `logical`
+location that a package resides, which may be different than ``prefix`` in
+the case of virtual packages or other special circumstances.  For most use
+cases inside a package, it's dependency locations can be accessed via either
+``self.spec['foo'].home`` or ``self.spec['foo'].prefix``.  Specific packages
+that should be consumed by dependents via ``.home`` instead of ``.prefix``
+should be noted in their respective documentation.
+
+See :ref:`custom-attributes` for more details and an example implementing
+a custom ``home`` attribute.
+
 ---------------------------
 Packaging workflow commands
 ---------------------------

lib/spack/docs/tables/system_prerequisites.csv

@@ -7,7 +7,7 @@ bash, , , Compiler wrappers
 tar, , , Extract/create archives
 gzip, , , Compress/Decompress archives
 unzip, , , Compress/Decompress archives
-bzip, , , Compress/Decompress archives
+bzip2, , , Compress/Decompress archives
 xz, , , Compress/Decompress archives
 zstd, , Optional, Compress/Decompress archives
 file, , , Create/Use Buildcaches
@@ -15,4 +15,4 @@ gnupg2, , , Sign/Verify Buildcaches
 git, , , Manage Software Repositories
 svn, , Optional, Manage Software Repositories
 hg, , Optional, Manage Software Repositories
-Python header files, , Optional (e.g. ``python3-dev`` on Debian), Bootstrapping from sources
+Python header files, , Optional (e.g. ``python3-dev`` on Debian), Bootstrapping from sources

lib/spack/llnl/util/filesystem.py

@@ -308,6 +308,68 @@ def change_sed_delimiter(old_delim, new_delim, *filenames):
         filter_file(double_quoted, '"%s"' % repl, f)
 
 
+@contextmanager
+def exploding_archive_catch(stage):
+    # Check for an exploding tarball, i.e. one that doesn't expand to
+    # a single directory.  If the tarball *didn't* explode, move its
+    # contents to the staging source directory & remove the container
+    # directory.  If the tarball did explode, just rename the tarball
+    # directory to the staging source directory.
+    #
+    # NOTE: The tar program on Mac OS X will encode HFS metadata in
+    # hidden files, which can end up *alongside* a single top-level
+    # directory.  We initially ignore presence of hidden files to
+    # accomodate these "semi-exploding" tarballs but ensure the files
+    # are copied to the source directory.
+
+    # Expand all tarballs in their own directory to contain
+    # exploding tarballs.
+    tarball_container = os.path.join(stage.path,
+                                     "spack-expanded-archive")
+    mkdirp(tarball_container)
+    orig_dir = os.getcwd()
+    os.chdir(tarball_container)
+    try:
+        yield
+        # catch an exploding archive on sucessful extraction
+        os.chdir(orig_dir)
+        exploding_archive_handler(tarball_container, stage)
+    except Exception as e:
+        # return current directory context to previous on failure
+        os.chdir(orig_dir)
+        raise e
+
+
+@system_path_filter
+def exploding_archive_handler(tarball_container, stage):
+    """
+    Args:
+        tarball_container: where the archive was expanded to
+        stage: Stage object referencing filesystem location
+            where archive is being expanded
+    """
+    files = os.listdir(tarball_container)
+    non_hidden = [f for f in files if not f.startswith('.')]
+    if len(non_hidden) == 1:
+        src = os.path.join(tarball_container, non_hidden[0])
+        if os.path.isdir(src):
+            stage.srcdir = non_hidden[0]
+            shutil.move(src, stage.source_path)
+            if len(files) > 1:
+                files.remove(non_hidden[0])
+                for f in files:
+                    src = os.path.join(tarball_container, f)
+                    dest = os.path.join(stage.path, f)
+                    shutil.move(src, dest)
+            os.rmdir(tarball_container)
+        else:
+            # This is a non-directory entry (e.g., a patch file) so simply
+            # rename the tarball container to be the source path.
+            shutil.move(tarball_container, stage.source_path)
+    else:
+        shutil.move(tarball_container, stage.source_path)
+
+
 @system_path_filter(arg_slice=slice(1))
 def get_owner_uid(path, err_msg=None):
     if not os.path.exists(path):

lib/spack/spack/__init__.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 #: (major, minor, micro, dev release) tuple
-spack_version_info = (0, 18, 1)
+spack_version_info = (0, 19, 0, 'dev0')
 
 #: PEP440 canonical <major>.<minor>.<micro>.<devN> string
 spack_version = '.'.join(str(s) for s in spack_version_info)

lib/spack/spack/analyzers/libabigail.py

@@ -12,7 +12,7 @@
 import spack.error
 import spack.hooks
 import spack.monitor
-import spack.package
+import spack.package_base
 import spack.repo
 import spack.util.executable
 

lib/spack/spack/bootstrap.py

@@ -80,32 +80,41 @@ def _try_import_from_store(module, query_spec, query_info=None):
 
     for candidate_spec in installed_specs:
         pkg = candidate_spec['python'].package
-        module_paths = {
+        module_paths = [
             os.path.join(candidate_spec.prefix, pkg.purelib),
             os.path.join(candidate_spec.prefix, pkg.platlib),
-        }
-        sys.path.extend(module_paths)
+        ]  # type: list[str]
+        path_before = list(sys.path)
+        # NOTE: try module_paths first and last, last allows an existing version in path
+        # to be picked up and used, possibly depending on something in the store, first
+        # allows the bootstrap version to work when an incompatible version is in
+        # sys.path
+        orders = [
+            module_paths + sys.path,
+            sys.path + module_paths,
+        ]
+        for path in orders:
+            sys.path = path
+            try:
+                _fix_ext_suffix(candidate_spec)
+                if _python_import(module):
+                    msg = ('[BOOTSTRAP MODULE {0}] The installed spec "{1}/{2}" '
+                           'provides the "{0}" Python module').format(
+                        module, query_spec, candidate_spec.dag_hash()
+                    )
+                    tty.debug(msg)
+                    if query_info is not None:
+                        query_info['spec'] = candidate_spec
+                    return True
+            except Exception as e:
+                msg = ('unexpected error while trying to import module '
+                       '"{0}" from spec "{1}" [error="{2}"]')
+                tty.warn(msg.format(module, candidate_spec, str(e)))
+            else:
+                msg = "Spec {0} did not provide module {1}"
+                tty.warn(msg.format(candidate_spec, module))
 
-        try:
-            _fix_ext_suffix(candidate_spec)
-            if _python_import(module):
-                msg = ('[BOOTSTRAP MODULE {0}] The installed spec "{1}/{2}" '
-                       'provides the "{0}" Python module').format(
-                    module, query_spec, candidate_spec.dag_hash()
-                )
-                tty.debug(msg)
-                if query_info is not None:
-                    query_info['spec'] = candidate_spec
-                return True
-        except Exception as e:
-            msg = ('unexpected error while trying to import module '
-                   '"{0}" from spec "{1}" [error="{2}"]')
-            tty.warn(msg.format(module, candidate_spec, str(e)))
-        else:
-            msg = "Spec {0} did not provide module {1}"
-            tty.warn(msg.format(candidate_spec, module))
-
-        sys.path = sys.path[:-3]
+        sys.path = path_before
 
     return False
 

lib/spack/spack/build_environment.py

@@ -55,7 +55,7 @@
 import spack.config
 import spack.install_test
 import spack.main
-import spack.package
+import spack.package_base
 import spack.paths
 import spack.platforms
 import spack.repo
@@ -722,7 +722,7 @@ def get_std_cmake_args(pkg):
         package were a CMakePackage instance.
 
     Args:
-        pkg (spack.package.PackageBase): package under consideration
+        pkg (spack.package_base.PackageBase): package under consideration
 
     Returns:
         list: arguments for cmake
@@ -738,7 +738,7 @@ def get_std_meson_args(pkg):
         package were a MesonPackage instance.
 
     Args:
-        pkg (spack.package.PackageBase): package under consideration
+        pkg (spack.package_base.PackageBase): package under consideration
 
     Returns:
         list: arguments for meson
@@ -748,12 +748,12 @@ def get_std_meson_args(pkg):
 
 def parent_class_modules(cls):
     """
-    Get list of superclass modules that descend from spack.package.PackageBase
+    Get list of superclass modules that descend from spack.package_base.PackageBase
 
     Includes cls.__module__
     """
-    if (not issubclass(cls, spack.package.PackageBase) or
-        issubclass(spack.package.PackageBase, cls)):
+    if (not issubclass(cls, spack.package_base.PackageBase) or
+        issubclass(spack.package_base.PackageBase, cls)):
         return []
     result = []
     module = sys.modules.get(cls.__module__)
@@ -771,7 +771,7 @@ def load_external_modules(pkg):
     associated with them.
 
     Args:
-        pkg (spack.package.PackageBase): package to load deps for
+        pkg (spack.package_base.PackageBase): package to load deps for
     """
     for dep in list(pkg.spec.traverse()):
         external_modules = dep.external_modules or []
@@ -1109,7 +1109,7 @@ def start_build_process(pkg, function, kwargs):
 
     Args:
 
-        pkg (spack.package.PackageBase): package whose environment we should set up the
+        pkg (spack.package_base.PackageBase): package whose environment we should set up the
             child process for.
         function (typing.Callable): argless function to run in the child
             process.
@@ -1234,7 +1234,7 @@ def make_stack(tb, stack=None):
         if 'self' in frame.f_locals:
             # Find the first proper subclass of PackageBase.
             obj = frame.f_locals['self']
-            if isinstance(obj, spack.package.PackageBase):
+            if isinstance(obj, spack.package_base.PackageBase):
                 break
 
     # We found obj, the Package implementation we care about.

lib/spack/spack/build_systems/aspell_dict.py

@@ -9,7 +9,7 @@
 
 from spack.build_systems.autotools import AutotoolsPackage
 from spack.directives import extends
-from spack.package import ExtensionError
+from spack.package_base import ExtensionError
 from spack.util.executable import which
 
 

lib/spack/spack/build_systems/autotools.py

@@ -16,7 +16,7 @@
 from spack.build_environment import InstallError
 from spack.directives import conflicts, depends_on
 from spack.operating_systems.mac_os import macos_version
-from spack.package import PackageBase, run_after, run_before
+from spack.package_base import PackageBase, run_after, run_before
 from spack.util.executable import Executable
 from spack.version import Version
 

lib/spack/spack/build_systems/cached_cmake.py

@@ -8,7 +8,7 @@
 from llnl.util.filesystem import install, mkdirp
 
 from spack.build_systems.cmake import CMakePackage
-from spack.package import run_after
+from spack.package_base import run_after
 
 
 def cmake_cache_path(name, value, comment=""):

lib/spack/spack/build_systems/cmake.py

@@ -18,7 +18,7 @@
 
 import spack.build_environment
 from spack.directives import conflicts, depends_on, variant
-from spack.package import InstallError, PackageBase, run_after
+from spack.package_base import InstallError, PackageBase, run_after
 from spack.util.path import convert_to_posix_path
 
 # Regex to extract the primary generator from the CMake generator

lib/spack/spack/build_systems/cuda.py

@@ -6,7 +6,7 @@
 import spack.variant
 from spack.directives import conflicts, depends_on, variant
 from spack.multimethod import when
-from spack.package import PackageBase
+from spack.package_base import PackageBase
 
 
 class CudaPackage(PackageBase):
@@ -37,6 +37,7 @@ class CudaPackage(PackageBase):
     variant('cuda_arch',
             description='CUDA architecture',
             values=spack.variant.any_combination_of(*cuda_arch_values),
+            sticky=True,
             when='+cuda')
 
     # https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#nvcc-examples

lib/spack/spack/build_systems/gnu.py

@@ -3,14 +3,16 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import spack.package
+from typing import Optional
+
+import spack.package_base
 import spack.util.url
 
 
-class GNUMirrorPackage(spack.package.PackageBase):
+class GNUMirrorPackage(spack.package_base.PackageBase):
     """Mixin that takes care of setting url and mirrors for GNU packages."""
     #: Path of the package in a GNU mirror
-    gnu_mirror_path = None
+    gnu_mirror_path = None  # type: Optional[str]
 
     #: List of GNU mirrors used by Spack
     base_mirrors = [

lib/spack/spack/build_systems/intel.py

@@ -26,7 +26,7 @@
 
 import spack.error
 from spack.build_environment import dso_suffix
-from spack.package import InstallError, PackageBase, run_after
+from spack.package_base import InstallError, PackageBase, run_after
 from spack.util.environment import EnvironmentModifications
 from spack.util.executable import Executable
 from spack.util.prefix import Prefix
@@ -1115,7 +1115,7 @@ def _setup_dependent_env_callback(
                 raise InstallError('compilers_of_client arg required for MPI')
 
     def setup_dependent_package(self, module, dep_spec):
-        # https://spack.readthedocs.io/en/latest/spack.html#spack.package.PackageBase.setup_dependent_package
+        # https://spack.readthedocs.io/en/latest/spack.html#spack.package_base.PackageBase.setup_dependent_package
         # Reminder: "module" refers to Python module.
         # Called before the install() method of dependents.
 
@@ -1259,6 +1259,14 @@ def install(self, spec, prefix):
         for f in glob.glob('%s/intel*log' % tmpdir):
             install(f, dst)
 
+    @run_after('install')
+    def validate_install(self):
+        # Sometimes the installer exits with an error but doesn't pass a
+        # non-zero exit code to spack. Check for the existence of a 'bin'
+        # directory to catch this error condition.
+        if not os.path.exists(self.prefix.bin):
+            raise InstallError('The installer has failed to install anything.')
+
     @run_after('install')
     def configure_rpath(self):
         if '+rpath' not in self.spec:

lib/spack/spack/build_systems/lua.py

@@ -10,7 +10,7 @@
 
 from spack.directives import depends_on, extends
 from spack.multimethod import when
-from spack.package import PackageBase
+from spack.package_base import PackageBase
 from spack.util.executable import Executable
 
 

lib/spack/spack/build_systems/makefile.py

@@ -11,7 +11,7 @@
 from llnl.util.filesystem import working_dir
 
 from spack.directives import conflicts
-from spack.package import PackageBase, run_after
+from spack.package_base import PackageBase, run_after
 
 
 class MakefilePackage(PackageBase):

lib/spack/spack/build_systems/maven.py

@@ -7,7 +7,7 @@
 from llnl.util.filesystem import install_tree, working_dir
 
 from spack.directives import depends_on
-from spack.package import PackageBase, run_after
+from spack.package_base import PackageBase, run_after
 from spack.util.executable import which
 
 

lib/spack/spack/build_systems/meson.py

@@ -11,7 +11,7 @@
 from llnl.util.filesystem import working_dir
 
 from spack.directives import depends_on, variant
-from spack.package import PackageBase, run_after
+from spack.package_base import PackageBase, run_after
 
 
 class MesonPackage(PackageBase):

lib/spack/spack/build_systems/octave.py

@@ -6,7 +6,7 @@
 import inspect
 
 from spack.directives import extends
-from spack.package import PackageBase, run_after
+from spack.package_base import PackageBase, run_after
 
 
 class OctavePackage(PackageBase):

lib/spack/spack/build_systems/oneapi.py

@@ -14,7 +14,7 @@
 
 from llnl.util.filesystem import find_headers, find_libraries, join_path
 
-from spack.package import Package
+from spack.package_base import Package
 from spack.util.environment import EnvironmentModifications
 from spack.util.executable import Executable
 

lib/spack/spack/build_systems/perl.py

@@ -10,7 +10,7 @@
 from llnl.util.filesystem import filter_file
 
 from spack.directives import extends
-from spack.package import PackageBase, run_after
+from spack.package_base import PackageBase, run_after
 from spack.util.executable import Executable
 
 

lib/spack/spack/build_systems/python.py

@@ -6,6 +6,7 @@
 import os
 import re
 import shutil
+from typing import Optional
 
 import llnl.util.tty as tty
 from llnl.util.filesystem import (
@@ -19,13 +20,13 @@
 from llnl.util.lang import match_predicate
 
 from spack.directives import depends_on, extends
-from spack.package import PackageBase, run_after
+from spack.package_base import PackageBase, run_after
 
 
 class PythonPackage(PackageBase):
     """Specialized class for packages that are built using pip."""
     #: Package name, version, and extension on PyPI
-    pypi = None
+    pypi = None  # type: Optional[str]
 
     maintainers = ['adamjstewart']
 
@@ -46,7 +47,7 @@ class PythonPackage(PackageBase):
     # package manually
     depends_on('py-wheel', type='build')
 
-    py_namespace = None
+    py_namespace = None  # type: Optional[str]
 
     @staticmethod
     def _std_args(cls):

lib/spack/spack/build_systems/qmake.py

@@ -9,7 +9,7 @@
 from llnl.util.filesystem import working_dir
 
 from spack.directives import depends_on
-from spack.package import PackageBase, run_after
+from spack.package_base import PackageBase, run_after
 
 
 class QMakePackage(PackageBase):

lib/spack/spack/build_systems/r.py

@@ -5,9 +5,10 @@
 
 
 import inspect
+from typing import Optional
 
 from spack.directives import extends
-from spack.package import PackageBase, run_after
+from spack.package_base import PackageBase, run_after
 
 
 class RPackage(PackageBase):
@@ -28,10 +29,10 @@ class RPackage(PackageBase):
     # package attributes that can be expanded to set the homepage, url,
     # list_url, and git values
     # For CRAN packages
-    cran = None
+    cran = None  # type: Optional[str]
 
     # For Bioconductor packages
-    bioc = None
+    bioc = None  # type: Optional[str]
 
     maintainers = ['glennpj']
 

lib/spack/spack/build_systems/racket.py

@@ -3,13 +3,14 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import os
+from typing import Optional
 
 import llnl.util.tty as tty
 from llnl.util.filesystem import working_dir
 
 from spack.build_environment import SPACK_NO_PARALLEL_MAKE, determine_number_of_jobs
 from spack.directives import extends
-from spack.package import PackageBase
+from spack.package_base import PackageBase
 from spack.util.environment import env_flag
 from spack.util.executable import Executable, ProcessError
 
@@ -36,8 +37,8 @@ class RacketPackage(PackageBase):
     extends('racket')
 
     pkgs = False
-    subdirectory = None
-    name = None
+    subdirectory = None  # type: Optional[str]
+    name = None  # type: Optional[str]
     parallel = True
 
     @property

lib/spack/spack/build_systems/rocm.py

@@ -77,7 +77,7 @@
 
 import spack.variant
 from spack.directives import conflicts, depends_on, variant
-from spack.package import PackageBase
+from spack.package_base import PackageBase
 
 
 class ROCmPackage(PackageBase):

lib/spack/spack/build_systems/ruby.py

@@ -7,7 +7,7 @@
 import inspect
 
 from spack.directives import extends
-from spack.package import PackageBase, run_after
+from spack.package_base import PackageBase, run_after
 
 
 class RubyPackage(PackageBase):

lib/spack/spack/build_systems/scons.py

@@ -7,7 +7,7 @@
 import inspect
 
 from spack.directives import depends_on
-from spack.package import PackageBase, run_after
+from spack.package_base import PackageBase, run_after
 
 
 class SConsPackage(PackageBase):

lib/spack/spack/build_systems/sip.py

@@ -11,7 +11,7 @@
 from llnl.util.filesystem import find, join_path, working_dir
 
 from spack.directives import depends_on, extends
-from spack.package import PackageBase, run_after
+from spack.package_base import PackageBase, run_after
 
 
 class SIPPackage(PackageBase):

lib/spack/spack/build_systems/sourceforge.py

@@ -3,15 +3,17 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import spack.package
+from typing import Optional
+
+import spack.package_base
 import spack.util.url
 
 
-class SourceforgePackage(spack.package.PackageBase):
+class SourceforgePackage(spack.package_base.PackageBase):
     """Mixin that takes care of setting url and mirrors for Sourceforge
        packages."""
     #: Path of the package in a Sourceforge mirror
-    sourceforge_mirror_path = None
+    sourceforge_mirror_path = None  # type: Optional[str]
 
     #: List of Sourceforge mirrors used by Spack
     base_mirrors = [

lib/spack/spack/build_systems/sourceware.py

@@ -2,16 +2,17 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from typing import Optional
 
-import spack.package
+import spack.package_base
 import spack.util.url
 
 
-class SourcewarePackage(spack.package.PackageBase):
+class SourcewarePackage(spack.package_base.PackageBase):
     """Mixin that takes care of setting url and mirrors for Sourceware.org
        packages."""
     #: Path of the package in a Sourceware mirror
-    sourceware_mirror_path = None
+    sourceware_mirror_path = None  # type: Optional[str]
 
     #: List of Sourceware mirrors used by Spack
     base_mirrors = [

lib/spack/spack/build_systems/waf.py

@@ -9,7 +9,7 @@
 from llnl.util.filesystem import working_dir
 
 from spack.directives import depends_on
-from spack.package import PackageBase, run_after
+from spack.package_base import PackageBase, run_after
 
 
 class WafPackage(PackageBase):

lib/spack/spack/build_systems/xorg.py

@@ -3,15 +3,17 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import spack.package
+from typing import Optional
+
+import spack.package_base
 import spack.util.url
 
 
-class XorgPackage(spack.package.PackageBase):
+class XorgPackage(spack.package_base.PackageBase):
     """Mixin that takes care of setting url and mirrors for x.org
        packages."""
     #: Path of the package in a x.org mirror
-    xorg_mirror_path = None
+    xorg_mirror_path = None  # type: Optional[str]
 
     #: List of x.org mirrors used by Spack
     #  Note: x.org mirrors are a bit tricky, since many are out-of-sync or off.

lib/spack/spack/cmd/checksum.py

@@ -14,7 +14,7 @@
 import spack.repo
 import spack.stage
 import spack.util.crypto
-from spack.package import preferred_version
+from spack.package_base import preferred_version
 from spack.util.naming import valid_fully_qualified_module_name
 from spack.version import Version, ver
 

lib/spack/spack/cmd/create.py

@@ -57,7 +57,7 @@
 # See the Spack documentation for more information on packaging.
 # ----------------------------------------------------------------------------
 
-from spack import *
+from spack.package import *
 
 
 class {class_name}({base_class_name}):
@@ -826,7 +826,7 @@ def get_versions(args, name):
         spack.util.url.require_url_format(args.url)
         if args.url.startswith('file://'):
             valid_url = False  # No point in spidering these
-    except ValueError:
+    except (ValueError, TypeError):
         valid_url = False
 
     if args.url is not None and args.template != 'bundle' and valid_url:

lib/spack/spack/cmd/dependencies.py

@@ -11,7 +11,7 @@
 import spack.cmd
 import spack.cmd.common.arguments as arguments
 import spack.environment as ev
-import spack.package
+import spack.package_base
 import spack.repo
 import spack.store
 
@@ -57,7 +57,7 @@ def dependencies(parser, args):
 
     else:
         spec = specs[0]
-        dependencies = spack.package.possible_dependencies(
+        dependencies = spack.package_base.possible_dependencies(
             spec,
             transitive=args.transitive,
             expand_virtuals=args.expand_virtuals,

lib/spack/spack/cmd/external.py

@@ -5,7 +5,6 @@
 from __future__ import print_function
 
 import argparse
-import errno
 import os
 import sys
 
@@ -94,21 +93,6 @@ def external_find(args):
             # It's fine to not find any manifest file if we are doing the
             # search implicitly (i.e. as part of 'spack external find')
             pass
-        except Exception as e:
-            # For most exceptions, just print a warning and continue.
-            # Note that KeyboardInterrupt does not subclass Exception
-            # (so CTRL-C will terminate the program as expected).
-            skip_msg = ("Skipping manifest and continuing with other external "
-                        "checks")
-            if ((isinstance(e, IOError) or isinstance(e, OSError)) and
-                    e.errno in [errno.EPERM, errno.EACCES]):
-                # The manifest file does not have sufficient permissions enabled:
-                # print a warning and keep going
-                tty.warn("Unable to read manifest due to insufficient "
-                         "permissions.", skip_msg)
-            else:
-                tty.warn("Unable to read manifest, unexpected error: {0}"
-                         .format(str(e)), skip_msg)
 
     # If the user didn't specify anything, search for build tools by default
     if not args.tags and not args.all and not args.packages:
@@ -220,7 +204,7 @@ def external_list(args):
     list(spack.repo.path.all_packages())
     # Print all the detectable packages
     tty.msg("Detectable packages per repository")
-    for namespace, pkgs in sorted(spack.package.detectable_packages.items()):
+    for namespace, pkgs in sorted(spack.package_base.detectable_packages.items()):
         print("Repository:", namespace)
         colify.colify(pkgs, indent=4, output=sys.stdout)
 

lib/spack/spack/cmd/info.py

@@ -18,7 +18,7 @@
 import spack.fetch_strategy as fs
 import spack.repo
 import spack.spec
-from spack.package import has_test_method, preferred_version
+from spack.package_base import has_test_method, preferred_version
 
 description = 'get detailed information on a particular package'
 section = 'basic'
@@ -269,14 +269,14 @@ def print_tests(pkg):
     names = []
     pkg_cls = pkg if inspect.isclass(pkg) else pkg.__class__
     if has_test_method(pkg_cls):
-        pkg_base = spack.package.PackageBase
+        pkg_base = spack.package_base.PackageBase
         test_pkgs = [str(cls.test) for cls in inspect.getmro(pkg_cls) if
                      issubclass(cls, pkg_base) and cls.test != pkg_base.test]
         test_pkgs = list(set(test_pkgs))
         names.extend([(test.split()[1]).lower() for test in test_pkgs])
 
     # TODO Refactor START
-    # Use code from package.py's test_process IF this functionality is
+    # Use code from package_base.py's test_process IF this functionality is
     # accepted.
     v_names = list(set([vspec.name for vspec in pkg.virtuals_provided]))
 

lib/spack/spack/cmd/install.py

@@ -302,7 +302,7 @@ def install(parser, args, **kwargs):
         )
 
     reporter = spack.report.collect_info(
-        spack.package.PackageInstaller, '_install_task', args.log_format, args)
+        spack.package_base.PackageInstaller, '_install_task', args.log_format, args)
     if args.log_file:
         reporter.filename = args.log_file
 

lib/spack/spack/cmd/mark.py

@@ -12,7 +12,7 @@
 import spack.cmd
 import spack.cmd.common.arguments as arguments
 import spack.error
-import spack.package
+import spack.package_base
 import spack.repo
 import spack.store
 from spack.database import InstallStatuses

lib/spack/spack/cmd/solve.py

@@ -18,7 +18,7 @@
 import spack.config
 import spack.environment
 import spack.hash_types as ht
-import spack.package
+import spack.package_base
 import spack.solver.asp as asp
 
 description = "concretize a specs using an ASP solver"

lib/spack/spack/cmd/style.py

@@ -65,7 +65,7 @@ def is_package(f):
     packages, since we allow `from spack import *` and poking globals
     into packages.
     """
-    return f.startswith("var/spack/repos/")
+    return f.startswith("var/spack/repos/") and f.endswith('package.py')
 
 
 #: decorator for adding tools to the list
@@ -94,16 +94,16 @@ def changed_files(base="develop", untracked=True, all_files=False, root=None):
     git = which("git", required=True)
 
     # ensure base is in the repo
-    git("show-ref", "--verify", "--quiet", "refs/heads/%s" % base,
-        fail_on_error=False)
+    base_sha = git("rev-parse", "--quiet", "--verify", "--revs-only", base,
+                   fail_on_error=False, output=str)
     if git.returncode != 0:
         tty.die(
-            "This repository does not have a '%s' branch." % base,
+            "This repository does not have a '%s' revision." % base,
             "spack style needs this branch to determine which files changed.",
             "Ensure that '%s' exists, or specify files to check explicitly." % base
         )
 
-    range = "{0}...".format(base)
+    range = "{0}...".format(base_sha.strip())
 
     git_args = [
         # Add changed files committed since branching off of develop
@@ -236,7 +236,7 @@ def translate(match):
             continue
         if not args.root_relative and re_obj:
             line = re_obj.sub(translate, line)
-        print("  " + line)
+        print(line)
 
 
 def print_style_header(file_list, args):
@@ -290,20 +290,26 @@ def run_flake8(flake8_cmd, file_list, args):
 @tool("mypy")
 def run_mypy(mypy_cmd, file_list, args):
     # always run with config from running spack prefix
-    mypy_args = [
+    common_mypy_args = [
         "--config-file", os.path.join(spack.paths.prefix, "pyproject.toml"),
-        "--package", "spack",
-        "--package", "llnl",
         "--show-error-codes",
     ]
-    # not yet, need other updates to enable this
-    # if any([is_package(f) for f in file_list]):
-    #     mypy_args.extend(["--package", "packages"])
+    mypy_arg_sets = [common_mypy_args + [
+        "--package", "spack",
+        "--package", "llnl",
+    ]]
+    if 'SPACK_MYPY_CHECK_PACKAGES' in os.environ:
+        mypy_arg_sets.append(common_mypy_args + [
+            '--package', 'packages',
+            '--disable-error-code', 'no-redef',
+        ])
 
-    output = mypy_cmd(*mypy_args, fail_on_error=False, output=str)
-    returncode = mypy_cmd.returncode
+    returncode = 0
+    for mypy_args in mypy_arg_sets:
+        output = mypy_cmd(*mypy_args, fail_on_error=False, output=str)
+        returncode |= mypy_cmd.returncode
 
-    rewrite_and_print_output(output, args)
+        rewrite_and_print_output(output, args)
 
     print_tool_result("mypy", returncode)
     return returncode
@@ -318,16 +324,29 @@ def run_isort(isort_cmd, file_list, args):
 
     pat = re.compile("ERROR: (.*) Imports are incorrectly sorted")
     replacement = "ERROR: {0} Imports are incorrectly sorted"
-    returncode = 0
-    for chunk in grouper(file_list, 100):
-        packed_args = isort_args + tuple(chunk)
-        output = isort_cmd(*packed_args, fail_on_error=False, output=str, error=str)
-        returncode |= isort_cmd.returncode
+    returncode = [0]
 
-        rewrite_and_print_output(output, args, pat, replacement)
+    def process_files(file_list, is_args):
+        for chunk in grouper(file_list, 100):
+            packed_args = is_args + tuple(chunk)
+            output = isort_cmd(*packed_args, fail_on_error=False, output=str, error=str)
+            returncode[0] |= isort_cmd.returncode
 
-    print_tool_result("isort", returncode)
-    return returncode
+            rewrite_and_print_output(output, args, pat, replacement)
+
+    packages_isort_args = ('--rm', 'spack', '--rm', 'spack.pkgkit', '--rm',
+                           'spack.package_defs', '-a', 'from spack.package import *')
+    packages_isort_args = packages_isort_args + isort_args
+
+    # packages
+    process_files(filter(is_package, file_list),
+                  packages_isort_args)
+    # non-packages
+    process_files(filter(lambda f: not is_package(f), file_list),
+                  isort_args)
+
+    print_tool_result("isort", returncode[0])
+    return returncode[0]
 
 
 @tool("black")

lib/spack/spack/cmd/test.py

@@ -20,7 +20,7 @@
 import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.install_test
-import spack.package
+import spack.package_base
 import spack.repo
 import spack.report
 
@@ -189,7 +189,7 @@ def test_run(args):
     # Set up reporter
     setattr(args, 'package', [s.format() for s in test_suite.specs])
     reporter = spack.report.collect_info(
-        spack.package.PackageBase, 'do_test', args.log_format, args)
+        spack.package_base.PackageBase, 'do_test', args.log_format, args)
     if not reporter.filename:
         if args.log_file:
             if os.path.isabs(args.log_file):
@@ -217,7 +217,7 @@ def test_list(args):
         else set()
 
     def has_test_and_tags(pkg_class):
-        return spack.package.has_test_method(pkg_class) and \
+        return spack.package_base.has_test_method(pkg_class) and \
             (not args.tag or pkg_class.name in tagged)
 
     if args.list_all:

lib/spack/spack/cmd/uninstall.py

@@ -15,7 +15,7 @@
 import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.error
-import spack.package
+import spack.package_base
 import spack.repo
 import spack.store
 from spack.database import InstallStatuses
@@ -221,7 +221,7 @@ def do_uninstall(env, specs, force):
         except spack.repo.UnknownEntityError:
             # The package.py file has gone away -- but still
             # want to uninstall.
-            spack.package.Package.uninstall_by_spec(item, force=True)
+            spack.package_base.Package.uninstall_by_spec(item, force=True)
 
     # A package is ready to be uninstalled when nothing else references it,
     # unless we are requested to force uninstall it.

lib/spack/spack/cmd/url.py

@@ -422,7 +422,7 @@ def url_list_parsing(args, urls, url, pkg):
         urls (set): List of URLs that have already been added
         url (str or None): A URL to potentially add to ``urls`` depending on
             ``args``
-        pkg (spack.package.PackageBase): The Spack package
+        pkg (spack.package_base.PackageBase): The Spack package
 
     Returns:
         set: The updated set of ``urls``
@@ -470,7 +470,7 @@ def name_parsed_correctly(pkg, name):
     """Determine if the name of a package was correctly parsed.
 
     Args:
-        pkg (spack.package.PackageBase): The Spack package
+        pkg (spack.package_base.PackageBase): The Spack package
         name (str): The name that was extracted from the URL
 
     Returns:
@@ -487,7 +487,7 @@ def version_parsed_correctly(pkg, version):
     """Determine if the version of a package was correctly parsed.
 
     Args:
-        pkg (spack.package.PackageBase): The Spack package
+        pkg (spack.package_base.PackageBase): The Spack package
         version (str): The version that was extracted from the URL
 
     Returns:

lib/spack/spack/compilers/__init__.py

@@ -252,13 +252,6 @@ def find_new_compilers(path_hints=None, scope=None):
             merged configuration.
     """
     compilers = find_compilers(path_hints)
-    return select_new_compilers(compilers, scope)
-
-
-def select_new_compilers(compilers, scope=None):
-    """Given a list of compilers, remove those that are already defined in
-    the configuration.
-    """
     compilers_not_in_config = []
     for c in compilers:
         arch_spec = spack.spec.ArchSpec((None, c.operating_system, c.target))

lib/spack/spack/container/writers/__init__.py

@@ -171,12 +171,11 @@ def strip(self):
     def paths(self):
         """Important paths in the image"""
         Paths = collections.namedtuple('Paths', [
-            'environment', 'store', 'hidden_view', 'view'
+            'environment', 'store', 'view'
         ])
         return Paths(
             environment='/opt/spack-environment',
             store='/opt/software',
-            hidden_view='/opt/._view',
             view='/opt/view'
         )
 

lib/spack/spack/cray_manifest.py

@@ -39,6 +39,10 @@ def translated_compiler_name(manifest_compiler_name):
     elif manifest_compiler_name in spack.compilers.supported_compilers():
         return manifest_compiler_name
     else:
+        # Try to fail quickly. This can occur in two cases: (1) the compiler
+        # definition (2) a spec can specify a compiler that doesn't exist; the
+        # first will be caught when creating compiler definition. The second
+        # will result in Specs with associated undefined compilers.
         raise spack.compilers.UnknownCompilerError(
             "Manifest parsing - unknown compiler: {0}"
             .format(manifest_compiler_name))
@@ -182,8 +186,6 @@ def read(path, apply_updates):
     tty.debug("{0}: {1} compilers read from manifest".format(
         path,
         str(len(compilers))))
-    # Filter out the compilers that already appear in the configuration
-    compilers = spack.compilers.select_new_compilers(compilers)
     if apply_updates and compilers:
         spack.compilers.add_compilers_to_config(
             compilers, init_config=False)

lib/spack/spack/detection/common.py

@@ -240,7 +240,7 @@ def compute_windows_program_path_for_package(pkg):
     program files location, return list of best guesses
 
     Args:
-        pkg (spack.package.Package): package for which
+        pkg (spack.package_base.Package): package for which
                            Program Files location is to be computed
     """
     if not is_windows:

lib/spack/spack/environment/environment.py

@@ -21,6 +21,7 @@
 from llnl.util.lang import dedupe
 from llnl.util.symlink import symlink
 
+import spack.binary_distribution
 import spack.bootstrap
 import spack.compilers
 import spack.concretize
@@ -1282,6 +1283,10 @@ def _concretize_separately(self, tests=False):
         # processes try to write the config file in parallel
         _ = spack.compilers.get_compiler_config()
 
+        # Ensure that buildcache index is updated if reuse is on
+        if spack.config.get('config:reuse', False):
+            spack.binary_distribution.binary_index.update()
+
         # Early return if there is nothing to do
         if len(arguments) == 0:
             return []
@@ -1612,10 +1617,9 @@ def install_specs(self, specs=None, **install_args):
         # ensure specs already installed are marked explicit
         all_specs = specs or [cs for _, cs in self.concretized_specs()]
         specs_installed = [s for s in all_specs if s.installed]
-        if specs_installed:
-            with spack.store.db.write_transaction():  # do all in one transaction
-                for spec in specs_installed:
-                    spack.store.db.update_explicit(spec, True)
+        with spack.store.db.write_transaction():  # do all in one transaction
+            for spec in specs_installed:
+                spack.store.db.update_explicit(spec, True)
 
         if not specs_to_install:
             tty.msg('All of the packages are already installed')

lib/spack/spack/fetch_strategy.py

@@ -35,6 +35,7 @@
 import six.moves.urllib.parse as urllib_parse
 
 import llnl.util
+import llnl.util.filesystem as fs
 import llnl.util.tty as tty
 from llnl.util.filesystem import (
     get_single_file,
@@ -119,6 +120,11 @@ def __init__(self, **kwargs):
         # 'no_cache' option from version directive.
         self.cache_enabled = not kwargs.pop('no_cache', False)
 
+        self.package = None
+
+    def set_package(self, package):
+        self.package = package
+
     # Subclasses need to implement these methods
     def fetch(self):
         """Fetch source code archive or repo.
@@ -242,6 +248,10 @@ def source_id(self):
         if all(component_ids):
             return component_ids
 
+    def set_package(self, package):
+        for item in self:
+            item.package = package
+
 
 @fetcher
 class URLFetchStrategy(FetchStrategy):
@@ -327,7 +337,9 @@ def fetch(self):
                 continue
 
             try:
-                self._fetch_from_url(url)
+                partial_file, save_file = self._fetch_from_url(url)
+                if save_file and (partial_file is not None):
+                    llnl.util.filesystem.rename(partial_file, save_file)
                 break
             except FailedDownloadError as e:
                 errors.append(str(e))
@@ -377,7 +389,9 @@ def _check_headers(self, headers):
 
     @_needs_stage
     def _fetch_urllib(self, url):
-        save_file = self.stage.save_filename
+        save_file = None
+        if self.stage.save_filename:
+            save_file = self.stage.save_filename
         tty.msg('Fetching {0}'.format(url))
 
         # Run urllib but grab the mime type from the http headers
@@ -387,18 +401,16 @@ def _fetch_urllib(self, url):
             # clean up archive on failure.
             if self.archive_file:
                 os.remove(self.archive_file)
-            if os.path.lexists(save_file):
+            if save_file and os.path.exists(save_file):
                 os.remove(save_file)
             msg = 'urllib failed to fetch with error {0}'.format(e)
             raise FailedDownloadError(url, msg)
 
-        if os.path.lexists(save_file):
-            os.remove(save_file)
-
         with open(save_file, 'wb') as _open_file:
             shutil.copyfileobj(response, _open_file)
 
         self._check_headers(str(headers))
+        return None, save_file
 
     @_needs_stage
     def _fetch_curl(self, url):
@@ -459,7 +471,7 @@ def _fetch_curl(self, url):
             if self.archive_file:
                 os.remove(self.archive_file)
 
-            if partial_file and os.path.lexists(partial_file):
+            if partial_file and os.path.exists(partial_file):
                 os.remove(partial_file)
 
             if curl.returncode == 22:
@@ -486,9 +498,7 @@ def _fetch_curl(self, url):
                     "Curl failed with error %d" % curl.returncode)
 
         self._check_headers(headers)
-
-        if save_file and (partial_file is not None):
-            os.rename(partial_file, save_file)
+        return partial_file, save_file
 
     @property  # type: ignore # decorated properties unsupported in mypy
     @_needs_stage
@@ -520,7 +530,7 @@ def expand(self):
                 "Failed on expand() for URL %s" % self.url)
 
         if not self.extension:
-            self.extension = extension(self.archive_file)
+            self.extension = extension(self.url)
 
         if self.stage.expanded:
             tty.debug('Source already staged to %s' % self.stage.source_path)
@@ -528,50 +538,11 @@ def expand(self):
 
         decompress = decompressor_for(self.archive_file, self.extension)
 
-        # Expand all tarballs in their own directory to contain
-        # exploding tarballs.
-        tarball_container = os.path.join(self.stage.path,
-                                         "spack-expanded-archive")
-
         # Below we assume that the command to decompress expand the
         # archive in the current working directory
-        mkdirp(tarball_container)
-        with working_dir(tarball_container):
+        with fs.exploding_archive_catch(self.stage):
             decompress(self.archive_file)
 
-        # Check for an exploding tarball, i.e. one that doesn't expand to
-        # a single directory.  If the tarball *didn't* explode, move its
-        # contents to the staging source directory & remove the container
-        # directory.  If the tarball did explode, just rename the tarball
-        # directory to the staging source directory.
-        #
-        # NOTE: The tar program on Mac OS X will encode HFS metadata in
-        # hidden files, which can end up *alongside* a single top-level
-        # directory.  We initially ignore presence of hidden files to
-        # accomodate these "semi-exploding" tarballs but ensure the files
-        # are copied to the source directory.
-        files = os.listdir(tarball_container)
-        non_hidden = [f for f in files if not f.startswith('.')]
-        if len(non_hidden) == 1:
-            src = os.path.join(tarball_container, non_hidden[0])
-            if os.path.isdir(src):
-                self.stage.srcdir = non_hidden[0]
-                shutil.move(src, self.stage.source_path)
-                if len(files) > 1:
-                    files.remove(non_hidden[0])
-                    for f in files:
-                        src = os.path.join(tarball_container, f)
-                        dest = os.path.join(self.stage.path, f)
-                        shutil.move(src, dest)
-                os.rmdir(tarball_container)
-            else:
-                # This is a non-directory entry (e.g., a patch file) so simply
-                # rename the tarball container to be the source path.
-                shutil.move(tarball_container, self.stage.source_path)
-
-        else:
-            shutil.move(tarball_container, self.stage.source_path)
-
     def archive(self, destination):
         """Just moves this archive to the destination."""
         if not self.archive_file:
@@ -642,7 +613,7 @@ def fetch(self):
 
         # remove old symlink if one is there.
         filename = self.stage.save_filename
-        if os.path.lexists(filename):
+        if os.path.exists(filename):
             os.remove(filename)
 
         # Symlink to local cached archive.
@@ -1014,9 +985,20 @@ def clone(self, dest=None, commit=None, branch=None, tag=None, bare=False):
                     git(*args)
 
         # Init submodules if the user asked for them.
-        if self.submodules:
-            with working_dir(dest):
-                args = ['submodule', 'update', '--init', '--recursive']
+        git_commands = []
+        submodules = self.submodules
+        if callable(submodules):
+            submodules = list(submodules(self.package))
+            git_commands.append(["submodule", "init", "--"] + submodules)
+            git_commands.append(['submodule', 'update', '--recursive'])
+        elif submodules:
+            git_commands.append(["submodule", "update", "--init", "--recursive"])
+
+        if not git_commands:
+            return
+
+        with working_dir(dest):
+            for args in git_commands:
                 if not spack.config.get('config:debug'):
                     args.insert(1, '--quiet')
                 git(*args)
@@ -1556,7 +1538,7 @@ def _extrapolate(pkg, version):
     try:
         return URLFetchStrategy(pkg.url_for_version(version),
                                 fetch_options=pkg.fetch_options)
-    except spack.package.NoURLError:
+    except spack.package_base.NoURLError:
         msg = ("Can't extrapolate a URL for version %s "
                "because package %s defines no URLs")
         raise ExtrapolationError(msg % (version, pkg.name))

lib/spack/spack/installer.py

@@ -50,7 +50,7 @@
 import spack.error
 import spack.hooks
 import spack.monitor
-import spack.package
+import spack.package_base
 import spack.package_prefs as prefs
 import spack.repo
 import spack.store
@@ -103,7 +103,7 @@ def _check_last_phase(pkg):
     package already.
 
     Args:
-        pkg (spack.package.PackageBase): the package being installed
+        pkg (spack.package_base.PackageBase): the package being installed
 
     Raises:
         ``BadInstallPhase`` if stop_before or last phase is invalid
@@ -125,7 +125,7 @@ def _handle_external_and_upstream(pkg, explicit):
     database if it is external package.
 
     Args:
-        pkg (spack.package.Package): the package whose installation is under
+        pkg (spack.package_base.Package): the package whose installation is under
             consideration
         explicit (bool): the package was explicitly requested by the user
     Return:
@@ -265,7 +265,7 @@ def _install_from_cache(pkg, cache_only, explicit, unsigned=False):
     Extract the package from binary cache
 
     Args:
-        pkg (spack.package.PackageBase): the package to install from the binary cache
+        pkg (spack.package_base.PackageBase): package to install from the binary cache
         cache_only (bool): only extract from binary cache
         explicit (bool): ``True`` if installing the package was explicitly
             requested by the user, otherwise, ``False``
@@ -355,7 +355,7 @@ def _process_binary_cache_tarball(pkg, binary_spec, explicit, unsigned,
     Process the binary cache tarball.
 
     Args:
-        pkg (spack.package.PackageBase): the package being installed
+        pkg (spack.package_base.PackageBase): the package being installed
         binary_spec (spack.spec.Spec): the spec  whose cache has been confirmed
         explicit (bool): the package was explicitly requested by the user
         unsigned (bool): ``True`` if binary package signatures to be checked,
@@ -394,7 +394,7 @@ def _try_install_from_binary_cache(pkg, explicit, unsigned=False):
     Try to extract the package from binary cache.
 
     Args:
-        pkg (spack.package.PackageBase): the package to be extracted from binary cache
+        pkg (spack.package_base.PackageBase): package to be extracted from binary cache
         explicit (bool): the package was explicitly requested by the user
         unsigned (bool): ``True`` if binary package signatures to be checked,
             otherwise, ``False``
@@ -530,7 +530,7 @@ def log(pkg):
     Copy provenance into the install directory on success
 
     Args:
-        pkg (spack.package.Package): the package that was built and installed
+        pkg (spack.package_base.Package): the package that was built and installed
     """
     packages_dir = spack.store.layout.build_packages_path(pkg.spec)
 
@@ -616,7 +616,7 @@ def package_id(pkg):
     and packages for combinatorial environments.
 
     Args:
-        pkg (spack.package.PackageBase): the package from which the identifier is
+        pkg (spack.package_base.PackageBase): the package from which the identifier is
             derived
     """
     if not pkg.spec.concrete:
@@ -769,7 +769,7 @@ def _add_bootstrap_compilers(
         Args:
             compiler: the compiler to boostrap
             architecture: the architecture for which to bootstrap the compiler
-            pkgs (spack.package.PackageBase): the package with possible compiler
+            pkgs (spack.package_base.PackageBase): the package with possible compiler
                 dependencies
             request (BuildRequest): the associated install request
             all_deps (defaultdict(set)): dictionary of all dependencies and
@@ -786,7 +786,7 @@ def _add_init_task(self, pkg, request, is_compiler, all_deps):
         Creates and queus the initial build task for the package.
 
         Args:
-            pkg (spack.package.Package): the package to be built and installed
+            pkg (spack.package_base.Package): the package to be built and installed
             request (BuildRequest or None): the associated install request
                  where ``None`` can be used to indicate the package was
                  explicitly requested by the user
@@ -968,7 +968,7 @@ def _cleanup_task(self, pkg):
         Cleanup the build task for the spec
 
         Args:
-            pkg (spack.package.PackageBase): the package being installed
+            pkg (spack.package_base.PackageBase): the package being installed
         """
         self._remove_task(package_id(pkg))
 
@@ -982,7 +982,7 @@ def _ensure_install_ready(self, pkg):
         already locked.
 
         Args:
-            pkg (spack.package.PackageBase): the package being locally installed
+            pkg (spack.package_base.PackageBase): the package being locally installed
         """
         pkg_id = package_id(pkg)
         pre = "{0} cannot be installed locally:".format(pkg_id)
@@ -1014,7 +1014,8 @@ def _ensure_locked(self, lock_type, pkg):
 
         Args:
             lock_type (str): 'read' for a read lock, 'write' for a write lock
-            pkg (spack.package.PackageBase): the package whose spec is being installed
+            pkg (spack.package_base.PackageBase): the package whose spec is being
+                                                  installed
 
         Return:
             (lock_type, lock) tuple where lock will be None if it could not
@@ -1228,7 +1229,7 @@ def _install_task(self, task):
 
             # Create a child process to do the actual installation.
             # Preserve verbosity settings across installs.
-            spack.package.PackageBase._verbose = (
+            spack.package_base.PackageBase._verbose = (
                 spack.build_environment.start_build_process(
                     pkg, build_process, install_args)
             )
@@ -1373,7 +1374,7 @@ def _setup_install_dir(self, pkg):
         Write a small metadata file with the current spack environment.
 
         Args:
-            pkg (spack.package.Package): the package to be built and installed
+            pkg (spack.package_base.Package): the package to be built and installed
         """
         if not os.path.exists(pkg.spec.prefix):
             tty.debug('Creating the installation directory {0}'.format(pkg.spec.prefix))
@@ -1447,7 +1448,7 @@ def _flag_installed(self, pkg, dependent_ids=None):
         known dependents.
 
         Args:
-            pkg (spack.package.Package): Package that has been installed locally,
+            pkg (spack.package_base.Package): Package that has been installed locally,
                 externally or upstream
             dependent_ids (list or None): list of the package's
                 dependent ids, or None if the dependent ids are limited to
@@ -1536,7 +1537,7 @@ def install(self):
         Install the requested package(s) and or associated dependencies.
 
         Args:
-            pkg (spack.package.Package): the package to be built and installed"""
+            pkg (spack.package_base.Package): the package to be built and installed"""
 
         self._init_queue()
         fail_fast_err = 'Terminating after first install failure'
@@ -1788,7 +1789,7 @@ def __init__(self, pkg, install_args):
         process in the build.
 
         Arguments:
-            pkg (spack.package.PackageBase) the package being installed.
+            pkg (spack.package_base.PackageBase) the package being installed.
             install_args (dict) arguments to do_install() from parent process.
 
         """
@@ -1848,8 +1849,8 @@ def run(self):
 
         # get verbosity from do_install() parameter or saved value
         self.echo = self.verbose
-        if spack.package.PackageBase._verbose is not None:
-            self.echo = spack.package.PackageBase._verbose
+        if spack.package_base.PackageBase._verbose is not None:
+            self.echo = spack.package_base.PackageBase._verbose
 
         self.pkg.stage.keep = self.keep_stage
 
@@ -2001,7 +2002,7 @@ def build_process(pkg, install_args):
     This function's return value is returned to the parent process.
 
     Arguments:
-        pkg (spack.package.PackageBase): the package being installed.
+        pkg (spack.package_base.PackageBase): the package being installed.
         install_args (dict): arguments to do_install() from parent process.
 
     """
@@ -2049,7 +2050,7 @@ def __init__(self, pkg, request, compiler, start, attempts, status,
         Instantiate a build task for a package.
 
         Args:
-            pkg (spack.package.Package): the package to be built and installed
+            pkg (spack.package_base.Package): the package to be built and installed
             request (BuildRequest or None): the associated install request
                  where ``None`` can be used to indicate the package was
                  explicitly requested by the user
@@ -2062,7 +2063,7 @@ def __init__(self, pkg, request, compiler, start, attempts, status,
         """
 
         # Ensure dealing with a package that has a concrete spec
-        if not isinstance(pkg, spack.package.PackageBase):
+        if not isinstance(pkg, spack.package_base.PackageBase):
             raise ValueError("{0} must be a package".format(str(pkg)))
 
         self.pkg = pkg
@@ -2240,11 +2241,11 @@ def __init__(self, pkg, install_args):
         Instantiate a build request for a package.
 
         Args:
-            pkg (spack.package.Package): the package to be built and installed
+            pkg (spack.package_base.Package): the package to be built and installed
             install_args (dict): the install arguments associated with ``pkg``
         """
         # Ensure dealing with a package that has a concrete spec
-        if not isinstance(pkg, spack.package.PackageBase):
+        if not isinstance(pkg, spack.package_base.PackageBase):
             raise ValueError("{0} must be a package".format(str(pkg)))
 
         self.pkg = pkg
@@ -2314,7 +2315,7 @@ def get_deptypes(self, pkg):
         """Determine the required dependency types for the associated package.
 
         Args:
-            pkg (spack.package.PackageBase): explicit or implicit package being
+            pkg (spack.package_base.PackageBase): explicit or implicit package being
                 installed
 
         Returns:
@@ -2337,7 +2338,7 @@ def run_tests(self, pkg):
         """Determine if the tests should be run for the provided packages
 
         Args:
-            pkg (spack.package.PackageBase): explicit or implicit package being
+            pkg (spack.package_base.PackageBase): explicit or implicit package being
                 installed
 
         Returns:

lib/spack/spack/modules/lmod.py

@@ -196,6 +196,14 @@ def provides(self):
         if self.spec.name == 'llvm-amdgpu':
             provides['compiler'] = spack.spec.CompilerSpec(str(self.spec))
             provides['compiler'].name = 'rocmcc'
+        # Special case for oneapi
+        if self.spec.name == 'intel-oneapi-compilers':
+            provides['compiler'] = spack.spec.CompilerSpec(str(self.spec))
+            provides['compiler'].name = 'oneapi'
+        # Special case for oneapi classic
+        if self.spec.name == 'intel-oneapi-compilers-classic':
+            provides['compiler'] = spack.spec.CompilerSpec(str(self.spec))
+            provides['compiler'].name = 'intel'
 
         # All the other tokens in the hierarchy must be virtual dependencies
         for x in self.hierarchy_tokens:

lib/spack/spack/patch.py

@@ -356,7 +356,7 @@ def patch_for_package(self, sha256, pkg):
 
         Arguments:
             sha256 (str): sha256 hash to look up
-            pkg (spack.package.Package): Package object to get patch for.
+            pkg (spack.package_base.Package): Package object to get patch for.
 
         We build patch objects lazily because building them requires that
         we have information about the package's location in its repo.

lib/spack/spack/pkgkit.py

@@ -1,83 +0,0 @@
-# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-# flake8: noqa: F401
-"""pkgkit is a set of useful build tools and directives for packages.
-
-Everything in this module is automatically imported into Spack package files.
-"""
-import llnl.util.filesystem
-from llnl.util.filesystem import *
-
-import spack.directives
-import spack.util.executable
-from spack.build_systems.aspell_dict import AspellDictPackage
-from spack.build_systems.autotools import AutotoolsPackage
-from spack.build_systems.cached_cmake import (
-    CachedCMakePackage,
-    cmake_cache_option,
-    cmake_cache_path,
-    cmake_cache_string,
-)
-from spack.build_systems.cmake import CMakePackage
-from spack.build_systems.cuda import CudaPackage
-from spack.build_systems.gnu import GNUMirrorPackage
-from spack.build_systems.intel import IntelPackage
-from spack.build_systems.lua import LuaPackage
-from spack.build_systems.makefile import MakefilePackage
-from spack.build_systems.maven import MavenPackage
-from spack.build_systems.meson import MesonPackage
-from spack.build_systems.octave import OctavePackage
-from spack.build_systems.oneapi import (
-    IntelOneApiLibraryPackage,
-    IntelOneApiPackage,
-    IntelOneApiStaticLibraryList,
-)
-from spack.build_systems.perl import PerlPackage
-from spack.build_systems.python import PythonPackage
-from spack.build_systems.qmake import QMakePackage
-from spack.build_systems.r import RPackage
-from spack.build_systems.racket import RacketPackage
-from spack.build_systems.rocm import ROCmPackage
-from spack.build_systems.ruby import RubyPackage
-from spack.build_systems.scons import SConsPackage
-from spack.build_systems.sip import SIPPackage
-from spack.build_systems.sourceforge import SourceforgePackage
-from spack.build_systems.sourceware import SourcewarePackage
-from spack.build_systems.waf import WafPackage
-from spack.build_systems.xorg import XorgPackage
-from spack.dependency import all_deptypes
-from spack.directives import *
-from spack.install_test import get_escaped_text_output
-from spack.installer import (
-    ExternalPackageError,
-    InstallError,
-    InstallLockError,
-    UpstreamPackageError,
-)
-from spack.mixins import filter_compiler_wrappers
-from spack.multimethod import when
-from spack.package import (
-    BundlePackage,
-    DependencyConflictError,
-    Package,
-    build_system_flags,
-    env_flags,
-    flatten_dependencies,
-    inject_flags,
-    install_dependency_symlinks,
-    on_package_attributes,
-    run_after,
-    run_before,
-)
-from spack.spec import InvalidSpecDetected, Spec
-from spack.util.executable import *
-from spack.variant import (
-    any_combination_of,
-    auto_or_any_combination_of,
-    conditional,
-    disjoint_sets,
-)
-from spack.version import Version, ver

lib/spack/spack/relocate.py

@@ -433,8 +433,9 @@ def needs_binary_relocation(m_type, m_subtype):
         m_type (str): MIME type of the file
         m_subtype (str): MIME subtype of the file
     """
+    subtypes = ('x-executable', 'x-sharedlib', 'x-mach-binary', 'x-pie-executable')
     if m_type == 'application':
-        if m_subtype in ('x-executable', 'x-sharedlib', 'x-mach-binary'):
+        if m_subtype in subtypes:
             return True
     return False
 

lib/spack/spack/repo.py

@@ -202,17 +202,11 @@ class RepoLoader(_PrependFileLoader):
     """Loads a Python module associated with a package in specific repository"""
     #: Code in ``_package_prepend`` is prepended to imported packages.
     #:
-    #: Spack packages were originally expected to call `from spack import *`
-    #: themselves, but it became difficult to manage and imports in the Spack
-    #: core the top-level namespace polluted by package symbols this way.  To
-    #: solve this, the top-level ``spack`` package contains very few symbols
-    #: of its own, and importing ``*`` is essentially a no-op.  The common
-    #: routines and directives that packages need are now in ``spack.pkgkit``,
-    #: and the import system forces packages to automatically include
-    #: this. This way, old packages that call ``from spack import *`` will
-    #: continue to work without modification, but it's no longer required.
+    #: Spack packages are expected to call `from spack.package import *`
+    #: themselves, but we are allowing a deprecation period before breaking
+    #: external repos that don't do this yet.
     _package_prepend = ('from __future__ import absolute_import;'
-                        'from spack.pkgkit import *')
+                        'from spack.package import *')
 
     def __init__(self, fullname, repo, package_name):
         self.repo = repo
@@ -450,10 +444,10 @@ def is_package_file(filename):
     # Package files are named `package.py` and are not in lib/spack/spack
     # We have to remove the file extension because it can be .py and can be
     # .pyc depending on context, and can differ between the files
-    import spack.package  # break cycle
+    import spack.package_base  # break cycle
     filename_noext = os.path.splitext(filename)[0]
     packagebase_filename_noext = os.path.splitext(
-        inspect.getfile(spack.package.PackageBase))[0]
+        inspect.getfile(spack.package_base.PackageBase))[0]
     return (filename_noext != packagebase_filename_noext and
             os.path.basename(filename_noext) == 'package')
 

lib/spack/spack/report.py

@@ -15,7 +15,7 @@
 
 import spack.build_environment
 import spack.fetch_strategy
-import spack.package
+import spack.package_base
 from spack.install_test import TestSuite
 from spack.reporter import Reporter
 from spack.reporters.cdash import CDash
@@ -131,7 +131,7 @@ def gather_info(do_fn):
             """
             @functools.wraps(do_fn)
             def wrapper(instance, *args, **kwargs):
-                if isinstance(instance, spack.package.PackageBase):
+                if isinstance(instance, spack.package_base.PackageBase):
                     pkg = instance
                 elif hasattr(args[0], 'pkg'):
                     pkg = args[0].pkg

lib/spack/spack/reporters/cdash.py

@@ -22,7 +22,7 @@
 
 import spack.build_environment
 import spack.fetch_strategy
-import spack.package
+import spack.package_base
 from spack.error import SpackError
 from spack.reporter import Reporter
 from spack.util.crypto import checksum

lib/spack/spack/reporters/junit.py

@@ -7,7 +7,7 @@
 
 import spack.build_environment
 import spack.fetch_strategy
-import spack.package
+import spack.package_base
 from spack.reporter import Reporter
 
 __all__ = ['JUnit']

lib/spack/spack/s3_handler.py

@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-from io import BufferedReader, IOBase
+from io import BufferedReader
 
 import six
 import six.moves.urllib.error as urllib_error
@@ -23,15 +23,11 @@
 # https://github.com/python/cpython/pull/3249
 class WrapStream(BufferedReader):
     def __init__(self, raw):
-        # In botocore >=1.23.47, StreamingBody inherits from IOBase, so we
-        # only add missing attributes in older versions.
-        # https://github.com/boto/botocore/commit/a624815eabac50442ed7404f3c4f2664cd0aa784
-        if not isinstance(raw, IOBase):
-            raw.readable = lambda: True
-            raw.writable = lambda: False
-            raw.seekable = lambda: False
-            raw.closed = False
-            raw.flush = lambda: None
+        raw.readable = lambda: True
+        raw.writable = lambda: False
+        raw.seekable = lambda: False
+        raw.closed = False
+        raw.flush = lambda: None
         super(WrapStream, self).__init__(raw)
 
     def detach(self):

lib/spack/spack/schema/cray_manifest.py

@@ -26,9 +26,6 @@
                 "cpe-version": {"type": "string", "minLength": 1},
                 "system-type": {"type": "string", "minLength": 1},
                 "schema-version": {"type": "string", "minLength": 1},
-                # Older schemas use did not have "cpe-version", just the
-                # schema version; in that case it was just called "version"
-                "version": {"type": "string", "minLength": 1},
             }
         },
         "compilers": {

lib/spack/spack/schema/env.py

@@ -198,9 +198,6 @@ def update(data):
                " [files={0}]")
         warnings.warn(msg.format(', '.join(data['include'])))
 
-    if 'packages' in data:
-        updated |= spack.schema.packages.update(data['packages'])
-
     # Spack 0.19 drops support for `spack:concretization` in favor of
     # `spack:concretizer:unify`. Here we provide an upgrade path that changes the former
     # into the latter, or warns when there's an ambiguity. Note that Spack 0.17 is not

lib/spack/spack/schema/packages.py

@@ -9,54 +9,6 @@
 """
 
 
-def deprecate_paths_and_modules(instance, deprecated_properties):
-    """Function to produce warning/error messages if "paths" and "modules" are
-    found in "packages.yaml"
-
-    Args:
-        instance: instance of the configuration file
-        deprecated_properties: deprecated properties in instance
-
-    Returns:
-        Warning/Error message to be printed
-    """
-    import copy
-    import os.path
-
-    import llnl.util.tty
-
-    import spack.util.spack_yaml as syaml
-
-    # Copy the instance to remove default attributes that are not related
-    # to the part that needs to be reported
-    instance_copy = copy.copy(instance)
-
-    # Check if this configuration comes from an environment or not
-    absolute_path = instance_copy._end_mark.name
-    command_to_suggest = '$ spack config update packages'
-    if os.path.basename(absolute_path) == 'spack.yaml':
-        command_to_suggest = '$ spack env update <environment>'
-
-    # Retrieve the relevant part of the configuration as YAML
-    keys_to_be_removed = [
-        x for x in instance_copy if x not in deprecated_properties
-    ]
-    for key in keys_to_be_removed:
-        instance_copy.pop(key)
-    yaml_as_str = syaml.dump_config(instance_copy, blame=True)
-
-    if llnl.util.tty.is_debug():
-        msg = 'OUTDATED CONFIGURATION FILE [file={0}]\n{1}'
-        llnl.util.tty.debug(msg.format(absolute_path, yaml_as_str))
-
-    msg = ('detected deprecated properties in {0}\nActivate the debug '
-           'flag to have more information on the deprecated parts or '
-           'run:\n\n\t{2}\n\nto update the file to the new format\n')
-    return msg.format(
-        absolute_path, yaml_as_str, command_to_suggest
-    )
-
-
 #: Properties for inclusion in other schemas
 properties = {
     'packages': {
@@ -136,16 +88,7 @@ def deprecate_paths_and_modules(instance, deprecated_properties):
                             'required': ['spec']
                         }
                     },
-                    # Deprecated properties, will trigger an error with a
-                    # message telling how to update.
-                    'paths': {'type': 'object'},
-                    'modules': {'type': 'object'},
                 },
-                'deprecatedProperties': {
-                    'properties': ['modules', 'paths'],
-                    'message': deprecate_paths_and_modules,
-                    'error': False
-                }
             },
         },
     },
@@ -160,41 +103,3 @@ def deprecate_paths_and_modules(instance, deprecated_properties):
     'additionalProperties': False,
     'properties': properties,
 }
-
-
-def update(data):
-    """Update the data in place to remove deprecated properties.
-
-    Args:
-        data (dict): dictionary to be updated
-
-    Returns:
-        True if data was changed, False otherwise
-    """
-    changed = False
-    for cfg_object in data.values():
-        externals = []
-
-        # If we don't have these deprecated attributes, continue
-        if not any(x in cfg_object for x in ('paths', 'modules')):
-            continue
-
-        # If we arrive here we need to make some changes i.e.
-        # we need to remove and eventually convert some attributes
-        changed = True
-        paths = cfg_object.pop('paths', {})
-        for spec, prefix in paths.items():
-            externals.append({
-                'spec': str(spec),
-                'prefix': str(prefix)
-            })
-        modules = cfg_object.pop('modules', {})
-        for spec, module in modules.items():
-            externals.append({
-                'spec': str(spec),
-                'modules': [str(module)]
-            })
-        if externals:
-            cfg_object['externals'] = externals
-
-    return changed

lib/spack/spack/solver/asp.py

@@ -41,7 +41,7 @@
 import spack.directives
 import spack.environment as ev
 import spack.error
-import spack.package
+import spack.package_base
 import spack.package_prefs
 import spack.platforms
 import spack.repo
@@ -1820,7 +1820,7 @@ def setup(self, driver, specs, reuse=None):
         self.possible_virtuals = set(
             x.name for x in specs if x.virtual
         )
-        possible = spack.package.possible_dependencies(
+        possible = spack.package_base.possible_dependencies(
             *specs,
             virtuals=self.possible_virtuals,
             deptype=spack.dependency.all_deptypes
@@ -1880,14 +1880,9 @@ def setup(self, driver, specs, reuse=None):
         # Inject dev_path from environment
         env = ev.active_environment()
         if env:
-            for name, info in env.dev_specs.items():
-                dev_spec = spack.spec.Spec(info['spec'])
-                dev_spec.constrain(
-                    'dev_path=%s' % spack.util.path.canonicalize_path(info['path'])
-                )
-
-                self.condition(spack.spec.Spec(name), dev_spec,
-                               msg="%s is a develop spec" % name)
+            for spec in sorted(specs):
+                for dep in spec.traverse():
+                    _develop_specs_from_env(dep, env)
 
         self.gen.h1('Spec Constraints')
         self.literal_specs(specs)

lib/spack/spack/spec.py

@@ -896,7 +896,7 @@ def clear(self):
 def _command_default_handler(descriptor, spec, cls):
     """Default handler when looking for the 'command' attribute.
 
-    Tries to search for ``spec.name`` in the ``spec.prefix.bin`` directory.
+    Tries to search for ``spec.name`` in the ``spec.home.bin`` directory.
 
     Parameters:
         descriptor (ForwardQueryToPackage): descriptor that triggered the call
@@ -910,20 +910,21 @@ def _command_default_handler(descriptor, spec, cls):
     Raises:
         RuntimeError: If the command is not found
     """
-    path = os.path.join(spec.prefix.bin, spec.name)
+    home = getattr(spec.package, 'home')
+    path = os.path.join(home.bin, spec.name)
 
     if fs.is_exe(path):
         return spack.util.executable.Executable(path)
     else:
         msg = 'Unable to locate {0} command in {1}'
-        raise RuntimeError(msg.format(spec.name, spec.prefix.bin))
+        raise RuntimeError(msg.format(spec.name, home.bin))
 
 
 def _headers_default_handler(descriptor, spec, cls):
     """Default handler when looking for the 'headers' attribute.
 
     Tries to search for ``*.h`` files recursively starting from
-    ``spec.prefix.include``.
+    ``spec.package.home.include``.
 
     Parameters:
         descriptor (ForwardQueryToPackage): descriptor that triggered the call
@@ -937,21 +938,22 @@ def _headers_default_handler(descriptor, spec, cls):
     Raises:
         NoHeadersError: If no headers are found
     """
-    headers = fs.find_headers('*', root=spec.prefix.include, recursive=True)
+    home = getattr(spec.package, 'home')
+    headers = fs.find_headers('*', root=home.include, recursive=True)
 
     if headers:
         return headers
     else:
         msg = 'Unable to locate {0} headers in {1}'
         raise spack.error.NoHeadersError(
-            msg.format(spec.name, spec.prefix.include))
+            msg.format(spec.name, home))
 
 
 def _libs_default_handler(descriptor, spec, cls):
     """Default handler when looking for the 'libs' attribute.
 
     Tries to search for ``lib{spec.name}`` recursively starting from
-    ``spec.prefix``. If ``spec.name`` starts with ``lib``, searches for
+    ``spec.package.home``. If ``spec.name`` starts with ``lib``, searches for
     ``{spec.name}`` instead.
 
     Parameters:
@@ -978,6 +980,7 @@ def _libs_default_handler(descriptor, spec, cls):
     # get something like 'libabcXabc.so, but for now we consider this
     # unlikely).
     name = spec.name.replace('-', '?')
+    home = getattr(spec.package, 'home')
 
     # Avoid double 'lib' for packages whose names already start with lib
     if not name.startswith('lib'):
@@ -990,12 +993,12 @@ def _libs_default_handler(descriptor, spec, cls):
 
     for shared in search_shared:
         libs = fs.find_libraries(
-            name, spec.prefix, shared=shared, recursive=True)
+            name, home, shared=shared, recursive=True)
         if libs:
             return libs
 
     msg = 'Unable to recursively locate {0} libraries in {1}'
-    raise spack.error.NoLibrariesError(msg.format(spec.name, spec.prefix))
+    raise spack.error.NoLibrariesError(msg.format(spec.name, home))
 
 
 class ForwardQueryToPackage(object):
@@ -1116,6 +1119,9 @@ def __set__(self, instance, value):
 
 
 class SpecBuildInterface(lang.ObjectWrapper):
+    # home is available in the base Package so no default is needed
+    home = ForwardQueryToPackage('home', default_handler=None)
+
     command = ForwardQueryToPackage(
         'command',
         default_handler=_command_default_handler

lib/spack/spack/stage.py

@@ -363,12 +363,13 @@ def __exit__(self, exc_type, exc_val, exc_tb):
     def expected_archive_files(self):
         """Possible archive file paths."""
         paths = []
-
         fnames = []
         expanded = True
         if isinstance(self.default_fetcher, fs.URLFetchStrategy):
             expanded = self.default_fetcher.expand_archive
-            fnames.append(os.path.basename(self.default_fetcher.url))
+            clean_url = os.path.basename(
+                sup.sanitize_file_path(self.default_fetcher.url))
+            fnames.append(clean_url)
 
         if self.mirror_paths:
             fnames.extend(os.path.basename(x) for x in self.mirror_paths)

lib/spack/spack/test/build_systems.py

@@ -128,7 +128,7 @@ def test_cmake_bad_generator(config, mock_packages):
     s.concretize()
     pkg = spack.repo.get(s)
     pkg.generator = 'Yellow Sticky Notes'
-    with pytest.raises(spack.package.InstallError):
+    with pytest.raises(spack.package_base.InstallError):
         get_std_cmake_args(pkg)
 
 

lib/spack/spack/test/cmd/analyze.py

@@ -10,7 +10,7 @@
 
 import spack.cmd.install
 import spack.config
-import spack.package
+import spack.package_base
 import spack.util.spack_json as sjson
 from spack.main import SpackCommand
 from spack.spec import Spec

lib/spack/spack/test/cmd/clean.py

@@ -9,7 +9,7 @@
 
 import spack.caches
 import spack.main
-import spack.package
+import spack.package_base
 import spack.stage
 
 clean = spack.main.SpackCommand('clean')
@@ -31,7 +31,7 @@ def __init__(self, name):
         def __call__(self, *args, **kwargs):
             counts[self.name] += 1
 
-    monkeypatch.setattr(spack.package.PackageBase, 'do_clean',
+    monkeypatch.setattr(spack.package_base.PackageBase, 'do_clean',
                         Counter('package'))
     monkeypatch.setattr(spack.stage, 'purge', Counter('stages'))
     monkeypatch.setattr(

lib/spack/spack/test/cmd/config.py

@@ -29,22 +29,6 @@ def _create_config(scope=None, data={}, section='packages'):
     return cfg_file
 
 
-@pytest.fixture()
-def packages_yaml_v015(mutable_config):
-    """Create a packages.yaml in the old format"""
-    old_data = {
-        'packages': {
-            'cmake': {
-                'paths': {'cmake@3.14.0': '/usr'}
-            },
-            'gcc': {
-                'modules': {'gcc@8.3.0': 'gcc-8'}
-            }
-        }
-    }
-    return functools.partial(_create_config, data=old_data, section='packages')
-
-
 @pytest.fixture()
 def config_yaml_v015(mutable_config):
     """Create a packages.yaml in the old format"""
@@ -493,19 +477,6 @@ def test_config_remove_from_env(mutable_empty_config, mutable_mock_env_path):
     assert output == expected
 
 
-def test_config_update_packages(packages_yaml_v015):
-    """Test Spack updating old packages.yaml format for externals
-    to new format. Ensure that data is preserved and converted
-    properly.
-    """
-    packages_yaml_v015()
-    config('update', '-y', 'packages')
-
-    # Check the entries have been transformed
-    data = spack.config.get('packages')
-    check_packages_updated(data)
-
-
 def test_config_update_config(config_yaml_v015):
     config_yaml_v015()
     config('update', '-y', 'config')
@@ -522,100 +493,26 @@ def test_config_update_not_needed(mutable_config):
     assert data_before == data_after
 
 
-def test_config_update_fail_on_permission_issue(
-        packages_yaml_v015, monkeypatch
-):
-    # The first time it will update and create the backup file
-    packages_yaml_v015()
-    # Mock a global scope where we cannot write
-    monkeypatch.setattr(
-        spack.cmd.config, '_can_update_config_file', lambda x, y: False
-    )
-    with pytest.raises(spack.main.SpackCommandError):
-        config('update', '-y', 'packages')
-
-
-def test_config_revert(packages_yaml_v015):
-    cfg_file = packages_yaml_v015()
-    bkp_file = cfg_file + '.bkp'
-
-    config('update', '-y', 'packages')
-
-    # Check that the backup file exists, compute its md5 sum
-    assert os.path.exists(bkp_file)
-    md5bkp = fs.md5sum(bkp_file)
-
-    config('revert', '-y', 'packages')
-
-    # Check that the backup file does not exist anymore and
-    # that the md5 sum of the configuration file is the same
-    # as that of the old backup file
-    assert not os.path.exists(bkp_file)
-    assert md5bkp == fs.md5sum(cfg_file)
-
-
-def test_config_revert_raise_if_cant_write(packages_yaml_v015, monkeypatch):
-    packages_yaml_v015()
-    config('update', '-y', 'packages')
-
-    # Mock a global scope where we cannot write
-    monkeypatch.setattr(
-        spack.cmd.config, '_can_revert_update', lambda x, y, z: False
-    )
-    # The command raises with an helpful error if a configuration
-    # file is to be deleted and we don't have sufficient permissions
-    with pytest.raises(spack.main.SpackCommandError):
-        config('revert', '-y', 'packages')
-
-
-def test_updating_config_implicitly_raises(packages_yaml_v015):
-    # Trying to write implicitly to a scope with a configuration file
-    # in the old format raises an exception
-    packages_yaml_v015()
-    with pytest.raises(RuntimeError):
-        config('add', 'packages:cmake:buildable:false')
-
-
-def test_updating_multiple_scopes_at_once(packages_yaml_v015):
-    # Create 2 config files in the old format
-    packages_yaml_v015(scope='user')
-    packages_yaml_v015(scope='site')
-
-    # Update both of them at once
-    config('update', '-y', 'packages')
-
-    for scope in ('user', 'site'):
-        data = spack.config.get('packages', scope=scope)
-        check_packages_updated(data)
-
-
 @pytest.mark.regression('18031')
 def test_config_update_can_handle_comments(mutable_config):
     # Create an outdated config file with comments
     scope = spack.config.default_modify_scope()
-    cfg_file = spack.config.config.get_config_filename(scope, 'packages')
+    cfg_file = spack.config.config.get_config_filename(scope, 'config')
     with open(cfg_file, mode='w') as f:
         f.write("""
-packages:
+config:
   # system cmake in /usr
-  cmake:
-    paths:
-      cmake@3.14.0:  /usr
-    # Another comment after the outdated section
-    buildable: False
+  install_tree: './foo'
+  # Another comment after the outdated section
+  install_hash_length: 7
 """)
 
     # Try to update it, it should not raise errors
-    config('update', '-y', 'packages')
+    config('update', '-y', 'config')
 
     # Check data
-    data = spack.config.get('packages', scope=scope)
-    assert 'paths' not in data['cmake']
-    assert 'externals' in data['cmake']
-    externals = data['cmake']['externals']
-    assert len(externals) == 1
-    assert externals[0]['spec'] == 'cmake@3.14.0'
-    assert externals[0]['prefix'] == '/usr'
+    data = spack.config.get('config', scope=scope)
+    assert 'root' in data['install_tree']
 
     # Check the comment is there
     with open(cfg_file) as f:
@@ -627,39 +524,21 @@ def test_config_update_can_handle_comments(mutable_config):
 
 @pytest.mark.regression('18050')
 def test_config_update_works_for_empty_paths(mutable_config):
-    # Create an outdated config file with empty "paths" and "modules"
     scope = spack.config.default_modify_scope()
-    cfg_file = spack.config.config.get_config_filename(scope, 'packages')
+    cfg_file = spack.config.config.get_config_filename(scope, 'config')
     with open(cfg_file, mode='w') as f:
         f.write("""
-packages:
-  cmake:
-    paths: {}
-    modules: {}
-    buildable: False
+config:
+    install_tree: ''
 """)
 
     # Try to update it, it should not raise errors
-    output = config('update', '-y', 'packages')
+    output = config('update', '-y', 'config')
 
     # This ensures that we updated the configuration
     assert '[backup=' in output
 
 
-def check_packages_updated(data):
-    """Check that the data from the packages_yaml_v015
-    has been updated.
-    """
-    assert 'externals' in data['cmake']
-    externals = data['cmake']['externals']
-    assert {'spec': 'cmake@3.14.0', 'prefix': '/usr'} in externals
-    assert 'paths' not in data['cmake']
-    assert 'externals' in data['gcc']
-    externals = data['gcc']['externals']
-    assert {'spec': 'gcc@8.3.0', 'modules': ['gcc-8']} in externals
-    assert 'modules' not in data['gcc']
-
-
 def check_config_updated(data):
     assert isinstance(data['install_tree'], dict)
     assert data['install_tree']['root'] == '/fake/path'

lib/spack/spack/test/cmd/create.py

@@ -160,3 +160,9 @@ def _parse_name_offset(path, v):
         spack.cmd.create.get_name(args)
     captured = capsys.readouterr()
     assert "Couldn't guess a name" in str(captured)
+
+
+def test_no_url(parser):
+    """Test creation of package without a URL."""
+    args = parser.parse_args(['--skip-editor', '-n', 'create-new-package'])
+    spack.cmd.create.create(parser, args)

lib/spack/spack/test/cmd/env.py

@@ -2288,76 +2288,6 @@ def test_env_write_only_non_default_nested(tmpdir):
     assert manifest == contents
 
 
-@pytest.fixture
-def packages_yaml_v015(tmpdir):
-    """Return the path to an existing manifest in the v0.15.x format
-    and the path to a non yet existing backup file.
-    """
-    raw_yaml = """
-spack:
-  specs:
-  - mpich
-  packages:
-    cmake:
-      paths:
-        cmake@3.17.3: /usr
-"""
-    manifest = tmpdir.ensure('spack.yaml')
-    backup_file = tmpdir.join('spack.yaml.bkp')
-    manifest.write(raw_yaml)
-    return manifest, backup_file
-
-
-def test_update_anonymous_env(packages_yaml_v015):
-    manifest, backup_file = packages_yaml_v015
-    env('update', '-y', str(manifest.dirname))
-
-    # The environment is now at the latest format
-    assert ev.is_latest_format(str(manifest))
-    # A backup file has been created and it's not at the latest format
-    assert os.path.exists(str(backup_file))
-    assert not ev.is_latest_format(str(backup_file))
-
-
-def test_double_update(packages_yaml_v015):
-    manifest, backup_file = packages_yaml_v015
-
-    # Update the environment
-    env('update', '-y', str(manifest.dirname))
-    # Try to read the environment (it should not error)
-    ev.create('test', str(manifest))
-    # Updating again does nothing since the manifest is up-to-date
-    env('update', '-y', str(manifest.dirname))
-
-    # The environment is at the latest format
-    assert ev.is_latest_format(str(manifest))
-    # A backup file has been created and it's not at the latest format
-    assert os.path.exists(str(backup_file))
-    assert not ev.is_latest_format(str(backup_file))
-
-
-def test_update_and_revert(packages_yaml_v015):
-    manifest, backup_file = packages_yaml_v015
-
-    # Update the environment
-    env('update', '-y', str(manifest.dirname))
-    assert os.path.exists(str(backup_file))
-    assert not ev.is_latest_format(str(backup_file))
-    assert ev.is_latest_format(str(manifest))
-
-    # Revert to previous state
-    env('revert', '-y', str(manifest.dirname))
-    assert not os.path.exists(str(backup_file))
-    assert not ev.is_latest_format(str(manifest))
-
-
-def test_old_format_cant_be_updated_implicitly(packages_yaml_v015):
-    manifest, backup_file = packages_yaml_v015
-    env('activate', str(manifest.dirname))
-    with pytest.raises(spack.main.SpackCommandError):
-        add('hdf5')
-
-
 @pytest.mark.parametrize('concretization,unify', [
     ('together', 'true'),
     ('separately', 'false')

lib/spack/spack/test/cmd/external.py

@@ -8,8 +8,6 @@
 
 import pytest
 
-from llnl.util.filesystem import getuid, touch
-
 import spack
 import spack.detection
 import spack.detection.path
@@ -196,53 +194,6 @@ def test_find_external_empty_default_manifest_dir(
     external('find')
 
 
-@pytest.mark.skipif(sys.platform == 'win32',
-                    reason="Can't chmod on Windows")
-@pytest.mark.skipif(getuid() == 0, reason='user is root')
-def test_find_external_manifest_with_bad_permissions(
-        mutable_config, working_env, mock_executable, mutable_mock_repo,
-        _platform_executables, tmpdir, monkeypatch):
-    """The user runs 'spack external find'; the default path for storing
-    manifest files exists but with insufficient permissions. Check that
-    the command does not fail.
-    """
-    test_manifest_dir = str(tmpdir.mkdir('manifest_dir'))
-    test_manifest_file_path = os.path.join(test_manifest_dir, 'badperms.json')
-    touch(test_manifest_file_path)
-    monkeypatch.setenv('PATH', '')
-    monkeypatch.setattr(spack.cray_manifest, 'default_path',
-                        test_manifest_dir)
-    try:
-        os.chmod(test_manifest_file_path, 0)
-        output = external('find')
-        assert 'insufficient permissions' in output
-        assert 'Skipping manifest and continuing' in output
-    finally:
-        os.chmod(test_manifest_file_path, 0o700)
-
-
-def test_find_external_manifest_failure(
-        mutable_config, mutable_mock_repo, tmpdir, monkeypatch):
-    """The user runs 'spack external find'; the manifest parsing fails with
-    some exception. Ensure that the command still succeeds (i.e. moves on
-    to other external detection mechanisms).
-    """
-    # First, create an empty manifest file (without a file to read, the
-    # manifest parsing is skipped)
-    test_manifest_dir = str(tmpdir.mkdir('manifest_dir'))
-    test_manifest_file_path = os.path.join(test_manifest_dir, 'test.json')
-    touch(test_manifest_file_path)
-
-    def fail():
-        raise Exception()
-
-    monkeypatch.setattr(
-        spack.cmd.external, '_collect_and_consume_cray_manifest_files', fail)
-    monkeypatch.setenv('PATH', '')
-    output = external('find')
-    assert 'Skipping manifest and continuing' in output
-
-
 def test_find_external_nonempty_default_manifest_dir(
         mutable_database, mutable_mock_repo,
         _platform_executables, tmpdir, monkeypatch,

lib/spack/spack/test/cmd/install.py

@@ -20,7 +20,7 @@
 import spack.config
 import spack.environment as ev
 import spack.hash_types as ht
-import spack.package
+import spack.package_base
 import spack.util.executable
 from spack.error import SpackError
 from spack.main import SpackCommand
@@ -66,7 +66,7 @@ def test_install_package_and_dependency(
 def test_install_runtests_notests(monkeypatch, mock_packages, install_mockery):
     def check(pkg):
         assert not pkg.run_tests
-    monkeypatch.setattr(spack.package.PackageBase, 'unit_test_check', check)
+    monkeypatch.setattr(spack.package_base.PackageBase, 'unit_test_check', check)
     install('-v', 'dttop')
 
 
@@ -75,7 +75,7 @@ def test_install_runtests_root(monkeypatch, mock_packages, install_mockery):
     def check(pkg):
         assert pkg.run_tests == (pkg.name == 'dttop')
 
-    monkeypatch.setattr(spack.package.PackageBase, 'unit_test_check', check)
+    monkeypatch.setattr(spack.package_base.PackageBase, 'unit_test_check', check)
     install('--test=root', 'dttop')
 
 
@@ -84,7 +84,7 @@ def test_install_runtests_all(monkeypatch, mock_packages, install_mockery):
     def check(pkg):
         assert pkg.run_tests
 
-    monkeypatch.setattr(spack.package.PackageBase, 'unit_test_check', check)
+    monkeypatch.setattr(spack.package_base.PackageBase, 'unit_test_check', check)
     install('--test=all', 'a')
 
 
@@ -257,7 +257,7 @@ def test_install_commit(
 
     """
     repo_path, filename, commits = mock_git_version_info
-    monkeypatch.setattr(spack.package.PackageBase,
+    monkeypatch.setattr(spack.package_base.PackageBase,
                         'git', 'file://%s' % repo_path,
                         raising=False)
 

lib/spack/spack/test/cmd/pkg.py

@@ -22,7 +22,7 @@
 
 #: new fake package template
 pkg_template = '''\
-from spack import *
+from spack.package import *
 
 class {name}(Package):
     homepage = "http://www.example.com"

lib/spack/spack/test/cmd/stage.py

@@ -28,7 +28,7 @@ def test_stage_spec(monkeypatch):
     def fake_stage(pkg, mirror_only=False):
         expected.remove(pkg.name)
 
-    monkeypatch.setattr(spack.package.PackageBase, 'do_stage', fake_stage)
+    monkeypatch.setattr(spack.package_base.PackageBase, 'do_stage', fake_stage)
 
     stage('trivial-install-test-package', 'mpileaks')
 
@@ -42,7 +42,7 @@ def check_stage_path(monkeypatch, tmpdir):
     def fake_stage(pkg, mirror_only=False):
         assert pkg.path == expected_path
 
-    monkeypatch.setattr(spack.package.PackageBase, 'do_stage', fake_stage)
+    monkeypatch.setattr(spack.package_base.PackageBase, 'do_stage', fake_stage)
 
     return expected_path
 
@@ -69,7 +69,7 @@ def fake_stage(pkg, mirror_only=False):
         assert pkg.name == 'trivial-install-test-package'
         assert pkg.path is None
 
-    monkeypatch.setattr(spack.package.PackageBase, 'do_stage', fake_stage)
+    monkeypatch.setattr(spack.package_base.PackageBase, 'do_stage', fake_stage)
 
     e = ev.create('test')
     e.add('mpileaks')
@@ -87,7 +87,7 @@ def fake_stage(pkg, mirror_only=False):
         assert pkg.name == 'mpileaks'
         assert pkg.version == Version('100.100')
 
-    monkeypatch.setattr(spack.package.PackageBase, 'do_stage', fake_stage)
+    monkeypatch.setattr(spack.package_base.PackageBase, 'do_stage', fake_stage)
 
     e = ev.create('test')
     e.add('mpileaks@100.100')
@@ -115,7 +115,7 @@ def test_stage_full_env(mutable_mock_env_path, monkeypatch):
     def fake_stage(pkg, mirror_only=False):
         expected.remove(pkg.name)
 
-    monkeypatch.setattr(spack.package.PackageBase, 'do_stage', fake_stage)
+    monkeypatch.setattr(spack.package_base.PackageBase, 'do_stage', fake_stage)
 
     with e:
         stage()

lib/spack/spack/test/cmd/style.py

@@ -76,7 +76,9 @@ def flake8_package_with_errors(scope="function"):
         package = FileFilter(filename)
         package.filter("state = 'unmodified'", "state    =    'modified'", string=True)
         package.filter(
-            "from spack import *", "from spack import *\nimport os", string=True
+            "from spack.package import *",
+            "from spack.package import *\nimport os",
+            string=True
         )
         yield filename
     finally:
@@ -96,6 +98,26 @@ def test_changed_files(flake8_package):
     assert flake8_package in files
 
 
+def test_changed_files_from_git_rev_base(tmpdir, capfd):
+    """Test arbitrary git ref as base."""
+    git = which("git", required=True)
+    with tmpdir.as_cwd():
+        git("init")
+        git("checkout", "-b", "main")
+        git("config", "user.name", "test user")
+        git("config", "user.email", "test@user.com")
+        git("commit", "--allow-empty", "-m", "initial commit")
+
+        tmpdir.ensure('bin/spack')
+        assert changed_files(base="HEAD") == ['bin/spack']
+        assert changed_files(base="main") == ['bin/spack']
+
+        git("add", 'bin/spack')
+        git("commit", "-m", "v1")
+        assert changed_files(base="HEAD") == []
+        assert changed_files(base="HEAD~") == ["bin/spack"]
+
+
 def test_changed_no_base(tmpdir, capfd):
     """Ensure that we fail gracefully with no base branch."""
     tmpdir.join("bin").ensure("spack")
@@ -111,7 +133,7 @@ def test_changed_no_base(tmpdir, capfd):
             changed_files(base="foobar")
 
         out, err = capfd.readouterr()
-        assert "This repository does not have a 'foobar' branch." in err
+        assert "This repository does not have a 'foobar'" in err
 
 
 def test_changed_files_all_files(flake8_package):
@@ -273,17 +295,17 @@ def test_style(flake8_package, tmpdir):
         relative = os.path.relpath(flake8_package)
 
         # no args
-        output = style()
+        output = style(fail_on_error=False)
         assert relative in output
         assert "spack style checks were clean" in output
 
         # one specific arg
-        output = style(flake8_package)
+        output = style(flake8_package, fail_on_error=False)
         assert relative in output
         assert "spack style checks were clean" in output
 
         # specific file that isn't changed
-        output = style(__file__)
+        output = style(__file__, fail_on_error=False)
         assert relative not in output
         assert __file__ in output
         assert "spack style checks were clean" in output

lib/spack/spack/test/cmd/test.py

@@ -13,7 +13,7 @@
 
 import spack.cmd.install
 import spack.config
-import spack.package
+import spack.package_base
 import spack.paths
 import spack.store
 from spack.main import SpackCommand
@@ -239,7 +239,7 @@ def test_test_list(
                     reason="Not supported on Windows (yet)")
 def test_has_test_method_fails(capsys):
     with pytest.raises(SystemExit):
-        spack.package.has_test_method('printing-package')
+        spack.package_base.has_test_method('printing-package')
 
     captured = capsys.readouterr()[1]
     assert 'is not a class' in captured

lib/spack/spack/test/conftest.py

@@ -35,7 +35,7 @@
 import spack.database
 import spack.directory_layout
 import spack.environment as ev
-import spack.package
+import spack.package_base
 import spack.package_prefs
 import spack.paths
 import spack.platforms
@@ -532,7 +532,7 @@ def _pkg_install_fn(pkg, spec, prefix):
 
 @pytest.fixture
 def mock_pkg_install(monkeypatch):
-    monkeypatch.setattr(spack.package.PackageBase, 'install',
+    monkeypatch.setattr(spack.package_base.PackageBase, 'install',
                         _pkg_install_fn, raising=False)
 
 
@@ -934,7 +934,7 @@ def mock_fetch(mock_archive, monkeypatch):
     mock_fetcher.append(URLFetchStrategy(mock_archive.url))
 
     monkeypatch.setattr(
-        spack.package.PackageBase, 'fetcher', mock_fetcher)
+        spack.package_base.PackageBase, 'fetcher', mock_fetcher)
 
 
 class MockLayout(object):

lib/spack/spack/test/cray_manifest.py

@@ -10,7 +10,6 @@
 logic needs to consume all related specs in a single pass).
 """
 import json
-import os
 
 import pytest
 
@@ -33,7 +32,7 @@
   },
   "compiler": {
     "name": "gcc",
-    "version": "10.2.0.cray"
+    "version": "10.2.0"
   },
   "dependencies": {
     "packagey": {
@@ -157,7 +156,7 @@ def spec_json(self):
 # Intended to match example_compiler_entry above
 _common_compiler = JsonCompilerEntry(
     name='gcc',
-    version='10.2.0.cray',
+    version='10.2.0',
     arch={
         "os": "centos8",
         "target": "x86_64"
@@ -310,16 +309,8 @@ def test_failed_translate_compiler_name():
 
 def create_manifest_content():
     return {
-        # Note: the cray_manifest module doesn't use the _meta section right
-        # now, but it is anticipated to be useful
-        '_meta': {
-            "file-type": "cray-pe-json",
-            "system-type": "test",
-            "schema-version": "1.3",
-            "cpe-version": "22.06"
-        },
         'specs': list(x.to_dict() for x in generate_openmpi_entries()),
-        'compilers': [_common_compiler.compiler_json()]
+        'compilers': []
     }
 
 
@@ -345,45 +336,3 @@ def test_read_cray_manifest(
             ' ^/openmpifakehasha'.split(),
             concretize=True)
         assert concretized_specs[0]['hwloc'].dag_hash() == 'hwlocfakehashaaa'
-
-
-def test_read_cray_manifest_twice_no_compiler_duplicates(
-        tmpdir, mutable_config, mock_packages, mutable_database):
-    if spack.config.get('config:concretizer') == 'clingo':
-        pytest.skip("The ASP-based concretizer is currently picky about "
-                    " OS matching and will fail.")
-
-    with tmpdir.as_cwd():
-        test_db_fname = 'external-db.json'
-        with open(test_db_fname, 'w') as db_file:
-            json.dump(create_manifest_content(), db_file)
-
-        # Read the manifest twice
-        cray_manifest.read(test_db_fname, True)
-        cray_manifest.read(test_db_fname, True)
-
-        compilers = spack.compilers.all_compilers()
-        filtered = list(c for c in compilers if
-                        c.spec == spack.spec.CompilerSpec('gcc@10.2.0.cray'))
-        assert(len(filtered) == 1)
-
-
-def test_read_old_manifest_v1_2(
-        tmpdir, mutable_config, mock_packages, mutable_database):
-    """Test reading a file using the older format
-    ('version' instead of 'schema-version').
-    """
-    manifest_dir = str(tmpdir.mkdir('manifest_dir'))
-    manifest_file_path = os.path.join(manifest_dir, 'test.json')
-    with open(manifest_file_path, 'w') as manifest_file:
-        manifest_file.write("""\
-{
-  "_meta": {
-    "file-type": "cray-pe-json",
-    "system-type": "EX",
-    "version": "1.3"
-  },
-  "specs": []
-}
-""")
-    cray_manifest.read(manifest_file_path, True)

lib/spack/spack/test/data/compression/Foo

@@ -0,0 +1 @@
+TEST