zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into f/env-location

Browse Source
This commit is contained in:
psakievich
2022-12-01 17:00:36 -07:00
committed by GitHub
parent e19cc2385e f63fb2f521
commit e773396747
1327 changed files with 9224 additions and 17199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/bootstrap.yml vendored
View File

@@ -214,7 +214,7 @@ jobs:
- name: Bootstrap clingo
run: |
set -ex
for ver in '2.7' '3.6' '3.7' '3.8' '3.9' '3.10' ; do
for ver in '3.6' '3.7' '3.8' '3.9' '3.10' ; do
not_found=1
ver_dir="$(find $RUNNER_TOOL_CACHE/Python -wholename "*/${ver}.*/*/bin" | grep . || true)"
echo "Testing $ver_dir"

8
.github/workflows/ci.yaml vendored
View File

@@ -20,12 +20,6 @@ jobs:
uses: ./.github/workflows/valid-style.yml
with:
with_coverage: ${{ needs.changes.outputs.core }}
audit-ancient-python:
uses: ./.github/workflows/audit.yaml
needs: [ changes ]
with:
with_coverage: ${{ needs.changes.outputs.core }}
python_version: 2.7
all-prechecks:
needs: [ prechecks ]
runs-on: ubuntu-latest
@@ -85,7 +79,7 @@ jobs:
needs: [ prechecks ]
uses: ./.github/workflows/windows_python.yml
all:
needs: [ windows, unit-tests, bootstrap, audit-ancient-python ]
needs: [ windows, unit-tests, bootstrap ]
runs-on: ubuntu-latest
steps:
- name: Success

44
.github/workflows/unit_tests.yaml vendored
View File

@@ -11,31 +11,38 @@ concurrency:
jobs:
# Run unit tests with different configurations on linux
ubuntu:
runs-on: ubuntu-latest
runs-on: ${{ matrix.os }}
strategy:
matrix:
python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11']
os: [ubuntu-latest]
python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
concretizer: ['clingo']
on_develop:
- ${{ github.ref == 'refs/heads/develop' }}
include:
- python-version: 2.7
- python-version: '3.11'
os: ubuntu-latest
concretizer: original
on_develop: ${{ github.ref == 'refs/heads/develop' }}
- python-version: '3.11'
concretizer: original
- python-version: '3.6'
os: ubuntu-20.04
concretizer: clingo
on_develop: ${{ github.ref == 'refs/heads/develop' }}
exclude:
- python-version: '3.7'
os: ubuntu-latest
concretizer: 'clingo'
on_develop: false
- python-version: '3.8'
os: ubuntu-latest
concretizer: 'clingo'
on_develop: false
- python-version: '3.9'
os: ubuntu-latest
concretizer: 'clingo'
on_develop: false
- python-version: '3.10'
os: ubuntu-latest
concretizer: 'clingo'
on_develop: false
@@ -52,24 +59,11 @@ jobs:
# Needed for unit tests
sudo apt-get -y install \
coreutils cvs gfortran graphviz gnupg2 mercurial ninja-build \
patchelf cmake bison libbison-dev kcov
cmake bison libbison-dev kcov
- name: Install Python packages
run: |
pip install --upgrade pip six setuptools pytest codecov[toml] pytest-xdist
# Install pytest-cov only on recent Python, to avoid stalling on Python 2.7 due
# to bugs on an unmaintained version of the package when used with xdist.
if [[ ${{ matrix.python-version }} != "2.7" ]]; then
pip install --upgrade pytest-cov
fi
# ensure style checks are not skipped in unit tests for python >= 3.6
# note that true/false (i.e., 1/0) are opposite in conditions in python and bash
if python -c 'import sys; sys.exit(not sys.version_info >= (3, 6))'; then
pip install --upgrade flake8 "isort>=4.3.5" "mypy>=0.900" "click==8.0.4" "black<=21.12b0"
fi
- name: Pin pathlib for Python 2.7
if: ${{ matrix.python-version == 2.7 }}
run: |
pip install -U pathlib2==2.3.6 toml
pip install --upgrade pip six setuptools pytest codecov[toml] pytest-xdist pytest-cov
pip install --upgrade flake8 "isort>=4.3.5" "mypy>=0.900" "click" "black"
- name: Setup git configuration
run: |
# Need this for the git tests to succeed.
@@ -82,6 +76,7 @@ jobs:
run: |
. share/spack/setup-env.sh
spack bootstrap disable spack-install
spack bootstrap now
spack -v solve zlib
- name: Run unit tests
env:
@@ -89,7 +84,7 @@ jobs:
SPACK_TEST_SOLVER: ${{ matrix.concretizer }}
SPACK_TEST_PARALLEL: 2
COVERAGE: true
UNIT_TEST_COVERAGE: ${{ (matrix.python-version == '3.11') }}
UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.11' }}
run: |
share/spack/qa/run-unit-tests
- uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70
@@ -165,10 +160,7 @@ jobs:
- name: Install System packages
run: |
sudo apt-get -y update
# Needed for unit tests
sudo apt-get -y install \
coreutils cvs gfortran graphviz gnupg2 mercurial ninja-build \
patchelf kcov
sudo apt-get -y install coreutils cvs gfortran graphviz gnupg2 mercurial ninja-build kcov
- name: Install Python packages
run: |
pip install --upgrade pip six setuptools pytest codecov coverage[toml] pytest-cov clingo pytest-xdist

6
.github/workflows/valid-style.yml vendored
View File

@@ -28,9 +28,9 @@ jobs:
pip install --upgrade pip
pip install --upgrade vermin
- name: vermin (Spack's Core)
run: vermin --backport argparse --violations --backport typing -t=2.7- -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/
run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/
- name: vermin (Repositories)
run: vermin --backport argparse --violations --backport typing -t=2.7- -t=3.6- -vvv var/spack/repos
run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv var/spack/repos
# Run style checks on the files that have been changed
style:
runs-on: ubuntu-latest
@@ -44,7 +44,7 @@ jobs:
cache: 'pip'
- name: Install Python packages
run: |
python3 -m pip install --upgrade pip six setuptools types-six click==8.0.2 'black==21.12b0' mypy isort clingo flake8
python3 -m pip install --upgrade pip six setuptools types-six black mypy isort clingo flake8
- name: Setup git configuration
run: |
# Need this for the git tests to succeed.

274
CHANGELOG.md
View File

@@ -1,16 +1,284 @@
# v0.19.0 (2022-11-11)
`v0.19.0` is a major feature release.
## Major features in this release
1. **Package requirements**
Spack's traditional [package preferences](
https://spack.readthedocs.io/en/latest/build_settings.html#package-preferences)
are soft, but we've added hard requriements to `packages.yaml` and `spack.yaml`
(#32528, #32369). Package requirements use the same syntax as specs:
```yaml
packages:
libfabric:
require: "@1.13.2"
mpich:
require:
- one_of: ["+cuda", "+rocm"]
```
More details in [the docs](
https://spack.readthedocs.io/en/latest/build_settings.html#package-requirements).
2. **Environment UI Improvements**
* Fewer surprising modifications to `spack.yaml` (#33711):
* `spack install` in an environment will no longer add to the `specs:` list; you'll
need to either use `spack add <spec>` or `spack install --add <spec>`.
* Similarly, `spack uninstall` will not remove from your environment's `specs:`
list; you'll need to use `spack remove` or `spack uninstall --remove`.
This will make it easier to manage an environment, as there is clear separation
between the stack to be installed (`spack.yaml`/`spack.lock`) and which parts of
it should be installed (`spack install` / `spack uninstall`).
* `concretizer:unify:true` is now the default mode for new environments (#31787)
We see more users creating `unify:true` environments now. Users who need
`unify:false` can add it to their environment to get the old behavior. This will
concretize every spec in the environment independently.
* Include environment configuration from URLs (#29026, [docs](
https://spack.readthedocs.io/en/latest/environments.html#included-configurations))
You can now include configuration in your environment directly from a URL:
```yaml
spack:
include:
- https://github.com/path/to/raw/config/compilers.yaml
```
4. **Multiple Build Systems**
An increasing number of packages in the ecosystem need the ability to support
multiple build systems (#30738, [docs](
https://spack.readthedocs.io/en/latest/packaging_guide.html#multiple-build-systems)),
either across versions, across platforms, or within the same version of the software.
This has been hard to support through multiple inheritance, as methods from different
build system superclasses would conflict. `package.py` files can now define separate
builder classes with installation logic for different build systems, e.g.:
```python
class ArpackNg(CMakePackage, AutotoolsPackage):
build_system(
conditional("cmake", when="@0.64:"),
conditional("autotools", when="@:0.63"),
default="cmake",
)
class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder):
def cmake_args(self):
pass
class Autotoolsbuilder(spack.build_systems.autotools.AutotoolsBuilder):
def configure_args(self):
pass
```
5. **Compiler and variant propagation**
Currently, compiler flags and variants are inconsistent: compiler flags set for a
package are inherited by its dependencies, while variants are not. We should have
these be consistent by allowing for inheritance to be enabled or disabled for both
variants and compiler flags.
Example syntax:
- `package ++variant`:
enabled variant that will be propagated to dependencies
- `package +variant`:
enabled variant that will NOT be propagated to dependencies
- `package ~~variant`:
disabled variant that will be propagated to dependencies
- `package ~variant`:
disabled variant that will NOT be propagated to dependencies
- `package cflags==-g`:
`cflags` will be propagated to dependencies
- `package cflags=-g`:
`cflags` will NOT be propagated to dependencies
Syntax for non-boolan variants is similar to compiler flags. More in the docs for
[variants](
https://spack.readthedocs.io/en/latest/basic_usage.html#variants) and [compiler flags](
https://spack.readthedocs.io/en/latest/basic_usage.html#compiler-flags).
6. **Enhancements to git version specifiers**
* `v0.18.0` added the ability to use git commits as versions. You can now use the
`git.` prefix to specify git tags or branches as versions. All of these are valid git
versions in `v0.19` (#31200):
```console
foo@abcdef1234abcdef1234abcdef1234abcdef1234 # raw commit
foo@git.abcdef1234abcdef1234abcdef1234abcdef1234 # commit with git prefix
foo@git.develop # the develop branch
foo@git.0.19 # use the 0.19 tag
```
* `v0.19` also gives you more control over how Spack interprets git versions, in case
Spack cannot detect the version from the git repository. You can suffix a git
version with `=<version>` to force Spack to concretize it as a particular version
(#30998, #31914, #32257):
```console
# use mybranch, but treat it as version 3.2 for version comparison
foo@git.mybranch=3.2
# use the given commit, but treat it as develop for version comparison
foo@git.abcdef1234abcdef1234abcdef1234abcdef1234=develop
```
More in [the docs](
https://spack.readthedocs.io/en/latest/basic_usage.html#version-specifier)
7. **Changes to Cray EX Support**
Cray machines have historically had their own "platform" within Spack, because we
needed to go through the module system to leverage compilers and MPI installations on
these machines. The Cray EX programming environment now provides standalone `craycc`
executables and proper `mpicc` wrappers, so Spack can treat EX machines like Linux
with extra packages (#29392).
We expect this to greatly reduce bugs, as external packages and compilers can now be
used by prefix instead of through modules. We will also no longer be subject to
reproducibility issues when modules change from Cray PE release to release and from
site to site. This also simplifies dealing with the underlying Linux OS on cray
systems, as Spack will properly model the machine's OS as either SuSE or RHEL.
8. **Improvements to tests and testing in CI**
* `spack ci generate --tests` will generate a `.gitlab-ci.yml` file that not only does
builds but also runs tests for built packages (#27877). Public GitHub pipelines now
also run tests in CI.
* `spack test run --explicit` will only run tests for packages that are explicitly
installed, instead of all packages.
9. **Experimental binding link model**
You can add a new option to `config.yaml` to make Spack embed absolute paths to
needed shared libraries in ELF executables and shared libraries on Linux (#31948, [docs](
https://spack.readthedocs.io/en/latest/config_yaml.html#shared-linking-bind)):
```yaml
config:
shared_linking:
type: rpath
bind: true
```
This can improve launch time at scale for parallel applications, and it can make
installations less susceptible to environment variables like `LD_LIBRARY_PATH`, even
especially when dealing with external libraries that use `RUNPATH`. You can think of
this as a faster, even higher-precedence version of `RPATH`.
## Other new features of note
* `spack spec` prints dependencies more legibly. Dependencies in the output now appear
at the *earliest* level of indentation possible (#33406)
* You can override `package.py` attributes like `url`, directly in `packages.yaml`
(#33275, [docs](
https://spack.readthedocs.io/en/latest/build_settings.html#assigning-package-attributes))
* There are a number of new architecture-related format strings you can use in Spack
configuration files to specify paths (#29810, [docs](
https://spack.readthedocs.io/en/latest/configuration.html#config-file-variables))
* Spack now supports bootstrapping Clingo on Windows (#33400)
* There is now support for an `RPATH`-like library model on Windows (#31930)
## Performance Improvements
* Major performance improvements for installation from binary caches (#27610, #33628,
#33636, #33608, #33590, #33496)
* Test suite can now be parallelized using `xdist` (used in GitHub Actions) (#32361)
* Reduce lock contention for parallel builds in environments (#31643)
## New binary caches and stacks
* We now build nearly all of E4S with `oneapi` in our buildcache (#31781, #31804,
#31804, #31803, #31840, #31991, #32117, #32107, #32239)
* Added 3 new machine learning-centric stacks to binary cache: `x86_64_v3`, CUDA, ROCm
(#31592, #33463)
## Removals and Deprecations
* Support for Python 3.5 is dropped (#31908). Only Python 2.7 and 3.6+ are officially
supported.
* This is the last Spack release that will support Python 2 (#32615). Spack `v0.19`
will emit a deprecation warning if you run it with Python 2, and Python 2 support will
soon be removed from the `develop` branch.
* `LD_LIBRARY_PATH` is no longer set by default by `spack load` or module loads.
Setting `LD_LIBRARY_PATH` in Spack environments/modules can cause binaries from
outside of Spack to crash, and Spack's own builds use `RPATH` and do not need
`LD_LIBRARY_PATH` set in order to run. If you still want the old behavior, you
can run these commands to configure Spack to set `LD_LIBRARY_PATH`:
```console
spack config add modules:prefix_inspections:lib64:[LD_LIBRARY_PATH]
spack config add modules:prefix_inspections:lib:[LD_LIBRARY_PATH]
```
* The `spack:concretization:[together|separately]` has been removed after being
deprecated in `v0.18`. Use `concretizer:unify:[true|false]`.
* `config:module_roots` is no longer supported after being deprecated in `v0.18`. Use
configuration in module sets instead (#28659, [docs](
https://spack.readthedocs.io/en/latest/module_file_support.html)).
* `spack activate` and `spack deactivate` are no longer supported, having been
deprecated in `v0.18`. Use an environment with a view instead of
activating/deactivating ([docs](
https://spack.readthedocs.io/en/latest/environments.html#configuration-in-spack-yaml)).
* The old YAML format for buildcaches is now deprecated (#33707). If you are using an
old buildcache with YAML metadata you will need to regenerate it with JSON metadata.
* `spack bootstrap trust` and `spack bootstrap untrust` are deprecated in favor of
`spack bootstrap enable` and `spack bootstrap disable` and will be removed in `v0.20`.
(#33600)
* The `graviton2` architecture has been renamed to `neoverse_n1`, and `graviton3`
is now `neoverse_v1`. Buildcaches using the old architecture names will need to be rebuilt.
* The terms `blacklist` and `whitelist` have been replaced with `include` and `exclude`
in all configuration files (#31569). You can use `spack config update` to
automatically fix your configuration files.
## Notable Bugfixes
* Permission setting on installation now handles effective uid properly (#19980)
* `buildable:true` for an MPI implementation now overrides `buildable:false` for `mpi` (#18269)
* Improved error messages when attempting to use an unconfigured compiler (#32084)
* Do not punish explicitly requested compiler mismatches in the solver (#30074)
* `spack stage`: add missing --fresh and --reuse (#31626)
* Fixes for adding build system executables like `cmake` to package scope (#31739)
* Bugfix for binary relocation with aliased strings produced by newer `binutils` (#32253)
## Spack community stats
* 6,751 total packages, 335 new since `v0.18.0`
* 141 new Python packages
* 89 new R packages
* 303 people contributed to this release
* 287 committers to packages
* 57 committers to core
# v0.18.1 (2022-07-19)
### Spack Bugfixes
* Fix several bugs related to bootstrapping (#30834,#31042,#31180)
* Fix a regression that was causing spec hashes to differ between
* Fix a regression that was causing spec hashes to differ between
Python 2 and Python 3 (#31092)
* Fixed compiler flags for oneAPI and DPC++ (#30856)
* Fixed several issues related to concretization (#31142,#31153,#31170,#31226)
* Improved support for Cray manifest file and `spack external find` (#31144,#31201,#31173,#31186)
* Assign a version to openSUSE Tumbleweed according to the GLIBC version
in the system (#19895)
in the system (#19895)
* Improved Dockerfile generation for `spack containerize` (#29741,#31321)
* Fixed a few bugs related to concurrent execution of commands (#31509,#31493,#31477)
* Fixed a few bugs related to concurrent execution of commands (#31509,#31493,#31477)
### Package updates
* WarpX: add v22.06, fixed libs property (#30866,#31102)

4
SECURITY.md
View File

@@ -10,8 +10,8 @@ For more on Spack's release structure, see
| Version | Supported |
| ------- | ------------------ |
| develop | :white_check_mark: |
| 0.17.x | :white_check_mark: |
| 0.16.x | :white_check_mark: |
| 0.19.x | :white_check_mark: |
| 0.18.x | :white_check_mark: |
## Reporting a Vulnerability

8
bin/spack
View File

@@ -31,13 +31,11 @@ import os
import os.path
import sys
min_python3 = (3, 5)
min_python3 = (3, 6)
if sys.version_info[:2] < (2, 7) or (
sys.version_info[:2] >= (3, 0) and sys.version_info[:2] < min_python3
):
if sys.version_info[:2] < min_python3:
v_info = sys.version_info[:3]
msg = "Spack requires Python 2.7 or %d.%d or higher " % min_python3
msg = "Spack requires Python %d.%d or higher " % min_python3
msg += "You are running spack with Python %d.%d.%d." % v_info
sys.exit(msg)

8
etc/spack/defaults/config.yaml
View File

@@ -19,7 +19,7 @@ config:
install_tree:
root: $spack/opt/spack
projections:
all: "${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}"
all: "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}"
# install_tree can include an optional padded length (int or boolean)
# default is False (do not pad)
# if padded_length is True, Spack will pad as close to the system max path
@@ -218,4 +218,8 @@ config:
# Number of seconds a buildcache's index.json is cached locally before probing
# for updates, within a single Spack invocation. Defaults to 10 minutes.
binary_index_ttl: 600
binary_index_ttl: 600
flags:
# Whether to keep -Werror flags active in package builds.
keep_werror: 'none'

222
lib/spack/docs/basic_usage.rst
View File

@@ -1114,21 +1114,21 @@ set of arbitrary versions, such as ``@1.0,1.5,1.7`` (``1.0``, ``1.5``,
or ``1.7``). When you supply such a specifier to ``spack install``,
it constrains the set of versions that Spack will install.
For packages with a ``git`` attribute, ``git`` references
may be specified instead of a numerical version i.e. branches, tags
and commits. Spack will stage and build based off the ``git``
For packages with a ``git`` attribute, ``git`` references
may be specified instead of a numerical version i.e. branches, tags
and commits. Spack will stage and build based off the ``git``
reference provided. Acceptable syntaxes for this are:
.. code-block:: sh
# branches and tags
foo@git.develop # use the develop branch
foo@git.0.19 # use the 0.19 tag
# commit hashes
foo@abcdef1234abcdef1234abcdef1234abcdef1234 # 40 character hashes are automatically treated as git commits
foo@git.abcdef1234abcdef1234abcdef1234abcdef1234
Spack versions from git reference either have an associated version supplied by the user,
or infer a relationship to known versions from the structure of the git repository. If an
associated version is supplied by the user, Spack treats the git version as equivalent to that
@@ -1244,8 +1244,8 @@ For example, for the ``stackstart`` variant:
.. code-block:: sh
mpileaks stackstart=4 # variant will be propagated to dependencies
mpileaks stackstart==4 # only mpileaks will have this variant value
mpileaks stackstart==4 # variant will be propagated to dependencies
mpileaks stackstart=4 # only mpileaks will have this variant value
^^^^^^^^^^^^^^
Compiler Flags
@@ -1672,9 +1672,13 @@ own install prefix. However, certain packages are typically installed
`Python <https://www.python.org>`_ packages are typically installed in the
``$prefix/lib/python-2.7/site-packages`` directory.
Spack has support for this type of installation as well. In Spack,
a package that can live inside the prefix of another package is called
an *extension*. Suppose you have Python installed like so:
In Spack, installation prefixes are immutable, so this type of installation
is not directly supported. However, it is possible to create views that
allow you to merge install prefixes of multiple packages into a single new prefix.
Views are a convenient way to get a more traditional filesystem structure.
Using *extensions*, you can ensure that Python packages always share the
same prefix in the view as Python itself. Suppose you have
Python installed like so:
.. code-block:: console
@@ -1712,8 +1716,6 @@ You can find extensions for your Python installation like this:
py-ipython@2.3.1 py-pygments@2.0.1 py-setuptools@11.3.1
py-matplotlib@1.4.2 py-pyparsing@2.0.3 py-six@1.9.0
==> None activated.
The extensions are a subset of what's returned by ``spack list``, and
they are packages like any other. They are installed into their own
prefixes, and you can see this with ``spack find --paths``:
@@ -1741,32 +1743,72 @@ directly when you run ``python``:
ImportError: No module named numpy
>>>
^^^^^^^^^^^^^^^^
Using Extensions
^^^^^^^^^^^^^^^^
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Using Extensions in Environments
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
There are four ways to get ``numpy`` working in Python. The first is
to use :ref:`shell-support`. You can simply ``load`` the extension,
and it will be added to the ``PYTHONPATH`` in your current shell:
The recommended way of working with extensions such as ``py-numpy``
above is through :ref:`Environments <environments>`. For example,
the following creates an environment in the current working directory
with a filesystem view in the ``./view`` directory:
.. code-block:: console
$ spack load python
$ spack load py-numpy
$ spack env create --with-view view --dir .
$ spack -e . add py-numpy
$ spack -e . concretize
$ spack -e . install
We recommend environments for two reasons. Firstly, environments
can be activated (requires :ref:`shell-support`):
.. code-block:: console
$ spack env activate .
which sets all the right environment variables such as ``PATH`` and
``PYTHONPATH``. This ensures that
.. code-block:: console
$ python
>>> import numpy
works. Secondly, even without shell support, the view ensures
that Python can locate its extensions:
.. code-block:: console
$ ./view/bin/python
>>> import numpy
See :ref:`environments` for a more in-depth description of Spack
environments and customizations to views.
^^^^^^^^^^^^^^^^^^^^
Using ``spack load``
^^^^^^^^^^^^^^^^^^^^
A more traditional way of using Spack and extensions is ``spack load``
(requires :ref:`shell-support`). This will add the extension to ``PYTHONPATH``
in your current shell, and Python itself will be available in the ``PATH``:
.. code-block:: console
$ spack load py-numpy
$ python
>>> import numpy
Now ``import numpy`` will succeed for as long as you keep your current
session open.
The loaded packages can be checked using ``spack find --loaded``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Loading Extensions via Modules
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Instead of using Spack's environment modification capabilities through
the ``spack load`` command, you can load numpy through your
environment modules (using ``environment-modules`` or ``lmod``). This
will also add the extension to the ``PYTHONPATH`` in your current
shell.
Apart from ``spack env activate`` and ``spack load``, you can load numpy
through your environment modules (using ``environment-modules`` or
``lmod``). This will also add the extension to the ``PYTHONPATH`` in
your current shell.
.. code-block:: console
@@ -1776,130 +1818,6 @@ If you do not know the name of the specific numpy module you wish to
load, you can use the ``spack module tcl|lmod loads`` command to get
the name of the module from the Spack spec.
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Activating Extensions in a View
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Another way to use extensions is to create a view, which merges the
python installation along with the extensions into a single prefix.
See :ref:`configuring_environment_views` for a more in-depth description
of views.
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Activating Extensions Globally
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
As an alternative to creating a merged prefix with Python and its extensions,
and prior to support for views, Spack has provided a means to install the
extension into the Spack installation prefix for the extendee. This has
typically been useful since extendable packages typically search their own
installation path for addons by default.
Global activations are performed with the ``spack activate`` command:
.. _cmd-spack-activate:
^^^^^^^^^^^^^^^^^^
``spack activate``
^^^^^^^^^^^^^^^^^^
.. code-block:: console
$ spack activate py-numpy
==> Activated extension py-setuptools@11.3.1%gcc@4.4.7 arch=linux-debian7-x86_64-3c74eb69 for python@2.7.8%gcc@4.4.7.
==> Activated extension py-nose@1.3.4%gcc@4.4.7 arch=linux-debian7-x86_64-5f70f816 for python@2.7.8%gcc@4.4.7.
==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=linux-debian7-x86_64-66733244 for python@2.7.8%gcc@4.4.7.
Several things have happened here. The user requested that
``py-numpy`` be activated in the ``python`` installation it was built
with. Spack knows that ``py-numpy`` depends on ``py-nose`` and
``py-setuptools``, so it activated those packages first. Finally,
once all dependencies were activated in the ``python`` installation,
``py-numpy`` was activated as well.
If we run ``spack extensions`` again, we now see the three new
packages listed as activated:
.. code-block:: console
$ spack extensions python
==> python@2.7.8%gcc@4.4.7 arch=linux-debian7-x86_64-703c7a96
==> 36 extensions:
geos py-ipython py-pexpect py-pyside py-sip
py-basemap py-libxml2 py-pil py-pytz py-six
py-biopython py-mako py-pmw py-rpy2 py-sympy
py-cython py-matplotlib py-pychecker py-scientificpython py-virtualenv
py-dateutil py-mpi4py py-pygments py-scikit-learn
py-epydoc py-mx py-pylint py-scipy
py-gnuplot py-nose py-pyparsing py-setuptools
py-h5py py-numpy py-pyqt py-shiboken
==> 12 installed:
-- linux-debian7-x86_64 / gcc@4.4.7 --------------------------------
py-dateutil@2.4.0 py-nose@1.3.4 py-pyside@1.2.2
py-dateutil@2.4.0 py-numpy@1.9.1 py-pytz@2014.10
py-ipython@2.3.1 py-pygments@2.0.1 py-setuptools@11.3.1
py-matplotlib@1.4.2 py-pyparsing@2.0.3 py-six@1.9.0
==> 3 currently activated:
-- linux-debian7-x86_64 / gcc@4.4.7 --------------------------------
py-nose@1.3.4 py-numpy@1.9.1 py-setuptools@11.3.1
Now, when a user runs python, ``numpy`` will be available for import
*without* the user having to explicitly load it. ``python@2.7.8`` now
acts like a system Python installation with ``numpy`` installed inside
of it.
Spack accomplishes this by symbolically linking the *entire* prefix of
the ``py-numpy`` package into the prefix of the ``python`` package. To the
python interpreter, it looks like ``numpy`` is installed in the
``site-packages`` directory.
The only limitation of global activation is that you can only have a *single*
version of an extension activated at a time. This is because multiple
versions of the same extension would conflict if symbolically linked
into the same prefix. Users who want a different version of a package
can still get it by using environment modules or views, but they will have to
explicitly load their preferred version.
^^^^^^^^^^^^^^^^^^^^^^^^^^
``spack activate --force``
^^^^^^^^^^^^^^^^^^^^^^^^^^
If, for some reason, you want to activate a package *without* its
dependencies, you can use ``spack activate --force``:
.. code-block:: console
$ spack activate --force py-numpy
==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=linux-debian7-x86_64-66733244 for python@2.7.8%gcc@4.4.7.
.. _cmd-spack-deactivate:
^^^^^^^^^^^^^^^^^^^^
``spack deactivate``
^^^^^^^^^^^^^^^^^^^^
We've seen how activating an extension can be used to set up a default
version of a Python module. Obviously, you may want to change that at
some point. ``spack deactivate`` is the command for this. There are
several variants:
* ``spack deactivate <extension>`` will deactivate a single
extension. If another activated extension depends on this one,
Spack will warn you and exit with an error.
* ``spack deactivate --force <extension>`` deactivates an extension
regardless of packages that depend on it.
* ``spack deactivate --all <extension>`` deactivates an extension and
all of its dependencies. Use ``--force`` to disregard dependents.
* ``spack deactivate --all <extendee>`` deactivates *all* activated
extensions of a package. For example, to deactivate *all* python
extensions, use:
.. code-block:: console
$ spack deactivate --all python
-----------------------
Filesystem requirements
-----------------------

6
lib/spack/docs/build_systems/cachedcmakepackage.rst
View File

@@ -5,9 +5,9 @@
.. _cachedcmakepackage:
------------------
CachedCMakePackage
------------------
-----------
CachedCMake
-----------
The CachedCMakePackage base class is used for CMake-based workflows
that create a CMake cache file prior to running ``cmake``. This is

8
lib/spack/docs/build_systems/cudapackage.rst
View File

@@ -5,9 +5,9 @@
.. _cudapackage:
-----------
CudaPackage
-----------
----
Cuda
----
Different from other packages, ``CudaPackage`` does not represent a build system.
Instead its goal is to simplify and unify usage of ``CUDA`` in other packages by providing a `mixin-class <https://en.wikipedia.org/wiki/Mixin>`_.
@@ -80,7 +80,7 @@ standard CUDA compiler flags.
**cuda_flags**
This built-in static method returns a list of command line flags
This built-in static method returns a list of command line flags
for the chosen ``cuda_arch`` value(s). The flags are intended to
be passed to the CUDA compiler driver (i.e., ``nvcc``).

8
lib/spack/docs/build_systems/inteloneapipackage.rst
View File

@@ -6,9 +6,9 @@
.. _inteloneapipackage:
====================
IntelOneapiPackage
====================
===========
IntelOneapi
===========
.. contents::
@@ -36,7 +36,7 @@ For more information on a specific package, do::
Intel no longer releases new versions of Parallel Studio, which can be
used in Spack via the :ref:`intelpackage`. All of its components can
now be found in oneAPI.
now be found in oneAPI.
Examples
========

6
lib/spack/docs/build_systems/intelpackage.rst
View File

@@ -5,9 +5,9 @@
.. _intelpackage:
------------
IntelPackage
------------
-----
Intel
-----
.. contents::

15
lib/spack/docs/build_systems/pythonpackage.rst
View File

@@ -5,9 +5,9 @@
.. _pythonpackage:
-------------
PythonPackage
-------------
------
Python
------
Python packages and modules have their own special build system. This
documentation covers everything you'll need to know in order to write
@@ -724,10 +724,9 @@ extends vs. depends_on
This is very similar to the naming dilemma above, with a slight twist.
As mentioned in the :ref:`Packaging Guide <packaging_extensions>`,
``extends`` and ``depends_on`` are very similar, but ``extends`` adds
the ability to *activate* the package. Activation involves symlinking
everything in the installation prefix of the package to the installation
prefix of Python. This allows the user to import a Python module without
``extends`` and ``depends_on`` are very similar, but ``extends`` ensures
that the extension and extendee share the same prefix in views.
This allows the user to import a Python module without
having to add that module to ``PYTHONPATH``.
When deciding between ``extends`` and ``depends_on``, the best rule of
@@ -735,7 +734,7 @@ thumb is to check the installation prefix. If Python libraries are
installed to ``<prefix>/lib/pythonX.Y/site-packages``, then you
should use ``extends``. If Python libraries are installed elsewhere
or the only files that get installed reside in ``<prefix>/bin``, then
don't use ``extends``, as symlinking the package wouldn't be useful.
don't use ``extends``.
^^^^^^^^^^^^^^^^^^^^^
Alternatives to Spack

8
lib/spack/docs/build_systems/rocmpackage.rst
View File

@@ -5,9 +5,9 @@
.. _rocmpackage:
-----------
ROCmPackage
-----------
----
ROCm
----
The ``ROCmPackage`` is not a build system but a helper package. Like ``CudaPackage``,
it provides standard variants, dependencies, and conflicts to facilitate building
@@ -25,7 +25,7 @@ This package provides the following variants:
* **rocm**
This variant is used to enable/disable building with ``rocm``.
This variant is used to enable/disable building with ``rocm``.
The default is disabled (or ``False``).
* **amdgpu_target**

14
lib/spack/docs/build_systems/rpackage.rst
View File

@@ -5,9 +5,9 @@
.. _rpackage:
--------
RPackage
--------
--
R
--
Like Python, R has its own built-in build system.
@@ -193,10 +193,10 @@ Build system dependencies
As an extension of the R ecosystem, your package will obviously depend
on R to build and run. Normally, we would use ``depends_on`` to express
this, but for R packages, we use ``extends``. ``extends`` is similar to
``depends_on``, but adds an additional feature: the ability to "activate"
the package by symlinking it to the R installation directory. Since
every R package needs this, the ``RPackage`` base class contains:
this, but for R packages, we use ``extends``. This implies a special
dependency on R, which is used to set environment variables such as
``R_LIBS`` uniformly. Since every R package needs this, the ``RPackage``
base class contains:
.. code-block:: python

12
lib/spack/docs/build_systems/sourceforgepackage.rst
View File

@@ -5,15 +5,15 @@
.. _sourceforgepackage:
------------------
SourceforgePackage
------------------
-----------
Sourceforge
-----------
``SourceforgePackage`` is a
``SourceforgePackage`` is a
`mixin-class <https://en.wikipedia.org/wiki/Mixin>`_. It automatically
sets the URL based on a list of Sourceforge mirrors listed in
`sourceforge_mirror_path`, which defaults to a half dozen known mirrors.
Refer to the package source
Refer to the package source
(`<https://github.com/spack/spack/blob/develop/lib/spack/spack/build_systems/sourceforge.py>`__) for the current list of mirrors used by Spack.
@@ -29,7 +29,7 @@ This package provides a method for populating mirror URLs.
It is decorated with `property` so its results are treated as
a package attribute.
Refer to
Refer to
`<https://spack.readthedocs.io/en/latest/packaging_guide.html#mirrors-of-the-main-url>`__
for information on how Spack uses the `urls` attribute during
fetching.

18
lib/spack/docs/conf.py
View File

@@ -37,12 +37,6 @@
os.symlink(os.path.abspath("../../.."), link_name, target_is_directory=True)
sys.path.insert(0, os.path.abspath("_spack_root/lib/spack/external"))
sys.path.insert(0, os.path.abspath("_spack_root/lib/spack/external/pytest-fallback"))
if sys.version_info[0] < 3:
sys.path.insert(0, os.path.abspath("_spack_root/lib/spack/external/yaml/lib"))
else:
sys.path.insert(0, os.path.abspath("_spack_root/lib/spack/external/yaml/lib3"))
sys.path.append(os.path.abspath("_spack_root/lib/spack/"))
# Add the Spack bin directory to the path so that we can use its output in docs.
@@ -160,8 +154,8 @@ def setup(sphinx):
master_doc = "index"
# General information about the project.
project = u"Spack"
copyright = u"2013-2021, Lawrence Livermore National Laboratory."
project = "Spack"
copyright = "2013-2021, Lawrence Livermore National Laboratory."
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@@ -350,7 +344,7 @@ class SpackStyle(DefaultStyle):
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
("index", "Spack.tex", u"Spack Documentation", u"Todd Gamblin", "manual"),
("index", "Spack.tex", "Spack Documentation", "Todd Gamblin", "manual"),
]
# The name of an image file (relative to this directory) to place at the top of
@@ -378,7 +372,7 @@ class SpackStyle(DefaultStyle):
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [("index", "spack", u"Spack Documentation", [u"Todd Gamblin"], 1)]
man_pages = [("index", "spack", "Spack Documentation", ["Todd Gamblin"], 1)]
# If true, show URL addresses after external links.
# man_show_urls = False
@@ -393,8 +387,8 @@ class SpackStyle(DefaultStyle):
(
"index",
"Spack",
u"Spack Documentation",
u"Todd Gamblin",
"Spack Documentation",
"Todd Gamblin",
"Spack",
"One line description of project.",
"Miscellaneous",

2
lib/spack/docs/configuration.rst
View File

@@ -416,6 +416,8 @@ Spack understands several special variables. These are:
ArchSpec. E.g. ``skylake`` or ``neoverse-n1``.
* ``$target_family``. The target family for the current host, as
detected by ArchSpec. E.g. ``x86_64`` or ``aarch64``.
* ``$date``: the current date in the format YYYY-MM-DD
Note that, as with shell variables, you can write these as ``$varname``
or with braces to distinguish the variable from surrounding characters:

29
lib/spack/docs/contribution_guide.rst
View File

@@ -253,27 +253,6 @@ to update them.
multiple runs of ``spack style`` just to re-compute line numbers and
makes it much easier to fix errors directly off of the CI output.
.. warning::
Flake8 and ``pep8-naming`` require a number of dependencies in order
to run. If you installed ``py-flake8`` and ``py-pep8-naming``, the
easiest way to ensure the right packages are on your ``PYTHONPATH`` is
to run::
spack activate py-flake8
spack activate pep8-naming
so that all of the dependencies are symlinked to a central
location. If you see an error message like:
.. code-block:: console
Traceback (most recent call last):
File: "/usr/bin/flake8", line 5, in <module>
from pkg_resources import load_entry_point
ImportError: No module named pkg_resources
that means Flake8 couldn't find setuptools in your ``PYTHONPATH``.
^^^^^^^^^^^^^^^^^^^
Documentation Tests
@@ -309,13 +288,9 @@ All of these can be installed with Spack, e.g.
.. code-block:: console
$ spack activate py-sphinx
$ spack activate py-sphinx-rtd-theme
$ spack activate py-sphinxcontrib-programoutput
$ spack load py-sphinx py-sphinx-rtd-theme py-sphinxcontrib-programoutput
so that all of the dependencies are symlinked into that Python's
tree. Alternatively, you could arrange for their library
directories to be added to PYTHONPATH. If you see an error message
so that all of the dependencies are added to PYTHONPATH. If you see an error message
like:
.. code-block:: console

20
lib/spack/docs/environments.rst
View File

@@ -234,8 +234,8 @@ packages will be listed as roots of the Environment.
All of the Spack commands that act on the list of installed specs are
Environment-sensitive in this way, including ``install``,
``uninstall``, ``activate``, ``deactivate``, ``find``, ``extensions``,
and more. In the :ref:`environment-configuration` section we will discuss
``uninstall``, ``find``, ``extensions``, and more. In the
:ref:`environment-configuration` section we will discuss
Environment-sensitive commands further.
^^^^^^^^^^^^^^^^^^^^^
@@ -1071,19 +1071,23 @@ the include is conditional.
Building a subset of the environment
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The generated ``Makefile``\s contain install targets for each spec. Given the hash
of a particular spec, you can use the ``.install/<hash>`` target to install the
spec with its dependencies. There is also ``.install-deps/<hash>`` to *only* install
The generated ``Makefile``\s contain install targets for each spec, identified
by ``<name>-<version>-<hash>``. This allows you to install only a subset of the
packages in the environment. When packages are unique in the environment, it's
enough to know the name and let tab-completion fill out the version and hash.
The following phony targets are available: ``install/<spec>`` to install the
spec with its dependencies, and ``install-deps/<spec>`` to *only* install
its dependencies. This can be useful when certain flags should only apply to
dependencies. Below we show a use case where a spec is installed with verbose
output (``spack install --verbose``) while its dependencies are installed silently:
.. code:: console
$ spack env depfile -o Makefile --make-target-prefix my_env
$ spack env depfile -o Makefile
# Install dependencies in parallel, only show a log on error.
$ make -j16 my_env/.install-deps/<hash> SPACK_INSTALL_FLAGS=--show-log-on-error
$ make -j16 install-deps/python-3.11.0-<hash> SPACK_INSTALL_FLAGS=--show-log-on-error
# Install the root spec with verbose output.
$ make -j16 my_env/.install/<hash> SPACK_INSTALL_FLAGS=--verbose
$ make -j16 install/python-3.11.0-<hash> SPACK_INSTALL_FLAGS=--verbose

7
lib/spack/docs/getting_started.rst
View File

@@ -21,8 +21,9 @@ be present on the machine where Spack is run:
:header-rows: 1
These requirements can be easily installed on most modern Linux systems;
on macOS, XCode is required. Spack is designed to run on HPC
platforms like Cray. Not all packages should be expected
on macOS, the Command Line Tools package is required, and a full XCode suite
may be necessary for some packages such as Qt and apple-gl. Spack is designed
to run on HPC platforms like Cray. Not all packages should be expected
to work on all platforms.
A build matrix showing which packages are working on which systems is shown below.
@@ -1704,9 +1705,11 @@ dependencies or incompatible build tools like autoconf. Here are several
packages known to work on Windows:
* abseil-cpp
* bzip2
* clingo
* cpuinfo
* cmake
* hdf5
* glm
* nasm
* netlib-lapack (requires Intel Fortran)

79
lib/spack/docs/packaging_guide.rst
View File

@@ -2634,9 +2634,12 @@ extendable package:
extends('python')
...
Now, the ``py-numpy`` package can be used as an argument to ``spack
activate``. When it is activated, all the files in its prefix will be
symbolically linked into the prefix of the python package.
This accomplishes a few things. Firstly, the Python package can set special
variables such as ``PYTHONPATH`` for all extensions when the run or build
environment is set up. Secondly, filesystem views can ensure that extensions
are put in the same prefix as their extendee. This ensures that Python in
a view can always locate its Python packages, even without environment
variables set.
A package can only extend one other package at a time. To support packages
that may extend one of a list of other packages, Spack supports multiple
@@ -2684,9 +2687,8 @@ variant(s) are selected. This may be accomplished with conditional
...
Sometimes, certain files in one package will conflict with those in
another, which means they cannot both be activated (symlinked) at the
same time. In this case, you can tell Spack to ignore those files
when it does the activation:
another, which means they cannot both be used in a view at the
same time. In this case, you can tell Spack to ignore those files:
.. code-block:: python
@@ -2698,7 +2700,7 @@ when it does the activation:
...
The code above will prevent everything in the ``$prefix/bin/`` directory
from being linked in at activation time.
from being linked in a view.
.. note::
@@ -2722,67 +2724,6 @@ extensions; as a consequence python extension packages (those inheriting from
``PythonPackage``) likewise override ``add_files_to_view`` in order to rewrite
shebang lines which point to the Python interpreter.
^^^^^^^^^^^^^^^^^^^^^^^^^
Activation & deactivation
^^^^^^^^^^^^^^^^^^^^^^^^^
Adding an extension to a view is referred to as an activation. If the view is
maintained in the Spack installation prefix of the extendee this is called a
global activation. Activations may involve updating some centralized state
that is maintained by the extendee package, so there can be additional work
for adding extensions compared with non-extension packages.
Spack's ``Package`` class has default ``activate`` and ``deactivate``
implementations that handle symbolically linking extensions' prefixes
into a specified view. Extendable packages can override these methods
to add custom activate/deactivate logic of their own. For example,
the ``activate`` and ``deactivate`` methods in the Python class handle
symbolic linking of extensions, but they also handle details surrounding
Python's ``.pth`` files, and other aspects of Python packaging.
Spack's extensions mechanism is designed to be extensible, so that
other packages (like Ruby, R, Perl, etc.) can provide their own
custom extension management logic, as they may not handle modules the
same way that Python does.
Let's look at Python's activate function:
.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/python/package.py
:pyobject: Python.activate
:linenos:
This function is called on the *extendee* (Python). It first calls
``activate`` in the superclass, which handles symlinking the
extension package's prefix into the specified view. It then does
some special handling of the ``easy-install.pth`` file, part of
Python's setuptools.
Deactivate behaves similarly to activate, but it unlinks files:
.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/python/package.py
:pyobject: Python.deactivate
:linenos:
Both of these methods call some custom functions in the Python
package. See the source for Spack's Python package for details.
^^^^^^^^^^^^^^^^^^^^
Activation arguments
^^^^^^^^^^^^^^^^^^^^
You may have noticed that the ``activate`` function defined above
takes keyword arguments. These are the keyword arguments from
``extends()``, and they are passed to both activate and deactivate.
This capability allows an extension to customize its own activation by
passing arguments to the extendee. Extendees can likewise implement
custom ``activate()`` and ``deactivate()`` functions to suit their
needs.
The only keyword argument supported by default is the ``ignore``
argument, which can take a regex, list of regexes, or a predicate to
determine which files *not* to symlink during activation.
.. _virtual-dependencies:
--------------------
@@ -3584,7 +3525,7 @@ will likely contain some overriding of default builder methods:
def cmake_args(self):
pass
class Autotoolsbuilder(spack.build_systems.autotools.AutotoolsBuilder):
class AutotoolsBuilder(spack.build_systems.autotools.AutotoolsBuilder):
def configure_args(self):
pass

2
lib/spack/docs/tables/system_prerequisites.csv
View File

@@ -1,5 +1,5 @@
Name, Supported Versions, Notes, Requirement Reason
Python, 2.7/3.6-3.11, , Interpreter for Spack
Python, 3.6--3.11, , Interpreter for Spack
C/C++ Compilers, , , Building software
make, , , Build software
patch, , , Build software
1 Name Supported Versions Notes Requirement Reason
2 Python 2.7/3.6-3.11 3.6--3.11 Interpreter for Spack
3 C/C++ Compilers Building software
4 make Build software
5 patch Build software

41
lib/spack/env/cc vendored
View File

@@ -440,6 +440,47 @@ while [ $# -ne 0 ]; do
continue
fi
if [ -n "${SPACK_COMPILER_FLAGS_KEEP}" ] ; then
# NOTE: the eval is required to allow `|` alternatives inside the variable
eval "\
case \"\$1\" in
$SPACK_COMPILER_FLAGS_KEEP)
append other_args_list \"\$1\"
shift
continue
;;
esac
"
fi
# the replace list is a space-separated list of pipe-separated pairs,
# the first in each pair is the original prefix to be matched, the
# second is the replacement prefix
if [ -n "${SPACK_COMPILER_FLAGS_REPLACE}" ] ; then
for rep in ${SPACK_COMPILER_FLAGS_REPLACE} ; do
before=${rep%|*}
after=${rep#*|}
eval "\
stripped=\"\${1##$before}\"
"
if [ "$stripped" = "$1" ] ; then
continue
fi
replaced="$after$stripped"
# it matched, remove it
shift
if [ -z "$replaced" ] ; then
# completely removed, continue OUTER loop
continue 2
fi
# re-build argument list with replacement
set -- "$replaced" "$@"
done
fi
case "$1" in
-isystem*)
arg="${1#-isystem}"

7
lib/spack/external/ctest_log_parser.py vendored
View File

@@ -71,13 +71,12 @@
import re
import math
import multiprocessing
import io
import sys
import threading
import time
from contextlib import contextmanager
from six import StringIO
from six import string_types
_error_matches = [
"^FAIL: ",
@@ -246,7 +245,7 @@ def __getitem__(self, line_no):
def __str__(self):
"""Returns event lines and context."""
out = StringIO()
out = io.StringIO()
for i in range(self.start, self.end):
if i == self.line_no:
out.write(' >> %-6d%s' % (i, self[i]))
@@ -386,7 +385,7 @@ def parse(self, stream, context=6, jobs=None):
(tuple): two lists containing ``BuildError`` and
``BuildWarning`` objects.
"""
if isinstance(stream, string_types):
if isinstance(stream, str):
with open(stream) as f:
return self.parse(f, context, jobs)

2392
lib/spack/external/py2/argparse.py vendored
View File

File diff suppressed because it is too large Load Diff

289
lib/spack/external/py2/functools32/LICENSE vendored
View File

@@ -1,289 +0,0 @@
A. HISTORY OF THE SOFTWARE
==========================
Python was created in the early 1990s by Guido van Rossum at Stichting
Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands
as a successor of a language called ABC. Guido remains Python's
principal author, although it includes many contributions from others.
In 1995, Guido continued his work on Python at the Corporation for
National Research Initiatives (CNRI, see http://www.cnri.reston.va.us)
in Reston, Virginia where he released several versions of the
software.
In May 2000, Guido and the Python core development team moved to
BeOpen.com to form the BeOpen PythonLabs team. In October of the same
year, the PythonLabs team moved to Digital Creations (now Zope
Corporation, see http://www.zope.com). In 2001, the Python Software
Foundation (PSF, see http://www.python.org/psf/) was formed, a
non-profit organization created specifically to own Python-related
Intellectual Property. Zope Corporation is a sponsoring member of
the PSF.
All Python releases are Open Source (see http://www.opensource.org for
the Open Source Definition). Historically, most, but not all, Python
releases have also been GPL-compatible; the table below summarizes
the various releases.
Release Derived Year Owner GPL-
from compatible? (1)
0.9.0 thru 1.2 1991-1995 CWI yes
1.3 thru 1.5.2 1.2 1995-1999 CNRI yes
1.6 1.5.2 2000 CNRI no
2.0 1.6 2000 BeOpen.com no
1.6.1 1.6 2001 CNRI yes (2)
2.1 2.0+1.6.1 2001 PSF no
2.0.1 2.0+1.6.1 2001 PSF yes
2.1.1 2.1+2.0.1 2001 PSF yes
2.2 2.1.1 2001 PSF yes
2.1.2 2.1.1 2002 PSF yes
2.1.3 2.1.2 2002 PSF yes
2.2.1 2.2 2002 PSF yes
2.2.2 2.2.1 2002 PSF yes
2.2.3 2.2.2 2003 PSF yes
2.3 2.2.2 2002-2003 PSF yes
2.3.1 2.3 2002-2003 PSF yes
2.3.2 2.3.1 2002-2003 PSF yes
2.3.3 2.3.2 2002-2003 PSF yes
2.3.4 2.3.3 2004 PSF yes
2.3.5 2.3.4 2005 PSF yes
2.4 2.3 2004 PSF yes
2.4.1 2.4 2005 PSF yes
2.4.2 2.4.1 2005 PSF yes
2.4.3 2.4.2 2006 PSF yes
2.4.4 2.4.3 2006 PSF yes
2.5 2.4 2006 PSF yes
2.5.1 2.5 2007 PSF yes
2.5.2 2.5.1 2008 PSF yes
2.5.3 2.5.2 2008 PSF yes
2.6 2.5 2008 PSF yes
2.6.1 2.6 2008 PSF yes
2.6.2 2.6.1 2009 PSF yes
2.6.3 2.6.2 2009 PSF yes
2.6.4 2.6.3 2009 PSF yes
2.6.5 2.6.4 2010 PSF yes
3.0 2.6 2008 PSF yes
3.0.1 3.0 2009 PSF yes
3.1 3.0.1 2009 PSF yes
3.1.1 3.1 2009 PSF yes
3.1.2 3.1.1 2010 PSF yes
3.1.3 3.1.2 2010 PSF yes
3.1.4 3.1.3 2011 PSF yes
3.2 3.1 2011 PSF yes
3.2.1 3.2 2011 PSF yes
3.2.2 3.2.1 2011 PSF yes
3.2.3 3.2.2 2012 PSF yes
Footnotes:
(1) GPL-compatible doesn't mean that we're distributing Python under
the GPL. All Python licenses, unlike the GPL, let you distribute
a modified version without making your changes open source. The
GPL-compatible licenses make it possible to combine Python with
other software that is released under the GPL; the others don't.
(2) According to Richard Stallman, 1.6.1 is not GPL-compatible,
because its license has a choice of law clause. According to
CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1
is "not incompatible" with the GPL.
Thanks to the many outside volunteers who have worked under Guido's
direction to make these releases possible.
B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
===============================================================
PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
--------------------------------------------
1. This LICENSE AGREEMENT is between the Python Software Foundation
("PSF"), and the Individual or Organization ("Licensee") accessing and
otherwise using this software ("Python") in source or binary form and
its associated documentation.
2. Subject to the terms and conditions of this License Agreement, PSF hereby
grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
analyze, test, perform and/or display publicly, prepare derivative works,
distribute, and otherwise use Python alone or in any derivative version,
provided, however, that PSF's License Agreement and PSF's notice of copyright,
i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
2011, 2012 Python Software Foundation; All Rights Reserved" are retained in Python
alone or in any derivative version prepared by Licensee.
3. In the event Licensee prepares a derivative work that is based on
or incorporates Python or any part thereof, and wants to make
the derivative work available to others as provided herein, then
Licensee hereby agrees to include in any such work a brief summary of
the changes made to Python.
4. PSF is making Python available to Licensee on an "AS IS"
basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
INFRINGE ANY THIRD PARTY RIGHTS.
5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
6. This License Agreement will automatically terminate upon a material
breach of its terms and conditions.
7. Nothing in this License Agreement shall be deemed to create any
relationship of agency, partnership, or joint venture between PSF and
Licensee. This License Agreement does not grant permission to use PSF
trademarks or trade name in a trademark sense to endorse or promote
products or services of Licensee, or any third party.
8. By copying, installing or otherwise using Python, Licensee
agrees to be bound by the terms and conditions of this License
Agreement.
BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0
-------------------------------------------
BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1
1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an
office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the
Individual or Organization ("Licensee") accessing and otherwise using
this software in source or binary form and its associated
documentation ("the Software").
2. Subject to the terms and conditions of this BeOpen Python License
Agreement, BeOpen hereby grants Licensee a non-exclusive,
royalty-free, world-wide license to reproduce, analyze, test, perform
and/or display publicly, prepare derivative works, distribute, and
otherwise use the Software alone or in any derivative version,
provided, however, that the BeOpen Python License is retained in the
Software, alone or in any derivative version prepared by Licensee.
3. BeOpen is making the Software available to Licensee on an "AS IS"
basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND
DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT
INFRINGE ANY THIRD PARTY RIGHTS.
4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE
SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY
DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
5. This License Agreement will automatically terminate upon a material
breach of its terms and conditions.
6. This License Agreement shall be governed by and interpreted in all
respects by the law of the State of California, excluding conflict of
law provisions. Nothing in this License Agreement shall be deemed to
create any relationship of agency, partnership, or joint venture
between BeOpen and Licensee. This License Agreement does not grant
permission to use BeOpen trademarks or trade names in a trademark
sense to endorse or promote products or services of Licensee, or any
third party. As an exception, the "BeOpen Python" logos available at
http://www.pythonlabs.com/logos.html may be used according to the
permissions granted on that web page.
7. By copying, installing or otherwise using the software, Licensee
agrees to be bound by the terms and conditions of this License
Agreement.
CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1
---------------------------------------
1. This LICENSE AGREEMENT is between the Corporation for National
Research Initiatives, having an office at 1895 Preston White Drive,
Reston, VA 20191 ("CNRI"), and the Individual or Organization
("Licensee") accessing and otherwise using Python 1.6.1 software in
source or binary form and its associated documentation.
2. Subject to the terms and conditions of this License Agreement, CNRI
hereby grants Licensee a nonexclusive, royalty-free, world-wide
license to reproduce, analyze, test, perform and/or display publicly,
prepare derivative works, distribute, and otherwise use Python 1.6.1
alone or in any derivative version, provided, however, that CNRI's
License Agreement and CNRI's notice of copyright, i.e., "Copyright (c)
1995-2001 Corporation for National Research Initiatives; All Rights
Reserved" are retained in Python 1.6.1 alone or in any derivative
version prepared by Licensee. Alternately, in lieu of CNRI's License
Agreement, Licensee may substitute the following text (omitting the
quotes): "Python 1.6.1 is made available subject to the terms and
conditions in CNRI's License Agreement. This Agreement together with
Python 1.6.1 may be located on the Internet using the following
unique, persistent identifier (known as a handle): 1895.22/1013. This
Agreement may also be obtained from a proxy server on the Internet
using the following URL: http://hdl.handle.net/1895.22/1013".
3. In the event Licensee prepares a derivative work that is based on
or incorporates Python 1.6.1 or any part thereof, and wants to make
the derivative work available to others as provided herein, then
Licensee hereby agrees to include in any such work a brief summary of
the changes made to Python 1.6.1.
4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS"
basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND
DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT
INFRINGE ANY THIRD PARTY RIGHTS.
5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1,
OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
6. This License Agreement will automatically terminate upon a material
breach of its terms and conditions.
7. This License Agreement shall be governed by the federal
intellectual property law of the United States, including without
limitation the federal copyright law, and, to the extent such
U.S. federal law does not apply, by the law of the Commonwealth of
Virginia, excluding Virginia's conflict of law provisions.
Notwithstanding the foregoing, with regard to derivative works based
on Python 1.6.1 that incorporate non-separable material that was
previously distributed under the GNU General Public License (GPL), the
law of the Commonwealth of Virginia shall govern this License
Agreement only as to issues arising under or with respect to
Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this
License Agreement shall be deemed to create any relationship of
agency, partnership, or joint venture between CNRI and Licensee. This
License Agreement does not grant permission to use CNRI trademarks or
trade name in a trademark sense to endorse or promote products or
services of Licensee, or any third party.
8. By clicking on the "ACCEPT" button where indicated, or by copying,
installing or otherwise using Python 1.6.1, Licensee agrees to be
bound by the terms and conditions of this License Agreement.
ACCEPT
CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2
--------------------------------------------------
Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam,
The Netherlands. All rights reserved.
Permission to use, copy, modify, and distribute this software and its
documentation for any purpose and without fee is hereby granted,
provided that the above copyright notice appear in all copies and that
both that copyright notice and this permission notice appear in
supporting documentation, and that the name of Stichting Mathematisch
Centrum or CWI not be used in advertising or publicity pertaining to
distribution of the software without specific, written prior
permission.
STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

1
lib/spack/external/py2/functools32/__init__.py vendored
View File

@@ -1 +0,0 @@
from .functools32 import *

158
lib/spack/external/py2/functools32/_dummy_thread32.py vendored
View File

@@ -1,158 +0,0 @@
"""Drop-in replacement for the thread module.
Meant to be used as a brain-dead substitute so that threaded code does
not need to be rewritten for when the thread module is not present.
Suggested usage is::
try:
try:
import _thread # Python >= 3
except:
import thread as _thread # Python < 3
except ImportError:
import _dummy_thread as _thread
"""
# Exports only things specified by thread documentation;
# skipping obsolete synonyms allocate(), start_new(), exit_thread().
__all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock',
'interrupt_main', 'LockType']
# A dummy value
TIMEOUT_MAX = 2**31
# NOTE: this module can be imported early in the extension building process,
# and so top level imports of other modules should be avoided. Instead, all
# imports are done when needed on a function-by-function basis. Since threads
# are disabled, the import lock should not be an issue anyway (??).
class error(Exception):
"""Dummy implementation of _thread.error."""
def __init__(self, *args):
self.args = args
def start_new_thread(function, args, kwargs={}):
"""Dummy implementation of _thread.start_new_thread().
Compatibility is maintained by making sure that ``args`` is a
tuple and ``kwargs`` is a dictionary. If an exception is raised
and it is SystemExit (which can be done by _thread.exit()) it is
caught and nothing is done; all other exceptions are printed out
by using traceback.print_exc().
If the executed function calls interrupt_main the KeyboardInterrupt will be
raised when the function returns.
"""
if type(args) != type(tuple()):
raise TypeError("2nd arg must be a tuple")
if type(kwargs) != type(dict()):
raise TypeError("3rd arg must be a dict")
global _main
_main = False
try:
function(*args, **kwargs)
except SystemExit:
pass
except:
import traceback
traceback.print_exc()
_main = True
global _interrupt
if _interrupt:
_interrupt = False
raise KeyboardInterrupt
def exit():
"""Dummy implementation of _thread.exit()."""
raise SystemExit
def get_ident():
"""Dummy implementation of _thread.get_ident().
Since this module should only be used when _threadmodule is not
available, it is safe to assume that the current process is the
only thread. Thus a constant can be safely returned.
"""
return -1
def allocate_lock():
"""Dummy implementation of _thread.allocate_lock()."""
return LockType()
def stack_size(size=None):
"""Dummy implementation of _thread.stack_size()."""
if size is not None:
raise error("setting thread stack size not supported")
return 0
class LockType(object):
"""Class implementing dummy implementation of _thread.LockType.
Compatibility is maintained by maintaining self.locked_status
which is a boolean that stores the state of the lock. Pickling of
the lock, though, should not be done since if the _thread module is
then used with an unpickled ``lock()`` from here problems could
occur from this class not having atomic methods.
"""
def __init__(self):
self.locked_status = False
def acquire(self, waitflag=None, timeout=-1):
"""Dummy implementation of acquire().
For blocking calls, self.locked_status is automatically set to
True and returned appropriately based on value of
``waitflag``. If it is non-blocking, then the value is
actually checked and not set if it is already acquired. This
is all done so that threading.Condition's assert statements
aren't triggered and throw a little fit.
"""
if waitflag is None or waitflag:
self.locked_status = True
return True
else:
if not self.locked_status:
self.locked_status = True
return True
else:
if timeout > 0:
import time
time.sleep(timeout)
return False
__enter__ = acquire
def __exit__(self, typ, val, tb):
self.release()
def release(self):
"""Release the dummy lock."""
# XXX Perhaps shouldn't actually bother to test? Could lead
# to problems for complex, threaded code.
if not self.locked_status:
raise error
self.locked_status = False
return True
def locked(self):
return self.locked_status
# Used to signal that interrupt_main was called in a "thread"
_interrupt = False
# True when not executing in a "thread"
_main = True
def interrupt_main():
"""Set _interrupt flag to True to have start_new_thread raise
KeyboardInterrupt upon exiting."""
if _main:
raise KeyboardInterrupt
else:
global _interrupt
_interrupt = True

423
lib/spack/external/py2/functools32/functools32.py vendored
View File

@@ -1,423 +0,0 @@
"""functools.py - Tools for working with functions and callable objects
"""
# Python module wrapper for _functools C module
# to allow utilities written in Python to be added
# to the functools module.
# Written by Nick Coghlan <ncoghlan at gmail.com>
# and Raymond Hettinger <python at rcn.com>
# Copyright (C) 2006-2010 Python Software Foundation.
# See C source code for _functools credits/copyright
__all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES',
'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial']
from _functools import partial, reduce
from collections import MutableMapping, namedtuple
from .reprlib32 import recursive_repr as _recursive_repr
from weakref import proxy as _proxy
import sys as _sys
try:
from thread import allocate_lock as Lock
except ImportError:
from ._dummy_thread32 import allocate_lock as Lock
################################################################################
### OrderedDict
################################################################################
class _Link(object):
__slots__ = 'prev', 'next', 'key', '__weakref__'
class OrderedDict(dict):
'Dictionary that remembers insertion order'
# An inherited dict maps keys to values.
# The inherited dict provides __getitem__, __len__, __contains__, and get.
# The remaining methods are order-aware.
# Big-O running times for all methods are the same as regular dictionaries.
# The internal self.__map dict maps keys to links in a doubly linked list.
# The circular doubly linked list starts and ends with a sentinel element.
# The sentinel element never gets deleted (this simplifies the algorithm).
# The sentinel is in self.__hardroot with a weakref proxy in self.__root.
# The prev links are weakref proxies (to prevent circular references).
# Individual links are kept alive by the hard reference in self.__map.
# Those hard references disappear when a key is deleted from an OrderedDict.
def __init__(self, *args, **kwds):
'''Initialize an ordered dictionary. The signature is the same as
regular dictionaries, but keyword arguments are not recommended because
their insertion order is arbitrary.
'''
if len(args) > 1:
raise TypeError('expected at most 1 arguments, got %d' % len(args))
try:
self.__root
except AttributeError:
self.__hardroot = _Link()
self.__root = root = _proxy(self.__hardroot)
root.prev = root.next = root
self.__map = {}
self.__update(*args, **kwds)
def __setitem__(self, key, value,
dict_setitem=dict.__setitem__, proxy=_proxy, Link=_Link):
'od.__setitem__(i, y) <==> od[i]=y'
# Setting a new item creates a new link at the end of the linked list,
# and the inherited dictionary is updated with the new key/value pair.
if key not in self:
self.__map[key] = link = Link()
root = self.__root
last = root.prev
link.prev, link.next, link.key = last, root, key
last.next = link
root.prev = proxy(link)
dict_setitem(self, key, value)
def __delitem__(self, key, dict_delitem=dict.__delitem__):
'od.__delitem__(y) <==> del od[y]'
# Deleting an existing item uses self.__map to find the link which gets
# removed by updating the links in the predecessor and successor nodes.
dict_delitem(self, key)
link = self.__map.pop(key)
link_prev = link.prev
link_next = link.next
link_prev.next = link_next
link_next.prev = link_prev
def __iter__(self):
'od.__iter__() <==> iter(od)'
# Traverse the linked list in order.
root = self.__root
curr = root.next
while curr is not root:
yield curr.key
curr = curr.next
def __reversed__(self):
'od.__reversed__() <==> reversed(od)'
# Traverse the linked list in reverse order.
root = self.__root
curr = root.prev
while curr is not root:
yield curr.key
curr = curr.prev
def clear(self):
'od.clear() -> None. Remove all items from od.'
root = self.__root
root.prev = root.next = root
self.__map.clear()
dict.clear(self)
def popitem(self, last=True):
'''od.popitem() -> (k, v), return and remove a (key, value) pair.
Pairs are returned in LIFO order if last is true or FIFO order if false.
'''
if not self:
raise KeyError('dictionary is empty')
root = self.__root
if last:
link = root.prev
link_prev = link.prev
link_prev.next = root
root.prev = link_prev
else:
link = root.next
link_next = link.next
root.next = link_next
link_next.prev = root
key = link.key
del self.__map[key]
value = dict.pop(self, key)
return key, value
def move_to_end(self, key, last=True):
'''Move an existing element to the end (or beginning if last==False).
Raises KeyError if the element does not exist.
When last=True, acts like a fast version of self[key]=self.pop(key).
'''
link = self.__map[key]
link_prev = link.prev
link_next = link.next
link_prev.next = link_next
link_next.prev = link_prev
root = self.__root
if last:
last = root.prev
link.prev = last
link.next = root
last.next = root.prev = link
else:
first = root.next
link.prev = root
link.next = first
root.next = first.prev = link
def __sizeof__(self):
sizeof = _sys.getsizeof
n = len(self) + 1 # number of links including root
size = sizeof(self.__dict__) # instance dictionary
size += sizeof(self.__map) * 2 # internal dict and inherited dict
size += sizeof(self.__hardroot) * n # link objects
size += sizeof(self.__root) * n # proxy objects
return size
update = __update = MutableMapping.update
keys = MutableMapping.keys
values = MutableMapping.values
items = MutableMapping.items
__ne__ = MutableMapping.__ne__
__marker = object()
def pop(self, key, default=__marker):
'''od.pop(k[,d]) -> v, remove specified key and return the corresponding
value. If key is not found, d is returned if given, otherwise KeyError
is raised.
'''
if key in self:
result = self[key]
del self[key]
return result
if default is self.__marker:
raise KeyError(key)
return default
def setdefault(self, key, default=None):
'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
if key in self:
return self[key]
self[key] = default
return default
@_recursive_repr()
def __repr__(self):
'od.__repr__() <==> repr(od)'
if not self:
return '%s()' % (self.__class__.__name__,)
return '%s(%r)' % (self.__class__.__name__, list(self.items()))
def __reduce__(self):
'Return state information for pickling'
items = [[k, self[k]] for k in self]
inst_dict = vars(self).copy()
for k in vars(OrderedDict()):
inst_dict.pop(k, None)
if inst_dict:
return (self.__class__, (items,), inst_dict)
return self.__class__, (items,)
def copy(self):
'od.copy() -> a shallow copy of od'
return self.__class__(self)
@classmethod
def fromkeys(cls, iterable, value=None):
'''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.
If not specified, the value defaults to None.
'''
self = cls()
for key in iterable:
self[key] = value
return self
def __eq__(self, other):
'''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive
while comparison to a regular mapping is order-insensitive.
'''
if isinstance(other, OrderedDict):
return len(self)==len(other) and \
all(p==q for p, q in zip(self.items(), other.items()))
return dict.__eq__(self, other)
# update_wrapper() and wraps() are tools to help write
# wrapper functions that can handle naive introspection
WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__doc__')
WRAPPER_UPDATES = ('__dict__',)
def update_wrapper(wrapper,
wrapped,
assigned = WRAPPER_ASSIGNMENTS,
updated = WRAPPER_UPDATES):
"""Update a wrapper function to look like the wrapped function
wrapper is the function to be updated
wrapped is the original function
assigned is a tuple naming the attributes assigned directly
from the wrapped function to the wrapper function (defaults to
functools.WRAPPER_ASSIGNMENTS)
updated is a tuple naming the attributes of the wrapper that
are updated with the corresponding attribute from the wrapped
function (defaults to functools.WRAPPER_UPDATES)
"""
wrapper.__wrapped__ = wrapped
for attr in assigned:
try:
value = getattr(wrapped, attr)
except AttributeError:
pass
else:
setattr(wrapper, attr, value)
for attr in updated:
getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
# Return the wrapper so this can be used as a decorator via partial()
return wrapper
def wraps(wrapped,
assigned = WRAPPER_ASSIGNMENTS,
updated = WRAPPER_UPDATES):
"""Decorator factory to apply update_wrapper() to a wrapper function
Returns a decorator that invokes update_wrapper() with the decorated
function as the wrapper argument and the arguments to wraps() as the
remaining arguments. Default arguments are as for update_wrapper().
This is a convenience function to simplify applying partial() to
update_wrapper().
"""
return partial(update_wrapper, wrapped=wrapped,
assigned=assigned, updated=updated)
def total_ordering(cls):
"""Class decorator that fills in missing ordering methods"""
convert = {
'__lt__': [('__gt__', lambda self, other: not (self < other or self == other)),
('__le__', lambda self, other: self < other or self == other),
('__ge__', lambda self, other: not self < other)],
'__le__': [('__ge__', lambda self, other: not self <= other or self == other),
('__lt__', lambda self, other: self <= other and not self == other),
('__gt__', lambda self, other: not self <= other)],
'__gt__': [('__lt__', lambda self, other: not (self > other or self == other)),
('__ge__', lambda self, other: self > other or self == other),
('__le__', lambda self, other: not self > other)],
'__ge__': [('__le__', lambda self, other: (not self >= other) or self == other),
('__gt__', lambda self, other: self >= other and not self == other),
('__lt__', lambda self, other: not self >= other)]
}
roots = set(dir(cls)) & set(convert)
if not roots:
raise ValueError('must define at least one ordering operation: < > <= >=')
root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__
for opname, opfunc in convert[root]:
if opname not in roots:
opfunc.__name__ = opname
opfunc.__doc__ = getattr(int, opname).__doc__
setattr(cls, opname, opfunc)
return cls
def cmp_to_key(mycmp):
"""Convert a cmp= function into a key= function"""
class K(object):
__slots__ = ['obj']
def __init__(self, obj):
self.obj = obj
def __lt__(self, other):
return mycmp(self.obj, other.obj) < 0
def __gt__(self, other):
return mycmp(self.obj, other.obj) > 0
def __eq__(self, other):
return mycmp(self.obj, other.obj) == 0
def __le__(self, other):
return mycmp(self.obj, other.obj) <= 0
def __ge__(self, other):
return mycmp(self.obj, other.obj) >= 0
def __ne__(self, other):
return mycmp(self.obj, other.obj) != 0
__hash__ = None
return K
_CacheInfo = namedtuple("CacheInfo", "hits misses maxsize currsize")
def lru_cache(maxsize=100):
"""Least-recently-used cache decorator.
If *maxsize* is set to None, the LRU features are disabled and the cache
can grow without bound.
Arguments to the cached function must be hashable.
View the cache statistics named tuple (hits, misses, maxsize, currsize) with
f.cache_info(). Clear the cache and statistics with f.cache_clear().
Access the underlying function with f.__wrapped__.
See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
"""
# Users should only access the lru_cache through its public API:
# cache_info, cache_clear, and f.__wrapped__
# The internals of the lru_cache are encapsulated for thread safety and
# to allow the implementation to change (including a possible C version).
def decorating_function(user_function,
tuple=tuple, sorted=sorted, len=len, KeyError=KeyError):
hits, misses = [0], [0]
kwd_mark = (object(),) # separates positional and keyword args
lock = Lock() # needed because OrderedDict isn't threadsafe
if maxsize is None:
cache = dict() # simple cache without ordering or size limit
@wraps(user_function)
def wrapper(*args, **kwds):
key = args
if kwds:
key += kwd_mark + tuple(sorted(kwds.items()))
try:
result = cache[key]
hits[0] += 1
return result
except KeyError:
pass
result = user_function(*args, **kwds)
cache[key] = result
misses[0] += 1
return result
else:
cache = OrderedDict() # ordered least recent to most recent
cache_popitem = cache.popitem
cache_renew = cache.move_to_end
@wraps(user_function)
def wrapper(*args, **kwds):
key = args
if kwds:
key += kwd_mark + tuple(sorted(kwds.items()))
with lock:
try:
result = cache[key]
cache_renew(key) # record recent use of this key
hits[0] += 1
return result
except KeyError:
pass
result = user_function(*args, **kwds)
with lock:
cache[key] = result # record recent use of this key
misses[0] += 1
if len(cache) > maxsize:
cache_popitem(0) # purge least recently used cache entry
return result
def cache_info():
"""Report cache statistics"""
with lock:
return _CacheInfo(hits[0], misses[0], maxsize, len(cache))
def cache_clear():
"""Clear the cache and cache statistics"""
with lock:
cache.clear()
hits[0] = misses[0] = 0
wrapper.cache_info = cache_info
wrapper.cache_clear = cache_clear
return wrapper
return decorating_function

157
lib/spack/external/py2/functools32/reprlib32.py vendored
View File

@@ -1,157 +0,0 @@
"""Redo the builtin repr() (representation) but with limits on most sizes."""
__all__ = ["Repr", "repr", "recursive_repr"]
import __builtin__ as builtins
from itertools import islice
try:
from thread import get_ident
except ImportError:
from _dummy_thread32 import get_ident
def recursive_repr(fillvalue='...'):
'Decorator to make a repr function return fillvalue for a recursive call'
def decorating_function(user_function):
repr_running = set()
def wrapper(self):
key = id(self), get_ident()
if key in repr_running:
return fillvalue
repr_running.add(key)
try:
result = user_function(self)
finally:
repr_running.discard(key)
return result
# Can't use functools.wraps() here because of bootstrap issues
wrapper.__module__ = getattr(user_function, '__module__')
wrapper.__doc__ = getattr(user_function, '__doc__')
wrapper.__name__ = getattr(user_function, '__name__')
wrapper.__annotations__ = getattr(user_function, '__annotations__', {})
return wrapper
return decorating_function
class Repr:
def __init__(self):
self.maxlevel = 6
self.maxtuple = 6
self.maxlist = 6
self.maxarray = 5
self.maxdict = 4
self.maxset = 6
self.maxfrozenset = 6
self.maxdeque = 6
self.maxstring = 30
self.maxlong = 40
self.maxother = 30
def repr(self, x):
return self.repr1(x, self.maxlevel)
def repr1(self, x, level):
typename = type(x).__name__
if ' ' in typename:
parts = typename.split()
typename = '_'.join(parts)
if hasattr(self, 'repr_' + typename):
return getattr(self, 'repr_' + typename)(x, level)
else:
return self.repr_instance(x, level)
def _repr_iterable(self, x, level, left, right, maxiter, trail=''):
n = len(x)
if level <= 0 and n:
s = '...'
else:
newlevel = level - 1
repr1 = self.repr1
pieces = [repr1(elem, newlevel) for elem in islice(x, maxiter)]
if n > maxiter: pieces.append('...')
s = ', '.join(pieces)
if n == 1 and trail: right = trail + right
return '%s%s%s' % (left, s, right)
def repr_tuple(self, x, level):
return self._repr_iterable(x, level, '(', ')', self.maxtuple, ',')
def repr_list(self, x, level):
return self._repr_iterable(x, level, '[', ']', self.maxlist)
def repr_array(self, x, level):
header = "array('%s', [" % x.typecode
return self._repr_iterable(x, level, header, '])', self.maxarray)
def repr_set(self, x, level):
x = _possibly_sorted(x)
return self._repr_iterable(x, level, 'set([', '])', self.maxset)
def repr_frozenset(self, x, level):
x = _possibly_sorted(x)
return self._repr_iterable(x, level, 'frozenset([', '])',
self.maxfrozenset)
def repr_deque(self, x, level):
return self._repr_iterable(x, level, 'deque([', '])', self.maxdeque)
def repr_dict(self, x, level):
n = len(x)
if n == 0: return '{}'
if level <= 0: return '{...}'
newlevel = level - 1
repr1 = self.repr1
pieces = []
for key in islice(_possibly_sorted(x), self.maxdict):
keyrepr = repr1(key, newlevel)
valrepr = repr1(x[key], newlevel)
pieces.append('%s: %s' % (keyrepr, valrepr))
if n > self.maxdict: pieces.append('...')
s = ', '.join(pieces)
return '{%s}' % (s,)
def repr_str(self, x, level):
s = builtins.repr(x[:self.maxstring])
if len(s) > self.maxstring:
i = max(0, (self.maxstring-3)//2)
j = max(0, self.maxstring-3-i)
s = builtins.repr(x[:i] + x[len(x)-j:])
s = s[:i] + '...' + s[len(s)-j:]
return s
def repr_int(self, x, level):
s = builtins.repr(x) # XXX Hope this isn't too slow...
if len(s) > self.maxlong:
i = max(0, (self.maxlong-3)//2)
j = max(0, self.maxlong-3-i)
s = s[:i] + '...' + s[len(s)-j:]
return s
def repr_instance(self, x, level):
try:
s = builtins.repr(x)
# Bugs in x.__repr__() can cause arbitrary
# exceptions -- then make up something
except Exception:
return '<%s instance at %x>' % (x.__class__.__name__, id(x))
if len(s) > self.maxother:
i = max(0, (self.maxother-3)//2)
j = max(0, self.maxother-3-i)
s = s[:i] + '...' + s[len(s)-j:]
return s
def _possibly_sorted(x):
# Since not all sequences of items can be sorted and comparison
# functions may raise arbitrary exceptions, return an unsorted
# sequence in that case.
try:
return sorted(x)
except Exception:
return list(x)
aRepr = Repr()
repr = aRepr.repr

103
lib/spack/external/py2/typing.py vendored
View File

@@ -1,103 +0,0 @@
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""
This is a fake set of symbols to allow spack to import typing in python
versions where we do not support type checking (<3)
"""
from collections import defaultdict
# (1) Unparameterized types.
Annotated = object
Any = object
AnyStr = object
ByteString = object
Counter = object
Final = object
Hashable = object
NoReturn = object
Sized = object
SupportsAbs = object
SupportsBytes = object
SupportsComplex = object
SupportsFloat = object
SupportsIndex = object
SupportsInt = object
SupportsRound = object
# (2) Parameterized types.
AbstractSet = defaultdict(lambda: object)
AsyncContextManager = defaultdict(lambda: object)
AsyncGenerator = defaultdict(lambda: object)
AsyncIterable = defaultdict(lambda: object)
AsyncIterator = defaultdict(lambda: object)
Awaitable = defaultdict(lambda: object)
Callable = defaultdict(lambda: object)
ChainMap = defaultdict(lambda: object)
ClassVar = defaultdict(lambda: object)
Collection = defaultdict(lambda: object)
Container = defaultdict(lambda: object)
ContextManager = defaultdict(lambda: object)
Coroutine = defaultdict(lambda: object)
DefaultDict = defaultdict(lambda: object)
Deque = defaultdict(lambda: object)
Dict = defaultdict(lambda: object)
ForwardRef = defaultdict(lambda: object)
FrozenSet = defaultdict(lambda: object)
Generator = defaultdict(lambda: object)
Generic = defaultdict(lambda: object)
ItemsView = defaultdict(lambda: object)
Iterable = defaultdict(lambda: object)
Iterator = defaultdict(lambda: object)
KeysView = defaultdict(lambda: object)
List = defaultdict(lambda: object)
Literal = defaultdict(lambda: object)
Mapping = defaultdict(lambda: object)
MappingView = defaultdict(lambda: object)
MutableMapping = defaultdict(lambda: object)
MutableSequence = defaultdict(lambda: object)
MutableSet = defaultdict(lambda: object)
NamedTuple = defaultdict(lambda: object)
Optional = defaultdict(lambda: object)
OrderedDict = defaultdict(lambda: object)
Reversible = defaultdict(lambda: object)
Sequence = defaultdict(lambda: object)
Set = defaultdict(lambda: object)
Tuple = defaultdict(lambda: object)
Type = defaultdict(lambda: object)
TypedDict = defaultdict(lambda: object)
Union = defaultdict(lambda: object)
ValuesView = defaultdict(lambda: object)
# (3) Type variable declarations.
TypeVar = lambda *args, **kwargs: None
# (4) Functions.
cast = lambda _type, x: x
get_args = None
get_origin = None
get_type_hints = None
no_type_check = None
no_type_check_decorator = None
## typing_extensions
# We get a ModuleNotFoundError when attempting to import anything from typing_extensions
# if we separate this into a separate typing_extensions.py file for some reason.
# (1) Unparameterized types.
IntVar = object
Literal = object
NewType = object
Text = object
# (2) Parameterized types.
Protocol = defaultdict(lambda: object)
# (3) Macro for avoiding evaluation except during type checking.
TYPE_CHECKING = False
# (4) Decorators.
final = lambda x: x
overload = lambda x: x
runtime_checkable = lambda x: x

5
lib/spack/llnl/util/argparsewriter.py
View File

@@ -7,11 +7,10 @@
import argparse
import errno
import io
import re
import sys
from six import StringIO
class Command(object):
"""Parsed representation of a command from argparse.
@@ -181,7 +180,7 @@ def __init__(self, prog, out=None, aliases=False, rst_levels=_rst_levels):
self.rst_levels = rst_levels
def format(self, cmd):
string = StringIO()
string = io.StringIO()
string.write(self.begin_command(cmd.prog))
if cmd.description:

39
lib/spack/llnl/util/compat.py
View File

@@ -1,39 +0,0 @@
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
# isort: off
import sys
if sys.version_info < (3,):
from itertools import ifilter as filter
from itertools import imap as map
from itertools import izip as zip
from itertools import izip_longest as zip_longest # novm
from urllib import urlencode as urlencode
from urllib import urlopen as urlopen
else:
filter = filter
map = map
zip = zip
from itertools import zip_longest as zip_longest # novm # noqa: F401
from urllib.parse import urlencode as urlencode # novm # noqa: F401
from urllib.request import urlopen as urlopen # novm # noqa: F401
if sys.version_info >= (3, 3):
from collections.abc import Hashable as Hashable # novm
from collections.abc import Iterable as Iterable # novm
from collections.abc import Mapping as Mapping # novm
from collections.abc import MutableMapping as MutableMapping # novm
from collections.abc import MutableSequence as MutableSequence # novm
from collections.abc import MutableSet as MutableSet # novm
from collections.abc import Sequence as Sequence # novm
else:
from collections import Hashable as Hashable # noqa: F401
from collections import Iterable as Iterable # noqa: F401
from collections import Mapping as Mapping # noqa: F401
from collections import MutableMapping as MutableMapping # noqa: F401
from collections import MutableSequence as MutableSequence # noqa: F401
from collections import MutableSet as MutableSet # noqa: F401
from collections import Sequence as Sequence # noqa: F401

98
lib/spack/llnl/util/filesystem.py
View File

@@ -3,6 +3,7 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import collections
import collections.abc
import errno
import glob
import hashlib
@@ -17,10 +18,7 @@
from contextlib import contextmanager
from sys import platform as _platform
import six
from llnl.util import tty
from llnl.util.compat import Sequence
from llnl.util.lang import dedupe, memoized
from llnl.util.symlink import islink, symlink
@@ -290,9 +288,7 @@ def groupid_to_group(x):
shutil.copy(filename, tmp_filename)
try:
extra_kwargs = {}
if sys.version_info > (3, 0):
extra_kwargs = {"errors": "surrogateescape"}
extra_kwargs = {"errors": "surrogateescape"}
# Open as a text file and filter until the end of the file is
# reached or we found a marker in the line if it was specified
@@ -522,7 +518,7 @@ def chgrp(path, group, follow_symlinks=True):
if is_windows:
raise OSError("Function 'chgrp' is not supported on Windows")
if isinstance(group, six.string_types):
if isinstance(group, str):
gid = grp.getgrnam(group).gr_gid
else:
gid = group
@@ -1019,7 +1015,7 @@ def open_if_filename(str_or_file, mode="r"):
If it's a file object, just yields the file object.
"""
if isinstance(str_or_file, six.string_types):
if isinstance(str_or_file, str):
with open(str_or_file, mode) as f:
yield f
else:
@@ -1309,46 +1305,34 @@ def visit_directory_tree(root, visitor, rel_path="", depth=0):
depth (str): current depth from the root
"""
dir = os.path.join(root, rel_path)
if sys.version_info >= (3, 5, 0):
dir_entries = sorted(os.scandir(dir), key=lambda d: d.name) # novermin
else:
dir_entries = os.listdir(dir)
dir_entries.sort()
dir_entries = sorted(os.scandir(dir), key=lambda d: d.name)
for f in dir_entries:
if sys.version_info >= (3, 5, 0):
rel_child = os.path.join(rel_path, f.name)
islink = f.is_symlink()
# On Windows, symlinks to directories are distinct from
# symlinks to files, and it is possible to create a
# broken symlink to a directory (e.g. using os.symlink
# without `target_is_directory=True`), invoking `isdir`
# on a symlink on Windows that is broken in this manner
# will result in an error. In this case we can work around
# the issue by reading the target and resolving the
# directory ourselves
try:
isdir = f.is_dir()
except OSError as e:
if is_windows and hasattr(e, "winerror") and e.winerror == 5 and islink:
# if path is a symlink, determine destination and
# evaluate file vs directory
link_target = resolve_link_target_relative_to_the_link(f)
# link_target might be relative but
# resolve_link_target_relative_to_the_link
# will ensure that if so, that it is relative
# to the CWD and therefore
# makes sense
isdir = os.path.isdir(link_target)
else:
raise e
else:
rel_child = os.path.join(rel_path, f)
lexists, islink, isdir = lexists_islink_isdir(os.path.join(dir, f))
if not lexists:
continue
rel_child = os.path.join(rel_path, f.name)
islink = f.is_symlink()
# On Windows, symlinks to directories are distinct from
# symlinks to files, and it is possible to create a
# broken symlink to a directory (e.g. using os.symlink
# without `target_is_directory=True`), invoking `isdir`
# on a symlink on Windows that is broken in this manner
# will result in an error. In this case we can work around
# the issue by reading the target and resolving the
# directory ourselves
try:
isdir = f.is_dir()
except OSError as e:
if is_windows and hasattr(e, "winerror") and e.winerror == 5 and islink:
# if path is a symlink, determine destination and
# evaluate file vs directory
link_target = resolve_link_target_relative_to_the_link(f)
# link_target might be relative but
# resolve_link_target_relative_to_the_link
# will ensure that if so, that it is relative
# to the CWD and therefore
# makes sense
isdir = os.path.isdir(link_target)
else:
raise e
if not isdir and not islink:
# handle non-symlink files
@@ -1609,14 +1593,14 @@ def find(root, files, recursive=True):
Parameters:
root (str): The root directory to start searching from
files (str or Sequence): Library name(s) to search for
files (str or collections.abc.Sequence): Library name(s) to search for
recursive (bool): if False search only root folder,
if True descends top-down from the root. Defaults to True.
Returns:
list: The files that have been found
"""
if isinstance(files, six.string_types):
if isinstance(files, str):
files = [files]
if recursive:
@@ -1673,14 +1657,14 @@ def _find_non_recursive(root, search_files):
# Utilities for libraries and headers
class FileList(Sequence):
class FileList(collections.abc.Sequence):
"""Sequence of absolute paths to files.
Provides a few convenience methods to manipulate file paths.
"""
def __init__(self, files):
if isinstance(files, six.string_types):
if isinstance(files, str):
files = [files]
self.files = list(dedupe(files))
@@ -1776,7 +1760,7 @@ def directories(self):
def directories(self, value):
value = value or []
# Accept a single directory as input
if isinstance(value, six.string_types):
if isinstance(value, str):
value = [value]
self._directories = [path_to_os_path(os.path.normpath(x))[0] for x in value]
@@ -1912,9 +1896,9 @@ def find_headers(headers, root, recursive=False):
Returns:
HeaderList: The headers that have been found
"""
if isinstance(headers, six.string_types):
if isinstance(headers, str):
headers = [headers]
elif not isinstance(headers, Sequence):
elif not isinstance(headers, collections.abc.Sequence):
message = "{0} expects a string or sequence of strings as the "
message += "first argument [got {1} instead]"
message = message.format(find_headers.__name__, type(headers))
@@ -2078,9 +2062,9 @@ def find_system_libraries(libraries, shared=True):
Returns:
LibraryList: The libraries that have been found
"""
if isinstance(libraries, six.string_types):
if isinstance(libraries, str):
libraries = [libraries]
elif not isinstance(libraries, Sequence):
elif not isinstance(libraries, collections.abc.Sequence):
message = "{0} expects a string or sequence of strings as the "
message += "first argument [got {1} instead]"
message = message.format(find_system_libraries.__name__, type(libraries))
@@ -2135,9 +2119,9 @@ def find_libraries(libraries, root, shared=True, recursive=False, runtime=True):
Returns:
LibraryList: The libraries that have been found
"""
if isinstance(libraries, six.string_types):
if isinstance(libraries, str):
libraries = [libraries]
elif not isinstance(libraries, Sequence):
elif not isinstance(libraries, collections.abc.Sequence):
message = "{0} expects a string or sequence of strings as the "
message += "first argument [got {1} instead]"
message = message.format(find_libraries.__name__, type(libraries))

64
lib/spack/llnl/util/lang.py
View File

@@ -5,9 +5,11 @@
from __future__ import division
import collections.abc
import contextlib
import functools
import inspect
import itertools
import os
import re
import sys
@@ -15,11 +17,6 @@
from datetime import datetime, timedelta
from typing import Any, Callable, Iterable, List, Tuple
import six
from six import string_types
from llnl.util.compat import MutableMapping, MutableSequence, zip_longest
# Ignore emacs backups when listing modules
ignore_modules = [r"^\.#", "~$"]
@@ -200,14 +197,9 @@ def _memoized_function(*args, **kwargs):
return ret
except TypeError as e:
# TypeError is raised when indexing into a dict if the key is unhashable.
raise six.raise_from(
UnhashableArguments(
"args + kwargs '{}' was not hashable for function '{}'".format(
key, func.__name__
),
),
e,
)
raise UnhashableArguments(
"args + kwargs '{}' was not hashable for function '{}'".format(key, func.__name__),
) from e
return _memoized_function
@@ -312,7 +304,7 @@ def lazy_eq(lseq, rseq):
# zip_longest is implemented in native code, so use it for speed.
# use zip_longest instead of zip because it allows us to tell
# which iterator was longer.
for left, right in zip_longest(liter, riter, fillvalue=done):
for left, right in itertools.zip_longest(liter, riter, fillvalue=done):
if (left is done) or (right is done):
return False
@@ -332,7 +324,7 @@ def lazy_lt(lseq, rseq):
liter = lseq()
riter = rseq()
for left, right in zip_longest(liter, riter, fillvalue=done):
for left, right in itertools.zip_longest(liter, riter, fillvalue=done):
if (left is done) or (right is done):
return left is done # left was shorter than right
@@ -482,7 +474,7 @@ def add_func_to_class(name, func):
@lazy_lexicographic_ordering
class HashableMap(MutableMapping):
class HashableMap(collections.abc.MutableMapping):
"""This is a hashable, comparable dictionary. Hash is performed on
a tuple of the values in the dictionary."""
@@ -574,7 +566,7 @@ def match_predicate(*args):
def match(string):
for arg in args:
if isinstance(arg, string_types):
if isinstance(arg, str):
if re.search(arg, string):
return True
elif isinstance(arg, list) or isinstance(arg, tuple):
@@ -887,32 +879,28 @@ def load_module_from_file(module_name, module_path):
ImportError: when the module can't be loaded
FileNotFoundError: when module_path doesn't exist
"""
import importlib.util
if module_name in sys.modules:
return sys.modules[module_name]
# This recipe is adapted from https://stackoverflow.com/a/67692/771663
if sys.version_info[0] == 3 and sys.version_info[1] >= 5:
import importlib.util
spec = importlib.util.spec_from_file_location(module_name, module_path) # novm
module = importlib.util.module_from_spec(spec) # novm
# The module object needs to exist in sys.modules before the
# loader executes the module code.
#
# See https://docs.python.org/3/reference/import.html#loading
sys.modules[spec.name] = module
spec = importlib.util.spec_from_file_location(module_name, module_path) # novm
module = importlib.util.module_from_spec(spec) # novm
# The module object needs to exist in sys.modules before the
# loader executes the module code.
#
# See https://docs.python.org/3/reference/import.html#loading
sys.modules[spec.name] = module
try:
spec.loader.exec_module(module)
except BaseException:
try:
spec.loader.exec_module(module)
except BaseException:
try:
del sys.modules[spec.name]
except KeyError:
pass
raise
elif sys.version_info[0] == 2:
import imp
module = imp.load_source(module_name, module_path)
del sys.modules[spec.name]
except KeyError:
pass
raise
return module
@@ -1030,7 +1018,7 @@ def ensure_last(lst, *elements):
lst.append(lst.pop(lst.index(elt)))
class TypedMutableSequence(MutableSequence):
class TypedMutableSequence(collections.abc.MutableSequence):
"""Base class that behaves like a list, just with a different type.
Client code can inherit from this base class:

34
lib/spack/llnl/util/lock.py
View File

@@ -9,7 +9,6 @@
import sys
import time
from datetime import datetime
from typing import Dict, Tuple # novm
import llnl.util.tty as tty
from llnl.util.lang import pretty_seconds
@@ -81,7 +80,7 @@ class OpenFileTracker(object):
def __init__(self):
"""Create a new ``OpenFileTracker``."""
self._descriptors = {} # type: Dict[Tuple[int, int], OpenFile]
self._descriptors = {}
def get_fh(self, path):
"""Get a filehandle for a lockfile.
@@ -103,7 +102,7 @@ def get_fh(self, path):
try:
# see whether we've seen this inode/pid before
stat = os.stat(path)
key = (stat.st_ino, pid)
key = (stat.st_dev, stat.st_ino, pid)
open_file = self._descriptors.get(key)
except OSError as e:
@@ -129,32 +128,32 @@ def get_fh(self, path):
# if we just created the file, we'll need to get its inode here
if not stat:
inode = os.fstat(fd).st_ino
key = (inode, pid)
stat = os.fstat(fd)
key = (stat.st_dev, stat.st_ino, pid)
self._descriptors[key] = open_file
open_file.refs += 1
return open_file.fh
def release_fh(self, path):
"""Release a filehandle, only closing it if there are no more references."""
try:
inode = os.stat(path).st_ino
except OSError as e:
if e.errno != errno.ENOENT: # only handle file not found
raise
inode = None # this will not be in self._descriptors
key = (inode, os.getpid())
def release_by_stat(self, stat):
key = (stat.st_dev, stat.st_ino, os.getpid())
open_file = self._descriptors.get(key)
assert open_file, "Attempted to close non-existing lock path: %s" % path
assert open_file, "Attempted to close non-existing inode: %s" % stat.st_inode
open_file.refs -= 1
if not open_file.refs:
del self._descriptors[key]
open_file.fh.close()
def release_by_fh(self, fh):
self.release_by_stat(os.fstat(fh.fileno()))
def purge(self):
for key in list(self._descriptors.keys()):
self._descriptors[key].fh.close()
del self._descriptors[key]
#: Open file descriptors for locks in this process. Used to prevent one process
#: from opening the sam file many times for different byte range locks
@@ -432,8 +431,7 @@ def _unlock(self):
"""
fcntl.lockf(self._file, fcntl.LOCK_UN, self._length, self._start, os.SEEK_SET)
file_tracker.release_fh(self.path)
file_tracker.release_by_fh(self._file)
self._file = None
self._reads = 0
self._writes = 0

2
lib/spack/llnl/util/symlink.py
View File

@@ -24,7 +24,7 @@ def symlink(real_path, link_path):
On Windows, use junctions if os.symlink fails.
"""
if not is_windows or _win32_can_symlink():
os.symlink(real_path, link_path)
os.symlink(real_path, link_path, target_is_directory=os.path.isdir(real_path))
else:
try:
# Try to use junctions

28
lib/spack/llnl/util/tty/__init__.py
View File

@@ -6,6 +6,7 @@
from __future__ import unicode_literals
import contextlib
import io
import os
import struct
import sys
@@ -14,10 +15,6 @@
from datetime import datetime
from sys import platform as _platform
import six
from six import StringIO
from six.moves import input
if _platform != "win32":
import fcntl
import termios
@@ -183,7 +180,7 @@ def msg(message, *args, **kwargs):
else:
cwrite("@*b{%s==>} %s%s" % (st_text, get_timestamp(), cescape(_output_filter(message))))
for arg in args:
print(indent + _output_filter(six.text_type(arg)))
print(indent + _output_filter(str(arg)))
def info(message, *args, **kwargs):
@@ -201,13 +198,13 @@ def info(message, *args, **kwargs):
st_text = process_stacktrace(st_countback)
cprint(
"@%s{%s==>} %s%s"
% (format, st_text, get_timestamp(), cescape(_output_filter(six.text_type(message)))),
% (format, st_text, get_timestamp(), cescape(_output_filter(str(message)))),
stream=stream,
)
for arg in args:
if wrap:
lines = textwrap.wrap(
_output_filter(six.text_type(arg)),
_output_filter(str(arg)),
initial_indent=indent,
subsequent_indent=indent,
break_long_words=break_long_words,
@@ -215,7 +212,7 @@ def info(message, *args, **kwargs):
for line in lines:
stream.write(line + "\n")
else:
stream.write(indent + _output_filter(six.text_type(arg)) + "\n")
stream.write(indent + _output_filter(str(arg)) + "\n")
def verbose(message, *args, **kwargs):
@@ -238,7 +235,7 @@ def error(message, *args, **kwargs):
kwargs.setdefault("format", "*r")
kwargs.setdefault("stream", sys.stderr)
info("Error: " + six.text_type(message), *args, **kwargs)
info("Error: " + str(message), *args, **kwargs)
def warn(message, *args, **kwargs):
@@ -247,7 +244,7 @@ def warn(message, *args, **kwargs):
kwargs.setdefault("format", "*Y")
kwargs.setdefault("stream", sys.stderr)
info("Warning: " + six.text_type(message), *args, **kwargs)
info("Warning: " + str(message), *args, **kwargs)
def die(message, *args, **kwargs):
@@ -271,7 +268,7 @@ def get_number(prompt, **kwargs):
while number is None:
msg(prompt, newline=False)
ans = input()
if ans == six.text_type(abort):
if ans == str(abort):
return None
if ans:
@@ -336,11 +333,11 @@ def hline(label=None, **kwargs):
cols -= 2
cols = min(max_width, cols)
label = six.text_type(label)
label = str(label)
prefix = char * 2 + " "
suffix = " " + (cols - len(prefix) - clen(label)) * char
out = StringIO()
out = io.StringIO()
out.write(prefix)
out.write(label)
out.write(suffix)
@@ -372,10 +369,5 @@ def ioctl_gwinsz(fd):
return int(rc[0]), int(rc[1])
else:
if sys.version_info[0] < 3:
raise RuntimeError(
"Terminal size not obtainable on Windows with a\
Python version older than 3"
)
rc = (os.environ.get("LINES", 25), os.environ.get("COLUMNS", 80))
return int(rc[0]), int(rc[1])

7
lib/spack/llnl/util/tty/colify.py
View File

@@ -8,11 +8,10 @@
"""
from __future__ import division, unicode_literals
import io
import os
import sys
from six import StringIO, text_type
from llnl.util.tty import terminal_size
from llnl.util.tty.color import cextra, clen
@@ -134,7 +133,7 @@ def colify(elts, **options):
)
# elts needs to be an array of strings so we can count the elements
elts = [text_type(elt) for elt in elts]
elts = [str(elt) for elt in elts]
if not elts:
return (0, ())
@@ -232,7 +231,7 @@ def transpose():
def colified(elts, **options):
"""Invokes the ``colify()`` function but returns the result as a string
instead of writing it to an output string."""
sio = StringIO()
sio = io.StringIO()
options["output"] = sio
colify(elts, **options)
return sio.getvalue()

4
lib/spack/llnl/util/tty/color.py
View File

@@ -65,8 +65,6 @@
import sys
from contextlib import contextmanager
import six
class ColorParseError(Exception):
"""Raised when a color format fails to parse."""
@@ -259,7 +257,7 @@ def cescape(string):
Returns:
(str): the string with color codes escaped
"""
string = six.text_type(string)
string = str(string)
string = string.replace("@", "@@")
string = string.replace("}", "}}")
return string

36
lib/spack/llnl/util/tty/log.py
View File

@@ -24,8 +24,6 @@
from types import ModuleType # novm
from typing import Optional # novm
from six import StringIO, string_types
import llnl.util.tty as tty
termios = None # type: Optional[ModuleType]
@@ -241,8 +239,7 @@ def __exit__(self, exc_type, exception, traceback):
"""If termios was available, restore old settings."""
if self.old_cfg:
self._restore_default_terminal_settings()
if sys.version_info >= (3,):
atexit.unregister(self._restore_default_terminal_settings)
atexit.unregister(self._restore_default_terminal_settings)
# restore SIGSTP and SIGCONT handlers
if self.old_handlers:
@@ -309,7 +306,7 @@ def __init__(self, file_like):
self.file_like = file_like
if isinstance(file_like, string_types):
if isinstance(file_like, str):
self.open = True
elif _file_descriptors_work(file_like):
self.open = False
@@ -323,12 +320,9 @@ def __init__(self, file_like):
def unwrap(self):
if self.open:
if self.file_like:
if sys.version_info < (3,):
self.file = open(self.file_like, "w")
else:
self.file = open(self.file_like, "w", encoding="utf-8") # novm
self.file = open(self.file_like, "w", encoding="utf-8")
else:
self.file = StringIO()
self.file = io.StringIO()
return self.file
else:
# We were handed an already-open file object. In this case we also
@@ -699,13 +693,10 @@ def __init__(self, sys_attr):
self.sys_attr = sys_attr
self.saved_stream = None
if sys.platform.startswith("win32"):
if sys.version_info < (3, 5):
libc = ctypes.CDLL(ctypes.util.find_library("c"))
if hasattr(sys, "gettotalrefcount"): # debug build
libc = ctypes.CDLL("ucrtbased")
else:
if hasattr(sys, "gettotalrefcount"): # debug build
libc = ctypes.CDLL("ucrtbased")
else:
libc = ctypes.CDLL("api-ms-win-crt-stdio-l1-1-0")
libc = ctypes.CDLL("api-ms-win-crt-stdio-l1-1-0")
kernel32 = ctypes.WinDLL("kernel32")
@@ -794,7 +785,7 @@ def __enter__(self):
raise RuntimeError("file argument must be set by __init__ ")
# Open both write and reading on logfile
if type(self.logfile) == StringIO:
if type(self.logfile) == io.StringIO:
self._ioflag = True
# cannot have two streams on tempfile, so we must make our own
sys.stdout = self.logfile
@@ -927,13 +918,10 @@ def _writer_daemon(
if sys.version_info < (3, 8) or sys.platform != "darwin":
os.close(write_fd)
# Use line buffering (3rd param = 1) since Python 3 has a bug
# 1. Use line buffering (3rd param = 1) since Python 3 has a bug
# that prevents unbuffered text I/O.
if sys.version_info < (3,):
in_pipe = os.fdopen(read_multiprocess_fd.fd, "r", 1)
else:
# Python 3.x before 3.7 does not open with UTF-8 encoding by default
in_pipe = os.fdopen(read_multiprocess_fd.fd, "r", 1, encoding="utf-8")
# 2. Python 3.x before 3.7 does not open with UTF-8 encoding by default
in_pipe = os.fdopen(read_multiprocess_fd.fd, "r", 1, encoding="utf-8")
if stdin_multiprocess_fd:
stdin = os.fdopen(stdin_multiprocess_fd.fd)
@@ -1023,7 +1011,7 @@ def _writer_daemon(
finally:
# send written data back to parent if we used a StringIO
if isinstance(log_file, StringIO):
if isinstance(log_file, io.StringIO):
control_pipe.send(log_file.getvalue())
log_file_wrapper.close()
close_connection_and_file(read_multiprocess_fd, in_pipe)

2
lib/spack/spack/__init__.py
View File

@@ -4,7 +4,7 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
#: PEP440 canonical <major>.<minor>.<micro>.<devN> string
__version__ = "0.19.0.dev0"
__version__ = "0.20.0.dev0"
spack_version = __version__

9
lib/spack/spack/audit.py
View File

@@ -37,15 +37,14 @@ def _search_duplicate_compilers(error_cls):
"""
import ast
import collections
import collections.abc
import inspect
import itertools
import pickle
import re
from six.moves.urllib.request import urlopen
from urllib.request import urlopen
import llnl.util.lang
from llnl.util.compat import Sequence
import spack.config
import spack.patch
@@ -81,7 +80,7 @@ def __hash__(self):
return hash(value)
class AuditClass(Sequence):
class AuditClass(collections.abc.Sequence):
def __init__(self, group, tag, description, kwargs):
"""Return an object that acts as a decorator to register functions
associated with a specific class of sanity checks.
@@ -288,7 +287,7 @@ def _check_build_test_callbacks(pkgs, error_cls):
errors = []
for pkg_name in pkgs:
pkg_cls = spack.repo.path.get_pkg_class(pkg_name)
test_callbacks = pkg_cls.build_time_test_callbacks
test_callbacks = getattr(pkg_cls, "build_time_test_callbacks", None)
if test_callbacks and "test" in test_callbacks:
msg = '{0} package contains "test" method in ' "build_time_test_callbacks"

6
lib/spack/spack/binary_distribution.py
View File

@@ -17,9 +17,9 @@
import traceback
import warnings
from contextlib import closing
from urllib.error import HTTPError, URLError
import ruamel.yaml as yaml
from six.moves.urllib.error import HTTPError, URLError
import llnl.util.filesystem as fsys
import llnl.util.lang
@@ -1634,7 +1634,7 @@ def make_package_relative(workdir, spec, allow_root):
if "elf" in platform.binary_formats:
relocate.make_elf_binaries_relative(cur_path_names, orig_path_names, old_layout_root)
relocate.raise_if_not_relocatable(cur_path_names, allow_root)
allow_root or relocate.ensure_binaries_are_relocatable(cur_path_names)
orig_path_names = list()
cur_path_names = list()
for linkname in buildinfo.get("relocate_links", []):
@@ -1652,7 +1652,7 @@ def check_package_relocatable(workdir, spec, allow_root):
cur_path_names = list()
for filename in buildinfo["relocate_binaries"]:
cur_path_names.append(os.path.join(workdir, filename))
relocate.raise_if_not_relocatable(cur_path_names, allow_root)
allow_root or relocate.ensure_binaries_are_relocatable(cur_path_names)
def dedupe_hardlinks_if_necessary(root, buildinfo):

30
lib/spack/spack/bootstrap.py
View File

@@ -17,8 +17,6 @@
import sysconfig
import uuid
import six
import archspec.cpu
import llnl.util.filesystem as fs
@@ -78,7 +76,7 @@ def _try_import_from_store(module, query_spec, query_info=None):
command found and the concrete spec providing it
"""
# If it is a string assume it's one of the root specs by this module
if isinstance(query_spec, six.string_types):
if isinstance(query_spec, str):
# We have to run as part of this python interpreter
query_spec += " ^" + spec_for_current_python()
@@ -92,13 +90,6 @@ def _try_import_from_store(module, query_spec, query_info=None):
] # type: list[str]
path_before = list(sys.path)
# Python 3.8+ on Windows does not search dependent DLLs in PATH,
# so we need to manually add it using os.add_dll_directory
# https://docs.python.org/3/whatsnew/3.8.html#bpo-36085-whatsnew
if sys.version_info[:2] >= (3, 8) and sys.platform == "win32":
if os.path.isdir(candidate_spec.prefix.bin):
os.add_dll_directory(candidate_spec.prefix.bin) # novermin
# NOTE: try module_paths first and last, last allows an existing version in path
# to be picked up and used, possibly depending on something in the store, first
# allows the bootstrap version to work when an incompatible version is in
@@ -476,21 +467,14 @@ def source_is_enabled_or_raise(conf):
def spec_for_current_python():
"""For bootstrapping purposes we are just interested in the Python
minor version (all patches are ABI compatible with the same minor)
and on whether ucs4 support has been enabled for Python 2.7
minor version (all patches are ABI compatible with the same minor).
See:
https://www.python.org/dev/peps/pep-0513/
https://stackoverflow.com/a/35801395/771663
"""
version_str = ".".join(str(x) for x in sys.version_info[:2])
variant_str = ""
if sys.version_info[0] == 2 and sys.version_info[1] == 7:
unicode_size = sysconfig.get_config_var("Py_UNICODE_SIZE")
variant_str = "+ucs4" if unicode_size == 4 else "~ucs4"
spec_fmt = "python@{0} {1}"
return spec_fmt.format(version_str, variant_str)
return "python@{0}".format(version_str)
@contextlib.contextmanager
@@ -873,9 +857,7 @@ def ensure_mypy_in_path_or_raise():
def black_root_spec():
# black v21 is the last version to support Python 2.7.
# Upgrade when we no longer support Python 2.7
return _root_spec("py-black@:21")
return _root_spec("py-black")
def ensure_black_in_path_or_raise():
@@ -932,7 +914,7 @@ def _missing(name, purpose, system_only=True):
def _required_system_executable(exes, msg):
"""Search for an executable is the system path only."""
if isinstance(exes, six.string_types):
if isinstance(exes, str):
exes = (exes,)
if spack.util.executable.which_string(*exes):
return True, None
@@ -950,7 +932,7 @@ def _required_python_module(module, query_spec, msg):
def _required_executable(exes, query_spec, msg):
"""Search for an executable in the system path or in the bootstrap store."""
if isinstance(exes, six.string_types):
if isinstance(exes, str):
exes = (exes,)
if spack.util.executable.which_string(*exes) or _executables_in_store(exes, query_spec):
return True, None

149
lib/spack/spack/build_environment.py
View File

@@ -33,6 +33,7 @@
calls you can make from within the install() function.
"""
import inspect
import io
import multiprocessing
import os
import re
@@ -41,8 +42,6 @@
import traceback
import types
from six import StringIO
import llnl.util.tty as tty
from llnl.util.filesystem import install, install_tree, mkdirp
from llnl.util.lang import dedupe
@@ -285,6 +284,23 @@ def clean_environment():
return env
def _add_werror_handling(keep_werror, env):
keep_flags = set()
# set of pairs
replace_flags = [] # type: List[Tuple[str,str]]
if keep_werror == "all":
keep_flags.add("-Werror*")
else:
if keep_werror == "specific":
keep_flags.add("-Werror-*")
keep_flags.add("-Werror=*")
# This extra case is to handle -Werror-implicit-function-declaration
replace_flags.append(("-Werror-", "-Wno-error="))
replace_flags.append(("-Werror", "-Wno-error"))
env.set("SPACK_COMPILER_FLAGS_KEEP", "|".join(keep_flags))
env.set("SPACK_COMPILER_FLAGS_REPLACE", " ".join(["|".join(item) for item in replace_flags]))
def set_compiler_environment_variables(pkg, env):
assert pkg.spec.concrete
compiler = pkg.compiler
@@ -331,6 +347,13 @@ def set_compiler_environment_variables(pkg, env):
env.set("SPACK_DTAGS_TO_STRIP", compiler.disable_new_dtags)
env.set("SPACK_DTAGS_TO_ADD", compiler.enable_new_dtags)
if pkg.keep_werror is not None:
keep_werror = pkg.keep_werror
else:
keep_werror = spack.config.get("config:flags:keep_werror")
_add_werror_handling(keep_werror, env)
# Set the target parameters that the compiler will add
# Don't set on cray platform because the targeting module handles this
if spec.satisfies("platform=cray"):
@@ -353,10 +376,8 @@ def set_compiler_environment_variables(pkg, env):
if isinstance(pkg.flag_handler, types.FunctionType):
handler = pkg.flag_handler
else:
if sys.version_info >= (3, 0):
handler = pkg.flag_handler.__func__
else:
handler = pkg.flag_handler.im_func
handler = pkg.flag_handler.__func__
injf, envf, bsf = handler(pkg, flag, spec.compiler_flags[flag][:])
inject_flags[flag] = injf or []
env_flags[flag] = envf or []
@@ -542,14 +563,18 @@ def determine_number_of_jobs(
return min(max_cpus, config_default)
def _set_variables_for_single_module(pkg, module):
"""Helper function to set module variables for single module."""
def set_module_variables_for_package(pkg):
"""Populate the Python module of a package with some useful global names.
This makes things easier for package writers.
"""
# Put a marker on this module so that it won't execute the body of this
# function again, since it is not needed
marker = "_set_run_already_called"
if getattr(module, marker, False):
if getattr(pkg.module, marker, False):
return
module = ModuleChangePropagator(pkg)
jobs = determine_number_of_jobs(parallel=pkg.parallel)
m = module
@@ -569,6 +594,7 @@ def _set_variables_for_single_module(pkg, module):
if sys.platform == "win32":
m.nmake = Executable("nmake")
m.msbuild = Executable("msbuild")
# Standard CMake arguments
m.std_cmake_args = spack.build_systems.cmake.CMakeBuilder.std_args(pkg)
m.std_meson_args = spack.build_systems.meson.MesonBuilder.std_args(pkg)
@@ -616,20 +642,7 @@ def static_to_shared_library(static_lib, shared_lib=None, **kwargs):
# Put a marker on this module so that it won't execute the body of this
# function again, since it is not needed
setattr(m, marker, True)
def set_module_variables_for_package(pkg):
"""Populate the module scope of install() with some useful functions.
This makes things easier for package writers.
"""
# If a user makes their own package repo, e.g.
# spack.pkg.mystuff.libelf.Libelf, and they inherit from an existing class
# like spack.pkg.original.libelf.Libelf, then set the module variables
# for both classes so the parent class can still use them if it gets
# called. parent_class_modules includes pkg.module.
modules = parent_class_modules(pkg.__class__)
for mod in modules:
_set_variables_for_single_module(pkg, mod)
module.propagate_changes_to_mro()
def _static_to_shared_library(arch, compiler, static_lib, shared_lib=None, **kwargs):
@@ -739,25 +752,6 @@ def get_rpaths(pkg):
return list(dedupe(filter_system_paths(rpaths)))
def parent_class_modules(cls):
"""
Get list of superclass modules that descend from spack.package_base.PackageBase
Includes cls.__module__
"""
if not issubclass(cls, spack.package_base.PackageBase) or issubclass(
spack.package_base.PackageBase, cls
):
return []
result = []
module = sys.modules.get(cls.__module__)
if module:
result = [module]
for c in cls.__bases__:
result.extend(parent_class_modules(c))
return result
def load_external_modules(pkg):
"""Traverse a package's spec DAG and load any external modules.
@@ -978,22 +972,9 @@ def add_modifications_for_dep(dep):
if set_package_py_globals:
set_module_variables_for_package(dpkg)
# Allow dependencies to modify the module
# Get list of modules that may need updating
modules = []
for cls in inspect.getmro(type(spec.package)):
module = cls.module
if module == spack.package_base:
break
modules.append(module)
# Execute changes as if on a single module
# copy dict to ensure prior changes are available
changes = spack.util.pattern.Bunch()
dpkg.setup_dependent_package(changes, spec)
for module in modules:
module.__dict__.update(changes.__dict__)
current_module = ModuleChangePropagator(spec.package)
dpkg.setup_dependent_package(current_module, spec)
current_module.propagate_changes_to_mro()
if context == "build":
builder = spack.builder.create(dpkg)
@@ -1271,6 +1252,8 @@ def make_stack(tb, stack=None):
obj = frame.f_locals["self"]
if isinstance(obj, spack.package_base.PackageBase):
break
else:
return None
# We found obj, the Package implementation we care about.
# Point out the location in the install method where we failed.
@@ -1352,7 +1335,7 @@ def __init__(self, msg, module, classname, traceback_string, log_name, log_type,
@property
def long_message(self):
out = StringIO()
out = io.StringIO()
out.write(self._long_message if self._long_message else "")
have_log = self.log_name and os.path.exists(self.log_name)
@@ -1437,3 +1420,51 @@ def write_log_summary(out, log_type, log, last=None):
# If no errors are found but warnings are, display warnings
out.write("\n%s found in %s log:\n" % (plural(nwar, "warning"), log_type))
out.write(make_log_context(warnings))
class ModuleChangePropagator:
"""Wrapper class to accept changes to a package.py Python module, and propagate them in the
MRO of the package.
It is mainly used as a substitute of the ``package.py`` module, when calling the
"setup_dependent_package" function during build environment setup.
"""
_PROTECTED_NAMES = ("package", "current_module", "modules_in_mro", "_set_attributes")
def __init__(self, package):
self._set_self_attributes("package", package)
self._set_self_attributes("current_module", package.module)
#: Modules for the classes in the MRO up to PackageBase
modules_in_mro = []
for cls in inspect.getmro(type(package)):
module = cls.module
if module == self.current_module:
continue
if module == spack.package_base:
break
modules_in_mro.append(module)
self._set_self_attributes("modules_in_mro", modules_in_mro)
self._set_self_attributes("_set_attributes", {})
def _set_self_attributes(self, key, value):
super().__setattr__(key, value)
def __getattr__(self, item):
return getattr(self.current_module, item)
def __setattr__(self, key, value):
if key in ModuleChangePropagator._PROTECTED_NAMES:
msg = f'Cannot set attribute "{key}" in ModuleMonkeyPatcher'
return AttributeError(msg)
setattr(self.current_module, key, value)
self._set_attributes[key] = value
def propagate_changes_to_mro(self):
for module_in_mro in self.modules_in_mro:
module_in_mro.__dict__.update(self._set_attributes)

8
lib/spack/spack/build_systems/_checks.py
View File

@@ -4,8 +4,6 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import six
import llnl.util.lang
import spack.builder
@@ -26,7 +24,7 @@ def sanity_check_prefix(builder):
pkg = builder.pkg
def check_paths(path_list, filetype, predicate):
if isinstance(path_list, six.string_types):
if isinstance(path_list, str):
path_list = [path_list]
for path in path_list:
@@ -89,11 +87,11 @@ def ensure_build_dependencies_or_raise(spec, dependencies, error_msg):
)
for dep in missing_deps:
msg += " depends_on('{0}', type='build', when='@{1} {2}')\n".format(
msg += ' depends_on("{0}", type="build", when="@{1} {2}")\n'.format(
dep, spec.version, "build_system=autotools"
)
msg += "\nUpdate the version (when='@{0}') as needed.".format(spec.version)
msg += '\nUpdate the version (when="@{0}") as needed.'.format(spec.version)
raise RuntimeError(msg)

6
lib/spack/spack/build_systems/cmake.py
View File

@@ -2,6 +2,7 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import collections.abc
import inspect
import os
import platform
@@ -9,10 +10,7 @@
import sys
from typing import List, Tuple
import six
import llnl.util.filesystem as fs
from llnl.util.compat import Sequence
import spack.build_environment
import spack.builder
@@ -302,7 +300,7 @@ def define(cmake_var, value):
value = "ON" if value else "OFF"
else:
kind = "STRING"
if isinstance(value, Sequence) and not isinstance(value, six.string_types):
if isinstance(value, collections.abc.Sequence) and not isinstance(value, str):
value = ";".join(str(v) for v in value)
else:
value = str(value)

42
lib/spack/spack/build_systems/rocm.py
View File

@@ -96,18 +96,33 @@ class ROCmPackage(PackageBase):
"gfx803",
"gfx900",
"gfx900:xnack-",
"gfx902",
"gfx904",
"gfx906",
"gfx908",
"gfx90a",
"gfx906:xnack-",
"gfx908",
"gfx908:xnack-",
"gfx909",
"gfx90a",
"gfx90a:xnack-",
"gfx90a:xnack+",
"gfx90c",
"gfx940",
"gfx1010",
"gfx1011",
"gfx1012",
"gfx1013",
"gfx1030",
"gfx1031",
"gfx1032",
"gfx1033",
"gfx1034",
"gfx1035",
"gfx1036",
"gfx1100",
"gfx1101",
"gfx1102",
"gfx1103",
)
variant("rocm", default=False, description="Enable ROCm support")
@@ -144,6 +159,29 @@ def hip_flags(amdgpu_target):
# depends_on('hip@:6.0', when='amdgpu_target=gfx701')
# to indicate minimum version for each architecture.
# Add compiler minimum versions based on the first release where the
# processor is included in llvm/lib/Support/TargetParser.cpp
depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx900:xnack-")
depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx906:xnack-")
depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx908:xnack-")
depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx90c")
depends_on("llvm-amdgpu@4.3.0:", when="amdgpu_target=gfx90a")
depends_on("llvm-amdgpu@4.3.0:", when="amdgpu_target=gfx90a:xnack-")
depends_on("llvm-amdgpu@4.3.0:", when="amdgpu_target=gfx90a:xnack+")
depends_on("llvm-amdgpu@5.2.0:", when="amdgpu_target=gfx940")
depends_on("llvm-amdgpu@4.5.0:", when="amdgpu_target=gfx1013")
depends_on("llvm-amdgpu@3.8.0:", when="amdgpu_target=gfx1030")
depends_on("llvm-amdgpu@3.9.0:", when="amdgpu_target=gfx1031")
depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx1032")
depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx1033")
depends_on("llvm-amdgpu@4.3.0:", when="amdgpu_target=gfx1034")
depends_on("llvm-amdgpu@4.5.0:", when="amdgpu_target=gfx1035")
depends_on("llvm-amdgpu@5.2.0:", when="amdgpu_target=gfx1036")
depends_on("llvm-amdgpu@5.3.0:", when="amdgpu_target=gfx1100")
depends_on("llvm-amdgpu@5.3.0:", when="amdgpu_target=gfx1101")
depends_on("llvm-amdgpu@5.3.0:", when="amdgpu_target=gfx1102")
depends_on("llvm-amdgpu@5.3.0:", when="amdgpu_target=gfx1103")
# Compiler conflicts
# TODO: add conflicts statements along the lines of

2
lib/spack/spack/build_systems/sip.py
View File

@@ -157,7 +157,7 @@ def configure(self, pkg, spec, prefix):
]
)
self.python(configure, *args)
self.pkg.python(configure, *args)
def configure_args(self):
"""Arguments to pass to configure."""

2
lib/spack/spack/build_systems/waf.py
View File

@@ -72,9 +72,9 @@ class WafBuilder(BaseBuilder):
#: Names associated with package attributes in the old build-system format
legacy_attributes = (
"build_time_test_callbacks",
"build_time_test_callbacks",
"build_directory",
"install_time_test_callbacks",
)
# Callback names for build-time test

11
lib/spack/spack/builder.py
View File

@@ -3,15 +3,12 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import collections
import collections.abc
import copy
import functools
import inspect
from typing import List, Optional, Tuple
import six
import llnl.util.compat
import spack.build_environment
#: Builder classes, as registered by the "builder" decorator
@@ -168,7 +165,7 @@ def __forward(self):
property(forward_property_to_getattr(attribute_name)),
)
class Adapter(six.with_metaclass(_PackageAdapterMeta, base_cls)):
class Adapter(base_cls, metaclass=_PackageAdapterMeta):
def __init__(self, pkg):
# Deal with custom phases in packages here
if hasattr(pkg, "phases"):
@@ -280,7 +277,7 @@ def _decorator(fn):
return _decorator
class BuilderMeta(PhaseCallbacksMeta, type(llnl.util.compat.Sequence)): # type: ignore
class BuilderMeta(PhaseCallbacksMeta, type(collections.abc.Sequence)): # type: ignore
pass
@@ -457,7 +454,7 @@ def copy(self):
return copy.deepcopy(self)
class Builder(six.with_metaclass(BuilderMeta, llnl.util.compat.Sequence)):
class Builder(collections.abc.Sequence, metaclass=BuilderMeta):
"""A builder is a class that, given a package object (i.e. associated with
concrete spec), knows how to install it.

43
lib/spack/spack/ci.py
View File

@@ -16,11 +16,9 @@
import tempfile
import time
import zipfile
from six import iteritems, string_types
from six.moves.urllib.error import HTTPError, URLError
from six.moves.urllib.parse import urlencode
from six.moves.urllib.request import HTTPHandler, Request, build_opener
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import HTTPHandler, Request, build_opener
import llnl.util.filesystem as fs
import llnl.util.tty as tty
@@ -51,6 +49,7 @@
TEMP_STORAGE_MIRROR_NAME = "ci_temporary_mirror"
SPACK_RESERVED_TAGS = ["public", "protected", "notary"]
SHARED_PR_MIRROR_URL = "s3://spack-binaries-prs/shared_pr_mirror"
spack_gpg = spack.main.SpackCommand("gpg")
spack_compiler = spack.main.SpackCommand("compiler")
@@ -215,7 +214,7 @@ def stage_spec_jobs(specs, check_index_only=False, mirrors_to_check=None):
def _remove_satisfied_deps(deps, satisfied_list):
new_deps = {}
for key, value in iteritems(deps):
for key, value in deps.items():
new_value = set([v for v in value if v not in satisfied_list])
if new_value:
new_deps[key] = new_value
@@ -731,6 +730,12 @@ def generate_gitlab_ci_yaml(
# won't fetch its index and include in our local cache.
spack.mirror.add("ci_pr_mirror", remote_mirror_override, cfg.default_modify_scope())
shared_pr_mirror = None
if spack_pipeline_type == "spack_pull_request":
stack_name = os.environ.get("SPACK_CI_STACK_NAME", "")
shared_pr_mirror = url_util.join(SHARED_PR_MIRROR_URL, stack_name)
spack.mirror.add("ci_shared_pr_mirror", shared_pr_mirror, cfg.default_modify_scope())
pipeline_artifacts_dir = artifacts_root
if not pipeline_artifacts_dir:
proj_dir = os.environ.get("CI_PROJECT_DIR", os.getcwd())
@@ -805,6 +810,8 @@ def generate_gitlab_ci_yaml(
# Clean up remote mirror override if enabled
if remote_mirror_override:
spack.mirror.remove("ci_pr_mirror", cfg.default_modify_scope())
if spack_pipeline_type == "spack_pull_request":
spack.mirror.remove("ci_shared_pr_mirror", cfg.default_modify_scope())
all_job_names = []
output_object = {}
@@ -1294,6 +1301,7 @@ def generate_gitlab_ci_yaml(
"SPACK_LOCAL_MIRROR_DIR": rel_local_mirror_dir,
"SPACK_PIPELINE_TYPE": str(spack_pipeline_type),
"SPACK_CI_STACK_NAME": os.environ.get("SPACK_CI_STACK_NAME", "None"),
"SPACK_CI_SHARED_PR_MIRROR_URL": shared_pr_mirror or "None",
"SPACK_REBUILD_CHECK_UP_TO_DATE": str(prune_dag),
"SPACK_REBUILD_EVERYTHING": str(rebuild_everything),
}
@@ -1769,9 +1777,9 @@ def reproduce_ci_job(url, work_dir):
download_and_extract_artifacts(url, work_dir)
lock_file = fs.find(work_dir, "spack.lock")[0]
concrete_env_dir = os.path.dirname(lock_file)
repro_lock_dir = os.path.dirname(lock_file)
tty.debug("Concrete environment directory: {0}".format(concrete_env_dir))
tty.debug("Found lock file in: {0}".format(repro_lock_dir))
yaml_files = fs.find(work_dir, ["*.yaml", "*.yml"])
@@ -1794,6 +1802,20 @@ def reproduce_ci_job(url, work_dir):
if pipeline_yaml:
tty.debug("\n{0} is likely your pipeline file".format(yf))
relative_concrete_env_dir = pipeline_yaml["variables"]["SPACK_CONCRETE_ENV_DIR"]
tty.debug("Relative environment path used by cloud job: {0}".format(relative_concrete_env_dir))
# Using the relative concrete environment path found in the generated
# pipeline variable above, copy the spack environment files so they'll
# be found in the same location as when the job ran in the cloud.
concrete_env_dir = os.path.join(work_dir, relative_concrete_env_dir)
os.makedirs(concrete_env_dir, exist_ok=True)
copy_lock_path = os.path.join(concrete_env_dir, "spack.lock")
orig_yaml_path = os.path.join(repro_lock_dir, "spack.yaml")
copy_yaml_path = os.path.join(concrete_env_dir, "spack.yaml")
shutil.copyfile(lock_file, copy_lock_path)
shutil.copyfile(orig_yaml_path, copy_yaml_path)
# Find the install script in the unzipped artifacts and make it executable
install_script = fs.find(work_dir, "install.sh")[0]
st = os.stat(install_script)
@@ -1849,6 +1871,7 @@ def reproduce_ci_job(url, work_dir):
if repro_details:
mount_as_dir = repro_details["ci_project_dir"]
mounted_repro_dir = os.path.join(mount_as_dir, rel_repro_dir)
mounted_env_dir = os.path.join(mount_as_dir, relative_concrete_env_dir)
# We will also try to clone spack from your local checkout and
# reproduce the state present during the CI build, and put that into
@@ -1932,7 +1955,7 @@ def reproduce_ci_job(url, work_dir):
inst_list.append(" $ source {0}/share/spack/setup-env.sh\n".format(spack_root))
inst_list.append(
" $ spack env activate --without-view {0}\n\n".format(
mounted_repro_dir if job_image else repro_dir
mounted_env_dir if job_image else repro_dir
)
)
inst_list.append(" - Run the install script\n\n")
@@ -1960,7 +1983,7 @@ def process_command(name, commands, repro_dir):
"""
tty.debug("spack {0} arguments: {1}".format(name, commands))
if len(commands) == 0 or isinstance(commands[0], string_types):
if len(commands) == 0 or isinstance(commands[0], str):
commands = [commands]
# Create a string [command 1] && [command 2] && ... && [command n] with commands

7
lib/spack/spack/ci_needs_workaround.py
View File

@@ -2,12 +2,11 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from llnl.util.compat import Mapping
import collections.abc
get_job_name = lambda needs_entry: (
needs_entry.get("job")
if (isinstance(needs_entry, Mapping) and needs_entry.get("artifacts", True))
if (isinstance(needs_entry, collections.abc.Mapping) and needs_entry.get("artifacts", True))
else needs_entry
if isinstance(needs_entry, str)
else None
@@ -15,7 +14,7 @@
def convert_job(job_entry):
if not isinstance(job_entry, Mapping):
if not isinstance(job_entry, collections.abc.Mapping):
return job_entry
needs = job_entry.get("needs")

28
lib/spack/spack/ci_optimization.py
View File

@@ -2,23 +2,21 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import collections
import collections.abc
import copy
import hashlib
from collections import defaultdict
from llnl.util.compat import Mapping, Sequence
import spack.util.spack_yaml as syaml
def sort_yaml_obj(obj):
if isinstance(obj, Mapping):
if isinstance(obj, collections.abc.Mapping):
return syaml.syaml_dict(
(k, sort_yaml_obj(v)) for k, v in sorted(obj.items(), key=(lambda item: str(item[0])))
)
if isinstance(obj, Sequence) and not isinstance(obj, str):
if isinstance(obj, collections.abc.Sequence) and not isinstance(obj, str):
return syaml.syaml_list(sort_yaml_obj(x) for x in obj)
return obj
@@ -38,15 +36,15 @@ def matches(obj, proto):
Precondition: proto must not have any reference cycles
"""
if isinstance(obj, Mapping):
if not isinstance(proto, Mapping):
if isinstance(obj, collections.abc.Mapping):
if not isinstance(proto, collections.abc.Mapping):
return False
return all((key in obj and matches(obj[key], val)) for key, val in proto.items())
if isinstance(obj, Sequence) and not isinstance(obj, str):
if isinstance(obj, collections.abc.Sequence) and not isinstance(obj, str):
if not (isinstance(proto, Sequence) and not isinstance(proto, str)):
if not (isinstance(proto, collections.abc.Sequence) and not isinstance(proto, str)):
return False
if len(obj) != len(proto):
@@ -76,7 +74,9 @@ def subkeys(obj, proto):
Otherwise, obj is returned.
"""
if not (isinstance(obj, Mapping) and isinstance(proto, Mapping)):
if not (
isinstance(obj, collections.abc.Mapping) and isinstance(proto, collections.abc.Mapping)
):
return obj
new_obj = {}
@@ -88,7 +88,7 @@ def subkeys(obj, proto):
if matches(value, proto[key]) and matches(proto[key], value):
continue
if isinstance(value, Mapping):
if isinstance(value, collections.abc.Mapping):
new_obj[key] = subkeys(value, proto[key])
continue
@@ -116,7 +116,7 @@ def add_extends(yaml, key):
has_key = "extends" in yaml
extends = yaml.get("extends")
if has_key and not isinstance(extends, (str, Sequence)):
if has_key and not isinstance(extends, (str, collections.abc.Sequence)):
return
if extends is None:
@@ -261,7 +261,7 @@ def build_histogram(iterator, key):
The list is sorted in descending order by count, yielding the most
frequently occuring hashes first.
"""
buckets = defaultdict(int)
buckets = collections.defaultdict(int)
values = {}
num_objects = 0

11
lib/spack/spack/cmd/__init__.py
View File

@@ -14,7 +14,6 @@
from typing import List, Tuple
import ruamel.yaml as yaml
import six
from ruamel.yaml.error import MarkedYAMLError
import llnl.util.tty as tty
@@ -30,6 +29,7 @@
import spack.paths
import spack.spec
import spack.store
import spack.traverse as traverse
import spack.user_environment as uenv
import spack.util.spack_json as sjson
import spack.util.string
@@ -216,7 +216,7 @@ def parse_specs(args, **kwargs):
tests = kwargs.get("tests", False)
sargs = args
if not isinstance(args, six.string_types):
if not isinstance(args, str):
sargs = " ".join(args)
unquoted_flags = _UnquotedFlags.extract(sargs)
@@ -464,11 +464,12 @@ def format_list(specs):
# create the final, formatted versions of all specs
formatted = []
for spec in specs:
formatted.append((fmt(spec), spec))
if deps:
for depth, dep in spec.traverse(root=False, depth=True):
formatted.append((fmt(dep, depth), dep))
for depth, dep in traverse.traverse_tree([spec], depth_first=False):
formatted.append((fmt(dep.spec, depth), dep.spec))
formatted.append(("", None)) # mark newlines
else:
formatted.append((fmt(spec), spec))
# unless any of these are set, we can just colify and be done.
if not any((deps, paths)):

53
lib/spack/spack/cmd/activate.py
View File

@@ -1,53 +0,0 @@
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import llnl.util.tty as tty
import spack.cmd
import spack.cmd.common.arguments as arguments
import spack.environment as ev
from spack.filesystem_view import YamlFilesystemView
description = "activate a package extension"
section = "extensions"
level = "long"
def setup_parser(subparser):
subparser.add_argument(
"-f", "--force", action="store_true", help="activate without first activating dependencies"
)
subparser.add_argument("-v", "--view", metavar="VIEW", type=str, help="the view to operate on")
arguments.add_common_arguments(subparser, ["installed_spec"])
def activate(parser, args):
tty.warn(
"spack activate is deprecated in favor of " "environments and will be removed in v0.19.0"
)
specs = spack.cmd.parse_specs(args.spec)
if len(specs) != 1:
tty.die("activate requires one spec. %d given." % len(specs))
spec = spack.cmd.disambiguate_spec(specs[0], ev.active_environment())
if not spec.package.is_extension:
tty.die("%s is not an extension." % spec.name)
if args.view:
target = args.view
else:
target = spec.package.extendee_spec.prefix
view = YamlFilesystemView(target, spack.store.layout)
if spec.package.is_activated(view):
tty.msg("Package %s is already activated." % specs[0].short_spec)
return
# TODO: refactor FilesystemView.add_extension and use that here (so there
# aren't two ways of activating extensions)
spec.package.do_activate(view, with_dependencies=not args.force)

53
lib/spack/spack/cmd/bootstrap.py
View File

@@ -8,7 +8,6 @@
import platform
import shutil
import tempfile
import warnings
import llnl.util.filesystem
import llnl.util.tty
@@ -112,18 +111,10 @@ def setup_parser(subparser):
list = sp.add_parser("list", help="list all the sources of software to bootstrap Spack")
_add_scope_option(list)
trust = sp.add_parser("trust", help="(DEPRECATED) trust a bootstrapping source")
_add_scope_option(trust)
trust.add_argument("name", help="name of the source to be trusted")
untrust = sp.add_parser("untrust", help="(DEPRECATED) untrust a bootstrapping source")
_add_scope_option(untrust)
untrust.add_argument("name", help="name of the source to be untrusted")
add = sp.add_parser("add", help="add a new source for bootstrapping")
_add_scope_option(add)
add.add_argument(
"--trust", action="store_true", help="trust the source immediately upon addition"
"--trust", action="store_true", help="enable the source immediately upon addition"
)
add.add_argument("name", help="name of the new source of software")
add.add_argument("metadata_dir", help="directory where to find metadata files")
@@ -156,9 +147,9 @@ def _enable_or_disable(args):
return
if value is True:
_trust(args)
_enable_source(args)
else:
_untrust(args)
_disable_source(args)
def _reset(args):
@@ -254,8 +245,14 @@ def sort_fn(x):
_print_method(s, trusted.get(s["name"], None))
def _write_trust_state(args, value):
name = args.name
def _write_bootstrapping_source_status(name, enabled, scope=None):
"""Write if a bootstrapping source is enable or disabled to config file.
Args:
name (str): name of the bootstrapping source.
enabled (bool): True if the source is enabled, False if it is disabled.
scope (None or str): configuration scope to modify. If none use the default scope.
"""
sources = spack.config.get("bootstrap:sources")
matches = [s for s in sources if s["name"] == name]
@@ -277,30 +274,18 @@ def _write_trust_state(args, value):
# Setting the scope explicitly is needed to not copy over to a new scope
# the entire default configuration for bootstrap.yaml
scope = args.scope or spack.config.default_modify_scope("bootstrap")
spack.config.add("bootstrap:trusted:{0}:{1}".format(name, str(value)), scope=scope)
scope = scope or spack.config.default_modify_scope("bootstrap")
spack.config.add("bootstrap:trusted:{0}:{1}".format(name, str(enabled)), scope=scope)
def _deprecate_command(deprecated_cmd, suggested_cmd):
msg = (
"the 'spack bootstrap {} ...' command is deprecated and will be "
"removed in v0.20, use 'spack bootstrap {} ...' instead"
)
warnings.warn(msg.format(deprecated_cmd, suggested_cmd))
def _trust(args):
if args.subcommand == "trust":
_deprecate_command("trust", "enable")
_write_trust_state(args, value=True)
def _enable_source(args):
_write_bootstrapping_source_status(args.name, enabled=True, scope=args.scope)
msg = '"{0}" is now enabled for bootstrapping'
llnl.util.tty.msg(msg.format(args.name))
def _untrust(args):
if args.subcommand == "untrust":
_deprecate_command("untrust", "disable")
_write_trust_state(args, value=False)
def _disable_source(args):
_write_bootstrapping_source_status(args.name, enabled=False, scope=args.scope)
msg = '"{0}" is now disabled and will not be used for bootstrapping'
llnl.util.tty.msg(msg.format(args.name))
@@ -364,7 +349,7 @@ def _add(args):
msg = 'New bootstrapping source "{0}" added in the "{1}" configuration scope'
llnl.util.tty.msg(msg.format(args.name, write_scope))
if args.trust:
_trust(args)
_enable_source(args)
def _remove(args):
@@ -465,8 +450,6 @@ def bootstrap(parser, args):
"reset": _reset,
"root": _root,
"list": _list,
"trust": _trust,
"untrust": _untrust,
"add": _add,
"remove": _remove,
"mirror": _mirror,

13
lib/spack/spack/cmd/buildcache.py
View File

@@ -454,7 +454,7 @@ def check_fn(args):
if not specs:
tty.msg("No specs provided, exiting.")
sys.exit(0)
return
for spec in specs:
spec.concretize()
@@ -467,9 +467,10 @@ def check_fn(args):
if not configured_mirrors:
tty.msg("No mirrors provided, exiting.")
sys.exit(0)
return
sys.exit(bindist.check_specs_against_mirrors(configured_mirrors, specs, args.output_file))
if bindist.check_specs_against_mirrors(configured_mirrors, specs, args.output_file) == 1:
sys.exit(1)
def download_fn(args):
@@ -479,11 +480,11 @@ def download_fn(args):
least one of the required buildcache components."""
if not args.spec and not args.spec_file:
tty.msg("No specs provided, exiting.")
sys.exit(0)
return
if not args.path:
tty.msg("No download path provided, exiting")
sys.exit(0)
return
spec = _concrete_spec_from_args(args)
result = bindist.download_single_spec(spec, args.path)
@@ -532,8 +533,6 @@ def save_specfile_fn(args):
root_spec_as_json, args.specfile_dir, args.specs.split(), spec_format
)
sys.exit(0)
def copy_buildcache_file(src_url, dest_url, local_path=None):
"""Copy from source url to destination url"""

52
lib/spack/spack/cmd/checksum.py
View File

@@ -6,6 +6,7 @@
from __future__ import print_function
import argparse
import sys
import llnl.util.tty as tty
@@ -16,6 +17,7 @@
import spack.stage
import spack.util.crypto
from spack.package_base import deprecated_version, preferred_version
from spack.util.editor import editor
from spack.util.naming import valid_fully_qualified_module_name
from spack.version import VersionBase, ver
@@ -53,6 +55,13 @@ def setup_parser(subparser):
default=False,
help="checksum the preferred version only",
)
subparser.add_argument(
"-a",
"--add-to-package",
action="store_true",
default=False,
help="add new versions to package",
)
arguments.add_common_arguments(subparser, ["package"])
subparser.add_argument(
"versions", nargs=argparse.REMAINDER, help="versions to generate checksums for"
@@ -118,3 +127,46 @@ def checksum(parser, args):
print()
print(version_lines)
print()
if args.add_to_package:
filename = spack.repo.path.filename_for_package_name(pkg.name)
# Make sure we also have a newline after the last version
versions = [v + "\n" for v in version_lines.splitlines()]
versions.append("\n")
# We need to insert the versions in reversed order
versions.reverse()
versions.append(" # FIXME: Added by `spack checksum`\n")
version_line = None
with open(filename, "r") as f:
lines = f.readlines()
for i in range(len(lines)):
# Black is drunk, so this is what it looks like for now
# See https://github.com/psf/black/issues/2156 for more information
if lines[i].startswith(" # FIXME: Added by `spack checksum`") or lines[
i
].startswith(" version("):
version_line = i
break
if version_line is not None:
for v in versions:
lines.insert(version_line, v)
with open(filename, "w") as f:
f.writelines(lines)
msg = "opening editor to verify"
if not sys.stdout.isatty():
msg = "please verify"
tty.info(
"Added {0} new versions to {1}, "
"{2}.".format(len(versions) - 2, args.package, msg)
)
if sys.stdout.isatty():
editor(filename)
else:
tty.warn("Could not add new versions to {0}.".format(args.package))

12
lib/spack/spack/cmd/ci.py
View File

@@ -6,7 +6,6 @@
import json
import os
import shutil
import sys
import llnl.util.filesystem as fs
import llnl.util.tty as tty
@@ -285,6 +284,7 @@ def ci_rebuild(args):
remote_mirror_override = get_env_var("SPACK_REMOTE_MIRROR_OVERRIDE")
remote_mirror_url = get_env_var("SPACK_REMOTE_MIRROR_URL")
spack_ci_stack_name = get_env_var("SPACK_CI_STACK_NAME")
shared_pr_mirror_url = get_env_var("SPACK_CI_SHARED_PR_MIRROR_URL")
rebuild_everything = get_env_var("SPACK_REBUILD_EVERYTHING")
# Construct absolute paths relative to current $CI_PROJECT_DIR
@@ -472,6 +472,10 @@ def ci_rebuild(args):
spack.mirror.add("mirror_override", remote_mirror_override, cfg.default_modify_scope())
pipeline_mirrors.append(remote_mirror_override)
if spack_pipeline_type == "spack_pull_request":
if shared_pr_mirror_url != "None":
pipeline_mirrors.append(shared_pr_mirror_url)
matches = (
None
if full_rebuild
@@ -498,7 +502,7 @@ def ci_rebuild(args):
bindist.download_single_spec(job_spec, build_cache_dir, mirror_url=matching_mirror)
# Now we are done and successful
sys.exit(0)
return 0
# Before beginning the install, if this is a "rebuild everything" pipeline, we
# only want to keep the mirror being used by the current pipeline as it's binary
@@ -566,8 +570,6 @@ def ci_rebuild(args):
"-o",
"Makefile",
"--use-buildcache=package:never,dependencies:only",
"--make-target-prefix",
"ci",
slash_hash, # limit to spec we're building
],
[
@@ -584,7 +586,7 @@ def ci_rebuild(args):
"SPACK_COLOR=always",
"SPACK_INSTALL_FLAGS={}".format(args_to_string(deps_install_args)),
"-j$(nproc)",
"ci/.install-deps/{}".format(job_spec.dag_hash()),
"install-deps/{}".format(job_spec.format("{name}-{version}-{hash}")),
],
spack_cmd + ["install"] + root_install_args,
]

6
lib/spack/spack/cmd/compiler.py
View File

@@ -8,8 +8,6 @@
import argparse
import sys
from six import iteritems
import llnl.util.tty as tty
from llnl.util.lang import index_by
from llnl.util.tty.colify import colify
@@ -138,13 +136,13 @@ def compiler_info(args):
print("\t\t%s = %s" % (cpath, getattr(c, cpath, None)))
if c.flags:
print("\tflags:")
for flag, flag_value in iteritems(c.flags):
for flag, flag_value in c.flags.items():
print("\t\t%s = %s" % (flag, flag_value))
if len(c.environment) != 0:
if len(c.environment.get("set", {})) != 0:
print("\tenvironment:")
print("\t set:")
for key, value in iteritems(c.environment["set"]):
for key, value in c.environment["set"].items():
print("\t %s = %s" % (key, value))
if c.extra_rpaths:
print("\tExtra rpaths:")

96
lib/spack/spack/cmd/deactivate.py
View File

@@ -1,96 +0,0 @@
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import llnl.util.tty as tty
import spack.cmd
import spack.cmd.common.arguments as arguments
import spack.environment as ev
import spack.graph
import spack.store
from spack.filesystem_view import YamlFilesystemView
description = "deactivate a package extension"
section = "extensions"
level = "long"
def setup_parser(subparser):
subparser.add_argument(
"-f",
"--force",
action="store_true",
help="run deactivation even if spec is NOT currently activated",
)
subparser.add_argument("-v", "--view", metavar="VIEW", type=str, help="the view to operate on")
subparser.add_argument(
"-a",
"--all",
action="store_true",
help="deactivate all extensions of an extendable package, or "
"deactivate an extension AND its dependencies",
)
arguments.add_common_arguments(subparser, ["installed_spec"])
def deactivate(parser, args):
tty.warn(
"spack deactivate is deprecated in favor of " "environments and will be removed in v0.19.0"
)
specs = spack.cmd.parse_specs(args.spec)
if len(specs) != 1:
tty.die("deactivate requires one spec. %d given." % len(specs))
env = ev.active_environment()
spec = spack.cmd.disambiguate_spec(specs[0], env)
pkg = spec.package
if args.view:
target = args.view
elif pkg.is_extension:
target = pkg.extendee_spec.prefix
elif pkg.extendable:
target = spec.prefix
view = YamlFilesystemView(target, spack.store.layout)
if args.all:
if pkg.extendable:
tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec)
ext_pkgs = spack.store.db.activated_extensions_for(spec, view.extensions_layout)
for ext_pkg in ext_pkgs:
ext_pkg.spec.normalize()
if ext_pkg.is_activated(view):
ext_pkg.do_deactivate(view, force=True)
elif pkg.is_extension:
if not args.force and not spec.package.is_activated(view):
tty.die("%s is not activated." % pkg.spec.short_spec)
tty.msg("Deactivating %s and all dependencies." % pkg.spec.short_spec)
nodes_in_topological_order = spack.graph.topological_sort(spec)
for espec in reversed(nodes_in_topological_order):
epkg = espec.package
if epkg.extends(pkg.extendee_spec):
if epkg.is_activated(view) or args.force:
epkg.do_deactivate(view, force=args.force)
else:
tty.die("spack deactivate --all requires an extendable package " "or an extension.")
else:
if not pkg.is_extension:
tty.die(
"spack deactivate requires an extension.", "Did you mean 'spack deactivate --all'?"
)
if not args.force and not spec.package.is_activated(view):
tty.die("Package %s is not activated." % spec.short_spec)
spec.package.do_deactivate(view, force=args.force)

49
lib/spack/spack/cmd/env.py
View File

@@ -4,13 +4,12 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import argparse
import io
import os
import shutil
import sys
import tempfile
import six
import llnl.util.filesystem as fs
import llnl.util.tty as tty
from llnl.util.tty.colify import colify
@@ -674,13 +673,16 @@ def build_cache_flag(self, depth):
return ""
def accept(self, node):
dag_hash = node.edge.spec.dag_hash()
fmt = "{name}-{version}-{hash}"
tgt = node.edge.spec.format(fmt)
spec_str = node.edge.spec.format(
"{name}{@version}{%compiler}{variants}{arch=architecture}"
)
buildcache_flag = self.build_cache_flag(node.depth)
prereqs = " ".join([self.target(dep.spec.dag_hash()) for dep in self.neighbors(node)])
self.adjacency_list.append((dag_hash, spec_str, buildcache_flag, prereqs))
prereqs = " ".join([self.target(dep.spec.format(fmt)) for dep in self.neighbors(node)])
self.adjacency_list.append(
(tgt, prereqs, node.edge.spec.dag_hash(), spec_str, buildcache_flag)
)
# We already accepted this
return True
@@ -691,6 +693,8 @@ def env_depfile(args):
spack.cmd.require_active_env(cmd_name="env depfile")
env = ev.active_environment()
# Special make targets are useful when including a makefile in another, and you
# need to "namespace" the targets to avoid conflicts.
if args.make_target_prefix is None:
target_prefix = os.path.join(env.env_subdir_path, "makedeps")
else:
@@ -707,10 +711,10 @@ def get_target(name):
return os.path.join(target_prefix, name)
def get_install_target(name):
return os.path.join(target_prefix, ".install", name)
return os.path.join(target_prefix, "install", name)
def get_install_deps_target(name):
return os.path.join(target_prefix, ".install-deps", name)
return os.path.join(target_prefix, "install-deps", name)
# What things do we build when running make? By default, we build the
# root specs. If specific specs are provided as input, we build those.
@@ -729,15 +733,24 @@ def get_install_deps_target(name):
)
# Root specs without deps are the prereqs for the environment target
root_install_targets = [get_install_target(h.dag_hash()) for h in roots]
root_install_targets = [get_install_target(h.format("{name}-{version}-{hash}")) for h in roots]
# Cleanable targets...
cleanable_targets = [get_install_target(h) for h, _, _, _ in make_targets.adjacency_list]
cleanable_targets.extend(
[get_install_deps_target(h) for h, _, _, _ in make_targets.adjacency_list]
)
# All install and install-deps targets
all_install_related_targets = []
buf = six.StringIO()
# Convenience shortcuts: ensure that `make install/pkg-version-hash` triggers
# <absolute path to env>/.spack-env/makedeps/install/pkg-version-hash in case
# we don't have a custom make target prefix.
phony_convenience_targets = []
for tgt, _, _, _, _ in make_targets.adjacency_list:
all_install_related_targets.append(get_install_target(tgt))
all_install_related_targets.append(get_install_deps_target(tgt))
if args.make_target_prefix is None:
phony_convenience_targets.append(os.path.join("install", tgt))
phony_convenience_targets.append(os.path.join("install-deps", tgt))
buf = io.StringIO()
template = spack.tengine.make_environment().get_template(os.path.join("depfile", "Makefile"))
@@ -746,15 +759,17 @@ def get_install_deps_target(name):
"all_target": get_target("all"),
"env_target": get_target("env"),
"clean_target": get_target("clean"),
"cleanable_targets": " ".join(cleanable_targets),
"all_install_related_targets": " ".join(all_install_related_targets),
"root_install_targets": " ".join(root_install_targets),
"dirs_target": get_target("dirs"),
"environment": env.path,
"install_target": get_target(".install"),
"install_deps_target": get_target(".install-deps"),
"install_target": get_target("install"),
"install_deps_target": get_target("install-deps"),
"any_hash_target": get_target("%"),
"jobserver_support": "+" if args.jobserver else "",
"adjacency_list": make_targets.adjacency_list,
"phony_convenience_targets": " ".join(phony_convenience_targets),
"target_prefix": target_prefix,
}
)

22
lib/spack/spack/cmd/extensions.py
View File

@@ -14,7 +14,6 @@
import spack.environment as ev
import spack.repo
import spack.store
from spack.filesystem_view import YamlFilesystemView
description = "list extensions for package"
section = "extensions"
@@ -38,10 +37,9 @@ def setup_parser(subparser):
"--show",
action="store",
default="all",
choices=("packages", "installed", "activated", "all"),
choices=("packages", "installed", "all"),
help="show only part of output",
)
subparser.add_argument("-v", "--view", metavar="VIEW", type=str, help="the view to operate on")
subparser.add_argument(
"spec",
@@ -91,13 +89,6 @@ def extensions(parser, args):
tty.msg("%d extensions:" % len(extensions))
colify(ext.name for ext in extensions)
if args.view:
target = args.view
else:
target = spec.prefix
view = YamlFilesystemView(target, spack.store.layout)
if args.show in ("installed", "all"):
# List specs of installed extensions.
installed = [s.spec for s in spack.store.db.installed_extensions_for(spec)]
@@ -109,14 +100,3 @@ def extensions(parser, args):
else:
tty.msg("%d installed:" % len(installed))
cmd.display_specs(installed, args)
if args.show in ("activated", "all"):
# List specs of activated extensions.
activated = view.extensions_layout.extension_map(spec)
if args.show == "all":
print
if not activated:
tty.msg("None activated.")
else:
tty.msg("%d activated:" % len(activated))
cmd.display_specs(activated.values(), args)

24
lib/spack/spack/cmd/find.py
View File

@@ -140,13 +140,6 @@ def setup_parser(subparser):
subparser.add_argument("--start-date", help="earliest date of installation [YYYY-MM-DD]")
subparser.add_argument("--end-date", help="latest date of installation [YYYY-MM-DD]")
subparser.add_argument(
"-b",
"--bootstrap",
action="store_true",
help="show software in the internal bootstrap store",
)
arguments.add_common_arguments(subparser, ["constraint"])
@@ -251,23 +244,6 @@ def display_env(env, args, decorator, results):
def find(parser, args):
if args.bootstrap:
tty.warn(
"`spack find --bootstrap` is deprecated and will be removed in v0.19.",
"Use `spack --bootstrap find` instead.",
)
if args.bootstrap:
bootstrap_store_path = spack.bootstrap.store_path()
with spack.bootstrap.ensure_bootstrap_configuration():
msg = 'Showing internal bootstrap store at "{0}"'
tty.msg(msg.format(bootstrap_store_path))
_find(parser, args)
return
_find(parser, args)
def _find(parser, args):
q_args = query_arguments(args)
results = args.specs(**q_args)

2
lib/spack/spack/cmd/gc.py
View File

@@ -43,4 +43,4 @@ def gc(parser, args):
if not args.yes_to_all:
spack.cmd.uninstall.confirm_removal(specs)
spack.cmd.uninstall.do_uninstall(None, specs, force=False)
spack.cmd.uninstall.do_uninstall(specs, force=False)

7
lib/spack/spack/cmd/info.py
View File

@@ -7,8 +7,7 @@
import inspect
import textwrap
from six.moves import zip_longest
from itertools import zip_longest
import llnl.util.tty as tty
import llnl.util.tty.color as color
@@ -242,8 +241,8 @@ def print_tests(pkg):
# So the presence of a callback in Spack does not necessarily correspond
# to the actual presence of built-time tests for a package.
for callbacks, phase in [
(pkg.build_time_test_callbacks, "Build"),
(pkg.install_time_test_callbacks, "Install"),
(getattr(pkg, "build_time_test_callbacks", None), "Build"),
(getattr(pkg, "install_time_test_callbacks", None), "Install"),
]:
color.cprint("")
color.cprint(section_title("Available {0} Phase Test Methods:".format(phase)))

6
lib/spack/spack/cmd/list.py
View File

@@ -12,6 +12,7 @@
import os
import re
import sys
from html import escape
import llnl.util.tty as tty
from llnl.util.tty.colify import colify
@@ -21,11 +22,6 @@
import spack.repo
from spack.version import VersionList
if sys.version_info > (3, 1):
from html import escape # novm
else:
from cgi import escape
description = "list and search available packages"
section = "basic"
level = "short"

6
lib/spack/spack/cmd/location.py
View File

@@ -9,6 +9,7 @@
import llnl.util.tty as tty
import spack.builder
import spack.cmd
import spack.cmd.common.arguments as arguments
import spack.environment as ev
@@ -134,6 +135,7 @@ def location(parser, args):
# Either concretize or filter from already concretized environment
spec = spack.cmd.matching_spec_from_env(spec)
pkg = spec.package
builder = spack.builder.create(pkg)
if args.stage_dir:
print(pkg.stage.path)
@@ -141,10 +143,10 @@ def location(parser, args):
if args.build_dir:
# Out of source builds have build_directory defined
if hasattr(pkg, "build_directory"):
if hasattr(builder, "build_directory"):
# build_directory can be either absolute or relative to the stage path
# in either case os.path.join makes it absolute
print(os.path.normpath(os.path.join(pkg.stage.path, pkg.build_directory)))
print(os.path.normpath(os.path.join(pkg.stage.path, builder.build_directory)))
return
# Otherwise assume in-source builds

5
lib/spack/spack/cmd/providers.py
View File

@@ -3,10 +3,9 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import io
import sys
import six
import llnl.util.tty.colify as colify
import spack.cmd
@@ -29,7 +28,7 @@ def setup_parser(subparser):
def providers(parser, args):
valid_virtuals = sorted(spack.repo.path.provider_index.providers.keys())
buffer = six.StringIO()
buffer = io.StringIO()
isatty = sys.stdout.isatty()
if isatty:
buffer.write("Virtual packages:\n")

23
lib/spack/spack/cmd/style.py
View File

@@ -9,6 +9,7 @@
import os
import re
import sys
from itertools import zip_longest
import llnl.util.tty as tty
import llnl.util.tty.color as color
@@ -18,14 +19,6 @@
import spack.paths
from spack.util.executable import which
if sys.version_info < (3, 0):
from itertools import izip_longest # novm
zip_longest = izip_longest
else:
from itertools import zip_longest # novm
description = "runs source code style checks on spack"
section = "developer"
level = "long"
@@ -267,7 +260,7 @@ def run_flake8(flake8_cmd, file_list, args):
"--config=%s" % os.path.join(spack.paths.prefix, ".flake8"),
*chunk,
fail_on_error=False,
output=str
output=str,
)
returncode |= flake8_cmd.returncode
@@ -375,14 +368,6 @@ def run_black(black_cmd, file_list, args):
packed_args = black_args + tuple(chunk)
output = black_cmd(*packed_args, fail_on_error=False, output=str, error=str)
returncode |= black_cmd.returncode
# ignore Python 2.7 deprecation error because we already know it's deprecated.
output = "\n".join(
line
for line in output.split("\n")
if "DEPRECATION: Python 2 support will be removed" not in line
)
rewrite_and_print_output(output, args, pat, replacement)
print_tool_result("black", returncode)
@@ -400,10 +385,6 @@ def validate_toolset(arg_value):
def style(parser, args):
# ensure python version is new enough
if sys.version_info < (3, 6):
tty.die("spack style requires Python 3.6 or later.")
# save initial working directory for relativizing paths later
args.initial_working_dir = os.getcwd()

8
lib/spack/spack/cmd/tags.py
View File

@@ -2,11 +2,9 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import io
import sys
import six
import llnl.util.tty as tty
import llnl.util.tty.colify as colify
@@ -20,7 +18,7 @@
def report_tags(category, tags):
buffer = six.StringIO()
buffer = io.StringIO()
isatty = sys.stdout.isatty()
if isatty:
@@ -88,7 +86,7 @@ def tags(parser, args):
return
# Report packages associated with tags
buffer = six.StringIO()
buffer = io.StringIO()
isatty = sys.stdout.isatty()
tags = args.tag if args.tag else available_tags

3
lib/spack/spack/cmd/tutorial.py
View File

@@ -11,6 +11,7 @@
import llnl.util.tty as tty
from llnl.util.filesystem import working_dir
import spack
import spack.cmd.common.arguments as arguments
import spack.config
import spack.paths
@@ -24,7 +25,7 @@
# tutorial configuration parameters
tutorial_branch = "releases/v0.18"
tutorial_branch = "releases/v0.19"
tutorial_mirror = "file:///mirror"
tutorial_key = os.path.join(spack.paths.share_path, "keys", "tutorial.pub")

100
lib/spack/spack/cmd/uninstall.py
View File

@@ -17,6 +17,7 @@
import spack.package_base
import spack.repo
import spack.store
import spack.traverse as traverse
from spack.database import InstallStatuses
description = "remove installed packages"
@@ -144,11 +145,7 @@ def installed_dependents(specs, env):
active environment, and one from specs to dependent installs outside of
the active environment.
Any of the input specs may appear in both mappings (if there are
dependents both inside and outside the current environment).
If a dependent spec is used both by the active environment and by
an inactive environment, it will only appear in the first mapping.
Every installed dependent spec is listed once.
If there is not current active environment, the first mapping will be
empty.
@@ -158,19 +155,27 @@ def installed_dependents(specs, env):
env_hashes = set(env.all_hashes()) if env else set()
all_specs_in_db = spack.store.db.query()
# Ensure we stop traversal at input specs.
visited = set(s.dag_hash() for s in specs)
for spec in specs:
installed = [x for x in all_specs_in_db if spec in x]
# separate installed dependents into dpts in this environment and
# dpts that are outside this environment
for dpt in installed:
if dpt not in specs:
if dpt.dag_hash() in env_hashes:
active_dpts.setdefault(spec, set()).add(dpt)
else:
outside_dpts.setdefault(spec, set()).add(dpt)
for dpt in traverse.traverse_nodes(
spec.dependents(deptype="all"),
direction="parents",
visited=visited,
deptype="all",
root=True,
key=lambda s: s.dag_hash(),
):
hash = dpt.dag_hash()
# Ensure that all the specs we get are installed
record = spack.store.db.query_local_by_spec_hash(hash)
if record is None or not record.installed:
continue
if hash in env_hashes:
active_dpts.setdefault(spec, set()).add(dpt)
else:
outside_dpts.setdefault(spec, set()).add(dpt)
return active_dpts, outside_dpts
@@ -225,54 +230,21 @@ def _remove_from_env(spec, env):
pass # ignore non-root specs
def do_uninstall(env, specs, force):
"""Uninstalls all the specs in a list.
def do_uninstall(specs, force=False):
# TODO: get rid of the call-sites that use this function,
# so that we don't have to do a dance of list -> set -> list -> set
hashes_to_remove = set(s.dag_hash() for s in specs)
Args:
env (spack.environment.Environment or None): active environment, or ``None``
if there is not one
specs (list): list of specs to be uninstalled
force (bool): force uninstallation (boolean)
"""
packages = []
for item in specs:
try:
# should work if package is known to spack
packages.append(item.package)
except spack.repo.UnknownEntityError:
# The package.py file has gone away -- but still
# want to uninstall.
spack.package_base.PackageBase.uninstall_by_spec(item, force=True)
# A package is ready to be uninstalled when nothing else references it,
# unless we are requested to force uninstall it.
def is_ready(dag_hash):
if force:
return True
_, record = spack.store.db.query_by_spec_hash(dag_hash)
if not record.ref_count:
return True
# If this spec is only used as a build dependency, we can uninstall
return all(
dspec.deptypes == ("build",) or not dspec.parent.installed
for dspec in record.spec.edges_from_dependents()
)
while packages:
ready = [x for x in packages if is_ready(x.spec.dag_hash())]
if not ready:
msg = (
"unexpected error [cannot proceed uninstalling specs with"
" remaining link or run dependents {0}]"
)
msg = msg.format(", ".join(x.name for x in packages))
raise spack.error.SpackError(msg)
packages = [x for x in packages if x not in ready]
for item in ready:
item.do_uninstall(force=force)
for s in traverse.traverse_nodes(
specs,
order="topo",
direction="children",
root=True,
cover="nodes",
deptype="all",
):
if s.dag_hash() in hashes_to_remove:
spack.package_base.PackageBase.uninstall_by_spec(s, force=force)
def get_uninstall_list(args, specs, env):
@@ -414,7 +386,7 @@ def uninstall_specs(args, specs):
confirm_removal(uninstall_list)
# Uninstall everything on the list
do_uninstall(env, uninstall_list, args.force)
do_uninstall(uninstall_list, args.force)
if env:
with env.write_transaction():

5
lib/spack/spack/cmd/unit_test.py
View File

@@ -7,6 +7,7 @@
import argparse
import collections
import io
import os.path
import re
import sys
@@ -16,8 +17,6 @@
except ImportError:
pytest = None # type: ignore
from six import StringIO
import llnl.util.filesystem
import llnl.util.tty.color as color
from llnl.util.tty.colify import colify
@@ -126,7 +125,7 @@ def colorize(c, prefix):
old_output = sys.stdout
try:
sys.stdout = output = StringIO()
sys.stdout = output = io.StringIO()
pytest.main(["--collect-only"] + extra_args)
finally:
sys.stdout = old_output

5
lib/spack/spack/cmd/url.py
View File

@@ -5,10 +5,9 @@
from __future__ import division, print_function
import urllib.parse
from collections import defaultdict
import six.moves.urllib.parse as urllib_parse
import llnl.util.tty.color as color
from llnl.util import tty
@@ -323,7 +322,7 @@ def add(self, pkg_name, fetcher):
md5_hashes[pkg_name].append(fetcher.url)
# parse out the URL scheme (https/http/ftp/etc.)
urlinfo = urllib_parse.urlparse(fetcher.url)
urlinfo = urllib.parse.urlparse(fetcher.url)
self.schemes[urlinfo.scheme] += 1
if urlinfo.scheme == "http":

10
lib/spack/spack/compilers/__init__.py
View File

@@ -12,8 +12,6 @@
import os
from typing import Dict # novm
import six
import archspec.cpu
import llnl.util.filesystem as fs
@@ -427,7 +425,7 @@ def compiler_from_dict(items):
environment,
extra_rpaths,
enable_implicit_rpaths=implicit_rpaths,
**compiler_flags
**compiler_flags,
)
@@ -677,18 +675,18 @@ def _default(fn_args):
try:
version = callback(path)
if version and six.text_type(version).strip() and version != "unknown":
if version and str(version).strip() and version != "unknown":
value = fn_args._replace(id=compiler_id._replace(version=version))
return value, None
error = "Couldn't get version for compiler {0}".format(path)
except spack.util.executable.ProcessError as e:
error = "Couldn't get version for compiler {0}\n".format(path) + six.text_type(e)
error = "Couldn't get version for compiler {0}\n".format(path) + str(e)
except Exception as e:
# Catching "Exception" here is fine because it just
# means something went wrong running a candidate executable.
error = "Error while executing candidate compiler {0}" "\n{1}: {2}".format(
path, e.__class__.__name__, six.text_type(e)
path, e.__class__.__name__, str(e)
)
return None, error

91
lib/spack/spack/compilers/msvc.py
View File

@@ -10,10 +10,13 @@
from distutils.version import StrictVersion
from typing import Dict, List, Set # novm
import spack.compiler
import spack.operating_systems.windows_os
import spack.platforms
import spack.util.executable
from spack.compiler import Compiler
from spack.error import SpackError
from spack.version import Version
avail_fc_version = set() # type: Set[str]
fc_path = dict() # type: Dict[str, str]
@@ -39,10 +42,10 @@ def get_valid_fortran_pth(comp_ver):
class Msvc(Compiler):
# Subclasses use possible names of C compiler
cc_names = ["cl.exe"]
cc_names = ["cl.exe"] # type: List[str]
# Subclasses use possible names of C++ compiler
cxx_names = ["cl.exe"]
cxx_names = ["cl.exe"] # type: List[str]
# Subclasses use possible names of Fortran 77 compiler
f77_names = ["ifx.exe"] # type: List[str]
@@ -91,45 +94,67 @@ def __init__(self, *args, **kwargs):
@property
def msvc_version(self):
ver = re.search(Msvc.version_regex, self.cc).group(1)
ver = "".join(ver.split(".")[:2])[:-1]
"""This is the VCToolset version *NOT* the actual version of the cl compiler
For CL version, query `Msvc.cl_version`"""
return Version(re.search(Msvc.version_regex, self.cc).group(1))
@property
def short_msvc_version(self):
"""
This is the shorthand VCToolset version of form
MSVC<short-ver> *NOT* the full version, for that see
Msvc.msvc_version
"""
ver = self.msvc_version[:2].joined.string[:3]
return "MSVC" + ver
@property
def cl_version(self):
"""Cl toolset version"""
return spack.compiler.get_compiler_version_output(self.cc)
def setup_custom_environment(self, pkg, env):
"""Set environment variables for MSVC using the
Microsoft-provided script."""
if sys.version_info[:2] > (2, 6):
# Set the build environment variables for spack. Just using
# subprocess.call() doesn't work since that operates in its own
# environment which is destroyed (along with the adjusted variables)
# once the process terminates. So go the long way around: examine
# output, sort into dictionary, use that to make the build
# environment.
out = subprocess.check_output( # novermin
'cmd /u /c "{}" {} && set'.format(self.setvarsfile, "amd64"),
stderr=subprocess.STDOUT,
)
if sys.version_info[0] >= 3:
out = out.decode("utf-16le", errors="replace") # novermin
# Set the build environment variables for spack. Just using
# subprocess.call() doesn't work since that operates in its own
# environment which is destroyed (along with the adjusted variables)
# once the process terminates. So go the long way around: examine
# output, sort into dictionary, use that to make the build
# environment.
int_env = dict(
(key.lower(), value)
for key, _, value in (line.partition("=") for line in out.splitlines())
if key and value
)
# get current platform architecture and format for vcvars argument
arch = spack.platforms.real_host().default.lower()
arch = arch.replace("-", "_")
# vcvars can target specific sdk versions, force it to pick up concretized sdk
# version, if needed by spec
sdk_ver = "" if "win-sdk" not in pkg.spec else pkg.spec["win-sdk"].version.string + ".0"
# provide vcvars with msvc version selected by concretization,
# not whatever it happens to pick up on the system (highest available version)
out = subprocess.check_output( # novermin
'cmd /u /c "{}" {} {} {} && set'.format(
self.setvarsfile, arch, sdk_ver, "-vcvars_ver=%s" % self.msvc_version
),
stderr=subprocess.STDOUT,
)
if sys.version_info[0] >= 3:
out = out.decode("utf-16le", errors="replace") # novermin
if "path" in int_env:
env.set_path("PATH", int_env["path"].split(";"))
env.set_path("INCLUDE", int_env.get("include", "").split(";"))
env.set_path("LIB", int_env.get("lib", "").split(";"))
int_env = dict(
(key.lower(), value)
for key, _, value in (line.partition("=") for line in out.splitlines())
if key and value
)
env.set("CC", self.cc)
env.set("CXX", self.cxx)
env.set("FC", self.fc)
env.set("F77", self.f77)
else:
# Should not this be an exception?
print("Cannot pull msvc compiler information in Python 2.6 or below")
if "path" in int_env:
env.set_path("PATH", int_env["path"].split(";"))
env.set_path("INCLUDE", int_env.get("include", "").split(";"))
env.set_path("LIB", int_env.get("lib", "").split(";"))
env.set("CC", self.cc)
env.set("CXX", self.cxx)
env.set("FC", self.fc)
env.set("F77", self.f77)
@classmethod
def fc_version(cls, fc):

12
lib/spack/spack/config.py
View File

@@ -39,9 +39,7 @@
from typing import List # novm
import ruamel.yaml as yaml
import six
from ruamel.yaml.error import MarkedYAMLError
from six import iteritems
import llnl.util.lang
import llnl.util.tty as tty
@@ -358,7 +356,7 @@ def clear(self):
def _process_dict_keyname_overrides(data):
"""Turn a trailing `:' in a key name into an override attribute."""
result = {}
for sk, sv in iteritems(data):
for sk, sv in data.items():
if sk.endswith(":"):
key = syaml.syaml_str(sk[:-1])
key.override = True
@@ -739,7 +737,7 @@ def override(path_or_scope, value=None):
#: configuration scopes added on the command line
#: set by ``spack.main.main()``.
command_line_scopes = [] # type: List[str]
command_line_scopes: List[str] = []
def _add_platform_scope(cfg, scope_type, name, path):
@@ -973,7 +971,7 @@ def validate(data, schema, filename=None):
line_number = e.instance.lc.line + 1
else:
line_number = None
raise six.raise_from(ConfigFormatError(e, data, filename, line_number), e)
raise ConfigFormatError(e, data, filename, line_number) from e
# return the validated data so that we can access the raw data
# mostly relevant for environments
return test_data
@@ -991,7 +989,7 @@ def read_config_file(filename, schema=None):
if not os.path.exists(filename):
# Ignore nonexistent files.
tty.debug("Skipping nonexistent config path {0}".format(filename))
tty.debug("Skipping nonexistent config path {0}".format(filename), level=3)
return None
elif not os.path.isfile(filename):
@@ -1140,7 +1138,7 @@ def they_are(t):
# come *before* dest in OrderdDicts
dest_keys = [dk for dk in dest.keys() if dk not in source]
for sk, sv in iteritems(source):
for sk, sv in source.items():
# always remove the dest items. Python dicts do not overwrite
# keys on insert, so this ensures that source keys are copied
# into dest along with mark provenance (i.e., file/line info).

15
lib/spack/spack/cray_manifest.py
View File

@@ -4,11 +4,9 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import json
import sys
import jsonschema
import jsonschema.exceptions
import six
import llnl.util.tty as tty
@@ -98,7 +96,7 @@ def spec_from_entry(entry):
continue
# Value could be a list (of strings), boolean, or string
if isinstance(value, six.string_types):
if isinstance(value, str):
variant_strs.append("{0}={1}".format(name, value))
else:
try:
@@ -163,21 +161,14 @@ def entries_to_specs(entries):
def read(path, apply_updates):
if sys.version_info >= (3, 0):
decode_exception_type = json.decoder.JSONDecodeError
else:
decode_exception_type = ValueError
decode_exception_type = json.decoder.JSONDecodeError
try:
with open(path, "r") as json_file:
json_data = json.load(json_file)
jsonschema.validate(json_data, manifest_schema)
except (jsonschema.exceptions.ValidationError, decode_exception_type) as e:
raise six.raise_from(
ManifestValidationError("error parsing manifest JSON:", str(e)),
e,
)
raise ManifestValidationError("error parsing manifest JSON:", str(e)) from e
specs = entries_to_specs(json_data["specs"])
tty.debug("{0}: {1} specs read from manifest".format(path, str(len(specs))))

34
lib/spack/spack/database.py
View File

@@ -28,8 +28,6 @@
import time
from typing import Dict # novm
import six
try:
import uuid
@@ -53,7 +51,6 @@
InconsistentInstallDirectoryError,
)
from spack.error import SpackError
from spack.filesystem_view import YamlFilesystemView
from spack.util.crypto import bit_length
from spack.version import Version
@@ -726,6 +723,15 @@ def query_by_spec_hash(self, hash_key, data=None):
return True, db._data[hash_key]
return False, None
def query_local_by_spec_hash(self, hash_key):
"""Get a spec by hash in the local database
Return:
(InstallRecord or None): InstallRecord when installed
locally, otherwise None."""
with self.read_transaction():
return self._data.get(hash_key, None)
def _assign_dependencies(self, hash_key, installs, data):
# Add dependencies from other records in the install DB to
# form a full spec.
@@ -771,10 +777,7 @@ def _read_from_file(self, filename):
with open(filename, "r") as f:
fdata = sjson.load(f)
except Exception as e:
raise six.raise_from(
CorruptDatabaseError("error parsing database:", str(e)),
e,
)
raise CorruptDatabaseError("error parsing database:", str(e)) from e
if fdata is None:
return
@@ -1379,23 +1382,6 @@ def installed_extensions_for(self, extendee_spec):
if spec.package.extends(extendee_spec):
yield spec.package
@_autospec
def activated_extensions_for(self, extendee_spec, extensions_layout=None):
"""
Return the specs of all packages that extend
the given spec
"""
if extensions_layout is None:
view = YamlFilesystemView(extendee_spec.prefix, spack.store.layout)
extensions_layout = view.extensions_layout
for spec in self.query():
try:
extensions_layout.check_activated(extendee_spec, spec)
yield spec.package
except spack.directory_layout.NoSuchExtensionError:
continue
# TODO: conditional way to do this instead of catching exceptions
def _get_by_hash_local(self, dag_hash, default=None, installed=any):
# hash is a full hash and is in the data somewhere
if dag_hash in self._data:

8
lib/spack/spack/dependency.py
View File

@@ -2,11 +2,7 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""Data structures that represent Spack's dependency relationships.
"""
from six import string_types
"""Data structures that represent Spack's dependency relationships."""
import spack.spec
#: The types of dependency relationships that Spack understands.
@@ -48,7 +44,7 @@ def canonical_deptype(deptype):
if deptype in ("all", all):
return all_deptypes
elif isinstance(deptype, string_types):
elif isinstance(deptype, str):
if deptype not in all_deptypes:
raise ValueError("Invalid dependency type: %s" % deptype)
return (deptype,)

175
lib/spack/spack/detection/common.py
View File

@@ -14,19 +14,20 @@
detection mechanisms.
"""
import collections
import glob
import itertools
import os
import os.path
import re
import sys
import six
import llnl.util.tty
import spack.config
import spack.operating_systems.windows_os as winOs
import spack.spec
import spack.util.spack_yaml
import spack.util.windows_registry
is_windows = sys.platform == "win32"
#: Information on a package that has been detected
@@ -106,6 +107,19 @@ def _spec_is_valid(spec):
return True
def path_to_dict(search_paths):
"""Return dictionary[fullpath]: basename from list of paths"""
path_to_lib = {}
# Reverse order of search directories so that a lib in the first
# entry overrides later entries
for search_path in reversed(search_paths):
for lib in os.listdir(search_path):
lib_path = os.path.join(search_path, lib)
if llnl.util.filesystem.is_readable_file(lib_path):
path_to_lib[lib_path] = lib
return path_to_lib
def is_executable(file_path):
"""Return True if the path passed as argument is that of an executable"""
return os.path.isfile(file_path) and os.access(file_path, os.X_OK)
@@ -115,7 +129,7 @@ def _convert_to_iterable(single_val_or_multiple):
x = single_val_or_multiple
if x is None:
return []
elif isinstance(x, six.string_types):
elif isinstance(x, str):
return [x]
elif isinstance(x, spack.spec.Spec):
# Specs are iterable, but a single spec should be converted to a list
@@ -141,9 +155,11 @@ def executable_prefix(executable_dir):
assert os.path.isdir(executable_dir)
components = executable_dir.split(os.sep)
if "bin" not in components:
# convert to lower to match Bin, BIN, bin
lowered_components = executable_dir.lower().split(os.sep)
if "bin" not in lowered_components:
return executable_dir
idx = components.index("bin")
idx = lowered_components.index("bin")
return os.sep.join(components[:idx])
@@ -160,11 +176,16 @@ def library_prefix(library_dir):
assert os.path.isdir(library_dir)
components = library_dir.split(os.sep)
if "lib64" in components:
idx = components.index("lib64")
# covert to lowercase to match lib, LIB, Lib, etc.
lowered_components = library_dir.lower().split(os.sep)
if "lib64" in lowered_components:
idx = lowered_components.index("lib64")
return os.sep.join(components[:idx])
elif "lib" in components:
idx = components.index("lib")
elif "lib" in lowered_components:
idx = lowered_components.index("lib")
return os.sep.join(components[:idx])
elif is_windows and "bin" in lowered_components:
idx = lowered_components.index("bin")
return os.sep.join(components[:idx])
else:
return library_dir
@@ -197,10 +218,117 @@ def update_configuration(detected_packages, scope=None, buildable=True):
return all_new_specs
def _windows_drive():
"""Return Windows drive string"""
return os.environ["HOMEDRIVE"]
class WindowsCompilerExternalPaths(object):
@staticmethod
def find_windows_compiler_root_paths():
"""Helper for Windows compiler installation root discovery
At the moment simply returns location of VS install paths from VSWhere
But should be extended to include more information as relevant"""
return list(winOs.WindowsOs.vs_install_paths)
@staticmethod
def find_windows_compiler_cmake_paths():
"""Semi hard-coded search path for cmake bundled with MSVC"""
return [
os.path.join(
path, "Common7", "IDE", "CommonExtensions", "Microsoft", "CMake", "CMake", "bin"
)
for path in WindowsCompilerExternalPaths.find_windows_compiler_root_paths()
]
@staticmethod
def find_windows_compiler_ninja_paths():
"""Semi hard-coded search heuristic for locating ninja bundled with MSVC"""
return [
os.path.join(path, "Common7", "IDE", "CommonExtensions", "Microsoft", "CMake", "Ninja")
for path in WindowsCompilerExternalPaths.find_windows_compiler_root_paths()
]
@staticmethod
def find_windows_compiler_bundled_packages():
"""Return all MSVC compiler bundled packages"""
return (
WindowsCompilerExternalPaths.find_windows_compiler_cmake_paths()
+ WindowsCompilerExternalPaths.find_windows_compiler_ninja_paths()
)
class WindowsKitExternalPaths(object):
if is_windows:
plat_major_ver = str(winOs.windows_version()[0])
@staticmethod
def find_windows_kit_roots():
"""Return Windows kit root, typically %programfiles%\\Windows Kits\\10|11\\"""
if not is_windows:
return []
program_files = os.environ["PROGRAMFILES(x86)"]
kit_base = os.path.join(
program_files, "Windows Kits", WindowsKitExternalPaths.plat_major_ver
)
return kit_base
@staticmethod
def find_windows_kit_bin_paths(kit_base=None):
"""Returns Windows kit bin directory per version"""
kit_base = WindowsKitExternalPaths.find_windows_kit_roots() if not kit_base else kit_base
kit_bin = os.path.join(kit_base, "bin")
return glob.glob(os.path.join(kit_bin, "[0-9]*", "*\\"))
@staticmethod
def find_windows_kit_lib_paths(kit_base=None):
"""Returns Windows kit lib directory per version"""
kit_base = WindowsKitExternalPaths.find_windows_kit_roots() if not kit_base else kit_base
kit_lib = os.path.join(kit_base, "Lib")
return glob.glob(os.path.join(kit_lib, "[0-9]*", "*", "*\\"))
@staticmethod
def find_windows_driver_development_kit_paths():
"""Provides a list of all installation paths
for the WDK by version and architecture
"""
wdk_content_root = os.getenv("WDKContentRoot")
return WindowsKitExternalPaths.find_windows_kit_lib_paths(wdk_content_root)
@staticmethod
def find_windows_kit_reg_installed_roots_paths():
reg = spack.util.windows_registry.WindowsRegistryView(
"SOFTWARE\\Microsoft\\Windows Kits\\Installed Roots",
root_key=spack.util.windows_registry.HKEY.HKEY_LOCAL_MACHINE,
)
if not reg:
# couldn't find key, return empty list
return []
return WindowsKitExternalPaths.find_windows_kit_lib_paths(
reg.get_value("KitsRoot%s" % WindowsKitExternalPaths.plat_major_ver).value
)
@staticmethod
def find_windows_kit_reg_sdk_paths():
reg = spack.util.windows_registry.WindowsRegistryView(
"SOFTWARE\\WOW6432Node\\Microsoft\\Microsoft SDKs\\Windows\\v%s.0"
% WindowsKitExternalPaths.plat_major_ver,
root_key=spack.util.windows_registry.HKEY.HKEY_LOCAL_MACHINE,
)
if not reg:
# couldn't find key, return empty list
return []
return WindowsKitExternalPaths.find_windows_kit_lib_paths(
reg.get_value("InstallationFolder").value
)
def find_win32_additional_install_paths():
"""Not all programs on Windows live on the PATH
Return a list of other potential install locations.
"""
drive_letter = _windows_drive()
windows_search_ext = []
cuda_re = r"CUDA_PATH[a-zA-Z1-9_]*"
# The list below should be expanded with other
@@ -213,7 +341,7 @@ def find_win32_additional_install_paths():
# to interact with Windows
# Add search path for default Chocolatey (https://github.com/chocolatey/choco)
# install directory
windows_search_ext.append("C:\\ProgramData\\chocolatey\\bin")
windows_search_ext.append("%s\\ProgramData\\chocolatey\\bin" % drive_letter)
# Add search path for NuGet package manager default install location
windows_search_ext.append(os.path.join(user, ".nuget", "packages"))
windows_search_ext.extend(
@@ -235,9 +363,32 @@ def compute_windows_program_path_for_package(pkg):
return []
# note windows paths are fine here as this method should only ever be invoked
# to interact with Windows
program_files = "C:\\Program Files{}\\{}"
program_files = "{}\\Program Files{}\\{}"
drive_letter = _windows_drive()
return [
program_files.format(arch, name)
program_files.format(drive_letter, arch, name)
for arch, name in itertools.product(("", " (x86)"), (pkg.name, pkg.name.capitalize()))
]
def compute_windows_user_path_for_package(pkg):
"""Given a package attempt to compute its user scoped
install location, return list of potential locations based
on common heuristics. For more info on Windows user specific
installs see:
https://learn.microsoft.com/en-us/dotnet/api/system.environment.specialfolder?view=netframework-4.8"""
if not is_windows:
return []
# Current user directory
user = os.environ["USERPROFILE"]
app_data = "AppData"
app_data_locations = ["Local", "Roaming"]
user_appdata_install_stubs = [os.path.join(app_data, x) for x in app_data_locations]
return [
os.path.join(user, app_data, name)
for app_data, name in list(
itertools.product(user_appdata_install_stubs, (pkg.name, pkg.name.capitalize()))
)
] + [os.path.join(user, name) for name in (pkg.name, pkg.name.capitalize())]

121
lib/spack/spack/detection/path.py
View File

@@ -15,21 +15,35 @@
import llnl.util.filesystem
import llnl.util.tty
import spack.operating_systems.windows_os as winOs
import spack.util.environment
import spack.util.ld_so_conf
from .common import (
from .common import ( # find_windows_compiler_bundled_packages,
DetectedPackage,
WindowsCompilerExternalPaths,
WindowsKitExternalPaths,
_convert_to_iterable,
compute_windows_program_path_for_package,
compute_windows_user_path_for_package,
executable_prefix,
find_win32_additional_install_paths,
is_executable,
library_prefix,
path_to_dict,
)
is_windows = sys.platform == "win32"
def executables_in_path(path_hints=None):
def common_windows_package_paths():
paths = WindowsCompilerExternalPaths.find_windows_compiler_bundled_packages()
paths.extend(find_win32_additional_install_paths())
paths.extend(WindowsKitExternalPaths.find_windows_kit_bin_paths())
paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_installed_roots_paths())
paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_sdk_paths())
return paths
def executables_in_path(path_hints):
"""Get the paths of all executables available from the current PATH.
For convenience, this is constructed as a dictionary where the keys are
@@ -43,41 +57,16 @@ def executables_in_path(path_hints=None):
path_hints (list): list of paths to be searched. If None the list will be
constructed based on the PATH environment variable.
"""
# If we're on a Windows box, run vswhere,
# steal the installationPath using windows_os.py logic,
# construct paths to CMake and Ninja, add to PATH
path_hints = path_hints or spack.util.environment.get_path("PATH")
if sys.platform == "win32":
msvc_paths = list(winOs.WindowsOs.vs_install_paths)
msvc_cmake_paths = [
os.path.join(
path, "Common7", "IDE", "CommonExtensions", "Microsoft", "CMake", "CMake", "bin"
)
for path in msvc_paths
]
path_hints = msvc_cmake_paths + path_hints
msvc_ninja_paths = [
os.path.join(path, "Common7", "IDE", "CommonExtensions", "Microsoft", "CMake", "Ninja")
for path in msvc_paths
]
path_hints = msvc_ninja_paths + path_hints
path_hints.extend(find_win32_additional_install_paths())
if is_windows:
path_hints.extend(common_windows_package_paths())
search_paths = llnl.util.filesystem.search_paths_for_executables(*path_hints)
path_to_exe = {}
# Reverse order of search directories so that an exe in the first PATH
# entry overrides later entries
for search_path in reversed(search_paths):
for exe in os.listdir(search_path):
exe_path = os.path.join(search_path, exe)
if is_executable(exe_path):
path_to_exe[exe_path] = exe
return path_to_exe
return path_to_dict(search_paths)
def libraries_in_ld_library_path(path_hints=None):
def libraries_in_ld_and_system_library_path(path_hints=None):
"""Get the paths of all libraries available from LD_LIBRARY_PATH,
LIBRARY_PATH, DYLD_LIBRARY_PATH, and DYLD_FALLBACK_LIBRARY_PATH.
LIBRARY_PATH, DYLD_LIBRARY_PATH, DYLD_FALLBACK_LIBRARY_PATH, and
standard system library paths.
For convenience, this is constructed as a dictionary where the keys are
the library paths and the values are the names of the libraries
@@ -90,26 +79,33 @@ def libraries_in_ld_library_path(path_hints=None):
path_hints (list): list of paths to be searched. If None the list will be
constructed based on the set of LD_LIBRARY_PATH, LIBRARY_PATH,
DYLD_LIBRARY_PATH, and DYLD_FALLBACK_LIBRARY_PATH environment
variables.
variables as well as the standard system library paths.
"""
path_hints = path_hints or spack.util.environment.get_path(
"LIBRARY_PATH"
) + spack.util.environment.get_path("LD_LIBRARY_PATH") + spack.util.environment.get_path(
"DYLD_LIBRARY_PATH"
) + spack.util.environment.get_path(
"DYLD_FALLBACK_LIBRARY_PATH"
path_hints = (
path_hints
or spack.util.environment.get_path("LD_LIBRARY_PATH")
+ spack.util.environment.get_path("DYLD_LIBRARY_PATH")
+ spack.util.environment.get_path("DYLD_FALLBACK_LIBRARY_PATH")
+ spack.util.ld_so_conf.host_dynamic_linker_search_paths()
)
search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints)
return path_to_dict(search_paths)
path_to_lib = {}
# Reverse order of search directories so that a lib in the first
# LD_LIBRARY_PATH entry overrides later entries
for search_path in reversed(search_paths):
for lib in os.listdir(search_path):
lib_path = os.path.join(search_path, lib)
if llnl.util.filesystem.is_readable_file(lib_path):
path_to_lib[lib_path] = lib
return path_to_lib
def libraries_in_windows_paths(path_hints):
path_hints.extend(spack.util.environment.get_path("PATH"))
search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints)
# on Windows, some libraries (.dlls) are found in the bin directory or sometimes
# at the search root. Add both of those options to the search scheme
search_paths.extend(llnl.util.filesystem.search_paths_for_executables(*path_hints))
search_paths.extend(WindowsKitExternalPaths.find_windows_kit_lib_paths())
search_paths.extend(WindowsKitExternalPaths.find_windows_kit_bin_paths())
search_paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_installed_roots_paths())
search_paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_sdk_paths())
# SDK and WGL should be handled by above, however on occasion the WDK is in an atypical
# location, so we handle that case specifically.
search_paths.extend(WindowsKitExternalPaths.find_windows_driver_development_kit_paths())
return path_to_dict(search_paths)
def _group_by_prefix(paths):
@@ -129,19 +125,33 @@ def by_library(packages_to_check, path_hints=None):
# Other libraries could use the strings function to extract it as described
# in https://unix.stackexchange.com/questions/58846/viewing-linux-library-executable-version-info
"""Return the list of packages that have been detected on the system,
searching by LD_LIBRARY_PATH.
searching by LD_LIBRARY_PATH, LIBRARY_PATH, DYLD_LIBRARY_PATH,
DYLD_FALLBACK_LIBRARY_PATH, and standard system library paths.
Args:
packages_to_check (list): list of packages to be detected
path_hints (list): list of paths to be searched. If None the list will be
constructed based on the LD_LIBRARY_PATH environment variable.
constructed based on the LD_LIBRARY_PATH, LIBRARY_PATH,
DYLD_LIBRARY_PATH, DYLD_FALLBACK_LIBRARY_PATH environment variables
and standard system library paths.
"""
path_to_lib_name = libraries_in_ld_library_path(path_hints=path_hints)
# If no path hints from command line, intialize to empty list so
# we can add default hints on a per package basis
path_hints = [] if path_hints is None else path_hints
lib_pattern_to_pkgs = collections.defaultdict(list)
for pkg in packages_to_check:
if hasattr(pkg, "libraries"):
for lib in pkg.libraries:
lib_pattern_to_pkgs[lib].append(pkg)
path_hints.extend(compute_windows_user_path_for_package(pkg))
path_hints.extend(compute_windows_program_path_for_package(pkg))
path_to_lib_name = (
libraries_in_ld_and_system_library_path(path_hints=path_hints)
if not is_windows
else libraries_in_windows_paths(path_hints)
)
pkg_to_found_libs = collections.defaultdict(set)
for lib_pattern, pkgs in lib_pattern_to_pkgs.items():
@@ -226,13 +236,14 @@ def by_executable(packages_to_check, path_hints=None):
path_hints (list): list of paths to be searched. If None the list will be
constructed based on the PATH environment variable.
"""
path_hints = [] if path_hints is None else path_hints
path_hints = spack.util.environment.get_path("PATH") if path_hints is None else path_hints
exe_pattern_to_pkgs = collections.defaultdict(list)
for pkg in packages_to_check:
if hasattr(pkg, "executables"):
for exe in pkg.platform_executables():
exe_pattern_to_pkgs[exe].append(pkg)
# Add Windows specific, package related paths to the search paths
path_hints.extend(compute_windows_user_path_for_package(pkg))
path_hints.extend(compute_windows_program_path_for_package(pkg))
path_to_exe_name = executables_in_path(path_hints=path_hints)

21
lib/spack/spack/directives.py
View File

@@ -28,16 +28,14 @@ class OpenMpi(Package):
* ``version``
"""
import collections.abc
import functools
import os.path
import re
from typing import List, Set # novm
import six
import llnl.util.lang
import llnl.util.tty.color
from llnl.util.compat import Sequence
import spack.error
import spack.patch
@@ -234,10 +232,10 @@ class Foo(Package):
"""
global directive_names
if isinstance(dicts, six.string_types):
if isinstance(dicts, str):
dicts = (dicts,)
if not isinstance(dicts, Sequence):
if not isinstance(dicts, collections.abc.Sequence):
message = "dicts arg must be list, tuple, or string. Found {0}"
raise TypeError(message.format(type(dicts)))
@@ -300,7 +298,7 @@ def remove_directives(arg):
# ...so if it is not a sequence make it so
values = result
if not isinstance(values, Sequence):
if not isinstance(values, collections.abc.Sequence):
values = (values,)
DirectiveMeta._directives_to_be_executed.extend(values)
@@ -391,7 +389,7 @@ def _depends_on(pkg, spec, when=None, type=default_deptype, patches=None):
patches = [patches]
# auto-call patch() directive on any strings in patch list
patches = [patch(p) if isinstance(p, six.string_types) else p for p in patches]
patches = [patch(p) if isinstance(p, str) else p for p in patches]
assert all(callable(p) for p in patches)
# this is where we actually add the dependency to this package
@@ -468,14 +466,7 @@ def _execute_depends_on(pkg):
@directive(("extendees", "dependencies"))
def extends(spec, type=("build", "run"), **kwargs):
"""Same as depends_on, but allows symlinking into dependency's
prefix tree.
This is for Python and other language modules where the module
needs to be installed into the prefix of the Python installation.
Spack handles this by installing modules into their own prefix,
but allowing ONE module version to be symlinked into a parent
Python install at a time, using ``spack activate``.
"""Same as depends_on, but also adds this package to the extendee list.
keyword arguments can be passed to extends() so that extension
packages can pass parameters to the extendee's extension

217
lib/spack/spack/directory_layout.py
View File

@@ -10,12 +10,8 @@
import re
import shutil
import sys
import tempfile
from contextlib import contextmanager
import ruamel.yaml as yaml
import six
import llnl.util.filesystem as fs
import llnl.util.tty as tty
@@ -365,12 +361,12 @@ def remove_install_directory(self, spec, deprecated=False):
os.unlink(path)
os.remove(metapath)
except OSError as e:
raise six.raise_from(RemoveFailedError(spec, path, e), e)
raise RemoveFailedError(spec, path, e) from e
elif os.path.exists(path):
try:
shutil.rmtree(path, **kwargs)
except OSError as e:
raise six.raise_from(RemoveFailedError(spec, path, e), e)
raise RemoveFailedError(spec, path, e) from e
path = os.path.dirname(path)
while path != self.root:
@@ -389,205 +385,6 @@ def remove_install_directory(self, spec, deprecated=False):
path = os.path.dirname(path)
class ExtensionsLayout(object):
"""A directory layout is used to associate unique paths with specs for
package extensions.
Keeps track of which extensions are activated for what package.
Depending on the use case, this can mean globally activated extensions
directly in the installation folder - or extensions activated in
filesystem views.
"""
def __init__(self, view, **kwargs):
self.view = view
def add_extension(self, spec, ext_spec):
"""Add to the list of currently installed extensions."""
raise NotImplementedError()
def check_activated(self, spec, ext_spec):
"""Ensure that ext_spec can be removed from spec.
If not, raise NoSuchExtensionError.
"""
raise NotImplementedError()
def check_extension_conflict(self, spec, ext_spec):
"""Ensure that ext_spec can be activated in spec.
If not, raise ExtensionAlreadyInstalledError or
ExtensionConflictError.
"""
raise NotImplementedError()
def extension_map(self, spec):
"""Get a dict of currently installed extension packages for a spec.
Dict maps { name : extension_spec }
Modifying dict does not affect internals of this layout.
"""
raise NotImplementedError()
def extendee_target_directory(self, extendee):
"""Specify to which full path extendee should link all files
from extensions."""
raise NotImplementedError
def remove_extension(self, spec, ext_spec):
"""Remove from the list of currently installed extensions."""
raise NotImplementedError()
class YamlViewExtensionsLayout(ExtensionsLayout):
"""Maintain extensions within a view."""
def __init__(self, view, layout):
"""layout is the corresponding YamlDirectoryLayout object for which
we implement extensions.
"""
super(YamlViewExtensionsLayout, self).__init__(view)
self.layout = layout
self.extension_file_name = "extensions.yaml"
# Cache of already written/read extension maps.
self._extension_maps = {}
def add_extension(self, spec, ext_spec):
_check_concrete(spec)
_check_concrete(ext_spec)
# Check whether it's already installed or if it's a conflict.
exts = self._extension_map(spec)
self.check_extension_conflict(spec, ext_spec)
# do the actual adding.
exts[ext_spec.name] = ext_spec
self._write_extensions(spec, exts)
def check_extension_conflict(self, spec, ext_spec):
exts = self._extension_map(spec)
if ext_spec.name in exts:
installed_spec = exts[ext_spec.name]
if ext_spec.dag_hash() == installed_spec.dag_hash():
raise ExtensionAlreadyInstalledError(spec, ext_spec)
else:
raise ExtensionConflictError(spec, ext_spec, installed_spec)
def check_activated(self, spec, ext_spec):
exts = self._extension_map(spec)
if (ext_spec.name not in exts) or (ext_spec != exts[ext_spec.name]):
raise NoSuchExtensionError(spec, ext_spec)
def extension_file_path(self, spec):
"""Gets full path to an installed package's extension file, which
keeps track of all the extensions for that package which have been
added to this view.
"""
_check_concrete(spec)
normalize_path = lambda p: (os.path.abspath(p).rstrip(os.path.sep))
view_prefix = self.view.get_projection_for_spec(spec)
if normalize_path(spec.prefix) == normalize_path(view_prefix):
# For backwards compatibility, when the view is the extended
# package's installation directory, do not include the spec name
# as a subdirectory.
components = [view_prefix, self.layout.metadata_dir, self.extension_file_name]
else:
components = [
view_prefix,
self.layout.metadata_dir,
spec.name,
self.extension_file_name,
]
return os.path.join(*components)
def extension_map(self, spec):
"""Defensive copying version of _extension_map() for external API."""
_check_concrete(spec)
return self._extension_map(spec).copy()
def remove_extension(self, spec, ext_spec):
_check_concrete(spec)
_check_concrete(ext_spec)
# Make sure it's installed before removing.
exts = self._extension_map(spec)
self.check_activated(spec, ext_spec)
# do the actual removing.
del exts[ext_spec.name]
self._write_extensions(spec, exts)
def _extension_map(self, spec):
"""Get a dict<name -> spec> for all extensions currently
installed for this package."""
_check_concrete(spec)
if spec not in self._extension_maps:
path = self.extension_file_path(spec)
if not os.path.exists(path):
self._extension_maps[spec] = {}
else:
by_hash = self.layout.specs_by_hash()
exts = {}
with open(path) as ext_file:
yaml_file = yaml.load(ext_file)
for entry in yaml_file["extensions"]:
name = next(iter(entry))
dag_hash = entry[name]["hash"]
prefix = entry[name]["path"]
if dag_hash not in by_hash:
raise InvalidExtensionSpecError(
"Spec %s not found in %s" % (dag_hash, prefix)
)
ext_spec = by_hash[dag_hash]
if prefix != ext_spec.prefix:
raise InvalidExtensionSpecError(
"Prefix %s does not match spec hash %s: %s"
% (prefix, dag_hash, ext_spec)
)
exts[ext_spec.name] = ext_spec
self._extension_maps[spec] = exts
return self._extension_maps[spec]
def _write_extensions(self, spec, extensions):
path = self.extension_file_path(spec)
if not extensions:
# Remove the empty extensions file
os.remove(path)
return
# Create a temp file in the same directory as the actual file.
dirname, basename = os.path.split(path)
fs.mkdirp(dirname)
tmp = tempfile.NamedTemporaryFile(prefix=basename, dir=dirname, delete=False)
# write tmp file
with tmp:
yaml.dump(
{
"extensions": [
{ext.name: {"hash": ext.dag_hash(), "path": str(ext.prefix)}}
for ext in sorted(extensions.values())
]
},
tmp,
default_flow_style=False,
encoding="utf-8",
)
# Atomic update by moving tmpfile on top of old one.
fs.rename(tmp.name, path)
class DirectoryLayoutError(SpackError):
"""Superclass for directory layout errors."""
@@ -644,13 +441,3 @@ def __init__(self, spec, ext_spec, conflict):
"%s cannot be installed in %s because it conflicts with %s"
% (ext_spec.short_spec, spec.short_spec, conflict.short_spec)
)
class NoSuchExtensionError(DirectoryLayoutError):
"""Raised when an extension isn't there on deactivate."""
def __init__(self, spec, ext_spec):
super(NoSuchExtensionError, self).__init__(
"%s cannot be removed from %s because it's not activated."
% (ext_spec.short_spec, spec.short_spec)
)

24
lib/spack/spack/environment/environment.py
View File

@@ -13,7 +13,6 @@
import time
import ruamel.yaml as yaml
import six
import llnl.util.filesystem as fs
import llnl.util.tty as tty
@@ -711,7 +710,7 @@ def __init__(self, path, init_file=None, with_view=None, keep_relative=False):
self.views = {}
elif with_view is True:
self.views = {default_view_name: ViewDescriptor(self.path, self.view_path_default)}
elif isinstance(with_view, six.string_types):
elif isinstance(with_view, str):
self.views = {default_view_name: ViewDescriptor(self.path, with_view)}
# If with_view is None, then defer to the view settings determined by
# the manifest file
@@ -808,7 +807,7 @@ def _read_manifest(self, f, raw_yaml=None):
# enable_view can be boolean, string, or None
if enable_view is True or enable_view is None:
self.views = {default_view_name: ViewDescriptor(self.path, self.view_path_default)}
elif isinstance(enable_view, six.string_types):
elif isinstance(enable_view, str):
self.views = {default_view_name: ViewDescriptor(self.path, enable_view)}
elif enable_view:
path = self.path
@@ -818,17 +817,12 @@ def _read_manifest(self, f, raw_yaml=None):
)
else:
self.views = {}
# Retrieve the current concretization strategy
configuration = config_dict(self.yaml)
# Let `concretization` overrule `concretize:unify` config for now,
# but use a translation table to have internally a representation
# as if we were using the new configuration
translation = {"separately": False, "together": True}
try:
self.unify = translation[configuration["concretization"]]
except KeyError:
self.unify = spack.config.get("concretizer:unify", False)
# Retrieve unification scheme for the concretizer
self.unify = spack.config.get("concretizer:unify", False)
# Retrieve dev-build packages:
self.dev_specs = configuration.get("develop", {})
@@ -2135,16 +2129,14 @@ def _update_and_write_manifest(self, raw_yaml_dict, yaml_dict):
ayl[name][:] = [
s
for s in ayl.setdefault(name, [])
if (not isinstance(s, six.string_types))
or s.startswith("$")
or Spec(s) in speclist.specs
if (not isinstance(s, str)) or s.startswith("$") or Spec(s) in speclist.specs
]
# Put the new specs into the first active list from the yaml
new_specs = [
entry
for entry in speclist.yaml_list
if isinstance(entry, six.string_types)
if isinstance(entry, str)
and not any(entry in ayl[name] for ayl in active_yaml_lists)
]
list_for_new_specs = active_yaml_lists[0].setdefault(name, [])
@@ -2220,7 +2212,7 @@ def yaml_equivalent(first, second):
elif isinstance(first, list):
return isinstance(second, list) and _equiv_list(first, second)
else: # it's a string
return isinstance(second, six.string_types) and first == second
return isinstance(second, str) and first == second
def _equiv_list(first, second):

10
lib/spack/spack/fetch_strategy.py
View File

@@ -29,11 +29,9 @@
import re
import shutil
import sys
import urllib.parse
from typing import List, Optional # novm
import six
import six.moves.urllib.parse as urllib_parse
import llnl.util
import llnl.util.filesystem as fs
import llnl.util.tty as tty
@@ -322,7 +320,7 @@ def candidate_urls(self):
# This must be skipped on Windows due to URL encoding
# of ':' characters on filepaths on Windows
if sys.platform != "win32" and url.startswith("file://"):
path = urllib_parse.quote(url[len("file://") :])
path = urllib.parse.quote(url[len("file://") :])
url = "file://" + path
urls.append(url)
@@ -620,7 +618,7 @@ def archive(self, destination, **kwargs):
patterns = kwargs.get("exclude", None)
if patterns is not None:
if isinstance(patterns, six.string_types):
if isinstance(patterns, str):
patterns = [patterns]
for p in patterns:
tar.add_default_arg("--exclude=%s" % p)
@@ -1607,7 +1605,7 @@ def from_url_scheme(url, *args, **kwargs):
in the given url."""
url = kwargs.get("url", url)
parsed_url = urllib_parse.urlparse(url, scheme="file")
parsed_url = urllib.parse.urlparse(url, scheme="file")
scheme_mapping = kwargs.get("scheme_mapping") or {
"file": "url",

118
lib/spack/spack/filesystem_view.py
View File

@@ -12,7 +12,6 @@
import sys
from llnl.util import tty
from llnl.util.compat import filter, map, zip
from llnl.util.filesystem import (
mkdirp,
remove_dead_links,
@@ -37,10 +36,6 @@
import spack.store
import spack.util.spack_json as s_json
import spack.util.spack_yaml as s_yaml
from spack.directory_layout import (
ExtensionAlreadyInstalledError,
YamlViewExtensionsLayout,
)
from spack.error import SpackError
__all__ = ["FilesystemView", "YamlFilesystemView"]
@@ -166,9 +161,6 @@ def add_specs(self, *specs, **kwargs):
"""
Add given specs to view.
The supplied specs might be standalone packages or extensions of
other packages.
Should accept `with_dependencies` as keyword argument (default
True) to indicate wether or not dependencies should be activated as
well.
@@ -176,13 +168,7 @@ def add_specs(self, *specs, **kwargs):
Should except an `exclude` keyword argument containing a list of
regexps that filter out matching spec names.
This method should make use of `activate_{extension,standalone}`.
"""
raise NotImplementedError
def add_extension(self, spec):
"""
Add (link) an extension in this view. Does not add dependencies.
This method should make use of `activate_standalone`.
"""
raise NotImplementedError
@@ -202,9 +188,6 @@ def remove_specs(self, *specs, **kwargs):
"""
Removes given specs from view.
The supplied spec might be a standalone package or an extension of
another package.
Should accept `with_dependencies` as keyword argument (default
True) to indicate wether or not dependencies should be deactivated
as well.
@@ -216,13 +199,7 @@ def remove_specs(self, *specs, **kwargs):
Should except an `exclude` keyword argument containing a list of
regexps that filter out matching spec names.
This method should make use of `deactivate_{extension,standalone}`.
"""
raise NotImplementedError
def remove_extension(self, spec):
"""
Remove (unlink) an extension from this view.
This method should make use of `deactivate_standalone`.
"""
raise NotImplementedError
@@ -296,8 +273,6 @@ def __init__(self, root, layout, **kwargs):
msg += " which does not match projections passed manually."
raise ConflictingProjectionsError(msg)
self.extensions_layout = YamlViewExtensionsLayout(self, layout)
self._croot = colorize_root(self._root) + " "
def write_projections(self):
@@ -332,38 +307,10 @@ def add_specs(self, *specs, **kwargs):
self.print_conflict(v, s)
return
extensions = set(filter(lambda s: s.package.is_extension, specs))
standalones = specs - extensions
set(map(self._check_no_ext_conflicts, extensions))
# fail on first error, otherwise link extensions as well
if all(map(self.add_standalone, standalones)):
all(map(self.add_extension, extensions))
def add_extension(self, spec):
if not spec.package.is_extension:
tty.error(self._croot + "Package %s is not an extension." % spec.name)
return False
if spec.external:
tty.warn(self._croot + "Skipping external package: %s" % colorize_spec(spec))
return True
if not spec.package.is_activated(self):
spec.package.do_activate(self, verbose=self.verbose, with_dependencies=False)
# make sure the meta folder is linked as well (this is not done by the
# extension-activation mechnism)
if not self.check_added(spec):
self.link_meta_folder(spec)
return True
for s in specs:
self.add_standalone(s)
def add_standalone(self, spec):
if spec.package.is_extension:
tty.error(self._croot + "Package %s is an extension." % spec.name)
return False
if spec.external:
tty.warn(self._croot + "Skipping external package: %s" % colorize_spec(spec))
return True
@@ -372,19 +319,6 @@ def add_standalone(self, spec):
tty.warn(self._croot + "Skipping already linked package: %s" % colorize_spec(spec))
return True
if spec.package.extendable:
# Check for globally activated extensions in the extendee that
# we're looking at.
activated = [p.spec for p in spack.store.db.activated_extensions_for(spec)]
if activated:
tty.error(
"Globally activated extensions cannot be used in "
"conjunction with filesystem views. "
"Please deactivate the following specs: "
)
spack.cmd.display_specs(activated, flags=True, variants=True, long=False)
return False
self.merge(spec)
self.link_meta_folder(spec)
@@ -533,27 +467,10 @@ def remove_specs(self, *specs, **kwargs):
# Remove the packages from the view
for spec in to_deactivate_sorted:
if spec.package.is_extension:
self.remove_extension(spec, with_dependents=with_dependents)
else:
self.remove_standalone(spec)
self.remove_standalone(spec)
self._purge_empty_directories()
def remove_extension(self, spec, with_dependents=True):
"""
Remove (unlink) an extension from this view.
"""
if not self.check_added(spec):
tty.warn(self._croot + "Skipping package not linked in view: %s" % spec.name)
return
if spec.package.is_activated(self):
spec.package.do_deactivate(
self, verbose=self.verbose, remove_dependents=with_dependents
)
self.unlink_meta_folder(spec)
def remove_standalone(self, spec):
"""
Remove (unlink) a standalone package from this view.
@@ -575,8 +492,8 @@ def get_projection_for_spec(self, spec):
Relies on the ordering of projections to avoid ambiguity.
"""
spec = spack.spec.Spec(spec)
# Extensions are placed by their extendee, not by their own spec
locator_spec = spec
if spec.package.extendee_spec:
locator_spec = spec.package.extendee_spec
@@ -712,18 +629,6 @@ def unlink_meta_folder(self, spec):
assert os.path.exists(path)
shutil.rmtree(path)
def _check_no_ext_conflicts(self, spec):
"""
Check that there is no extension conflict for specs.
"""
extendee = spec.package.extendee_spec
try:
self.extensions_layout.check_extension_conflict(extendee, spec)
except ExtensionAlreadyInstalledError:
# we print the warning here because later on the order in which
# packages get activated is not clear (set-sorting)
tty.warn(self._croot + "Skipping already activated package: %s" % spec.name)
class SimpleFilesystemView(FilesystemView):
"""A simple and partial implementation of FilesystemView focused on
@@ -842,14 +747,13 @@ def get_projection_for_spec(self, spec):
Relies on the ordering of projections to avoid ambiguity.
"""
spec = spack.spec.Spec(spec)
# Extensions are placed by their extendee, not by their own spec
locator_spec = spec
if spec.package.extendee_spec:
locator_spec = spec.package.extendee_spec
proj = spack.projections.get_projection(self.projections, locator_spec)
if spec.package.extendee_spec:
spec = spec.package.extendee_spec
proj = spack.projections.get_projection(self.projections, spec)
if proj:
return os.path.join(self._root, locator_spec.format(proj))
return os.path.join(self._root, spec.format(proj))
return self._root

5
lib/spack/spack/gcs_handler.py
View File

@@ -2,8 +2,7 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import six.moves.urllib.response as urllib_response
import urllib.response
import spack.util.url as url_util
import spack.util.web as web_util
@@ -21,4 +20,4 @@ def gcs_open(req, *args, **kwargs):
stream = gcsblob.get_blob_byte_stream()
headers = gcsblob.get_blob_headers()
return urllib_response.addinfourl(stream, headers, url)
return urllib.response.addinfourl(stream, headers, url)

20
lib/spack/spack/hooks/extensions.py
View File

@@ -1,20 +0,0 @@
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import spack
from spack.filesystem_view import YamlFilesystemView
def pre_uninstall(spec):
pkg = spec.package
assert spec.concrete
if pkg.is_extension:
target = pkg.extendee_spec.prefix
view = YamlFilesystemView(target, spack.store.layout)
if pkg.is_activated(view):
# deactivate globally
pkg.do_deactivate(force=True)

11
lib/spack/spack/install_test.py
View File

@@ -7,9 +7,6 @@
import os
import re
import shutil
import sys
import six
import llnl.util.filesystem as fs
import llnl.util.tty as tty
@@ -163,8 +160,7 @@ def content_hash(self):
json_text = sjson.dump(self.to_dict())
sha = hashlib.sha1(json_text.encode("utf-8"))
b32_hash = base64.b32encode(sha.digest()).lower()
if sys.version_info[0] >= 3:
b32_hash = b32_hash.decode("utf-8")
b32_hash = b32_hash.decode("utf-8")
self._hash = b32_hash
return self._hash
@@ -436,10 +432,7 @@ def from_file(filename):
test_suite._hash = content_hash
return test_suite
except Exception as e:
raise six.raise_from(
sjson.SpackJSONError("error parsing JSON TestSuite:", str(e)),
e,
)
raise sjson.SpackJSONError("error parsing JSON TestSuite:", str(e)) from e
def _add_msg_to_file(filename, msg):

65
lib/spack/spack/installer.py
View File

@@ -2,7 +2,6 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""
This module encapsulates package installation functionality.
@@ -30,6 +29,7 @@
import copy
import glob
import heapq
import io
import itertools
import os
import shutil
@@ -37,8 +37,6 @@
import time
from collections import defaultdict
import six
import llnl.util.filesystem as fs
import llnl.util.lock as lk
import llnl.util.tty as tty
@@ -56,9 +54,9 @@
import spack.store
import spack.util.executable
import spack.util.path
import spack.util.timer as timer
from spack.util.environment import EnvironmentModifications, dump_environment
from spack.util.executable import which
from spack.util.timer import Timer
#: Counter to support unique spec sequencing that is used to ensure packages
#: with the same priority are (initially) processed in the order in which they
@@ -281,11 +279,10 @@ def _print_installed_pkg(message):
print(colorize("@*g{[+]} ") + spack.util.path.debug_padded_filter(message))
def _print_timer(pre, pkg_id, fetch, build, total):
tty.msg(
"{0} Successfully installed {1}".format(pre, pkg_id),
"Fetch: {0}. Build: {1}. Total: {2}.".format(_hms(fetch), _hms(build), _hms(total)),
)
def _print_timer(pre, pkg_id, timer):
phases = ["{}: {}.".format(p.capitalize(), _hms(timer.duration(p))) for p in timer.phases]
phases.append("Total: {}".format(_hms(timer.duration())))
tty.msg("{0} Successfully installed {1}".format(pre, pkg_id), " ".join(phases))
def _install_from_cache(pkg, cache_only, explicit, unsigned=False):
@@ -304,9 +301,9 @@ def _install_from_cache(pkg, cache_only, explicit, unsigned=False):
bool: ``True`` if the package was extract from binary cache,
``False`` otherwise
"""
timer = Timer()
t = timer.Timer()
installed_from_cache = _try_install_from_binary_cache(
pkg, explicit, unsigned=unsigned, timer=timer
pkg, explicit, unsigned=unsigned, timer=t
)
pkg_id = package_id(pkg)
if not installed_from_cache:
@@ -316,15 +313,9 @@ def _install_from_cache(pkg, cache_only, explicit, unsigned=False):
tty.msg("{0}: installing from source".format(pre))
return False
timer.stop()
t.stop()
tty.debug("Successfully extracted {0} from binary cache".format(pkg_id))
_print_timer(
pre=_log_prefix(pkg.name),
pkg_id=pkg_id,
fetch=timer.phases.get("search", 0) + timer.phases.get("fetch", 0),
build=timer.phases.get("install", 0),
total=timer.total,
)
_print_timer(pre=_log_prefix(pkg.name), pkg_id=pkg_id, timer=t)
_print_installed_pkg(pkg.spec.prefix)
spack.hooks.post_install(pkg.spec)
return True
@@ -372,7 +363,7 @@ def _process_external_package(pkg, explicit):
def _process_binary_cache_tarball(
pkg, binary_spec, explicit, unsigned, mirrors_for_spec=None, timer=None
pkg, binary_spec, explicit, unsigned, mirrors_for_spec=None, timer=timer.NULL_TIMER
):
"""
Process the binary cache tarball.
@@ -391,11 +382,11 @@ def _process_binary_cache_tarball(
bool: ``True`` if the package was extracted from binary cache,
else ``False``
"""
timer.start("fetch")
download_result = binary_distribution.download_tarball(
binary_spec, unsigned, mirrors_for_spec=mirrors_for_spec
)
if timer:
timer.phase("fetch")
timer.stop("fetch")
# see #10063 : install from source if tarball doesn't exist
if download_result is None:
tty.msg("{0} exists in binary cache but with different hash".format(pkg.name))
@@ -405,6 +396,7 @@ def _process_binary_cache_tarball(
tty.msg("Extracting {0} from binary cache".format(pkg_id))
# don't print long padded paths while extracting/relocating binaries
timer.start("install")
with spack.util.path.filter_padding():
binary_distribution.extract_tarball(
binary_spec, download_result, allow_root=False, unsigned=unsigned, force=False
@@ -412,12 +404,11 @@ def _process_binary_cache_tarball(
pkg.installed_from_binary_cache = True
spack.store.db.add(pkg.spec, spack.store.layout, explicit=explicit)
if timer:
timer.phase("install")
timer.stop("install")
return True
def _try_install_from_binary_cache(pkg, explicit, unsigned=False, timer=None):
def _try_install_from_binary_cache(pkg, explicit, unsigned=False, timer=timer.NULL_TIMER):
"""
Try to extract the package from binary cache.
@@ -430,10 +421,10 @@ def _try_install_from_binary_cache(pkg, explicit, unsigned=False, timer=None):
"""
pkg_id = package_id(pkg)
tty.debug("Searching for binary cache of {0}".format(pkg_id))
matches = binary_distribution.get_mirrors_for_spec(pkg.spec)
if timer:
timer.phase("search")
timer.start("search")
matches = binary_distribution.get_mirrors_for_spec(pkg.spec)
timer.stop("search")
if not matches:
return False
@@ -594,7 +585,7 @@ def log(pkg):
# Finally, archive files that are specific to each package
with fs.working_dir(pkg.stage.path):
errors = six.StringIO()
errors = io.StringIO()
target_dir = os.path.join(spack.store.layout.metadata_path(pkg.spec), "archived-files")
for glob_expr in pkg.builder.archive_files:
@@ -1906,7 +1897,7 @@ def __init__(self, pkg, install_args):
self.env_mods = install_args.get("env_modifications", EnvironmentModifications())
# timer for build phases
self.timer = Timer()
self.timer = timer.Timer()
# If we are using a padded path, filter the output to compress padded paths
# The real log still has full-length paths.
@@ -1920,12 +1911,16 @@ def __init__(self, pkg, install_args):
def run(self):
"""Main entry point from ``build_process`` to kick off install in child."""
self.timer.start("stage")
if not self.fake:
if not self.skip_patch:
self.pkg.do_patch()
else:
self.pkg.do_stage()
self.timer.stop("stage")
tty.debug(
"{0} Building {1} [{2}]".format(self.pre, self.pkg_id, self.pkg.build_system_class)
)
@@ -1960,9 +1955,7 @@ def run(self):
_print_timer(
pre=self.pre,
pkg_id=self.pkg_id,
fetch=self.pkg._fetch_time,
build=self.timer.total - self.pkg._fetch_time,
total=self.timer.total,
timer=self.timer,
)
_print_installed_pkg(self.pkg.prefix)
@@ -2035,6 +2028,7 @@ def _real_install(self):
)
with log_contextmanager as logger:
# Redirect stdout and stderr to daemon pipe
with logger.force_echo():
inner_debug_level = tty.debug_level()
tty.set_debug(debug_level)
@@ -2042,12 +2036,11 @@ def _real_install(self):
tty.msg(msg.format(self.pre, phase_fn.name))
tty.set_debug(inner_debug_level)
# Redirect stdout and stderr to daemon pipe
self.timer.phase(phase_fn.name)
# Catch any errors to report to logging
self.timer.start(phase_fn.name)
phase_fn.execute()
spack.hooks.on_phase_success(pkg, phase_fn.name, log_file)
self.timer.stop(phase_fn.name)
except BaseException:
combine_phase_logs(pkg.phase_log_files, pkg.log_path)

22
lib/spack/spack/main.py
View File

@@ -12,6 +12,7 @@
import argparse
import inspect
import io
import operator
import os
import os.path
@@ -23,8 +24,6 @@
import traceback
import warnings
from six import StringIO
import archspec.cpu
import llnl.util.lang
@@ -320,9 +319,9 @@ def add_subparsers(self, **kwargs):
kwargs.setdefault("required", True)
sp = super(SpackArgumentParser, self).add_subparsers(**kwargs)
# This monkey patching is needed for Python 3.5 and 3.6, which support
# This monkey patching is needed for Python 3.6, which supports
# having a required subparser but don't expose the API used above
if sys.version_info[:2] == (3, 5) or sys.version_info[:2] == (3, 6):
if sys.version_info[:2] == (3, 6):
sp.required = True
old_add_parser = sp.add_parser
@@ -388,7 +387,7 @@ def make_argument_parser(**kwargs):
"A flexible package manager that supports multiple versions,\n"
"configurations, platforms, and compilers."
),
**kwargs
**kwargs,
)
# stat names in groups of 7, for nice wrapping.
@@ -560,12 +559,6 @@ def setup_main_options(args):
# Assign a custom function to show warnings
warnings.showwarning = send_warning_to_tty
if sys.version_info[:2] == (2, 7):
warnings.warn(
"Python 2.7 support is deprecated and will be removed in Spack v0.20.\n"
" Please move to Python 3.6 or higher."
)
# Set up environment based on args.
tty.set_verbose(args.verbose)
tty.set_debug(args.debug)
@@ -706,7 +699,7 @@ def __call__(self, *argv, **kwargs):
prepend + [self.command_name] + list(argv)
)
out = StringIO()
out = io.StringIO()
try:
with log_output(out):
self.returncode = _invoke_command(self.command, self.parser, args, unknown)
@@ -1015,10 +1008,7 @@ def main(argv=None):
raise
sys.stderr.write("\n")
tty.error("Keyboard interrupt.")
if sys.version_info >= (3, 5):
return signal.SIGINT.value
else:
return signal.SIGINT
return signal.SIGINT.value
except SystemExit as e:
if spack.config.get("config:debug") or SHOW_BACKTRACE:

29
lib/spack/spack/mirror.py
View File

@@ -11,6 +11,7 @@
to download packages directly from a mirror (e.g., on an intranet).
"""
import collections
import collections.abc
import operator
import os
import os.path
@@ -18,10 +19,8 @@
import traceback
import ruamel.yaml.error as yaml_error
import six
import llnl.util.tty as tty
from llnl.util.compat import Mapping
from llnl.util.filesystem import mkdirp
import spack.config
@@ -37,7 +36,7 @@
def _is_string(url):
return isinstance(url, six.string_types)
return isinstance(url, str)
def _display_mirror_entry(size, name, url, type_=None):
@@ -78,10 +77,7 @@ def from_yaml(stream, name=None):
data = syaml.load(stream)
return Mirror.from_dict(data, name)
except yaml_error.MarkedYAMLError as e:
raise six.raise_from(
syaml.SpackYAMLError("error parsing YAML mirror:", str(e)),
e,
)
raise syaml.SpackYAMLError("error parsing YAML mirror:", str(e)) from e
@staticmethod
def from_json(stream, name=None):
@@ -89,10 +85,7 @@ def from_json(stream, name=None):
d = sjson.load(stream)
return Mirror.from_dict(d, name)
except Exception as e:
raise six.raise_from(
sjson.SpackJSONError("error parsing JSON mirror:", str(e)),
e,
)
raise sjson.SpackJSONError("error parsing JSON mirror:", str(e)) from e
def to_dict(self):
if self._push_url is None:
@@ -102,7 +95,7 @@ def to_dict(self):
@staticmethod
def from_dict(d, name=None):
if isinstance(d, six.string_types):
if isinstance(d, str):
return Mirror(d, name=name)
else:
return Mirror(d["fetch"], d["push"], name=name)
@@ -228,7 +221,7 @@ def _normalize(self):
self._push_url = None
class MirrorCollection(Mapping):
class MirrorCollection(collections.abc.Mapping):
"""A mapping of mirror names to mirrors."""
def __init__(self, mirrors=None, scope=None):
@@ -257,10 +250,7 @@ def from_yaml(stream, name=None):
data = syaml.load(stream)
return MirrorCollection(data)
except yaml_error.MarkedYAMLError as e:
raise six.raise_from(
syaml.SpackYAMLError("error parsing YAML mirror collection:", str(e)),
e,
)
raise syaml.SpackYAMLError("error parsing YAML mirror collection:", str(e)) from e
@staticmethod
def from_json(stream, name=None):
@@ -268,10 +258,7 @@ def from_json(stream, name=None):
d = sjson.load(stream)
return MirrorCollection(d)
except Exception as e:
raise six.raise_from(
sjson.SpackJSONError("error parsing JSON mirror collection:", str(e)),
e,
)
raise sjson.SpackJSONError("error parsing JSON mirror collection:", str(e)) from e
def to_dict(self, recursive=False):
return syaml_dict(

7
lib/spack/spack/mixins.py
View File

@@ -7,13 +7,6 @@
package.
"""
import os
import sys
from typing import Callable, DefaultDict, List # novm
if sys.version_info >= (3, 5):
CallbackDict = DefaultDict[str, List[Callable]]
else:
CallbackDict = None
import llnl.util.filesystem

3
lib/spack/spack/modules/lmod.py
View File

@@ -26,9 +26,6 @@
def configuration(module_set_name):
config_path = "modules:%s:lmod" % module_set_name
config = spack.config.get(config_path, {})
if not config and module_set_name == "default":
# return old format for backward compatibility
return spack.config.get("modules:lmod", {})
return config

Some files were not shown because too many files have changed in this diff Show More